X-Git-Url: https://scripts.mit.edu/gitweb/autoinstallsdev/mediawiki.git/blobdiff_plain/19e297c21b10b1b8a3acad5e73fc71dcb35db44a..6932310fd58ebef145fa01eb76edf7150284d8ea:/maintenance/purgeChangedFiles.php

diff --git a/maintenance/purgeChangedFiles.php b/maintenance/purgeChangedFiles.php
new file mode 100644
index 00000000..3c0fc7e5
--- /dev/null
+++ b/maintenance/purgeChangedFiles.php
@@ -0,0 +1,262 @@
+<?php
+/**
+ * Scan the logging table and purge affected files within a timeframe.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ */
+
+require_once __DIR__ . '/Maintenance.php';
+
+/**
+ * Maintenance script that scans the deletion log and purges affected files
+ * within a timeframe.
+ *
+ * @ingroup Maintenance
+ */
+class PurgeChangedFiles extends Maintenance {
+	/**
+	 * Mapping from type option to log type and actions.
+	 * @var array
+	 */
+	private static $typeMappings = [
+		'created' => [
+			'upload' => [ 'upload' ],
+			'import' => [ 'upload', 'interwiki' ],
+		],
+		'deleted' => [
+			'delete' => [ 'delete', 'revision' ],
+			'suppress' => [ 'delete', 'revision' ],
+		],
+		'modified' => [
+			'upload' => [ 'overwrite', 'revert' ],
+			'move' => [ 'move', 'move_redir' ],
+		],
+	];
+
+	/**
+	 * @var string
+	 */
+	private $startTimestamp;
+
+	/**
+	 * @var string
+	 */
+	private $endTimestamp;
+
+	public function __construct() {
+		parent::__construct();
+		$this->addDescription( 'Scan the logging table and purge files and thumbnails.' );
+		$this->addOption( 'starttime', 'Starting timestamp', true, true );
+		$this->addOption( 'endtime', 'Ending timestamp', true, true );
+		$this->addOption( 'type', 'Comma-separated list of types of changes to send purges for (' .
+			implode( ',', array_keys( self::$typeMappings ) ) . ',all)', false, true );
+		$this->addOption( 'htcp-dest', 'HTCP announcement destination (IP:port)', false, true );
+		$this->addOption( 'dry-run', 'Do not send purge requests' );
+		$this->addOption( 'sleep-per-batch', 'Milliseconds to sleep between batches', false, true );
+		$this->addOption( 'verbose', 'Show more output', false, false, 'v' );
+		$this->setBatchSize( 100 );
+	}
+
+	public function execute() {
+		global $wgHTCPRouting;
+
+		if ( $this->hasOption( 'htcp-dest' ) ) {
+			$parts = explode( ':', $this->getOption( 'htcp-dest' ) );
+			if ( count( $parts ) < 2 ) {
+				// Add default htcp port
+				$parts[] = '4827';
+			}
+
+			// Route all HTCP messages to provided host:port
+			$wgHTCPRouting = [
+				'' => [ 'host' => $parts[0], 'port' => $parts[1] ],
+			];
+			$this->verbose( "HTCP broadcasts to {$parts[0]}:{$parts[1]}\n" );
+		}
+
+		// Find out which actions we should be concerned with
+		$typeOpt = $this->getOption( 'type', 'all' );
+		$validTypes = array_keys( self::$typeMappings );
+		if ( $typeOpt === 'all' ) {
+			// Convert 'all' to all registered types
+			$typeOpt = implode( ',', $validTypes );
+		}
+		$typeList = explode( ',', $typeOpt );
+		foreach ( $typeList as $type ) {
+			if ( !in_array( $type, $validTypes ) ) {
+				$this->error( "\nERROR: Unknown type: {$type}\n" );
+				$this->maybeHelp( true );
+			}
+		}
+
+		// Validate the timestamps
+		$dbr = $this->getDB( DB_REPLICA );
+		$this->startTimestamp = $dbr->timestamp( $this->getOption( 'starttime' ) );
+		$this->endTimestamp = $dbr->timestamp( $this->getOption( 'endtime' ) );
+
+		if ( $this->startTimestamp > $this->endTimestamp ) {
+			$this->error( "\nERROR: starttime after endtime\n" );
+			$this->maybeHelp( true );
+		}
+
+		// Turn on verbose when dry-run is enabled
+		if ( $this->hasOption( 'dry-run' ) ) {
+			$this->mOptions['verbose'] = 1;
+		}
+
+		$this->verbose( 'Purging files that were: ' . implode( ', ', $typeList ) . "\n" );
+		foreach ( $typeList as $type ) {
+			$this->verbose( "Checking for {$type} files...\n" );
+			$this->purgeFromLogType( $type );
+			if ( !$this->hasOption( 'dry-run' ) ) {
+				$this->verbose( "...{$type} files purged.\n\n" );
+			}
+		}
+	}
+
+	/**
+	 * Purge cache and thumbnails for changes of the given type.
+	 *
+	 * @param string $type Type of change to find
+	 */
+	protected function purgeFromLogType( $type ) {
+		$repo = RepoGroup::singleton()->getLocalRepo();
+		$dbr = $this->getDB( DB_REPLICA );
+
+		foreach ( self::$typeMappings[$type] as $logType => $logActions ) {
+			$this->verbose( "Scanning for {$logType}/" . implode( ',', $logActions ) . "\n" );
+
+			$res = $dbr->select(
+				'logging',
+				[ 'log_title', 'log_timestamp', 'log_params' ],
+				[
+					'log_namespace' => NS_FILE,
+					'log_type' => $logType,
+					'log_action' => $logActions,
+					'log_timestamp >= ' . $dbr->addQuotes( $this->startTimestamp ),
+					'log_timestamp <= ' . $dbr->addQuotes( $this->endTimestamp ),
+				],
+				__METHOD__
+			);
+
+			$bSize = 0;
+			foreach ( $res as $row ) {
+				$file = $repo->newFile( Title::makeTitle( NS_FILE, $row->log_title ) );
+
+				if ( $this->hasOption( 'dry-run' ) ) {
+					$this->verbose( "{$type}[{$row->log_timestamp}]: {$row->log_title}\n" );
+					continue;
+				}
+
+				// Purge current version and its thumbnails
+				$file->purgeCache();
+				// Purge the old versions and their thumbnails
+				foreach ( $file->getHistory() as $oldFile ) {
+					$oldFile->purgeCache();
+				}
+
+				if ( $logType === 'delete' ) {
+					// If there is an orphaned storage file... delete it
+					if ( !$file->exists() && $repo->fileExists( $file->getPath() ) ) {
+						$dpath = $this->getDeletedPath( $repo, $file );
+						if ( $repo->fileExists( $dpath ) ) {
+							// Sanity check to avoid data loss
+							$repo->getBackend()->delete( [ 'src' => $file->getPath() ] );
+							$this->verbose( "Deleted orphan file: {$file->getPath()}.\n" );
+						} else {
+							$this->error( "File was not deleted: {$file->getPath()}.\n" );
+						}
+					}
+
+					// Purge items from fileachive table (rows are likely here)
+					$this->purgeFromArchiveTable( $repo, $file );
+				} elseif ( $logType === 'move' ) {
+					// Purge the target file as well
+
+					$params = unserialize( $row->log_params );
+					if ( isset( $params['4::target'] ) ) {
+						$target = $params['4::target'];
+						$targetFile = $repo->newFile( Title::makeTitle( NS_FILE, $target ) );
+						$targetFile->purgeCache();
+						$this->verbose( "Purged file {$target}; move target @{$row->log_timestamp}.\n" );
+					}
+				}
+
+				$this->verbose( "Purged file {$row->log_title}; {$type} @{$row->log_timestamp}.\n" );
+
+				if ( $this->hasOption( 'sleep-per-batch' ) && ++$bSize > $this->mBatchSize ) {
+					$bSize = 0;
+					// sleep-per-batch is milliseconds, usleep wants micro seconds.
+					usleep( 1000 * (int)$this->getOption( 'sleep-per-batch' ) );
+				}
+			}
+		}
+	}
+
+	protected function purgeFromArchiveTable( LocalRepo $repo, LocalFile $file ) {
+		$dbr = $repo->getReplicaDB();
+		$res = $dbr->select(
+			'filearchive',
+			[ 'fa_archive_name' ],
+			[ 'fa_name' => $file->getName() ],
+			__METHOD__
+		);
+
+		foreach ( $res as $row ) {
+			if ( $row->fa_archive_name === null ) {
+				// Was not an old version (current version names checked already)
+				continue;
+			}
+			$ofile = $repo->newFromArchiveName( $file->getTitle(), $row->fa_archive_name );
+			// If there is an orphaned storage file still there...delete it
+			if ( !$file->exists() && $repo->fileExists( $ofile->getPath() ) ) {
+				$dpath = $this->getDeletedPath( $repo, $ofile );
+				if ( $repo->fileExists( $dpath ) ) {
+					// Sanity check to avoid data loss
+					$repo->getBackend()->delete( [ 'src' => $ofile->getPath() ] );
+					$this->output( "Deleted orphan file: {$ofile->getPath()}.\n" );
+				} else {
+					$this->error( "File was not deleted: {$ofile->getPath()}.\n" );
+				}
+			}
+			$file->purgeOldThumbnails( $row->fa_archive_name );
+		}
+	}
+
+	protected function getDeletedPath( LocalRepo $repo, LocalFile $file ) {
+		$hash = $repo->getFileSha1( $file->getPath() );
+		$key = "{$hash}.{$file->getExtension()}";
+
+		return $repo->getDeletedHashPath( $key ) . $key;
+	}
+
+	/**
+	 * Send an output message iff the 'verbose' option has been provided.
+	 *
+	 * @param string $msg Message to output
+	 */
+	protected function verbose( $msg ) {
+		if ( $this->hasOption( 'verbose' ) ) {
+			$this->output( $msg );
+		}
+	}
+}
+
+$maintClass = "PurgeChangedFiles";
+require_once RUN_MAINTENANCE_IF_MAIN;