maintenance/findOrphanedFiles.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  */
  20
  21 require_once __DIR__ . '/Maintenance.php';
  22
  23 class FindOrphanedFiles extends Maintenance {
  24         function __construct() {
  25                 parent::__construct();
  26
  27                 $this->addDescription( "Find unregistered files in the 'public' repo zone." );
  28                 $this->addOption( 'subdir',
  29                         'Only scan files in this subdirectory (e.g. "a/a0")', false, true );
  30                 $this->addOption( 'verbose', "Mention file paths checked" );
  31                 $this->setBatchSize( 500 );
  32         }
  33
  34         function execute() {
  35                 $subdir = $this->getOption( 'subdir', '' );
  36                 $verbose = $this->hasOption( 'verbose' );
  37
  38                 $repo = RepoGroup::singleton()->getLocalRepo();
  39                 if ( $repo->hasSha1Storage() ) {
  40                         $this->error( "Local repo uses SHA-1 file storage names; aborting.", 1 );
  41                 }
  42
  43                 $directory = $repo->getZonePath( 'public' );
  44                 if ( $subdir != '' ) {
  45                         $directory .= "/$subdir/";
  46                 }
  47
  48                 if ( $verbose ) {
  49                         $this->output( "Scanning files under $directory:\n" );
  50                 }
  51
  52                 $list = $repo->getBackend()->getFileList( [ 'dir' => $directory ] );
  53                 if ( $list === null ) {
  54                         $this->error( "Could not get file listing.", 1 );
  55                 }
  56
  57                 $pathBatch = [];
  58                 foreach ( $list as $path ) {
  59                         if ( preg_match( '#^(thumb|deleted)/#', $path ) ) {
  60                                 continue; // handle ugly nested containers on stock installs
  61                         }
  62
  63                         $pathBatch[] = $path;
  64                         if ( count( $pathBatch ) >= $this->mBatchSize ) {
  65                                 $this->checkFiles( $repo, $pathBatch, $verbose );
  66                                 $pathBatch = [];
  67                         }
  68                 }
  69                 $this->checkFiles( $repo, $pathBatch, $verbose );
  70         }
  71
  72         protected function checkFiles( LocalRepo $repo, array $paths, $verbose ) {
  73                 if ( !count( $paths ) ) {
  74                         return;
  75                 }
  76
  77                 $dbr = $repo->getReplicaDB();
  78
  79                 $curNames = [];
  80                 $oldNames = [];
  81                 $imgIN = [];
  82                 $oiWheres = [];
  83                 foreach ( $paths as $path ) {
  84                         $name = basename( $path );
  85                         if ( preg_match( '#^archive/#', $path ) ) {
  86                                 if ( $verbose ) {
  87                                         $this->output( "Checking old file $name\n" );
  88                                 }
  89
  90                                 $oldNames[] = $name;
  91                                 list( , $base ) = explode( '!', $name, 2 ); // <TS_MW>!<img_name>
  92                                 $oiWheres[] = $dbr->makeList(
  93                                         [ 'oi_name' => $base, 'oi_archive_name' => $name ],
  94                                         LIST_AND
  95                                 );
  96                         } else {
  97                                 if ( $verbose ) {
  98                                         $this->output( "Checking current file $name\n" );
  99                                 }
 100
 101                                 $curNames[] = $name;
 102                                 $imgIN[] = $name;
 103                         }
 104                 }
 105
 106                 $res = $dbr->query(
 107                         $dbr->unionQueries(
 108                                 [
 109                                         $dbr->selectSQLText(
 110                                                 'image',
 111                                                 [ 'name' => 'img_name', 'old' => 0 ],
 112                                                 $imgIN ? [ 'img_name' => $imgIN ] : '1=0'
 113                                         ),
 114                                         $dbr->selectSQLText(
 115                                                 'oldimage',
 116                                                 [ 'name' => 'oi_archive_name', 'old' => 1 ],
 117                                                 $oiWheres ? $dbr->makeList( $oiWheres, LIST_OR ) : '1=0'
 118                                         )
 119                                 ],
 120                                 true // UNION ALL (performance)
 121                         ),
 122                         __METHOD__
 123                 );
 124
 125                 $curNamesFound = [];
 126                 $oldNamesFound = [];
 127                 foreach ( $res as $row ) {
 128                         if ( $row->old ) {
 129                                 $oldNamesFound[] = $row->name;
 130                         } else {
 131                                 $curNamesFound[] = $row->name;
 132                         }
 133                 }
 134
 135                 foreach ( array_diff( $curNames, $curNamesFound ) as $name ) {
 136                         $file = $repo->newFile( $name );
 137                         // Print name and public URL to ease recovery
 138                         if ( $file ) {
 139                                 $this->output( $name . "\n" . $file->getCanonicalUrl() . "\n\n" );
 140                         } else {
 141                                 $this->error( "Cannot get URL for bad file title '$name'" );
 142                         }
 143                 }
 144
 145                 foreach ( array_diff( $oldNames, $oldNamesFound ) as $name ) {
 146                         list( , $base ) = explode( '!', $name, 2 ); // <TS_MW>!<img_name>
 147                         $file = $repo->newFromArchiveName( Title::makeTitle( NS_FILE, $base ), $name );
 148                         // Print name and public URL to ease recovery
 149                         $this->output( $name . "\n" . $file->getCanonicalUrl() . "\n\n" );
 150                 }
 151         }
 152 }
 153
 154 $maintClass = 'FindOrphanedFiles';
 155 require_once RUN_MAINTENANCE_IF_MAIN;