includes/specials/SpecialExport.php

   1 <?php
   2 /**
   3  * Implements Special:Export
   4  *
   5  * Copyright © 2003-2008 Brion Vibber <brion@pobox.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup SpecialPage
  24  */
  25
  26 use Mediawiki\MediaWikiServices;
  27
  28 /**
  29  * A special page that allows users to export pages in a XML file
  30  *
  31  * @ingroup SpecialPage
  32  */
  33 class SpecialExport extends SpecialPage {
  34         private $curonly, $doExport, $pageLinkDepth, $templates;
  35
  36         public function __construct() {
  37                 parent::__construct( 'Export' );
  38         }
  39
  40         public function execute( $par ) {
  41                 $this->setHeaders();
  42                 $this->outputHeader();
  43                 $config = $this->getConfig();
  44
  45                 // Set some variables
  46                 $this->curonly = true;
  47                 $this->doExport = false;
  48                 $request = $this->getRequest();
  49                 $this->templates = $request->getCheck( 'templates' );
  50                 $this->pageLinkDepth = $this->validateLinkDepth(
  51                         $request->getIntOrNull( 'pagelink-depth' )
  52                 );
  53                 $nsindex = '';
  54                 $exportall = false;
  55
  56                 if ( $request->getCheck( 'addcat' ) ) {
  57                         $page = $request->getText( 'pages' );
  58                         $catname = $request->getText( 'catname' );
  59
  60                         if ( $catname !== '' && $catname !== null && $catname !== false ) {
  61                                 $t = Title::makeTitleSafe( NS_MAIN, $catname );
  62                                 if ( $t ) {
  63                                         /**
  64                                          * @todo FIXME: This can lead to hitting memory limit for very large
  65                                          * categories. Ideally we would do the lookup synchronously
  66                                          * during the export in a single query.
  67                                          */
  68                                         $catpages = $this->getPagesFromCategory( $t );
  69                                         if ( $catpages ) {
  70                                                 if ( $page !== '' ) {
  71                                                         $page .= "\n";
  72                                                 }
  73                                                 $page .= implode( "\n", $catpages );
  74                                         }
  75                                 }
  76                         }
  77                 } elseif ( $request->getCheck( 'addns' ) && $config->get( 'ExportFromNamespaces' ) ) {
  78                         $page = $request->getText( 'pages' );
  79                         $nsindex = $request->getText( 'nsindex', '' );
  80
  81                         if ( strval( $nsindex ) !== '' ) {
  82                                 /**
  83                                  * Same implementation as above, so same @todo
  84                                  */
  85                                 $nspages = $this->getPagesFromNamespace( $nsindex );
  86                                 if ( $nspages ) {
  87                                         $page .= "\n" . implode( "\n", $nspages );
  88                                 }
  89                         }
  90                 } elseif ( $request->getCheck( 'exportall' ) && $config->get( 'ExportAllowAll' ) ) {
  91                         $this->doExport = true;
  92                         $exportall = true;
  93
  94                         /* Although $page and $history are not used later on, we
  95                         nevertheless set them to avoid that PHP notices about using
  96                         undefined variables foul up our XML output (see call to
  97                         doExport(...) further down) */
  98                         $page = '';
  99                         $history = '';
 100                 } elseif ( $request->wasPosted() && $par == '' ) {
 101                         $page = $request->getText( 'pages' );
 102                         $this->curonly = $request->getCheck( 'curonly' );
 103                         $rawOffset = $request->getVal( 'offset' );
 104
 105                         if ( $rawOffset ) {
 106                                 $offset = wfTimestamp( TS_MW, $rawOffset );
 107                         } else {
 108                                 $offset = null;
 109                         }
 110
 111                         $maxHistory = $config->get( 'ExportMaxHistory' );
 112                         $limit = $request->getInt( 'limit' );
 113                         $dir = $request->getVal( 'dir' );
 114                         $history = [
 115                                 'dir' => 'asc',
 116                                 'offset' => false,
 117                                 'limit' => $maxHistory,
 118                         ];
 119                         $historyCheck = $request->getCheck( 'history' );
 120
 121                         if ( $this->curonly ) {
 122                                 $history = WikiExporter::CURRENT;
 123                         } elseif ( !$historyCheck ) {
 124                                 if ( $limit > 0 && ( $maxHistory == 0 || $limit < $maxHistory ) ) {
 125                                         $history['limit'] = $limit;
 126                                 }
 127
 128                                 if ( !is_null( $offset ) ) {
 129                                         $history['offset'] = $offset;
 130                                 }
 131
 132                                 if ( strtolower( $dir ) == 'desc' ) {
 133                                         $history['dir'] = 'desc';
 134                                 }
 135                         }
 136
 137                         if ( $page != '' ) {
 138                                 $this->doExport = true;
 139                         }
 140                 } else {
 141                         // Default to current-only for GET requests.
 142                         $page = $request->getText( 'pages', $par );
 143                         $historyCheck = $request->getCheck( 'history' );
 144
 145                         if ( $historyCheck ) {
 146                                 $history = WikiExporter::FULL;
 147                         } else {
 148                                 $history = WikiExporter::CURRENT;
 149                         }
 150
 151                         if ( $page != '' ) {
 152                                 $this->doExport = true;
 153                         }
 154                 }
 155
 156                 if ( !$config->get( 'ExportAllowHistory' ) ) {
 157                         // Override
 158                         $history = WikiExporter::CURRENT;
 159                 }
 160
 161                 $list_authors = $request->getCheck( 'listauthors' );
 162                 if ( !$this->curonly || !$config->get( 'ExportAllowListContributors' ) ) {
 163                         $list_authors = false;
 164                 }
 165
 166                 if ( $this->doExport ) {
 167                         $this->getOutput()->disable();
 168
 169                         // Cancel output buffering and gzipping if set
 170                         // This should provide safer streaming for pages with history
 171                         wfResetOutputBuffers();
 172                         $request->response()->header( "Content-type: application/xml; charset=utf-8" );
 173                         $request->response()->header( "X-Robots-Tag: noindex,nofollow" );
 174
 175                         if ( $request->getCheck( 'wpDownload' ) ) {
 176                                 // Provide a sane filename suggestion
 177                                 $filename = urlencode( $config->get( 'Sitename' ) . '-' . wfTimestampNow() . '.xml' );
 178                                 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
 179                         }
 180
 181                         $this->doExport( $page, $history, $list_authors, $exportall );
 182
 183                         return;
 184                 }
 185
 186                 $out = $this->getOutput();
 187                 $out->addWikiMsg( 'exporttext' );
 188
 189                 if ( $page == '' ) {
 190                         $categoryName = $request->getText( 'catname' );
 191                 } else {
 192                         $categoryName = '';
 193                 }
 194
 195                 $formDescriptor = [
 196                         'catname' => [
 197                                 'type' => 'textwithbutton',
 198                                 'name' => 'catname',
 199                                 'horizontal-label' => true,
 200                                 'label-message' => 'export-addcattext',
 201                                 'default' => $categoryName,
 202                                 'size' => 40,
 203                                 'buttontype' => 'submit',
 204                                 'buttonname' => 'addcat',
 205                                 'buttondefault' => $this->msg( 'export-addcat' )->text(),
 206                                 'hide-if' => [ '===', 'exportall', '1' ],
 207                         ],
 208                 ];
 209                 if ( $config->get( 'ExportFromNamespaces' ) ) {
 210                         $formDescriptor += [
 211                                 'nsindex' => [
 212                                         'type' => 'namespaceselectwithbutton',
 213                                         'default' => $nsindex,
 214                                         'label-message' => 'export-addnstext',
 215                                         'horizontal-label' => true,
 216                                         'name' => 'nsindex',
 217                                         'id' => 'namespace',
 218                                         'cssclass' => 'namespaceselector',
 219                                         'buttontype' => 'submit',
 220                                         'buttonname' => 'addns',
 221                                         'buttondefault' => $this->msg( 'export-addns' )->text(),
 222                                         'hide-if' => [ '===', 'exportall', '1' ],
 223                                 ],
 224                         ];
 225                 }
 226
 227                 if ( $config->get( 'ExportAllowAll' ) ) {
 228                         $formDescriptor += [
 229                                 'exportall' => [
 230                                         'type' => 'check',
 231                                         'label-message' => 'exportall',
 232                                         'name' => 'exportall',
 233                                         'id' => 'exportall',
 234                                         'default' => $request->wasPosted() ? $request->getCheck( 'exportall' ) : false,
 235                                 ],
 236                         ];
 237                 }
 238
 239                 $formDescriptor += [
 240                         'textarea' => [
 241                                 'class' => 'HTMLTextAreaField',
 242                                 'name' => 'pages',
 243                                 'label-message' => 'export-manual',
 244                                 'nodata' => true,
 245                                 'rows' => 10,
 246                                 'default' => $page,
 247                                 'hide-if' => [ '===', 'exportall', '1' ],
 248                         ],
 249                 ];
 250
 251                 if ( $config->get( 'ExportAllowHistory' ) ) {
 252                         $formDescriptor += [
 253                                 'curonly' => [
 254                                         'type' => 'check',
 255                                         'label-message' => 'exportcuronly',
 256                                         'name' => 'curonly',
 257                                         'id' => 'curonly',
 258                                         'default' => $request->wasPosted() ? $request->getCheck( 'curonly' ) : true,
 259                                 ],
 260                         ];
 261                 } else {
 262                         $out->addWikiMsg( 'exportnohistory' );
 263                 }
 264
 265                 $formDescriptor += [
 266                         'templates' => [
 267                                 'type' => 'check',
 268                                 'label-message' => 'export-templates',
 269                                 'name' => 'templates',
 270                                 'id' => 'wpExportTemplates',
 271                                 'default' => $request->wasPosted() ? $request->getCheck( 'templates' ) : false,
 272                         ],
 273                 ];
 274
 275                 if ( $config->get( 'ExportMaxLinkDepth' ) || $this->userCanOverrideExportDepth() ) {
 276                         $formDescriptor += [
 277                                 'pagelink-depth' => [
 278                                         'type' => 'text',
 279                                         'name' => 'pagelink-depth',
 280                                         'id' => 'pagelink-depth',
 281                                         'label-message' => 'export-pagelinks',
 282                                         'default' => '0',
 283                                         'size' => 20,
 284                                 ],
 285                         ];
 286                 }
 287
 288                 $formDescriptor += [
 289                         'wpDownload' => [
 290                                 'type' => 'check',
 291                                 'name' => 'wpDownload',
 292                                 'id' => 'wpDownload',
 293                                 'default' => $request->wasPosted() ? $request->getCheck( 'wpDownload' ) : true,
 294                                 'label-message' => 'export-download',
 295                         ],
 296                 ];
 297
 298                 if ( $config->get( 'ExportAllowListContributors' ) ) {
 299                         $formDescriptor += [
 300                                 'listauthors' => [
 301                                         'type' => 'check',
 302                                         'label-message' => 'exportlistauthors',
 303                                         'default' => $request->wasPosted() ? $request->getCheck( 'listauthors' ) : false,
 304                                         'name' => 'listauthors',
 305                                         'id' => 'listauthors',
 306                                 ],
 307                         ];
 308                 }
 309
 310                 $htmlForm = HTMLForm::factory( 'ooui', $formDescriptor, $this->getContext() );
 311                 $htmlForm->setSubmitTextMsg( 'export-submit' );
 312                 $htmlForm->prepareForm()->displayForm( false );
 313                 $this->addHelpLink( 'Help:Export' );
 314         }
 315
 316         /**
 317          * @return bool
 318          */
 319         private function userCanOverrideExportDepth() {
 320                 return $this->getUser()->isAllowed( 'override-export-depth' );
 321         }
 322
 323         /**
 324          * Do the actual page exporting
 325          *
 326          * @param string $page User input on what page(s) to export
 327          * @param int $history One of the WikiExporter history export constants
 328          * @param bool $list_authors Whether to add distinct author list (when
 329          *   not returning full history)
 330          * @param bool $exportall Whether to export everything
 331          */
 332         private function doExport( $page, $history, $list_authors, $exportall ) {
 333                 // If we are grabbing everything, enable full history and ignore the rest
 334                 if ( $exportall ) {
 335                         $history = WikiExporter::FULL;
 336                 } else {
 337                         $pageSet = []; // Inverted index of all pages to look up
 338
 339                         // Split up and normalize input
 340                         foreach ( explode( "\n", $page ) as $pageName ) {
 341                                 $pageName = trim( $pageName );
 342                                 $title = Title::newFromText( $pageName );
 343                                 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
 344                                         // Only record each page once!
 345                                         $pageSet[$title->getPrefixedText()] = true;
 346                                 }
 347                         }
 348
 349                         // Set of original pages to pass on to further manipulation...
 350                         $inputPages = array_keys( $pageSet );
 351
 352                         // Look up any linked pages if asked...
 353                         if ( $this->templates ) {
 354                                 $pageSet = $this->getTemplates( $inputPages, $pageSet );
 355                         }
 356                         $linkDepth = $this->pageLinkDepth;
 357                         if ( $linkDepth ) {
 358                                 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
 359                         }
 360
 361                         $pages = array_keys( $pageSet );
 362
 363                         // Normalize titles to the same format and remove dupes, see T19374
 364                         foreach ( $pages as $k => $v ) {
 365                                 $pages[$k] = str_replace( " ", "_", $v );
 366                         }
 367
 368                         $pages = array_unique( $pages );
 369                 }
 370
 371                 /* Ok, let's get to it... */
 372                 if ( $history == WikiExporter::CURRENT ) {
 373                         $lb = false;
 374                         $db = wfGetDB( DB_REPLICA );
 375                         $buffer = WikiExporter::BUFFER;
 376                 } else {
 377                         // Use an unbuffered query; histories may be very long!
 378                         $lb = MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->newMainLB();
 379                         $db = $lb->getConnection( DB_REPLICA );
 380                         $buffer = WikiExporter::STREAM;
 381
 382                         // This might take a while... :D
 383                         MediaWiki\suppressWarnings();
 384                         set_time_limit( 0 );
 385                         MediaWiki\restoreWarnings();
 386                 }
 387
 388                 $exporter = new WikiExporter( $db, $history, $buffer );
 389                 $exporter->list_authors = $list_authors;
 390                 $exporter->openStream();
 391
 392                 if ( $exportall ) {
 393                         $exporter->allPages();
 394                 } else {
 395                         foreach ( $pages as $page ) {
 396                                 # T10824: Only export pages the user can read
 397                                 $title = Title::newFromText( $page );
 398                                 if ( is_null( $title ) ) {
 399                                         // @todo Perhaps output an <error> tag or something.
 400                                         continue;
 401                                 }
 402
 403                                 if ( !$title->userCan( 'read', $this->getUser() ) ) {
 404                                         // @todo Perhaps output an <error> tag or something.
 405                                         continue;
 406                                 }
 407
 408                                 $exporter->pageByTitle( $title );
 409                         }
 410                 }
 411
 412                 $exporter->closeStream();
 413
 414                 if ( $lb ) {
 415                         $lb->closeAll();
 416                 }
 417         }
 418
 419         /**
 420          * @param Title $title
 421          * @return array
 422          */
 423         private function getPagesFromCategory( $title ) {
 424                 global $wgContLang;
 425
 426                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 427
 428                 $name = $title->getDBkey();
 429
 430                 $dbr = wfGetDB( DB_REPLICA );
 431                 $res = $dbr->select(
 432                         [ 'page', 'categorylinks' ],
 433                         [ 'page_namespace', 'page_title' ],
 434                         [ 'cl_from=page_id', 'cl_to' => $name ],
 435                         __METHOD__,
 436                         [ 'LIMIT' => $maxPages ]
 437                 );
 438
 439                 $pages = [];
 440
 441                 foreach ( $res as $row ) {
 442                         $n = $row->page_title;
 443                         if ( $row->page_namespace ) {
 444                                 $ns = $wgContLang->getNsText( $row->page_namespace );
 445                                 $n = $ns . ':' . $n;
 446                         }
 447
 448                         $pages[] = $n;
 449                 }
 450
 451                 return $pages;
 452         }
 453
 454         /**
 455          * @param int $nsindex
 456          * @return array
 457          */
 458         private function getPagesFromNamespace( $nsindex ) {
 459                 global $wgContLang;
 460
 461                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 462
 463                 $dbr = wfGetDB( DB_REPLICA );
 464                 $res = $dbr->select(
 465                         'page',
 466                         [ 'page_namespace', 'page_title' ],
 467                         [ 'page_namespace' => $nsindex ],
 468                         __METHOD__,
 469                         [ 'LIMIT' => $maxPages ]
 470                 );
 471
 472                 $pages = [];
 473
 474                 foreach ( $res as $row ) {
 475                         $n = $row->page_title;
 476
 477                         if ( $row->page_namespace ) {
 478                                 $ns = $wgContLang->getNsText( $row->page_namespace );
 479                                 $n = $ns . ':' . $n;
 480                         }
 481
 482                         $pages[] = $n;
 483                 }
 484
 485                 return $pages;
 486         }
 487
 488         /**
 489          * Expand a list of pages to include templates used in those pages.
 490          * @param array $inputPages List of titles to look up
 491          * @param array $pageSet Associative array indexed by titles for output
 492          * @return array Associative array index by titles
 493          */
 494         private function getTemplates( $inputPages, $pageSet ) {
 495                 return $this->getLinks( $inputPages, $pageSet,
 496                         'templatelinks',
 497                         [ 'namespace' => 'tl_namespace', 'title' => 'tl_title' ],
 498                         [ 'page_id=tl_from' ]
 499                 );
 500         }
 501
 502         /**
 503          * Validate link depth setting, if available.
 504          * @param int $depth
 505          * @return int
 506          */
 507         private function validateLinkDepth( $depth ) {
 508                 if ( $depth < 0 ) {
 509                         return 0;
 510                 }
 511
 512                 if ( !$this->userCanOverrideExportDepth() ) {
 513                         $maxLinkDepth = $this->getConfig()->get( 'ExportMaxLinkDepth' );
 514                         if ( $depth > $maxLinkDepth ) {
 515                                 return $maxLinkDepth;
 516                         }
 517                 }
 518
 519                 /*
 520                  * There's a HARD CODED limit of 5 levels of recursion here to prevent a
 521                  * crazy-big export from being done by someone setting the depth
 522                  * number too high. In other words, last resort safety net.
 523                  */
 524
 525                 return intval( min( $depth, 5 ) );
 526         }
 527
 528         /**
 529          * Expand a list of pages to include pages linked to from that page.
 530          * @param array $inputPages
 531          * @param array $pageSet
 532          * @param int $depth
 533          * @return array
 534          */
 535         private function getPageLinks( $inputPages, $pageSet, $depth ) {
 536                 // @codingStandardsIgnoreStart Squiz.WhiteSpace.SemicolonSpacing.Incorrect
 537                 for ( ; $depth > 0; --$depth ) {
 538                         // @codingStandardsIgnoreEnd
 539                         $pageSet = $this->getLinks(
 540                                 $inputPages, $pageSet, 'pagelinks',
 541                                 [ 'namespace' => 'pl_namespace', 'title' => 'pl_title' ],
 542                                 [ 'page_id=pl_from' ]
 543                         );
 544                         $inputPages = array_keys( $pageSet );
 545                 }
 546
 547                 return $pageSet;
 548         }
 549
 550         /**
 551          * Expand a list of pages to include items used in those pages.
 552          * @param array $inputPages Array of page titles
 553          * @param array $pageSet
 554          * @param string $table
 555          * @param array $fields Array of field names
 556          * @param array $join
 557          * @return array
 558          */
 559         private function getLinks( $inputPages, $pageSet, $table, $fields, $join ) {
 560                 $dbr = wfGetDB( DB_REPLICA );
 561
 562                 foreach ( $inputPages as $page ) {
 563                         $title = Title::newFromText( $page );
 564
 565                         if ( $title ) {
 566                                 $pageSet[$title->getPrefixedText()] = true;
 567                                 /// @todo FIXME: May or may not be more efficient to batch these
 568                                 ///        by namespace when given multiple input pages.
 569                                 $result = $dbr->select(
 570                                         [ 'page', $table ],
 571                                         $fields,
 572                                         array_merge(
 573                                                 $join,
 574                                                 [
 575                                                         'page_namespace' => $title->getNamespace(),
 576                                                         'page_title' => $title->getDBkey()
 577                                                 ]
 578                                         ),
 579                                         __METHOD__
 580                                 );
 581
 582                                 foreach ( $result as $row ) {
 583                                         $template = Title::makeTitle( $row->namespace, $row->title );
 584                                         $pageSet[$template->getPrefixedText()] = true;
 585                                 }
 586                         }
 587                 }
 588
 589                 return $pageSet;
 590         }
 591
 592         protected function getGroupName() {
 593                 return 'pagetools';
 594         }
 595 }