3 * Performs transformations of HTML by wrapping around libxml2 and working
4 * around its countless bugs.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 * http://www.gnu.org/copyleft/gpl.html
24 namespace HtmlFormatter;
33 private $itemsToRemove = [];
34 private $elementsToFlatten = [];
35 protected $removeMedia = false;
40 * @param string $html Text to process
42 public function __construct( $html ) {
47 * Turns a chunk of HTML into a proper document
51 public static function wrapHTML( $html ) {
52 return '<!doctype html><html><head></head><body>' . $html . '</body></html>';
56 * Override this in descendant class to modify HTML after it has been converted from DOM tree
57 * @param string $html HTML to process
58 * @return string Processed HTML
60 protected function onHtmlReady( $html ) {
65 * @return DOMDocument DOM to manipulate
67 public function getDoc() {
69 // DOMDocument::loadHTML apparently isn't very good with encodings, so
70 // convert input to ASCII by encoding everything above 128 as entities.
71 $html = \mb_convert_encoding( $this->html, 'HTML-ENTITIES', 'UTF-8' );
73 // Workaround for bug that caused spaces before references
74 // to disappear during processing: https://phabricator.wikimedia.org/T55086
75 $html = str_replace( ' <', ' <', $html );
77 \libxml_use_internal_errors( true );
78 $loader = \libxml_disable_entity_loader();
79 $this->doc = new \DOMDocument();
80 $this->doc->strictErrorChecking = false;
81 $this->doc->loadHTML( $html );
82 \libxml_disable_entity_loader( $loader );
83 \libxml_use_internal_errors( false );
84 $this->doc->encoding = 'UTF-8';
90 * Sets whether images/videos/sounds should be removed from output
93 public function setRemoveMedia( $flag = true ) {
94 $this->removeMedia = $flag;
98 * Adds one or more selector of content to remove. A subset of CSS selector
99 * syntax is supported:
106 * @param array|string $selectors Selector(s) of stuff to remove
108 public function remove( $selectors ) {
109 $this->itemsToRemove = array_merge( $this->itemsToRemove, (array)$selectors );
113 * Adds one or more element name to the list to flatten (remove tag, but not its content)
114 * Can accept undelimited regexes
116 * Note this interface may fail in surprising unexpected ways due to usage of regexes,
117 * so should not be relied on for HTML markup security measures.
119 * @param array|string $elements Name(s) of tag(s) to flatten
121 public function flatten( $elements ) {
122 $this->elementsToFlatten = array_merge( $this->elementsToFlatten, (array)$elements );
126 * Instructs the formatter to flatten all tags
128 public function flattenAllTags() {
129 $this->flatten( '[?!]?[a-z0-9]+' );
133 * Removes content we've chosen to remove. The text of the removed elements can be
134 * extracted with the getText method.
135 * @return array Array of removed DOMElements
137 public function filterContent() {
138 $removals = $this->parseItemsToRemove();
140 // Bail out early if nothing to do
141 if ( \array_reduce( $removals,
142 function ( $carry, $item ) {
143 return $carry && !$item;
150 $doc = $this->getDoc();
154 // You can't remove DOMNodes from a DOMNodeList as you're iterating
155 // over them in a foreach loop. It will seemingly leave the internal
156 // iterator on the foreach out of wack and results will be quite
157 // strange. Though, making a queue of items to remove seems to work.
158 $domElemsToRemove = [];
159 foreach ( $removals['TAG'] as $tagToRemove ) {
160 $tagToRemoveNodes = $doc->getElementsByTagName( $tagToRemove );
161 foreach ( $tagToRemoveNodes as $tagToRemoveNode ) {
162 if ( $tagToRemoveNode ) {
163 $domElemsToRemove[] = $tagToRemoveNode;
167 $removed = $this->removeElements( $domElemsToRemove );
169 // Elements with named IDs
170 $domElemsToRemove = [];
171 foreach ( $removals['ID'] as $itemToRemove ) {
172 $itemToRemoveNode = $doc->getElementById( $itemToRemove );
173 if ( $itemToRemoveNode ) {
174 $domElemsToRemove[] = $itemToRemoveNode;
177 $removed = array_merge( $removed, $this->removeElements( $domElemsToRemove ) );
180 $domElemsToRemove = [];
181 $xpath = new \DOMXPath( $doc );
182 foreach ( $removals['CLASS'] as $classToRemove ) {
183 $elements = $xpath->query( '//*[contains(@class, "' . $classToRemove . '")]' );
185 /** @var $element DOMElement */
186 foreach ( $elements as $element ) {
187 $classes = $element->getAttribute( 'class' );
188 if ( \preg_match( "/\b$classToRemove\b/", $classes ) && $element->parentNode ) {
189 $domElemsToRemove[] = $element;
193 $removed = \array_merge( $removed, $this->removeElements( $domElemsToRemove ) );
195 // Tags with CSS Classes
196 foreach ( $removals['TAG_CLASS'] as $classToRemove ) {
197 $parts = explode( '.', $classToRemove );
199 $elements = $xpath->query(
200 '//' . $parts[0] . '[@class="' . $parts[1] . '"]'
202 $removed = array_merge( $removed, $this->removeElements( $elements ) );
209 * Removes a list of elelments from DOMDocument
210 * @param array|DOMNodeList $elements
211 * @return array Array of removed elements
213 private function removeElements( $elements ) {
215 if ( $elements instanceof \DOMNodeList ) {
217 foreach ( $elements as $element ) {
221 /** @var $element DOMElement */
222 foreach ( $list as $element ) {
223 if ( $element->parentNode ) {
224 $element->parentNode->removeChild( $element );
231 * libxml in its usual pointlessness converts many chars to entities - this function
232 * perfoms a reverse conversion
233 * @param string $html
236 private function fixLibXML( $html ) {
237 // We don't include rules like '"' => '&quot;' because entities had already been
238 // normalized by libxml. Using this function with input not sanitized by libxml is UNSAFE!
240 '"' => '&quot;',
241 '&' => '&amp;',
242 '<' => '&lt;',
243 '>' => '&gt;',
245 $html = strtr( $html, $replacements );
247 if ( \function_exists( 'mb_convert_encoding' ) ) {
248 // Just in case the conversion in getDoc() above used named
249 // entities that aren't known to html_entity_decode().
250 $html = \mb_convert_encoding( $html, 'UTF-8', 'HTML-ENTITIES' );
252 $html = \html_entity_decode( $html, ENT_COMPAT, 'utf-8' );
258 * Performs final transformations and returns resulting HTML. Note that if you want to call this
259 * both without an element and with an element you should call it without an element first. If you
260 * specify the $element in the method it'll change the underlying dom and you won't be able to get
263 * @param DOMElement|string|null $element ID of element to get HTML from or
264 * false to get it from the whole tree
265 * @return string Processed HTML
267 public function getText( $element = null ) {
270 if ( $element !== null && !( $element instanceof \DOMElement ) ) {
271 $element = $this->doc->getElementById( $element );
274 $body = $this->doc->getElementsByTagName( 'body' )->item( 0 );
276 foreach ( $body->childNodes as $node ) {
277 $nodesArray[] = $node;
279 foreach ( $nodesArray as $nodeArray ) {
280 $body->removeChild( $nodeArray );
282 $body->appendChild( $element );
284 $html = $this->doc->saveHTML();
286 $html = $this->fixLibXml( $html );
287 if ( PHP_EOL === "\r\n" ) {
288 // Cleanup for CRLF misprocessing of unknown origin on Windows.
289 $html = str_replace( ' ', '', $html );
294 // Remove stuff added by wrapHTML()
295 $html = \preg_replace( '/<!--.*?-->|^.*?<body>|<\/body>.*$/s', '', $html );
296 $html = $this->onHtmlReady( $html );
298 if ( $this->elementsToFlatten ) {
299 $elements = \implode( '|', $this->elementsToFlatten );
300 $html = \preg_replace( "#</?($elements)\\b[^>]*>#is", '', $html );
307 * Helper function for parseItemsToRemove(). This function extracts the selector type
308 * and the raw name of a selector from a CSS-style selector string and assigns those
309 * values to parameters passed by reference. For example, if given '#toc' as the
310 * $selector parameter, it will assign 'ID' as the $type and 'toc' as the $rawName.
311 * @param string $selector CSS selector to parse
312 * @param string $type The type of selector (ID, CLASS, TAG_CLASS, or TAG)
313 * @param string $rawName The raw name of the selector
314 * @return bool Whether the selector was successfully recognised
315 * @throws MWException
317 protected function parseSelector( $selector, &$type, &$rawName ) {
318 if ( strpos( $selector, '.' ) === 0 ) {
320 $rawName = substr( $selector, 1 );
321 } elseif ( strpos( $selector, '#' ) === 0 ) {
323 $rawName = substr( $selector, 1 );
324 } elseif ( strpos( $selector, '.' ) !== 0 && strpos( $selector, '.' ) !== false ) {
326 $rawName = $selector;
327 } elseif ( strpos( $selector, '[' ) === false && strpos( $selector, ']' ) === false ) {
329 $rawName = $selector;
331 throw new \Exception( __METHOD__ . "(): unrecognized selector '$selector'" );
338 * Transforms CSS-style selectors into an internal representation suitable for
339 * processing by filterContent()
342 protected function parseItemsToRemove() {
350 foreach ( $this->itemsToRemove as $itemToRemove ) {
353 if ( $this->parseSelector( $itemToRemove, $type, $rawName ) ) {
354 $removals[$type][] = $rawName;
358 if ( $this->removeMedia ) {
359 $removals['TAG'][] = 'img';
360 $removals['TAG'][] = 'audio';
361 $removals['TAG'][] = 'video';