4 * Utility class for working with blacklists
8 * Build a set of regular expressions matching URLs with the list of regex fragments.
9 * Returns an empty list if the input list is empty.
11 * @param array $lines list of fragments which will match in URLs
12 * @param BaseBlacklist $blacklist
13 * @param int $batchSize largest allowed batch regex;
14 * if 0, will produce one regex per line
17 static function buildRegexes( $lines, BaseBlacklist $blacklist, $batchSize=4096 ) {
19 # It's faster using the S modifier even though it will usually only be run once
20 // $regex = 'https?://+[a-z0-9_\-.]*(' . implode( '|', $lines ) . ')';
21 // return '/' . str_replace( '/', '\/', preg_replace('|\\\*/|', '/', $regex) ) . '/Sim';
23 $regexStart = $blacklist->getRegexStart();
24 $regexEnd = $blacklist->getRegexEnd( $batchSize );
26 foreach ( $lines as $line ) {
27 if ( substr( $line, -1, 1 ) == "\\" ) {
28 // Final \ will break silently on the batched regexes.
29 // Skip it here to avoid breaking the next line;
30 // warnings from getBadLines() will still trigger on
31 // edit to keep new ones from floating in.
34 // FIXME: not very robust size check, but should work. :)
35 if ( $build === false ) {
37 } elseif ( strlen( $build ) + strlen( $line ) > $batchSize ) {
38 $regexes[] = $regexStart .
39 str_replace( '/', '\/', preg_replace( '|\\\*/|u', '/', $build ) ) .
47 if ( $build !== false ) {
48 $regexes[] = $regexStart .
49 str_replace( '/', '\/', preg_replace( '|\\\*/|u', '/', $build ) ) .
56 * Confirm that a set of regexes is either empty or valid.
58 * @param $regexes array set of regexes
59 * @return bool true if ok, false if contains invalid lines
61 static function validateRegexes( $regexes ) {
62 foreach ( $regexes as $regex ) {
64 $ok = preg_match( $regex, '' );
67 if ( $ok === false ) {
75 * Strip comments and whitespace, then remove blanks
80 static function stripLines( $lines ) {
83 preg_replace( '/#.*$/', '',
88 * Do a sanity check on the batch regex.
90 * @param $lines string unsanitized input lines
91 * @param $blacklist BaseBlacklist
92 * @param $fileName bool|string optional for debug reporting
93 * @return array of regexes
95 static function buildSafeRegexes( $lines, BaseBlacklist $blacklist, $fileName=false ) {
96 $lines = self::stripLines( $lines );
97 $regexes = self::buildRegexes( $lines, $blacklist );
98 if ( self::validateRegexes( $regexes ) ) {
101 // _Something_ broke... rebuild line-by-line; it'll be
102 // slower if there's a lot of blacklist lines, but one
103 // broken line won't take out hundreds of its brothers.
105 wfDebugLog( 'SpamBlacklist', "Spam blacklist warning: bogus line in $fileName\n" );
107 return self::buildRegexes( $lines, $blacklist, 0 );
112 * Returns an array of invalid lines
114 * @param array $lines
115 * @param $blacklist BaseBlacklist
116 * @return array of input lines which produce invalid input, or empty array if no problems
118 static function getBadLines( $lines, BaseBlacklist $blacklist ) {
119 $lines = self::stripLines( $lines );
122 foreach ( $lines as $line ) {
123 if ( substr( $line, -1, 1 ) == "\\" ) {
124 // Final \ will break silently on the batched regexes.
129 $regexes = self::buildRegexes( $lines, $blacklist );
130 if ( self::validateRegexes( $regexes ) ) {
131 // No other problems!
135 // Something failed in the batch, so check them one by one.
136 foreach ( $lines as $line ) {
137 $regexes = self::buildRegexes( [ $line ], $blacklist );
138 if ( !self::validateRegexes( $regexes ) ) {
146 * Build a set of regular expressions from the given multiline input text,
147 * with empty lines and comments stripped.
149 * @param $source string
150 * @param $blacklist BaseBlacklist
151 * @param $fileName bool|string optional, for reporting of bad files
152 * @return array of regular expressions, potentially empty
154 static function regexesFromText( $source, BaseBlacklist $blacklist, $fileName=false ) {
155 $lines = explode( "\n", $source );
156 return self::buildSafeRegexes( $lines, $blacklist, $fileName );
160 * Build a set of regular expressions from a MediaWiki message.
161 * Will be correctly empty if the message isn't present.
163 * @param $message string
164 * @param $blacklist BaseBlacklist
165 * @return array of regular expressions, potentially empty
167 static function regexesFromMessage( $message, BaseBlacklist $blacklist ) {
168 $source = wfMessage( $message )->inContentLanguage();
169 if ( !$source->isDisabled() ) {
170 return self::regexesFromText( $source->plain(), $blacklist );