2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
3 # http://www.mediawiki.org/
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 # http://www.gnu.org/copyleft/gpl.html
21 if( php_sapi_name() != 'cli' ) {
22 die( "Run me from the command line please.\n" );
26 if( isset( $_SERVER['argv'] ) && in_array( '--icu', $_SERVER['argv'] ) ) {
27 dl( 'php_utfnormal.so' );
30 #ini_set( 'memory_limit', '40M' );
32 require_once 'PHPUnit/Framework.php';
33 require_once 'PHPUnit/TextUI/TestRunner.php';
35 require_once 'UtfNormal.php';
38 * Additional tests for UtfNormal::cleanUp() function, inclusion
39 * regression checks for known problems.
45 class CleanUpTest extends PHPUnit_Framework_TestCase {
55 function testAscii() {
56 $text = 'This is plain ASCII text.';
57 $this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
62 $text = "a \x00 null";
63 $expect = "a \xef\xbf\xbd null";
66 bin2hex( UtfNormal::cleanUp( $text ) ) );
70 function testLatin() {
71 $text = "L'\xc3\xa9cole";
72 $this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
76 function testLatinNormal() {
77 $text = "L'e\xcc\x81cole";
78 $expect = "L'\xc3\xa9cole";
79 $this->assertEquals( $expect, UtfNormal::cleanUp( $text ) );
83 * This test is *very* expensive!
86 function XtestAllChars() {
87 $rep = UTF8_REPLACEMENT;
88 global $utfCanonicalComp, $utfCanonicalDecomp;
89 for( $i = 0x0; $i < UNICODE_MAX; $i++ ) {
90 $char = codepointToUtf8( $i );
91 $clean = UtfNormal::cleanUp( $char );
92 $x = sprintf( "%04X", $i );
93 if( $i % 0x1000 == 0 ) echo "U+$x\n";
97 ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST) ||
98 ($i > UNICODE_SURROGATE_LAST && $i < 0xfffe ) ||
99 ($i > 0xffff && $i <= UNICODE_MAX ) ) {
100 if( isset( $utfCanonicalComp[$char] ) || isset( $utfCanonicalDecomp[$char] ) ) {
101 $comp = UtfNormal::NFC( $char );
105 "U+$x should be decomposed" );
110 "U+$x should be intact" );
113 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
118 /** @todo document */
119 function testAllBytes() {
120 $this->doTestBytes( '', '' );
121 $this->doTestBytes( 'x', '' );
122 $this->doTestBytes( '', 'x' );
123 $this->doTestBytes( 'x', 'x' );
126 /** @todo document */
127 function doTestBytes( $head, $tail ) {
128 for( $i = 0x0; $i < 256; $i++ ) {
129 $char = $head . chr( $i ) . $tail;
130 $clean = UtfNormal::cleanUp( $char );
131 $x = sprintf( "%02X", $i );
135 ($i > 0x001f && $i < 0x80) ) {
139 "ASCII byte $x should be intact" );
140 if( $char != $clean ) return;
142 $norm = $head . UTF8_REPLACEMENT . $tail;
146 "Forbidden byte $x should be rejected" );
147 if( $norm != $clean ) return;
152 /** @todo document */
153 function testDoubleBytes() {
154 $this->doTestDoubleBytes( '', '' );
155 $this->doTestDoubleBytes( 'x', '' );
156 $this->doTestDoubleBytes( '', 'x' );
157 $this->doTestDoubleBytes( 'x', 'x' );
163 function doTestDoubleBytes( $head, $tail ) {
164 for( $first = 0xc0; $first < 0x100; $first++ ) {
165 for( $second = 0x80; $second < 0x100; $second++ ) {
166 $char = $head . chr( $first ) . chr( $second ) . $tail;
167 $clean = UtfNormal::cleanUp( $char );
168 $x = sprintf( "%02X,%02X", $first, $second );
172 $norm = UtfNormal::NFC( $char );
176 "Pair $x should be intact" );
177 if( $norm != $clean ) return;
178 } elseif( $first > 0xfd || $second > 0xbf ) {
179 # fe and ff are not legal head bytes -- expect two replacement chars
180 $norm = $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail;
184 "Forbidden pair $x should be rejected" );
185 if( $norm != $clean ) return;
187 $norm = $head . UTF8_REPLACEMENT . $tail;
191 "Forbidden pair $x should be rejected" );
192 if( $norm != $clean ) return;
198 /** @todo document */
199 function testTripleBytes() {
200 $this->doTestTripleBytes( '', '' );
201 $this->doTestTripleBytes( 'x', '' );
202 $this->doTestTripleBytes( '', 'x' );
203 $this->doTestTripleBytes( 'x', 'x' );
206 /** @todo document */
207 function doTestTripleBytes( $head, $tail ) {
208 for( $first = 0xc0; $first < 0x100; $first++ ) {
209 for( $second = 0x80; $second < 0x100; $second++ ) {
210 #for( $third = 0x80; $third < 0x100; $third++ ) {
211 for( $third = 0x80; $third < 0x81; $third++ ) {
212 $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
213 $clean = UtfNormal::cleanUp( $char );
214 $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
215 if( $first >= 0xe0 &&
219 if( $first == 0xe0 && $second < 0xa0 ) {
221 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
223 "Overlong triplet $x should be rejected" );
224 } elseif( $first == 0xed &&
225 ( chr( $first ) . chr( $second ) . chr( $third )) >= UTF8_SURROGATE_FIRST ) {
227 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
229 "Surrogate triplet $x should be rejected" );
232 bin2hex( UtfNormal::NFC( $char ) ),
234 "Triplet $x should be intact" );
236 } elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
238 bin2hex( UtfNormal::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT . $tail ),
240 "Valid 2-byte $x + broken tail" );
241 } elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
243 bin2hex( $head . UTF8_REPLACEMENT . UtfNormal::NFC( chr( $second ) . chr( $third ) . $tail ) ),
245 "Broken head + valid 2-byte $x" );
246 } elseif( ( $first > 0xfd || $second > 0xfd ) &&
247 ( ( $second > 0xbf && $third > 0xbf ) ||
248 ( $second < 0xc0 && $third < 0xc0 ) ||
249 ( $second > 0xfd ) ||
250 ( $third > 0xfd ) ) ) {
251 # fe and ff are not legal head bytes -- expect three replacement chars
253 bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
255 "Forbidden triplet $x should be rejected" );
256 } elseif( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
258 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
260 "Forbidden triplet $x should be rejected" );
263 bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
265 "Forbidden triplet $x should be rejected" );
272 /** @todo document */
273 function testChunkRegression() {
274 # Check for regression against a chunking bug
275 $text = "\x46\x55\xb8" .
282 $expect = "\x46\x55\xef\xbf\xbd" .
292 bin2hex( UtfNormal::cleanUp( $text ) ) );
295 /** @todo document */
296 function testInterposeRegression() {
311 $expect = "\x4e\x30" .
327 bin2hex( UtfNormal::cleanUp( $text ) ) );
330 /** @todo document */
331 function testOverlongRegression() {
333 "\x1a" . # forbidden ascii
335 "\xc1\xa6" . # overlong sequence
337 "\x1c" . # forbidden ascii
352 bin2hex( UtfNormal::cleanUp( $text ) ) );
355 /** @todo document */
356 function testSurrogateRegression() {
357 $text = "\xed\xb4\x96" . # surrogate 0xDD16
361 $expect = "\xef\xbf\xbd" .
367 bin2hex( UtfNormal::cleanUp( $text ) ) );
370 /** @todo document */
371 function testBomRegression() {
372 $text = "\xef\xbf\xbe" . # U+FFFE, illegal char
376 $expect = "\xef\xbf\xbd" .
382 bin2hex( UtfNormal::cleanUp( $text ) ) );
385 /** @todo document */
386 function testForbiddenRegression() {
387 $text = "\xef\xbf\xbf"; # U+FFFF, illegal char
388 $expect = "\xef\xbf\xbd";
391 bin2hex( UtfNormal::cleanUp( $text ) ) );
394 /** @todo document */
395 function testHangulRegression() {
396 $text = "\xed\x9c\xaf" . # Hangul char
397 "\xe1\x87\x81"; # followed by another final jamo
398 $expect = $text; # Should *not* change.
401 bin2hex( UtfNormal::cleanUp( $text ) ) );
406 $suite = new PHPUnit_Framework_TestSuite( 'CleanUpTest' );
407 $result = PHPUnit_TextUI_TestRunner::run( $suite );
409 if( !$result->wasSuccessful() ) {