2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
3 # http://www.mediawiki.org/
5 # This program is free software; you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation; either version 2 of the License, or
8 # (at your option) any later version.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License along
16 # with this program; if not, write to the Free Software Foundation, Inc.,
17 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 # http://www.gnu.org/copyleft/gpl.html
21 * This script generates UniNormalData.inc from the Unicode Character Database
22 * and supplementary files.
30 if( php_sapi_name() != 'cli' ) {
31 die( "Run me from the command line please.\n" );
34 require_once 'UtfNormalUtil.php';
36 $in = fopen("DerivedNormalizationProps.txt", "rt" );
38 print "Can't open DerivedNormalizationProps.txt for reading.\n";
39 print "If necessary, fetch this file from the internet:\n";
40 print "http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt\n";
43 print "Initializing normalization quick check tables...\n";
45 while( false !== ($line = fgets( $in ) ) ) {
47 if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
48 list( $junk, $first, $last, $prop, $value ) = $matches;
49 #print "$first $last $prop $value\n";
50 if( !$last ) $last = $first;
51 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++) {
52 $char = codepointToUtf8( $i );
53 $checkNFC[$char] = $value;
59 $in = fopen("CompositionExclusions.txt", "rt" );
61 print "Can't open CompositionExclusions.txt for reading.\n";
62 print "If necessary, fetch this file from the internet:\n";
63 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
67 while( false !== ($line = fgets( $in ) ) ) {
68 if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
69 $codepoint = $matches[1];
70 $source = codepointToUtf8( hexdec( $codepoint ) );
71 $exclude[$source] = true;
76 $in = fopen("UnicodeData.txt", "rt" );
78 print "Can't open UnicodeData.txt for reading.\n";
79 print "If necessary, fetch this file from the internet:\n";
80 print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
84 $compatibilityDecomp = array();
85 $canonicalDecomp = array();
86 $canonicalComp = array();
87 $combiningClass = array();
92 print "Reading character definitions...\n";
93 while( false !== ($line = fgets( $in ) ) ) {
94 $columns = explode(';', $line);
95 $codepoint = $columns[0];
97 $canonicalCombiningClass = $columns[3];
98 $decompositionMapping = $columns[5];
100 $source = codepointToUtf8( hexdec( $codepoint ) );
102 if( $canonicalCombiningClass != 0 ) {
103 $combiningClass[$source] = intval( $canonicalCombiningClass );
106 if( $decompositionMapping === '' ) continue;
107 if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
108 # Compatibility decomposition
110 $decompositionMapping = $matches[2];
117 $dest = hexSequenceToUtf8( $decompositionMapping );
119 $compatibilityDecomp[$source] = $dest;
121 $canonicalDecomp[$source] = $dest;
122 if( empty( $exclude[$source] ) ) {
123 $canonicalComp[$dest] = $source;
126 #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
130 print "Recursively expanding canonical mappings...\n";
133 while( $changed > 0 ) {
134 print "pass $pass\n";
136 foreach( $canonicalDecomp as $source => $dest ) {
137 $newDest = preg_replace_callback(
138 '/([\xc0-\xff][\x80-\xbf]+)/',
141 if( $newDest === $dest ) continue;
143 $canonicalDecomp[$source] = $newDest;
148 print "Recursively expanding compatibility mappings...\n";
151 while( $changed > 0 ) {
152 print "pass $pass\n";
154 foreach( $compatibilityDecomp as $source => $dest ) {
155 $newDest = preg_replace_callback(
156 '/([\xc0-\xff][\x80-\xbf]+)/',
159 if( $newDest === $dest ) continue;
161 $compatibilityDecomp[$source] = $newDest;
166 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
168 $out = fopen("UtfNormalData.inc", "wt");
170 $serCombining = escapeSingleString( serialize( $combiningClass ) );
171 $serComp = escapeSingleString( serialize( $canonicalComp ) );
172 $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
173 $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
174 $outdata = "<" . "?php
176 * This file was automatically generated -- do not edit!
177 * Run UtfNormalGenerate.php to create this file again (make clean && make)
180 global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp, \$utfCheckNFC;
181 \$utfCombiningClass = unserialize( '$serCombining' );
182 \$utfCanonicalComp = unserialize( '$serComp' );
183 \$utfCanonicalDecomp = unserialize( '$serCanon' );
184 \$utfCheckNFC = unserialize( '$serCheckNFC' );
186 fputs( $out, $outdata );
188 print "Wrote out UtfNormalData.inc\n";
190 print "Can't create file UtfNormalData.inc\n";
195 $out = fopen("UtfNormalDataK.inc", "wt");
197 $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
198 $outdata = "<" . "?php
200 * This file was automatically generated -- do not edit!
201 * Run UtfNormalGenerate.php to create this file again (make clean && make)
204 global \$utfCompatibilityDecomp;
205 \$utfCompatibilityDecomp = unserialize( '$serCompat' );
207 fputs( $out, $outdata );
209 print "Wrote out UtfNormalDataK.inc\n";
212 print "Can't create file UtfNormalDataK.inc\n";
218 function callbackCanonical( $matches ) {
219 global $canonicalDecomp;
220 if( isset( $canonicalDecomp[$matches[1]] ) ) {
221 return $canonicalDecomp[$matches[1]];
226 function callbackCompat( $matches ) {
227 global $compatibilityDecomp;
228 if( isset( $compatibilityDecomp[$matches[1]] ) ) {
229 return $compatibilityDecomp[$matches[1]];