unicode_gen.c - quickjs-tart - quickjs-based runtime for wallet-core logic

unicode_gen.c (105279B)
      1 /*
      2  * Generation of Unicode tables
      3  *
      4  * Copyright (c) 2017-2018 Fabrice Bellard
      5  * Copyright (c) 2017-2018 Charlie Gordon
      6  *
      7  * Permission is hereby granted, free of charge, to any person obtaining a copy
      8  * of this software and associated documentation files (the "Software"), to deal
      9  * in the Software without restriction, including without limitation the rights
     10  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
     11  * copies of the Software, and to permit persons to whom the Software is
     12  * furnished to do so, subject to the following conditions:
     13  *
     14  * The above copyright notice and this permission notice shall be included in
     15  * all copies or substantial portions of the Software.
     16  *
     17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
     20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
     23  * THE SOFTWARE.
     24  */
     25 #include <stdlib.h>
     26 #include <stdio.h>
     27 #include <stdarg.h>
     28 #include <inttypes.h>
     29 #include <string.h>
     30 #include <assert.h>
     31 #include <ctype.h>
     32 #include <time.h>
     33 
     34 #include "cutils.h"
     35 
     36 uint32_t total_tables;
     37 uint32_t total_table_bytes;
     38 uint32_t total_index;
     39 uint32_t total_index_bytes;
     40 
     41 /* define it to be able to test unicode.c */
     42 //#define USE_TEST
     43 /* profile tests */
     44 //#define PROFILE
     45 
     46 //#define DUMP_CASE_CONV_TABLE
     47 //#define DUMP_TABLE_SIZE
     48 //#define DUMP_CC_TABLE
     49 //#define DUMP_DECOMP_TABLE
     50 //#define DUMP_CASE_FOLDING_SPECIAL_CASES
     51 
     52 /* Ideas:
     53    - Generalize run length encoding + index for all tables
     54    - remove redundant tables for ID_start, ID_continue, Case_Ignorable, Cased
     55 
     56    Case conversion:
     57    - use a single entry for consecutive U/LF runs
     58    - allow EXT runs of length > 1
     59 
     60    Decomposition:
     61    - Greek lower case (+1f10/1f10) ?
     62    - allow holes in B runs
     63    - suppress more upper / lower case redundancy
     64 */
     65 
     66 #ifdef USE_TEST
     67 #include "libunicode.c"
     68 #endif
     69 
     70 #define CHARCODE_MAX 0x10ffff
     71 #define CC_LEN_MAX 3
     72 
     73 void *mallocz(size_t size)
     74 {
     75     void *ptr;
     76     ptr = malloc(size);
     77     memset(ptr, 0, size);
     78     return ptr;
     79 }
     80 
     81 const char *get_field(const char *p, int n)
     82 {
     83     int i;
     84     for(i = 0; i < n; i++) {
     85         while (*p != ';' && *p != '\0')
     86             p++;
     87         if (*p == '\0')
     88             return NULL;
     89         p++;
     90     }
     91     return p;
     92 }
     93 
     94 const char *get_field_buf(char *buf, size_t buf_size, const char *p, int n)
     95 {
     96     char *q;
     97     p = get_field(p, n);
     98     q = buf;
     99     while (*p != ';' && *p != '\0') {
    100         if ((q - buf) < buf_size - 1)
    101             *q++ = *p;
    102         p++;
    103     }
    104     *q = '\0';
    105     return buf;
    106 }
    107 
    108 void add_char(int **pbuf, int *psize, int *plen, int c)
    109 {
    110     int len, size, *buf;
    111     buf = *pbuf;
    112     size = *psize;
    113     len = *plen;
    114     if (len >= size) {
    115         size = *psize;
    116         size = max_int(len + 1, size * 3 / 2);
    117         buf = realloc(buf, sizeof(buf[0]) * size);
    118         *pbuf = buf;
    119         *psize = size;
    120     }
    121     buf[len++] = c;
    122     *plen = len;
    123 }
    124 
    125 int *get_field_str(int *plen, const char *str, int n)
    126 {
    127     const char *p;
    128     int *buf, len, size;
    129     p = get_field(str, n);
    130     if (!p) {
    131         *plen = 0;
    132         return NULL;
    133     }
    134     len = 0;
    135     size = 0;
    136     buf = NULL;
    137     for(;;) {
    138         while (isspace(*p))
    139             p++;
    140         if (!isxdigit(*p))
    141             break;
    142         add_char(&buf, &size, &len, strtoul(p, (char **)&p, 16));
    143     }
    144     *plen = len;
    145     return buf;
    146 }
    147 
    148 char *get_line(char *buf, int buf_size, FILE *f)
    149 {
    150     int len;
    151     if (!fgets(buf, buf_size, f))
    152         return NULL;
    153     len = strlen(buf);
    154     if (len > 0 && buf[len - 1] == '\n')
    155         buf[len - 1] = '\0';
    156     return buf;
    157 }
    158 
    159 typedef struct REString {
    160     struct REString *next;
    161     uint32_t hash;
    162     uint32_t len;
    163     uint32_t flags;
    164     uint32_t buf[];
    165 } REString;
    166 
    167 typedef struct {
    168     uint32_t n_strings;
    169     uint32_t hash_size;
    170     int hash_bits;
    171     REString **hash_table;
    172 } REStringList;
    173 
    174 static uint32_t re_string_hash(int len, const uint32_t *buf)
    175 {
    176     int i;
    177     uint32_t h;
    178     h = 1;
    179     for(i = 0; i < len; i++)
    180         h = h * 263 + buf[i];
    181     return h * 0x61C88647;
    182 }
    183 
    184 static void re_string_list_init(REStringList *s)
    185 {
    186     s->n_strings = 0;
    187     s->hash_size = 0;
    188     s->hash_bits = 0;
    189     s->hash_table = NULL;
    190 }
    191 
    192 static  __maybe_unused void re_string_list_free(REStringList *s)
    193 {
    194     REString *p, *p_next;
    195     int i;
    196     for(i = 0; i < s->hash_size; i++) {
    197         for(p = s->hash_table[i]; p != NULL; p = p_next) {
    198             p_next = p->next;
    199             free(p);
    200         }
    201     }
    202     free(s->hash_table);
    203 }
    204 
    205 static void lre_print_char(int c, BOOL is_range)
    206 {
    207     if (c == '\'' || c == '\\' ||
    208         (is_range && (c == '-' || c == ']'))) {
    209         printf("\\%c", c);
    210     } else if (c >= ' ' && c <= 126) {
    211         printf("%c", c);
    212     } else {
    213         printf("\\u{%04x}", c);
    214     }
    215 }
    216 
    217 static __maybe_unused void re_string_list_dump(const char *str, const REStringList *s)
    218 {
    219     REString *p;
    220     int i, j, k;
    221 
    222     printf("%s:\n", str);
    223     
    224     j = 0;
    225     for(i = 0; i < s->hash_size; i++) {
    226         for(p = s->hash_table[i]; p != NULL; p = p->next) {
    227             printf("  %d/%d: '", j, s->n_strings);
    228             for(k = 0; k < p->len; k++) {
    229                 lre_print_char(p->buf[k], FALSE);
    230             }
    231             printf("'\n");
    232             j++;
    233         }
    234     }
    235 }
    236 
    237 static REString *re_string_find2(REStringList *s, int len, const uint32_t *buf,
    238                                  uint32_t h0, BOOL add_flag)
    239 {
    240     uint32_t h = 0; /* avoid warning */
    241     REString *p;
    242     if (s->n_strings != 0) {
    243         h = h0 >> (32 - s->hash_bits);
    244         for(p = s->hash_table[h]; p != NULL; p = p->next) {
    245             if (p->hash == h0 && p->len == len &&
    246                 !memcmp(p->buf, buf, len * sizeof(buf[0]))) {
    247                 return p;
    248             }
    249         }
    250     }
    251     /* not found */
    252     if (!add_flag)
    253         return NULL;
    254     /* increase the size of the hash table if needed */
    255     if (unlikely((s->n_strings + 1) > s->hash_size)) {
    256         REString **new_hash_table, *p_next;
    257         int new_hash_bits, i;
    258         uint32_t new_hash_size;
    259         new_hash_bits = max_int(s->hash_bits + 1, 4);
    260         new_hash_size = 1 << new_hash_bits;
    261         new_hash_table = malloc(sizeof(new_hash_table[0]) * new_hash_size);
    262         if (!new_hash_table)
    263             return NULL;
    264         memset(new_hash_table, 0, sizeof(new_hash_table[0]) * new_hash_size);
    265         for(i = 0; i < s->hash_size; i++) {
    266             for(p = s->hash_table[i]; p != NULL; p = p_next) {
    267                 p_next = p->next;
    268                 h = p->hash >> (32 - new_hash_bits);
    269                 p->next = new_hash_table[h];
    270                 new_hash_table[h] = p;
    271             }
    272         }
    273         free(s->hash_table);
    274         s->hash_bits = new_hash_bits;
    275         s->hash_size = new_hash_size;
    276         s->hash_table = new_hash_table;
    277         h = h0 >> (32 - s->hash_bits);
    278     }
    279 
    280     p = malloc(sizeof(REString) + len * sizeof(buf[0]));
    281     if (!p)
    282         return NULL;
    283     p->next = s->hash_table[h];
    284     s->hash_table[h] = p;
    285     s->n_strings++;
    286     p->hash = h0;
    287     p->len = len;
    288     p->flags = 0;
    289     memcpy(p->buf, buf, sizeof(buf[0]) * len);
    290     return p;
    291 }
    292 
    293 static REString *re_string_find(REStringList *s, int len, const uint32_t *buf,
    294                                 BOOL add_flag)
    295 {
    296     uint32_t h0;
    297     h0 = re_string_hash(len, buf);
    298     return re_string_find2(s, len, buf, h0, add_flag);
    299 }
    300 
    301 static void re_string_add(REStringList *s, int len, const uint32_t *buf)
    302 {
    303     re_string_find(s, len, buf, TRUE);
    304 }
    305 
    306 #define UNICODE_GENERAL_CATEGORY
    307 
    308 typedef enum {
    309 #define DEF(id, str) GCAT_ ## id,
    310 #include "unicode_gen_def.h"
    311 #undef DEF
    312     GCAT_COUNT,
    313 } UnicodeGCEnum1;
    314 
    315 static const char *unicode_gc_name[] = {
    316 #define DEF(id, str) #id,
    317 #include "unicode_gen_def.h"
    318 #undef DEF
    319 };
    320 
    321 static const char *unicode_gc_short_name[] = {
    322 #define DEF(id, str) str,
    323 #include "unicode_gen_def.h"
    324 #undef DEF
    325 };
    326 
    327 #undef UNICODE_GENERAL_CATEGORY
    328 
    329 #define UNICODE_SCRIPT
    330 
    331 typedef enum {
    332 #define DEF(id, str) SCRIPT_ ## id,
    333 #include "unicode_gen_def.h"
    334 #undef DEF
    335     SCRIPT_COUNT,
    336 } UnicodeScriptEnum1;
    337 
    338 static const char *unicode_script_name[] = {
    339 #define DEF(id, str) #id,
    340 #include "unicode_gen_def.h"
    341 #undef DEF
    342 };
    343 
    344 const char *unicode_script_short_name[] = {
    345 #define DEF(id, str) str,
    346 #include "unicode_gen_def.h"
    347 #undef DEF
    348 };
    349 
    350 #undef UNICODE_SCRIPT
    351 
    352 #define UNICODE_PROP_LIST
    353 
    354 typedef enum {
    355 #define DEF(id, str) PROP_ ## id,
    356 #include "unicode_gen_def.h"
    357 #undef DEF
    358     PROP_COUNT,
    359 } UnicodePropEnum1;
    360 
    361 static const char *unicode_prop_name[] = {
    362 #define DEF(id, str) #id,
    363 #include "unicode_gen_def.h"
    364 #undef DEF
    365 };
    366 
    367 static const char *unicode_prop_short_name[] = {
    368 #define DEF(id, str) str,
    369 #include "unicode_gen_def.h"
    370 #undef DEF
    371 };
    372 
    373 #undef UNICODE_PROP_LIST
    374 
    375 #define UNICODE_SEQUENCE_PROP_LIST
    376 
    377 typedef enum {
    378 #define DEF(id) SEQUENCE_PROP_ ## id,
    379 #include "unicode_gen_def.h"
    380 #undef DEF
    381     SEQUENCE_PROP_COUNT,
    382 } UnicodeSequencePropEnum1;
    383 
    384 static const char *unicode_sequence_prop_name[] = {
    385 #define DEF(id) #id,
    386 #include "unicode_gen_def.h"
    387 #undef DEF
    388 };
    389 
    390 #undef UNICODE_SEQUENCE_PROP_LIST
    391 
    392 typedef struct {
    393     /* case conv */
    394     uint8_t u_len;
    395     uint8_t l_len;
    396     uint8_t f_len;
    397     int u_data[CC_LEN_MAX]; /* to upper case */
    398     int l_data[CC_LEN_MAX]; /* to lower case */
    399     int f_data[CC_LEN_MAX]; /* to case folding */
    400 
    401     uint8_t combining_class;
    402     uint8_t is_compat:1;
    403     uint8_t is_excluded:1;
    404     uint8_t general_category;
    405     uint8_t script;
    406     uint8_t script_ext_len;
    407     uint8_t *script_ext;
    408     uint32_t prop_bitmap_tab[3];
    409     /* decomposition */
    410     int decomp_len;
    411     int *decomp_data;
    412 } CCInfo;
    413 
    414 typedef struct {
    415     int count;
    416     int size;
    417     int *tab;
    418 } UnicodeSequenceProperties;
    419 
    420 CCInfo *unicode_db;
    421 REStringList rgi_emoji_zwj_sequence;
    422 DynBuf rgi_emoji_tag_sequence;
    423 
    424 int find_name(const char **tab, int tab_len, const char *name)
    425 {
    426     int i, len, name_len;
    427     const char *p, *r;
    428 
    429     name_len = strlen(name);
    430     for(i = 0; i < tab_len; i++) {
    431         p = tab[i];
    432         for(;;) {
    433             r = strchr(p, ',');
    434             if (!r)
    435                 len = strlen(p);
    436             else
    437                 len = r - p;
    438             if (len == name_len && memcmp(p, name, len) == 0)
    439                 return i;
    440             if (!r)
    441                 break;
    442             p = r + 1;
    443         }
    444     }
    445     return -1;
    446 }
    447 
    448 static BOOL get_prop(uint32_t c, int prop_idx)
    449 {
    450     return (unicode_db[c].prop_bitmap_tab[prop_idx >> 5] >> (prop_idx & 0x1f)) & 1;
    451 }
    452 
    453 static void set_prop(uint32_t c, int prop_idx, int val)
    454 {
    455     uint32_t mask;
    456     mask = 1U << (prop_idx & 0x1f);
    457     if (val)
    458         unicode_db[c].prop_bitmap_tab[prop_idx >> 5] |= mask;
    459     else
    460         unicode_db[c].prop_bitmap_tab[prop_idx >> 5]  &= ~mask;
    461 }
    462 
    463 void parse_unicode_data(const char *filename)
    464 {
    465     FILE *f;
    466     char line[1024];
    467     char buf1[256];
    468     const char *p;
    469     int code, lc, uc, last_code;
    470     CCInfo *ci, *tab = unicode_db;
    471 
    472     f = fopen(filename, "rb");
    473     if (!f) {
    474         perror(filename);
    475         exit(1);
    476     }
    477 
    478     last_code = 0;
    479     for(;;) {
    480         if (!get_line(line, sizeof(line), f))
    481             break;
    482         p = line;
    483         while (isspace(*p))
    484             p++;
    485         if (*p == '#')
    486             continue;
    487 
    488         p = get_field(line, 0);
    489         if (!p)
    490             continue;
    491         code = strtoul(p, NULL, 16);
    492         lc = 0;
    493         uc = 0;
    494 
    495         p = get_field(line, 12);
    496         if (p && *p != ';') {
    497             uc = strtoul(p, NULL, 16);
    498         }
    499 
    500         p = get_field(line, 13);
    501         if (p && *p != ';') {
    502             lc = strtoul(p, NULL, 16);
    503         }
    504         ci = &tab[code];
    505         if (uc > 0 || lc > 0) {
    506             assert(code <= CHARCODE_MAX);
    507             if (uc > 0) {
    508                 assert(ci->u_len == 0);
    509                 ci->u_len = 1;
    510                 ci->u_data[0] = uc;
    511             }
    512             if (lc > 0) {
    513                 assert(ci->l_len == 0);
    514                 ci->l_len = 1;
    515                 ci->l_data[0] = lc;
    516             }
    517         }
    518 
    519         {
    520             int i;
    521             get_field_buf(buf1, sizeof(buf1), line, 2);
    522             i = find_name(unicode_gc_name, countof(unicode_gc_name), buf1);
    523             if (i < 0) {
    524                 fprintf(stderr, "General category '%s' not found\n",
    525                         buf1);
    526                 exit(1);
    527             }
    528             ci->general_category = i;
    529         }
    530 
    531         p = get_field(line, 3);
    532         if (p && *p != ';' && *p != '\0') {
    533             int cc;
    534             cc = strtoul(p, NULL, 0);
    535             if (cc != 0) {
    536                 assert(code <= CHARCODE_MAX);
    537                 ci->combining_class = cc;
    538                 //                printf("%05x: %d\n", code, ci->combining_class);
    539             }
    540         }
    541 
    542         p = get_field(line, 5);
    543         if (p && *p != ';' && *p != '\0') {
    544             int size;
    545             assert(code <= CHARCODE_MAX);
    546             ci->is_compat = 0;
    547             if (*p == '<') {
    548                 while (*p != '\0' && *p != '>')
    549                     p++;
    550                 if (*p == '>')
    551                     p++;
    552                 ci->is_compat = 1;
    553             }
    554             size = 0;
    555             for(;;) {
    556                 while (isspace(*p))
    557                     p++;
    558                 if (!isxdigit(*p))
    559                     break;
    560                 add_char(&ci->decomp_data, &size, &ci->decomp_len, strtoul(p, (char **)&p, 16));
    561             }
    562 #if 0
    563             {
    564                 int i;
    565                 static int count, d_count;
    566 
    567                 printf("%05x: %c", code, ci->is_compat ? 'C': ' ');
    568                 for(i = 0; i < ci->decomp_len; i++)
    569                     printf(" %05x", ci->decomp_data[i]);
    570                 printf("\n");
    571                 count++;
    572                 d_count += ci->decomp_len;
    573                 //                printf("%d %d\n", count, d_count);
    574             }
    575 #endif
    576         }
    577 
    578         p = get_field(line, 9);
    579         if (p && *p == 'Y') {
    580             set_prop(code, PROP_Bidi_Mirrored, 1);
    581         }
    582 
    583         /* handle ranges */
    584         get_field_buf(buf1, sizeof(buf1), line, 1);
    585         if (strstr(buf1, " Last>")) {
    586             int i;
    587             //            printf("range: 0x%x-%0x\n", last_code, code);
    588             assert(ci->decomp_len == 0);
    589             assert(ci->script_ext_len == 0);
    590             for(i = last_code + 1; i < code; i++) {
    591                 unicode_db[i] = *ci;
    592             }
    593         }
    594         last_code = code;
    595     }
    596 
    597     fclose(f);
    598 }
    599 
    600 void parse_special_casing(CCInfo *tab, const char *filename)
    601 {
    602     FILE *f;
    603     char line[1024];
    604     const char *p;
    605     int code;
    606     CCInfo *ci;
    607 
    608     f = fopen(filename, "rb");
    609     if (!f) {
    610         perror(filename);
    611         exit(1);
    612     }
    613 
    614     for(;;) {
    615         if (!get_line(line, sizeof(line), f))
    616             break;
    617         p = line;
    618         while (isspace(*p))
    619             p++;
    620         if (*p == '#')
    621             continue;
    622 
    623         p = get_field(line, 0);
    624         if (!p)
    625             continue;
    626         code = strtoul(p, NULL, 16);
    627         assert(code <= CHARCODE_MAX);
    628         ci = &tab[code];
    629 
    630         p = get_field(line, 4);
    631         if (p) {
    632             /* locale dependent casing */
    633             while (isspace(*p))
    634                 p++;
    635             if (*p != '#' && *p != '\0')
    636                 continue;
    637         }
    638 
    639 
    640         p = get_field(line, 1);
    641         if (p && *p != ';') {
    642             ci->l_len = 0;
    643             for(;;) {
    644                 while (isspace(*p))
    645                     p++;
    646                 if (*p == ';')
    647                     break;
    648                 assert(ci->l_len < CC_LEN_MAX);
    649                 ci->l_data[ci->l_len++] = strtoul(p, (char **)&p, 16);
    650             }
    651 
    652             if (ci->l_len == 1 && ci->l_data[0] == code)
    653                 ci->l_len = 0;
    654         }
    655 
    656         p = get_field(line, 3);
    657         if (p && *p != ';') {
    658             ci->u_len = 0;
    659             for(;;) {
    660                 while (isspace(*p))
    661                     p++;
    662                 if (*p == ';')
    663                     break;
    664                 assert(ci->u_len < CC_LEN_MAX);
    665                 ci->u_data[ci->u_len++] = strtoul(p, (char **)&p, 16);
    666             }
    667 
    668             if (ci->u_len == 1 && ci->u_data[0] == code)
    669                 ci->u_len = 0;
    670         }
    671     }
    672 
    673     fclose(f);
    674 }
    675 
    676 void parse_case_folding(CCInfo *tab, const char *filename)
    677 {
    678     FILE *f;
    679     char line[1024];
    680     const char *p;
    681     int code, status;
    682     CCInfo *ci;
    683 
    684     f = fopen(filename, "rb");
    685     if (!f) {
    686         perror(filename);
    687         exit(1);
    688     }
    689 
    690     for(;;) {
    691         if (!get_line(line, sizeof(line), f))
    692             break;
    693         p = line;
    694         while (isspace(*p))
    695             p++;
    696         if (*p == '#')
    697             continue;
    698 
    699         p = get_field(line, 0);
    700         if (!p)
    701             continue;
    702         code = strtoul(p, NULL, 16);
    703         assert(code <= CHARCODE_MAX);
    704         ci = &tab[code];
    705 
    706         p = get_field(line, 1);
    707         if (!p)
    708             continue;
    709         /* locale dependent casing */
    710         while (isspace(*p))
    711             p++;
    712         status = *p;
    713         if (status != 'C' && status != 'S' && status != 'F')
    714             continue;
    715 
    716         p = get_field(line, 2);
    717         assert(p != NULL);
    718         if (status == 'S') {
    719             /* we always select the simple case folding and assume it
    720              * comes after the full case folding case */
    721             assert(ci->f_len >= 2);
    722             ci->f_len = 0;
    723         } else {
    724             assert(ci->f_len == 0);
    725         }
    726         for(;;) {
    727             while (isspace(*p))
    728                 p++;
    729             if (*p == ';')
    730                 break;
    731             assert(ci->l_len < CC_LEN_MAX);
    732             ci->f_data[ci->f_len++] = strtoul(p, (char **)&p, 16);
    733         }
    734     }
    735 
    736     fclose(f);
    737 }
    738 
    739 void parse_composition_exclusions(const char *filename)
    740 {
    741     FILE *f;
    742     char line[4096], *p;
    743     uint32_t c0;
    744 
    745     f = fopen(filename, "rb");
    746     if (!f) {
    747         perror(filename);
    748         exit(1);
    749     }
    750 
    751     for(;;) {
    752         if (!get_line(line, sizeof(line), f))
    753             break;
    754         p = line;
    755         while (isspace(*p))
    756             p++;
    757         if (*p == '#' || *p == '@' || *p == '\0')
    758             continue;
    759         c0 = strtoul(p, (char **)&p, 16);
    760         assert(c0 > 0 && c0 <= CHARCODE_MAX);
    761         unicode_db[c0].is_excluded = TRUE;
    762     }
    763     fclose(f);
    764 }
    765 
    766 void parse_derived_core_properties(const char *filename)
    767 {
    768     FILE *f;
    769     char line[4096], *p, buf[256], *q;
    770     uint32_t c0, c1, c;
    771     int i;
    772 
    773     f = fopen(filename, "rb");
    774     if (!f) {
    775         perror(filename);
    776         exit(1);
    777     }
    778 
    779     for(;;) {
    780         if (!get_line(line, sizeof(line), f))
    781             break;
    782         p = line;
    783         while (isspace(*p))
    784             p++;
    785         if (*p == '#' || *p == '@' || *p == '\0')
    786             continue;
    787         c0 = strtoul(p, (char **)&p, 16);
    788         if (*p == '.' && p[1] == '.') {
    789             p += 2;
    790             c1 = strtoul(p, (char **)&p, 16);
    791         } else {
    792             c1 = c0;
    793         }
    794         assert(c1 <= CHARCODE_MAX);
    795         p += strspn(p, " \t");
    796         if (*p == ';') {
    797             p++;
    798             p += strspn(p, " \t");
    799             q = buf;
    800             while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t' && *p != ';') {
    801                 if ((q - buf) < sizeof(buf) - 1)
    802                     *q++ = *p;
    803                 p++;
    804             }
    805             *q = '\0';
    806             i = find_name(unicode_prop_name,
    807                           countof(unicode_prop_name), buf);
    808             if (i < 0) {
    809                 if (!strcmp(buf, "Grapheme_Link"))
    810                     goto next;
    811                 fprintf(stderr, "Property not found: %s\n", buf);
    812                 exit(1);
    813             }
    814             for(c = c0; c <= c1; c++) {
    815                 set_prop(c, i, 1);
    816             }
    817 next: ;
    818         }
    819     }
    820     fclose(f);
    821 }
    822 
    823 void parse_derived_norm_properties(const char *filename)
    824 {
    825     FILE *f;
    826     char line[4096], *p, buf[256], *q;
    827     uint32_t c0, c1, c;
    828 
    829     f = fopen(filename, "rb");
    830     if (!f) {
    831         perror(filename);
    832         exit(1);
    833     }
    834 
    835     for(;;) {
    836         if (!get_line(line, sizeof(line), f))
    837             break;
    838         p = line;
    839         while (isspace(*p))
    840             p++;
    841         if (*p == '#' || *p == '@' || *p == '\0')
    842             continue;
    843         c0 = strtoul(p, (char **)&p, 16);
    844         if (*p == '.' && p[1] == '.') {
    845             p += 2;
    846             c1 = strtoul(p, (char **)&p, 16);
    847         } else {
    848             c1 = c0;
    849         }
    850         assert(c1 <= CHARCODE_MAX);
    851         p += strspn(p, " \t");
    852         if (*p == ';') {
    853             p++;
    854             p += strspn(p, " \t");
    855             q = buf;
    856             while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
    857                 if ((q - buf) < sizeof(buf) - 1)
    858                     *q++ = *p;
    859                 p++;
    860             }
    861             *q = '\0';
    862             if (!strcmp(buf, "Changes_When_NFKC_Casefolded")) {
    863                 for(c = c0; c <= c1; c++) {
    864                     set_prop(c, PROP_Changes_When_NFKC_Casefolded, 1);
    865                 }
    866             }
    867         }
    868     }
    869     fclose(f);
    870 }
    871 
    872 void parse_prop_list(const char *filename)
    873 {
    874     FILE *f;
    875     char line[4096], *p, buf[256], *q;
    876     uint32_t c0, c1, c;
    877     int i;
    878 
    879     f = fopen(filename, "rb");
    880     if (!f) {
    881         perror(filename);
    882         exit(1);
    883     }
    884 
    885     for(;;) {
    886         if (!get_line(line, sizeof(line), f))
    887             break;
    888         p = line;
    889         while (isspace(*p))
    890             p++;
    891         if (*p == '#' || *p == '@' || *p == '\0')
    892             continue;
    893         c0 = strtoul(p, (char **)&p, 16);
    894         if (*p == '.' && p[1] == '.') {
    895             p += 2;
    896             c1 = strtoul(p, (char **)&p, 16);
    897         } else {
    898             c1 = c0;
    899         }
    900         assert(c1 <= CHARCODE_MAX);
    901         p += strspn(p, " \t");
    902         if (*p == ';') {
    903             p++;
    904             p += strspn(p, " \t");
    905             q = buf;
    906             while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
    907                 if ((q - buf) < sizeof(buf) - 1)
    908                     *q++ = *p;
    909                 p++;
    910             }
    911             *q = '\0';
    912             i = find_name(unicode_prop_name,
    913                           countof(unicode_prop_name), buf);
    914             if (i < 0) {
    915                 fprintf(stderr, "Property not found: %s\n", buf);
    916                 exit(1);
    917             }
    918             for(c = c0; c <= c1; c++) {
    919                 set_prop(c, i, 1);
    920             }
    921         }
    922     }
    923     fclose(f);
    924 }
    925 
    926 #define SEQ_MAX_LEN 16
    927 
    928 static BOOL is_emoji_modifier(uint32_t c)
    929 {
    930     return (c >= 0x1f3fb && c <= 0x1f3ff);
    931 }
    932 
    933 static void add_sequence_prop(int idx, int seq_len, int *seq)
    934 {
    935     int i;
    936     
    937     assert(idx < SEQUENCE_PROP_COUNT);
    938     switch(idx) {
    939     case SEQUENCE_PROP_Basic_Emoji:
    940         /* convert to 2 properties lists */
    941         if (seq_len == 1) {
    942             set_prop(seq[0], PROP_Basic_Emoji1, 1);
    943         } else if (seq_len == 2 && seq[1] == 0xfe0f) {
    944             set_prop(seq[0], PROP_Basic_Emoji2, 1);
    945         } else {
    946             abort();
    947         }
    948         break;
    949     case SEQUENCE_PROP_RGI_Emoji_Modifier_Sequence:
    950         assert(seq_len == 2);
    951         assert(is_emoji_modifier(seq[1]));
    952         assert(get_prop(seq[0], PROP_Emoji_Modifier_Base));
    953         set_prop(seq[0], PROP_RGI_Emoji_Modifier_Sequence, 1);
    954         break;
    955     case SEQUENCE_PROP_RGI_Emoji_Flag_Sequence:
    956         {
    957             int code;
    958             assert(seq_len == 2);
    959             assert(seq[0] >= 0x1F1E6 && seq[0] <= 0x1F1FF);
    960             assert(seq[1] >= 0x1F1E6 && seq[1] <= 0x1F1FF);
    961             code = (seq[0] - 0x1F1E6) * 26 + (seq[1] - 0x1F1E6);
    962             /* XXX: would be more compact with a simple bitmap -> 676 bits */
    963             set_prop(code, PROP_RGI_Emoji_Flag_Sequence, 1);
    964         }
    965         break;
    966     case SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence:
    967         re_string_add(&rgi_emoji_zwj_sequence, seq_len, (uint32_t *)seq);
    968         break;
    969     case SEQUENCE_PROP_RGI_Emoji_Tag_Sequence:
    970         {
    971             assert(seq_len >= 3);
    972             assert(seq[0] == 0x1F3F4);
    973             assert(seq[seq_len - 1] == 0xE007F);
    974             for(i = 1; i < seq_len - 1; i++) {
    975                 assert(seq[i] >= 0xe0001 && seq[i] <= 0xe007e);
    976                 dbuf_putc(&rgi_emoji_tag_sequence, seq[i] - 0xe0000);
    977             }
    978             dbuf_putc(&rgi_emoji_tag_sequence, 0);
    979         }
    980         break;
    981     case SEQUENCE_PROP_Emoji_Keycap_Sequence:
    982         assert(seq_len == 3);
    983         assert(seq[1] == 0xfe0f);
    984         assert(seq[2] == 0x20e3);
    985         set_prop(seq[0], PROP_Emoji_Keycap_Sequence, 1);
    986         break;
    987     default:
    988         assert(0);
    989     }
    990 }
    991 
    992 void parse_sequence_prop_list(const char *filename)
    993 {
    994     FILE *f;
    995     char line[4096], *p, buf[256], *q, *p_start;
    996     uint32_t c0, c1, c;
    997     int idx, seq_len;
    998     int seq[SEQ_MAX_LEN];
    999     
   1000     f = fopen(filename, "rb");
   1001     if (!f) {
   1002         perror(filename);
   1003         exit(1);
   1004     }
   1005 
   1006     for(;;) {
   1007         if (!get_line(line, sizeof(line), f))
   1008             break;
   1009         p = line;
   1010         while (isspace(*p))
   1011             p++;
   1012         if (*p == '#' || *p == '@' || *p == '\0')
   1013             continue;
   1014         p_start = p;
   1015 
   1016         /* find the sequence property name */
   1017         p = strchr(p, ';');
   1018         if (!p)
   1019             continue;
   1020         p++;
   1021         p += strspn(p, " \t");
   1022         q = buf;
   1023         while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t' && *p != ';') {
   1024             if ((q - buf) < sizeof(buf) - 1)
   1025                 *q++ = *p;
   1026             p++;
   1027         }
   1028         *q = '\0';
   1029         idx = find_name(unicode_sequence_prop_name,
   1030                       countof(unicode_sequence_prop_name), buf);
   1031         if (idx < 0) {
   1032             fprintf(stderr, "Property not found: %s\n", buf);
   1033             exit(1);
   1034         }
   1035         
   1036         p = p_start;
   1037         c0 = strtoul(p, (char **)&p, 16);
   1038         assert(c0 <= CHARCODE_MAX);
   1039         
   1040         if (*p == '.' && p[1] == '.') {
   1041             p += 2;
   1042             c1 = strtoul(p, (char **)&p, 16);
   1043             assert(c1 <= CHARCODE_MAX);
   1044             for(c = c0; c <= c1; c++) {
   1045                 seq[0] = c;
   1046                 add_sequence_prop(idx, 1, seq);
   1047             }
   1048         } else {
   1049             seq_len = 0;
   1050             seq[seq_len++] = c0;
   1051             for(;;) {
   1052                 while (isspace(*p))
   1053                     p++;
   1054                 if (*p == ';' || *p == '\0')
   1055                     break;
   1056                 c0 = strtoul(p, (char **)&p, 16);
   1057                 assert(c0 <= CHARCODE_MAX);
   1058                 assert(seq_len < countof(seq));
   1059                 seq[seq_len++] = c0;
   1060             }
   1061             add_sequence_prop(idx, seq_len, seq);
   1062         }
   1063     }
   1064     fclose(f);
   1065 }
   1066 
   1067 void parse_scripts(const char *filename)
   1068 {
   1069     FILE *f;
   1070     char line[4096], *p, buf[256], *q;
   1071     uint32_t c0, c1, c;
   1072     int i;
   1073 
   1074     f = fopen(filename, "rb");
   1075     if (!f) {
   1076         perror(filename);
   1077         exit(1);
   1078     }
   1079 
   1080     for(;;) {
   1081         if (!get_line(line, sizeof(line), f))
   1082             break;
   1083         p = line;
   1084         while (isspace(*p))
   1085             p++;
   1086         if (*p == '#' || *p == '@' || *p == '\0')
   1087             continue;
   1088         c0 = strtoul(p, (char **)&p, 16);
   1089         if (*p == '.' && p[1] == '.') {
   1090             p += 2;
   1091             c1 = strtoul(p, (char **)&p, 16);
   1092         } else {
   1093             c1 = c0;
   1094         }
   1095         assert(c1 <= CHARCODE_MAX);
   1096         p += strspn(p, " \t");
   1097         if (*p == ';') {
   1098             p++;
   1099             p += strspn(p, " \t");
   1100             q = buf;
   1101             while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
   1102                 if ((q - buf) < sizeof(buf) - 1)
   1103                     *q++ = *p;
   1104                 p++;
   1105             }
   1106             *q = '\0';
   1107             i = find_name(unicode_script_name,
   1108                           countof(unicode_script_name), buf);
   1109             if (i < 0) {
   1110                 fprintf(stderr, "Unknown script: '%s'\n", buf);
   1111                 exit(1);
   1112             }
   1113             for(c = c0; c <= c1; c++)
   1114                 unicode_db[c].script = i;
   1115         }
   1116     }
   1117     fclose(f);
   1118 }
   1119 
   1120 void parse_script_extensions(const char *filename)
   1121 {
   1122     FILE *f;
   1123     char line[4096], *p, buf[256], *q;
   1124     uint32_t c0, c1, c;
   1125     int i;
   1126     uint8_t script_ext[255];
   1127     int script_ext_len;
   1128 
   1129     f = fopen(filename, "rb");
   1130     if (!f) {
   1131         perror(filename);
   1132         exit(1);
   1133     }
   1134 
   1135     for(;;) {
   1136         if (!get_line(line, sizeof(line), f))
   1137             break;
   1138         p = line;
   1139         while (isspace(*p))
   1140             p++;
   1141         if (*p == '#' || *p == '@' || *p == '\0')
   1142             continue;
   1143         c0 = strtoul(p, (char **)&p, 16);
   1144         if (*p == '.' && p[1] == '.') {
   1145             p += 2;
   1146             c1 = strtoul(p, (char **)&p, 16);
   1147         } else {
   1148             c1 = c0;
   1149         }
   1150         assert(c1 <= CHARCODE_MAX);
   1151         p += strspn(p, " \t");
   1152         script_ext_len = 0;
   1153         if (*p == ';') {
   1154             p++;
   1155             for(;;) {
   1156                 p += strspn(p, " \t");
   1157                 q = buf;
   1158                 while (*p != '\0' && *p != ' ' && *p != '#' && *p != '\t') {
   1159                     if ((q - buf) < sizeof(buf) - 1)
   1160                         *q++ = *p;
   1161                     p++;
   1162                 }
   1163                 *q = '\0';
   1164                 if (buf[0] == '\0')
   1165                     break;
   1166                 i = find_name(unicode_script_short_name,
   1167                               countof(unicode_script_short_name), buf);
   1168                 if (i < 0) {
   1169                     fprintf(stderr, "Script not found: %s\n", buf);
   1170                     exit(1);
   1171                 }
   1172                 assert(script_ext_len < sizeof(script_ext));
   1173                 script_ext[script_ext_len++] = i;
   1174             }
   1175             for(c = c0; c <= c1; c++) {
   1176                 CCInfo *ci = &unicode_db[c];
   1177                 ci->script_ext_len = script_ext_len;
   1178                 ci->script_ext = malloc(sizeof(ci->script_ext[0]) * script_ext_len);
   1179                 for(i = 0; i < script_ext_len; i++)
   1180                     ci->script_ext[i] = script_ext[i];
   1181             }
   1182         }
   1183     }
   1184     fclose(f);
   1185 }
   1186 
   1187 void dump_cc_info(CCInfo *ci, int i)
   1188 {
   1189     int j;
   1190     printf("%05x:", i);
   1191     if (ci->u_len != 0) {
   1192         printf(" U:");
   1193         for(j = 0; j < ci->u_len; j++)
   1194             printf(" %05x", ci->u_data[j]);
   1195     }
   1196     if (ci->l_len != 0) {
   1197         printf(" L:");
   1198         for(j = 0; j < ci->l_len; j++)
   1199             printf(" %05x", ci->l_data[j]);
   1200     }
   1201     if (ci->f_len != 0) {
   1202         printf(" F:");
   1203         for(j = 0; j < ci->f_len; j++)
   1204             printf(" %05x", ci->f_data[j]);
   1205     }
   1206     printf("\n");
   1207 }
   1208 
   1209 void dump_unicode_data(CCInfo *tab)
   1210 {
   1211     int i;
   1212     CCInfo *ci;
   1213     for(i = 0; i <= CHARCODE_MAX; i++) {
   1214         ci = &tab[i];
   1215         if (ci->u_len != 0 || ci->l_len != 0 || ci->f_len != 0) {
   1216             dump_cc_info(ci, i);
   1217         }
   1218     }
   1219 }
   1220 
   1221 BOOL is_complicated_case(const CCInfo *ci)
   1222 {
   1223     return (ci->u_len > 1 || ci->l_len > 1 ||
   1224             (ci->u_len > 0 && ci->l_len > 0) ||
   1225             (ci->f_len != ci->l_len) ||
   1226             (memcmp(ci->f_data, ci->l_data, ci->f_len * sizeof(ci->f_data[0])) != 0));
   1227 }
   1228 
   1229 #ifndef USE_TEST
   1230 enum {
   1231     RUN_TYPE_U,
   1232     RUN_TYPE_L,
   1233     RUN_TYPE_UF,
   1234     RUN_TYPE_LF,
   1235     RUN_TYPE_UL,
   1236     RUN_TYPE_LSU,
   1237     RUN_TYPE_U2L_399_EXT2,
   1238     RUN_TYPE_UF_D20,
   1239     RUN_TYPE_UF_D1_EXT,
   1240     RUN_TYPE_U_EXT,
   1241     RUN_TYPE_LF_EXT,
   1242     RUN_TYPE_UF_EXT2,
   1243     RUN_TYPE_LF_EXT2,
   1244     RUN_TYPE_UF_EXT3,
   1245 };
   1246 #endif
   1247 
   1248 const char *run_type_str[] = {
   1249     "U",
   1250     "L",
   1251     "UF",
   1252     "LF",
   1253     "UL",
   1254     "LSU",
   1255     "U2L_399_EXT2",
   1256     "UF_D20",
   1257     "UF_D1_EXT",
   1258     "U_EXT",
   1259     "LF_EXT",
   1260     "UF_EXT2",
   1261     "LF_EXT2",
   1262     "UF_EXT3",
   1263 };
   1264 
   1265 typedef struct {
   1266     int code;
   1267     int len;
   1268     int type;
   1269     int data;
   1270     int ext_len;
   1271     int ext_data[3];
   1272     int data_index; /* 'data' coming from the table */
   1273 } TableEntry;
   1274 
   1275 static int simple_to_lower(CCInfo *tab, int c)
   1276 {
   1277     if (tab[c].l_len != 1)
   1278         return c;
   1279     return tab[c].l_data[0];
   1280 }
   1281 
   1282 /* code (17), len (7), type (4) */
   1283 
   1284 void find_run_type(TableEntry *te, CCInfo *tab, int code)
   1285 {
   1286     int is_lower, len;
   1287     CCInfo *ci, *ci1, *ci2;
   1288 
   1289     ci = &tab[code];
   1290     ci1 = &tab[code + 1];
   1291     ci2 = &tab[code + 2];
   1292     te->code = code;
   1293 
   1294     if (ci->l_len == 1 && ci->l_data[0] == code + 2 &&
   1295         ci->f_len == 1 && ci->f_data[0] == ci->l_data[0] &&
   1296         ci->u_len == 0 &&
   1297 
   1298         ci1->l_len == 1 && ci1->l_data[0] == code + 2 &&
   1299         ci1->f_len == 1 && ci1->f_data[0] == ci1->l_data[0] &&
   1300         ci1->u_len == 1 && ci1->u_data[0] == code &&
   1301 
   1302         ci2->l_len == 0 &&
   1303         ci2->f_len == 0 &&
   1304         ci2->u_len == 1 && ci2->u_data[0] == code) {
   1305         te->len = 3;
   1306         te->data = 0;
   1307         te->type = RUN_TYPE_LSU;
   1308         return;
   1309     }
   1310 
   1311     if (is_complicated_case(ci)) {
   1312         len = 1;
   1313         while (code + len <= CHARCODE_MAX) {
   1314             ci1 = &tab[code + len];
   1315             if (ci1->u_len != 1 ||
   1316                 ci1->u_data[0] != ci->u_data[0] + len ||
   1317                 ci1->l_len != 0 ||
   1318                 ci1->f_len != 1 || ci1->f_data[0] != ci1->u_data[0])
   1319                 break;
   1320             len++;
   1321         }
   1322         if (len > 1) {
   1323             te->len = len;
   1324             te->type = RUN_TYPE_UF;
   1325             te->data = ci->u_data[0];
   1326             return;
   1327         }
   1328 
   1329         if (ci->l_len == 0 &&
   1330             ci->u_len == 2 && ci->u_data[1] == 0x399 &&
   1331             ci->f_len == 2 && ci->f_data[1] == 0x3B9 &&
   1332             ci->f_data[0] == simple_to_lower(tab, ci->u_data[0])) {
   1333             len = 1;
   1334             while (code + len <= CHARCODE_MAX) {
   1335                 ci1 = &tab[code + len];
   1336                 if (!(ci1->u_len == 2 &&
   1337                       ci1->u_data[1] == ci->u_data[1] &&
   1338                       ci1->u_data[0] == ci->u_data[0] + len &&
   1339                       ci1->f_len == 2 &&
   1340                       ci1->f_data[1] == ci->f_data[1] &&
   1341                       ci1->f_data[0] == ci->f_data[0] + len &&
   1342                       ci1->l_len == 0))
   1343                     break;
   1344                 len++;
   1345             }
   1346             te->len = len;
   1347             te->type = RUN_TYPE_UF_EXT2;
   1348             te->ext_data[0] = ci->u_data[0];
   1349             te->ext_data[1] = ci->u_data[1];
   1350             te->ext_len = 2;
   1351             return;
   1352         }
   1353 
   1354         if (ci->u_len == 2 && ci->u_data[1] == 0x399 &&
   1355             ci->l_len == 1 &&
   1356             ci->f_len == 1 && ci->f_data[0] == ci->l_data[0]) {
   1357             len = 1;
   1358             while (code + len <= CHARCODE_MAX) {
   1359                 ci1 = &tab[code + len];
   1360                 if (!(ci1->u_len == 2 &&
   1361                       ci1->u_data[1] == 0x399 &&
   1362                       ci1->u_data[0] == ci->u_data[0] + len &&
   1363                       ci1->l_len == 1 &&
   1364                       ci1->l_data[0] == ci->l_data[0] + len &&
   1365                       ci1->f_len == 1 && ci1->f_data[0] == ci1->l_data[0]))
   1366                     break;
   1367                 len++;
   1368             }
   1369             te->len = len;
   1370             te->type = RUN_TYPE_U2L_399_EXT2;
   1371             te->ext_data[0] = ci->u_data[0];
   1372             te->ext_data[1] = ci->l_data[0];
   1373             te->ext_len = 2;
   1374             return;
   1375         }
   1376 
   1377         if (ci->l_len == 1 && ci->u_len == 0 && ci->f_len == 0) {
   1378             len = 1;
   1379             while (code + len <= CHARCODE_MAX) {
   1380                 ci1 = &tab[code + len];
   1381                 if (!(ci1->l_len == 1 &&
   1382                       ci1->l_data[0] == ci->l_data[0] + len &&
   1383                       ci1->u_len == 0 && ci1->f_len == 0))
   1384                     break;
   1385                 len++;
   1386             }
   1387             te->len = len;
   1388             te->type = RUN_TYPE_L;
   1389             te->data = ci->l_data[0];
   1390             return;
   1391         }
   1392 
   1393         if (ci->l_len == 0 &&
   1394             ci->u_len == 1 &&
   1395             ci->u_data[0] < 0x1000 &&
   1396             ci->f_len == 1 && ci->f_data[0] == ci->u_data[0] + 0x20) {
   1397             te->len = 1;
   1398             te->type = RUN_TYPE_UF_D20;
   1399             te->data = ci->u_data[0];
   1400         } else if (ci->l_len == 0 &&
   1401                    ci->u_len == 1 &&
   1402                    ci->f_len == 1 && ci->f_data[0] == ci->u_data[0] + 1) {
   1403             te->len = 1;
   1404             te->type = RUN_TYPE_UF_D1_EXT;
   1405             te->ext_data[0] = ci->u_data[0];
   1406             te->ext_len = 1;
   1407         } else if (ci->l_len == 2 && ci->u_len == 0 && ci->f_len == 2 &&
   1408                    ci->l_data[0] == ci->f_data[0] &&
   1409                    ci->l_data[1] == ci->f_data[1]) {
   1410             te->len = 1;
   1411             te->type = RUN_TYPE_LF_EXT2;
   1412             te->ext_data[0] = ci->l_data[0];
   1413             te->ext_data[1] = ci->l_data[1];
   1414             te->ext_len = 2;
   1415         } else if (ci->u_len == 2 && ci->l_len == 0 && ci->f_len == 2 &&
   1416                    ci->f_data[0] == simple_to_lower(tab, ci->u_data[0]) &&
   1417                    ci->f_data[1] == simple_to_lower(tab, ci->u_data[1])) {
   1418             te->len = 1;
   1419             te->type = RUN_TYPE_UF_EXT2;
   1420             te->ext_data[0] = ci->u_data[0];
   1421             te->ext_data[1] = ci->u_data[1];
   1422             te->ext_len = 2;
   1423         } else if (ci->u_len == 3 && ci->l_len == 0 && ci->f_len == 3 &&
   1424                    ci->f_data[0] == simple_to_lower(tab, ci->u_data[0]) &&
   1425                    ci->f_data[1] == simple_to_lower(tab, ci->u_data[1]) &&
   1426                    ci->f_data[2] == simple_to_lower(tab, ci->u_data[2])) {
   1427             te->len = 1;
   1428             te->type = RUN_TYPE_UF_EXT3;
   1429             te->ext_data[0] = ci->u_data[0];
   1430             te->ext_data[1] = ci->u_data[1];
   1431             te->ext_data[2] = ci->u_data[2];
   1432             te->ext_len = 3;
   1433         } else if (ci->u_len == 2 && ci->l_len == 0 && ci->f_len == 1) {
   1434             // U+FB05 LATIN SMALL LIGATURE LONG S T
   1435             assert(code == 0xFB05);
   1436             te->len = 1;
   1437             te->type = RUN_TYPE_UF_EXT2;
   1438             te->ext_data[0] = ci->u_data[0];
   1439             te->ext_data[1] = ci->u_data[1];
   1440             te->ext_len = 2;
   1441         } else if (ci->u_len == 3 && ci->l_len == 0 && ci->f_len == 1) {
   1442             // U+1FD3 GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA or
   1443             // U+1FE3 GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
   1444             assert(code == 0x1FD3 || code == 0x1FE3);
   1445             te->len = 1;
   1446             te->type = RUN_TYPE_UF_EXT3;
   1447             te->ext_data[0] = ci->u_data[0];
   1448             te->ext_data[1] = ci->u_data[1];
   1449             te->ext_data[2] = ci->u_data[2];
   1450             te->ext_len = 3;
   1451         } else {
   1452             printf("unsupported encoding case:\n");
   1453             dump_cc_info(ci, code);
   1454             abort();
   1455         }
   1456     } else {
   1457         /* look for a run of identical conversions */
   1458         len = 0;
   1459         for(;;) {
   1460             if (code >= CHARCODE_MAX || len >= 126)
   1461                 break;
   1462             ci = &tab[code + len];
   1463             ci1 = &tab[code + len + 1];
   1464             if (is_complicated_case(ci) || is_complicated_case(ci1)) {
   1465                 break;
   1466             }
   1467             if (ci->l_len != 1 || ci->l_data[0] != code + len + 1)
   1468                 break;
   1469             if (ci1->u_len != 1 || ci1->u_data[0] != code + len)
   1470                 break;
   1471             len += 2;
   1472         }
   1473         if (len > 0) {
   1474             te->len = len;
   1475             te->type = RUN_TYPE_UL;
   1476             te->data = 0;
   1477             return;
   1478         }
   1479 
   1480         ci = &tab[code];
   1481         is_lower = ci->l_len > 0;
   1482         len = 1;
   1483         while (code + len <= CHARCODE_MAX) {
   1484             ci1 = &tab[code + len];
   1485             if (is_complicated_case(ci1))
   1486                 break;
   1487             if (is_lower) {
   1488                 if (ci1->l_len != 1 ||
   1489                     ci1->l_data[0] != ci->l_data[0] + len)
   1490                     break;
   1491             } else {
   1492                 if (ci1->u_len != 1 ||
   1493                     ci1->u_data[0] != ci->u_data[0] + len)
   1494                     break;
   1495             }
   1496             len++;
   1497         }
   1498         te->len = len;
   1499         if (is_lower) {
   1500             te->type = RUN_TYPE_LF;
   1501             te->data = ci->l_data[0];
   1502         } else {
   1503             te->type = RUN_TYPE_U;
   1504             te->data = ci->u_data[0];
   1505         }
   1506     }
   1507 }
   1508 
   1509 TableEntry conv_table[1000];
   1510 int conv_table_len;
   1511 int ext_data[1000];
   1512 int ext_data_len;
   1513 
   1514 void dump_case_conv_table1(void)
   1515 {
   1516     int i, j;
   1517     const TableEntry *te;
   1518 
   1519     for(i = 0; i < conv_table_len; i++) {
   1520         te = &conv_table[i];
   1521         printf("%05x %02x %-10s %05x",
   1522                te->code, te->len, run_type_str[te->type], te->data);
   1523         for(j = 0; j < te->ext_len; j++) {
   1524             printf(" %05x", te->ext_data[j]);
   1525         }
   1526         printf("\n");
   1527     }
   1528     printf("table_len=%d ext_len=%d\n", conv_table_len, ext_data_len);
   1529 }
   1530 
   1531 int find_data_index(const TableEntry *conv_table, int len, int data)
   1532 {
   1533     int i;
   1534     const TableEntry *te;
   1535     for(i = 0; i < len; i++) {
   1536         te = &conv_table[i];
   1537         if (te->code == data)
   1538             return i;
   1539     }
   1540     return -1;
   1541 }
   1542 
   1543 int find_ext_data_index(int data)
   1544 {
   1545     int i;
   1546     for(i = 0; i < ext_data_len; i++) {
   1547         if (ext_data[i] == data)
   1548             return i;
   1549     }
   1550     assert(ext_data_len < countof(ext_data));
   1551     ext_data[ext_data_len++] = data;
   1552     return ext_data_len - 1;
   1553 }
   1554 
   1555 void build_conv_table(CCInfo *tab)
   1556 {
   1557     int code, i, j;
   1558     CCInfo *ci;
   1559     TableEntry *te;
   1560 
   1561     te = conv_table;
   1562     for(code = 0; code <= CHARCODE_MAX; code++) {
   1563         ci = &tab[code];
   1564         if (ci->u_len == 0 && ci->l_len == 0 && ci->f_len == 0)
   1565             continue;
   1566         assert(te - conv_table < countof(conv_table));
   1567         find_run_type(te, tab, code);
   1568 #if 0
   1569         if (te->type == RUN_TYPE_TODO) {
   1570             printf("TODO: ");
   1571             dump_cc_info(ci, code);
   1572         }
   1573 #endif
   1574         assert(te->len <= 127);
   1575         code += te->len - 1;
   1576         te++;
   1577     }
   1578     conv_table_len = te - conv_table;
   1579 
   1580     /* find the data index */
   1581     for(i = 0; i < conv_table_len; i++) {
   1582         int data_index;
   1583         te = &conv_table[i];
   1584 
   1585         switch(te->type) {
   1586         case RUN_TYPE_U:
   1587         case RUN_TYPE_L:
   1588         case RUN_TYPE_UF:
   1589         case RUN_TYPE_LF:
   1590             data_index = find_data_index(conv_table, conv_table_len, te->data);
   1591             if (data_index < 0) {
   1592                 switch(te->type) {
   1593                 case RUN_TYPE_U:
   1594                     te->type = RUN_TYPE_U_EXT;
   1595                     te->ext_len = 1;
   1596                     te->ext_data[0] = te->data;
   1597                     break;
   1598                 case RUN_TYPE_LF:
   1599                     te->type = RUN_TYPE_LF_EXT;
   1600                     te->ext_len = 1;
   1601                     te->ext_data[0] = te->data;
   1602                     break;
   1603                 default:
   1604                     printf("%05x: index not found\n", te->code);
   1605                     exit(1);
   1606                 }
   1607             } else {
   1608                 te->data_index = data_index;
   1609             }
   1610             break;
   1611         case RUN_TYPE_UF_D20:
   1612             te->data_index = te->data;
   1613             break;
   1614         }
   1615     }
   1616 
   1617     /* find the data index for ext_data */
   1618     for(i = 0; i < conv_table_len; i++) {
   1619         te = &conv_table[i];
   1620         if (te->type == RUN_TYPE_UF_EXT3) {
   1621             int p, v;
   1622             v = 0;
   1623             for(j = 0; j < 3; j++) {
   1624                 p = find_ext_data_index(te->ext_data[j]);
   1625                 assert(p < 16);
   1626                 v = (v << 4) | p;
   1627             }
   1628             te->data_index = v;
   1629         }
   1630     }
   1631 
   1632     for(i = 0; i < conv_table_len; i++) {
   1633         te = &conv_table[i];
   1634         if (te->type == RUN_TYPE_LF_EXT2 ||
   1635             te->type == RUN_TYPE_UF_EXT2 ||
   1636             te->type == RUN_TYPE_U2L_399_EXT2) {
   1637             int p, v;
   1638             v = 0;
   1639             for(j = 0; j < 2; j++) {
   1640                 p = find_ext_data_index(te->ext_data[j]);
   1641                 assert(p < 64);
   1642                 v = (v << 6) | p;
   1643             }
   1644             te->data_index = v;
   1645         }
   1646     }
   1647 
   1648     for(i = 0; i < conv_table_len; i++) {
   1649         te = &conv_table[i];
   1650         if (te->type == RUN_TYPE_UF_D1_EXT ||
   1651             te->type == RUN_TYPE_U_EXT ||
   1652             te->type == RUN_TYPE_LF_EXT) {
   1653             te->data_index = find_ext_data_index(te->ext_data[0]);
   1654         }
   1655     }
   1656 #ifdef DUMP_CASE_CONV_TABLE
   1657     dump_case_conv_table1();
   1658 #endif
   1659 }
   1660 
   1661 void dump_case_conv_table(FILE *f)
   1662 {
   1663     int i;
   1664     uint32_t v;
   1665     const TableEntry *te;
   1666 
   1667     total_tables++;
   1668     total_table_bytes += conv_table_len * sizeof(uint32_t);
   1669     fprintf(f, "static const uint32_t case_conv_table1[%d] = {", conv_table_len);
   1670     for(i = 0; i < conv_table_len; i++) {
   1671         if (i % 4 == 0)
   1672             fprintf(f, "\n   ");
   1673         te = &conv_table[i];
   1674         v = te->code << (32 - 17);
   1675         v |= te->len << (32 - 17 - 7);
   1676         v |= te->type << (32 - 17 - 7 - 4);
   1677         v |= te->data_index >> 8;
   1678         fprintf(f, " 0x%08x,", v);
   1679     }
   1680     fprintf(f, "\n};\n\n");
   1681 
   1682     total_tables++;
   1683     total_table_bytes += conv_table_len;
   1684     fprintf(f, "static const uint8_t case_conv_table2[%d] = {", conv_table_len);
   1685     for(i = 0; i < conv_table_len; i++) {
   1686         if (i % 8 == 0)
   1687             fprintf(f, "\n   ");
   1688         te = &conv_table[i];
   1689         fprintf(f, " 0x%02x,", te->data_index & 0xff);
   1690     }
   1691     fprintf(f, "\n};\n\n");
   1692 
   1693     total_tables++;
   1694     total_table_bytes += ext_data_len * sizeof(uint16_t);
   1695     fprintf(f, "static const uint16_t case_conv_ext[%d] = {", ext_data_len);
   1696     for(i = 0; i < ext_data_len; i++) {
   1697         if (i % 8 == 0)
   1698             fprintf(f, "\n   ");
   1699         fprintf(f, " 0x%04x,", ext_data[i]);
   1700     }
   1701     fprintf(f, "\n};\n\n");
   1702 }
   1703 
   1704 
   1705 static CCInfo *global_tab;
   1706 
   1707 static int sp_cc_cmp(const void *p1, const void *p2)
   1708 {
   1709     CCInfo *c1 = &global_tab[*(const int *)p1];
   1710     CCInfo *c2 = &global_tab[*(const int *)p2];
   1711     if (c1->f_len < c2->f_len) {
   1712         return -1;
   1713     } else if (c2->f_len < c1->f_len) {
   1714         return 1;
   1715     } else {
   1716         return memcmp(c1->f_data, c2->f_data, sizeof(c1->f_data[0]) * c1->f_len);
   1717     }
   1718 }
   1719 
   1720 /* dump the case special cases (multi character results which are
   1721    identical and need specific handling in lre_canonicalize() */
   1722 void dump_case_folding_special_cases(CCInfo *tab)
   1723 {
   1724     int i, len, j;
   1725     int *perm;
   1726 
   1727     perm = malloc(sizeof(perm[0]) * (CHARCODE_MAX + 1));
   1728     for(i = 0; i <= CHARCODE_MAX; i++)
   1729         perm[i] = i;
   1730     global_tab = tab;
   1731     qsort(perm, CHARCODE_MAX + 1, sizeof(perm[0]), sp_cc_cmp);
   1732     for(i = 0; i <= CHARCODE_MAX;) {
   1733         if (tab[perm[i]].f_len <= 1) {
   1734             i++;
   1735         } else {
   1736             len = 1;
   1737             while ((i + len) <= CHARCODE_MAX && !sp_cc_cmp(&perm[i], &perm[i + len]))
   1738                 len++;
   1739 
   1740             if (len > 1) {
   1741                 for(j = i; j < i + len; j++)
   1742                     dump_cc_info(&tab[perm[j]], perm[j]);
   1743             }
   1744             i += len;
   1745         }
   1746     }
   1747     free(perm);
   1748     global_tab = NULL;
   1749 }
   1750 
   1751 
   1752 int tabcmp(const int *tab1, const int *tab2, int n)
   1753 {
   1754     int i;
   1755     for(i = 0; i < n; i++) {
   1756         if (tab1[i] != tab2[i])
   1757             return -1;
   1758     }
   1759     return 0;
   1760 }
   1761 
   1762 void dump_str(const char *str, const int *buf, int len)
   1763 {
   1764     int i;
   1765     printf("%s=", str);
   1766     for(i = 0; i < len; i++)
   1767         printf(" %05x", buf[i]);
   1768     printf("\n");
   1769 }
   1770 
   1771 void compute_internal_props(void)
   1772 {
   1773     int i;
   1774     BOOL has_ul;
   1775 
   1776     for(i = 0; i <= CHARCODE_MAX; i++) {
   1777         CCInfo *ci = &unicode_db[i];
   1778         has_ul = (ci->u_len != 0 || ci->l_len != 0 || ci->f_len != 0);
   1779         if (has_ul) {
   1780             assert(get_prop(i, PROP_Cased));
   1781         } else {
   1782             set_prop(i, PROP_Cased1, get_prop(i, PROP_Cased));
   1783         }
   1784         set_prop(i, PROP_ID_Continue1,
   1785                  get_prop(i, PROP_ID_Continue) & (get_prop(i, PROP_ID_Start) ^ 1));
   1786         set_prop(i, PROP_XID_Start1,
   1787                  get_prop(i, PROP_ID_Start) ^ get_prop(i, PROP_XID_Start));
   1788         set_prop(i, PROP_XID_Continue1,
   1789                  get_prop(i, PROP_ID_Continue) ^ get_prop(i, PROP_XID_Continue));
   1790         set_prop(i, PROP_Changes_When_Titlecased1,
   1791                  get_prop(i, PROP_Changes_When_Titlecased) ^ (ci->u_len != 0));
   1792         set_prop(i, PROP_Changes_When_Casefolded1,
   1793                  get_prop(i, PROP_Changes_When_Casefolded) ^ (ci->f_len != 0));
   1794         /* XXX: reduce table size (438 bytes) */
   1795         set_prop(i, PROP_Changes_When_NFKC_Casefolded1,
   1796                  get_prop(i, PROP_Changes_When_NFKC_Casefolded) ^ (ci->f_len != 0));
   1797 #if 0
   1798         /* TEST */
   1799 #define M(x) (1U << GCAT_ ## x)
   1800         {
   1801             int b;
   1802             b = ((M(Mn) | M(Cf) | M(Lm) | M(Sk)) >>
   1803                  unicode_db[i].general_category) & 1;
   1804             set_prop(i, PROP_Cased1,
   1805                      get_prop(i, PROP_Case_Ignorable) ^ b);
   1806         }
   1807 #undef M
   1808 #endif
   1809     }
   1810 }
   1811 
   1812 void dump_byte_table(FILE *f, const char *cname, const uint8_t *tab, int len)
   1813 {
   1814     int i;
   1815 
   1816     total_tables++;
   1817     total_table_bytes += len;
   1818     fprintf(f, "static const uint8_t %s[%d] = {", cname, len);
   1819     for(i = 0; i < len; i++) {
   1820         if (i % 8 == 0)
   1821             fprintf(f, "\n   ");
   1822         fprintf(f, " 0x%02x,", tab[i]);
   1823     }
   1824     fprintf(f, "\n};\n\n");
   1825 }
   1826 
   1827 void dump_index_table(FILE *f, const char *cname, const uint8_t *tab, int len)
   1828 {
   1829     int i, code, offset;
   1830 
   1831     total_index++;
   1832     total_index_bytes += len;
   1833     fprintf(f, "static const uint8_t %s[%d] = {\n", cname, len);
   1834     for(i = 0; i < len; i += 3) {
   1835         code = tab[i] + (tab[i+1] << 8) + ((tab[i+2] & 0x1f) << 16);
   1836         offset = ((i / 3) + 1) * 32 + (tab[i+2] >> 5);
   1837         fprintf(f, "    0x%02x, 0x%02x, 0x%02x,", tab[i], tab[i+1], tab[i+2]);
   1838         fprintf(f, "  // %6.5X at %d%s\n", code, offset,
   1839                 i == len - 3 ? " (upper bound)" : "");
   1840     }
   1841     fprintf(f, "};\n\n");
   1842 }
   1843 
   1844 #define PROP_BLOCK_LEN 32
   1845 
   1846 void build_prop_table(FILE *f, const char *name, int prop_index, BOOL add_index)
   1847 {
   1848     int i, j, n, v, offset, code;
   1849     DynBuf dbuf_s, *dbuf = &dbuf_s;
   1850     DynBuf dbuf1_s, *dbuf1 = &dbuf1_s;
   1851     DynBuf dbuf2_s, *dbuf2 = &dbuf2_s;
   1852     const uint32_t *buf;
   1853     int buf_len, block_end_pos, bit;
   1854     char cname[128];
   1855 
   1856     dbuf_init(dbuf1);
   1857 
   1858     for(i = 0; i <= CHARCODE_MAX;) {
   1859         v = get_prop(i, prop_index);
   1860         j = i + 1;
   1861         while (j <= CHARCODE_MAX && get_prop(j, prop_index) == v) {
   1862             j++;
   1863         }
   1864         n = j - i;
   1865         if (j == (CHARCODE_MAX + 1) && v == 0)
   1866             break; /* no need to encode last zero run */
   1867         //printf("%05x: %d %d\n", i, n, v);
   1868         dbuf_put_u32(dbuf1, n - 1);
   1869         i += n;
   1870     }
   1871 
   1872     dbuf_init(dbuf);
   1873     dbuf_init(dbuf2);
   1874     buf = (uint32_t *)dbuf1->buf;
   1875     buf_len = dbuf1->size / sizeof(buf[0]);
   1876 
   1877     /* the first value is assumed to be 0 */
   1878     assert(get_prop(0, prop_index) == 0);
   1879 
   1880     block_end_pos = PROP_BLOCK_LEN;
   1881     i = 0;
   1882     code = 0;
   1883     bit = 0;
   1884     while (i < buf_len) {
   1885         if (add_index && dbuf->size >= block_end_pos && bit == 0) {
   1886             offset = (dbuf->size - block_end_pos);
   1887             /* XXX: offset could be larger in case of runs of small
   1888                lengths. Could add code to change the encoding to
   1889                prevent it at the expense of one byte loss */
   1890             assert(offset <= 7);
   1891             v = code | (offset << 21);
   1892             dbuf_putc(dbuf2, v);
   1893             dbuf_putc(dbuf2, v >> 8);
   1894             dbuf_putc(dbuf2, v >> 16);
   1895             block_end_pos += PROP_BLOCK_LEN;
   1896         }
   1897 
   1898         /* Compressed byte encoding:
   1899            00..3F: 2 packed lengths: 3-bit + 3-bit
   1900            40..5F: 5-bits plus extra byte for length
   1901            60..7F: 5-bits plus 2 extra bytes for length
   1902            80..FF: 7-bit length
   1903            lengths must be incremented to get character count
   1904            Ranges alternate between false and true return value.
   1905          */
   1906         v = buf[i];
   1907         code += v + 1;
   1908         bit ^= 1;
   1909         if (v < 8 && (i + 1) < buf_len && buf[i + 1] < 8) {
   1910             code += buf[i + 1] + 1;
   1911             bit ^= 1;
   1912             dbuf_putc(dbuf, (v << 3) | buf[i + 1]);
   1913             i += 2;
   1914         } else if (v < 128) {
   1915             dbuf_putc(dbuf, 0x80 + v);
   1916             i++;
   1917         } else if (v < (1 << 13)) {
   1918             dbuf_putc(dbuf, 0x40 + (v >> 8));
   1919             dbuf_putc(dbuf, v);
   1920             i++;
   1921         } else {
   1922             assert(v < (1 << 21));
   1923             dbuf_putc(dbuf, 0x60 + (v >> 16));
   1924             dbuf_putc(dbuf, v >> 8);
   1925             dbuf_putc(dbuf, v);
   1926             i++;
   1927         }
   1928     }
   1929 
   1930     if (add_index) {
   1931         /* last index entry */
   1932         v = code;
   1933         dbuf_putc(dbuf2, v);
   1934         dbuf_putc(dbuf2, v >> 8);
   1935         dbuf_putc(dbuf2, v >> 16);
   1936     }
   1937 
   1938 #ifdef DUMP_TABLE_SIZE
   1939     printf("prop %s: length=%d bytes\n", unicode_prop_name[prop_index],
   1940            (int)(dbuf->size + dbuf2->size));
   1941 #endif
   1942     snprintf(cname, sizeof(cname), "unicode_prop_%s_table", unicode_prop_name[prop_index]);
   1943     dump_byte_table(f, cname, dbuf->buf, dbuf->size);
   1944     if (add_index) {
   1945         snprintf(cname, sizeof(cname), "unicode_prop_%s_index", unicode_prop_name[prop_index]);
   1946         dump_index_table(f, cname, dbuf2->buf, dbuf2->size);
   1947     }
   1948 
   1949     dbuf_free(dbuf);
   1950     dbuf_free(dbuf1);
   1951     dbuf_free(dbuf2);
   1952 }
   1953 
   1954 void build_flags_tables(FILE *f)
   1955 {
   1956     build_prop_table(f, "Cased1", PROP_Cased1, TRUE);
   1957     build_prop_table(f, "Case_Ignorable", PROP_Case_Ignorable, TRUE);
   1958     build_prop_table(f, "ID_Start", PROP_ID_Start, TRUE);
   1959     build_prop_table(f, "ID_Continue1", PROP_ID_Continue1, TRUE);
   1960 }
   1961 
   1962 void dump_name_table(FILE *f, const char *cname, const char **tab_name, int len,
   1963                      const char **tab_short_name)
   1964 {
   1965     int i, w, maxw;
   1966 
   1967     maxw = 0;
   1968     for(i = 0; i < len; i++) {
   1969         w = strlen(tab_name[i]);
   1970         if (tab_short_name && tab_short_name[i][0] != '\0') {
   1971             w += 1 + strlen(tab_short_name[i]);
   1972         }
   1973         if (maxw < w)
   1974             maxw = w;
   1975     }
   1976 
   1977     /* generate a sequence of strings terminated by an empty string */
   1978     fprintf(f, "static const char %s[] =\n", cname);
   1979     for(i = 0; i < len; i++) {
   1980         fprintf(f, "    \"");
   1981         w = fprintf(f, "%s", tab_name[i]);
   1982         if (tab_short_name && tab_short_name[i][0] != '\0') {
   1983             w += fprintf(f, ",%s", tab_short_name[i]);
   1984         }
   1985         fprintf(f, "\"%*s\"\\0\"\n", 1 + maxw - w, "");
   1986     }
   1987     fprintf(f, ";\n\n");
   1988 }
   1989 
   1990 void build_general_category_table(FILE *f)
   1991 {
   1992     int i, v, j, n, n1;
   1993     DynBuf dbuf_s, *dbuf = &dbuf_s;
   1994 #ifdef DUMP_TABLE_SIZE
   1995     int cw_count, cw_len_count[4], cw_start;
   1996 #endif
   1997 
   1998     fprintf(f, "typedef enum {\n");
   1999     for(i = 0; i < GCAT_COUNT; i++)
   2000         fprintf(f, "    UNICODE_GC_%s,\n", unicode_gc_name[i]);
   2001     fprintf(f, "    UNICODE_GC_COUNT,\n");
   2002     fprintf(f, "} UnicodeGCEnum;\n\n");
   2003 
   2004     dump_name_table(f, "unicode_gc_name_table",
   2005                     unicode_gc_name, GCAT_COUNT,
   2006                     unicode_gc_short_name);
   2007 
   2008 
   2009     dbuf_init(dbuf);
   2010 #ifdef DUMP_TABLE_SIZE
   2011     cw_count = 0;
   2012     for(i = 0; i < 4; i++)
   2013         cw_len_count[i] = 0;
   2014 #endif
   2015     for(i = 0; i <= CHARCODE_MAX;) {
   2016         v = unicode_db[i].general_category;
   2017         j = i + 1;
   2018         while (j <= CHARCODE_MAX && unicode_db[j].general_category == v)
   2019             j++;
   2020         n = j - i;
   2021         /* compress Lu/Ll runs */
   2022         if (v == GCAT_Lu) {
   2023             n1 = 1;
   2024             while ((i + n1) <= CHARCODE_MAX && unicode_db[i + n1].general_category == (v + (n1 & 1))) {
   2025                 n1++;
   2026             }
   2027             if (n1 > n) {
   2028                 v = 31;
   2029                 n = n1;
   2030             }
   2031         }
   2032         //        printf("%05x %05x %d\n", i, n, v);
   2033         n--;
   2034 #ifdef DUMP_TABLE_SIZE
   2035         cw_count++;
   2036         cw_start = dbuf->size;
   2037 #endif
   2038         if (n < 7) {
   2039             dbuf_putc(dbuf, (n << 5) | v);
   2040         } else if (n < 7 + 128) {
   2041             n1 = n - 7;
   2042             assert(n1 < 128);
   2043             dbuf_putc(dbuf, (0xf << 5) | v);
   2044             dbuf_putc(dbuf, n1);
   2045         } else if (n < 7 + 128 + (1 << 14)) {
   2046             n1 = n - (7 + 128);
   2047             assert(n1 < (1 << 14));
   2048             dbuf_putc(dbuf, (0xf << 5) | v);
   2049             dbuf_putc(dbuf, (n1 >> 8) + 128);
   2050             dbuf_putc(dbuf, n1);
   2051         } else {
   2052             n1 = n - (7 + 128 + (1 << 14));
   2053             assert(n1 < (1 << 22));
   2054             dbuf_putc(dbuf, (0xf << 5) | v);
   2055             dbuf_putc(dbuf, (n1 >> 16) + 128 + 64);
   2056             dbuf_putc(dbuf, n1 >> 8);
   2057             dbuf_putc(dbuf, n1);
   2058         }
   2059 #ifdef DUMP_TABLE_SIZE
   2060         cw_len_count[dbuf->size - cw_start - 1]++;
   2061 #endif
   2062         i += n + 1;
   2063     }
   2064 #ifdef DUMP_TABLE_SIZE
   2065     printf("general category: %d entries [", cw_count);
   2066     for(i = 0; i < 4; i++)
   2067         printf(" %d", cw_len_count[i]);
   2068     printf(" ], length=%d bytes\n", (int)dbuf->size);
   2069 #endif
   2070 
   2071     dump_byte_table(f, "unicode_gc_table", dbuf->buf, dbuf->size);
   2072 
   2073     dbuf_free(dbuf);
   2074 }
   2075 
   2076 void build_script_table(FILE *f)
   2077 {
   2078     int i, v, j, n, n1, type;
   2079     DynBuf dbuf_s, *dbuf = &dbuf_s;
   2080 #ifdef DUMP_TABLE_SIZE
   2081     int cw_count, cw_len_count[4], cw_start;
   2082 #endif
   2083 
   2084     fprintf(f, "typedef enum {\n");
   2085     for(i = 0; i < SCRIPT_COUNT; i++)
   2086         fprintf(f, "    UNICODE_SCRIPT_%s,\n", unicode_script_name[i]);
   2087     fprintf(f, "    UNICODE_SCRIPT_COUNT,\n");
   2088     fprintf(f, "} UnicodeScriptEnum;\n\n");
   2089 
   2090     dump_name_table(f, "unicode_script_name_table",
   2091                     unicode_script_name, SCRIPT_COUNT,
   2092                     unicode_script_short_name);
   2093 
   2094     dbuf_init(dbuf);
   2095 #ifdef DUMP_TABLE_SIZE
   2096     cw_count = 0;
   2097     for(i = 0; i < 4; i++)
   2098         cw_len_count[i] = 0;
   2099 #endif
   2100     for(i = 0; i <= CHARCODE_MAX;) {
   2101         v = unicode_db[i].script;
   2102         j = i + 1;
   2103         while (j <= CHARCODE_MAX && unicode_db[j].script == v)
   2104             j++;
   2105         n = j - i;
   2106         if (v == 0 && j == (CHARCODE_MAX + 1))
   2107             break;
   2108         //        printf("%05x %05x %d\n", i, n, v);
   2109         n--;
   2110 #ifdef DUMP_TABLE_SIZE
   2111         cw_count++;
   2112         cw_start = dbuf->size;
   2113 #endif
   2114         if (v == 0)
   2115             type = 0;
   2116         else
   2117             type = 1;
   2118         if (n < 96) {
   2119             dbuf_putc(dbuf, n | (type << 7));
   2120         } else if (n < 96 + (1 << 12)) {
   2121             n1 = n - 96;
   2122             assert(n1 < (1 << 12));
   2123             dbuf_putc(dbuf, ((n1 >> 8) + 96) | (type << 7));
   2124             dbuf_putc(dbuf, n1);
   2125         } else {
   2126             n1 = n - (96 + (1 << 12));
   2127             assert(n1 < (1 << 20));
   2128             dbuf_putc(dbuf, ((n1 >> 16) + 112) | (type << 7));
   2129             dbuf_putc(dbuf, n1 >> 8);
   2130             dbuf_putc(dbuf, n1);
   2131         }
   2132         if (type != 0)
   2133             dbuf_putc(dbuf, v);
   2134 
   2135 #ifdef DUMP_TABLE_SIZE
   2136         cw_len_count[dbuf->size - cw_start - 1]++;
   2137 #endif
   2138         i += n + 1;
   2139     }
   2140 #ifdef DUMP_TABLE_SIZE
   2141     printf("script: %d entries [", cw_count);
   2142     for(i = 0; i < 4; i++)
   2143         printf(" %d", cw_len_count[i]);
   2144     printf(" ], length=%d bytes\n", (int)dbuf->size);
   2145 #endif
   2146 
   2147     dump_byte_table(f, "unicode_script_table", dbuf->buf, dbuf->size);
   2148 
   2149     dbuf_free(dbuf);
   2150 }
   2151 
   2152 void build_script_ext_table(FILE *f)
   2153 {
   2154     int i, j, n, n1, script_ext_len;
   2155     DynBuf dbuf_s, *dbuf = &dbuf_s;
   2156 #if defined(DUMP_TABLE_SIZE)
   2157     int cw_count = 0;
   2158 #endif
   2159 
   2160     dbuf_init(dbuf);
   2161     for(i = 0; i <= CHARCODE_MAX;) {
   2162         script_ext_len = unicode_db[i].script_ext_len;
   2163         j = i + 1;
   2164         while (j <= CHARCODE_MAX &&
   2165                unicode_db[j].script_ext_len == script_ext_len &&
   2166                !memcmp(unicode_db[j].script_ext, unicode_db[i].script_ext,
   2167                        script_ext_len)) {
   2168             j++;
   2169         }
   2170         n = j - i;
   2171 #if defined(DUMP_TABLE_SIZE)
   2172         cw_count++;
   2173 #endif
   2174         n--;
   2175         if (n < 128) {
   2176             dbuf_putc(dbuf, n);
   2177         } else if (n < 128 + (1 << 14)) {
   2178             n1 = n - 128;
   2179             assert(n1 < (1 << 14));
   2180             dbuf_putc(dbuf, (n1 >> 8) + 128);
   2181             dbuf_putc(dbuf, n1);
   2182         } else {
   2183             n1 = n - (128 + (1 << 14));
   2184             assert(n1 < (1 << 22));
   2185             dbuf_putc(dbuf, (n1 >> 16) + 128 + 64);
   2186             dbuf_putc(dbuf, n1 >> 8);
   2187             dbuf_putc(dbuf, n1);
   2188         }
   2189         dbuf_putc(dbuf, script_ext_len);
   2190         for(j = 0; j < script_ext_len; j++)
   2191             dbuf_putc(dbuf, unicode_db[i].script_ext[j]);
   2192         i += n + 1;
   2193     }
   2194 #ifdef DUMP_TABLE_SIZE
   2195     printf("script_ext: %d entries", cw_count);
   2196     printf(", length=%d bytes\n", (int)dbuf->size);
   2197 #endif
   2198 
   2199     dump_byte_table(f, "unicode_script_ext_table", dbuf->buf, dbuf->size);
   2200 
   2201     dbuf_free(dbuf);
   2202 }
   2203 
   2204 /* the following properties are synthetized so no table is necessary */
   2205 #define PROP_TABLE_COUNT PROP_ASCII
   2206 
   2207 void build_prop_list_table(FILE *f)
   2208 {
   2209     int i;
   2210 
   2211     for(i = 0; i < PROP_TABLE_COUNT; i++) {
   2212         if (i == PROP_ID_Start ||
   2213             i == PROP_Case_Ignorable ||
   2214             i == PROP_ID_Continue1) {
   2215             /* already generated */
   2216         } else {
   2217             build_prop_table(f, unicode_prop_name[i], i, FALSE);
   2218         }
   2219     }
   2220 
   2221     fprintf(f, "typedef enum {\n");
   2222     for(i = 0; i < PROP_COUNT; i++)
   2223         fprintf(f, "    UNICODE_PROP_%s,\n", unicode_prop_name[i]);
   2224     fprintf(f, "    UNICODE_PROP_COUNT,\n");
   2225     fprintf(f, "} UnicodePropertyEnum;\n\n");
   2226 
   2227     i = PROP_ASCII_Hex_Digit;
   2228     dump_name_table(f, "unicode_prop_name_table",
   2229                     unicode_prop_name + i, PROP_XID_Start - i + 1,
   2230                     unicode_prop_short_name + i);
   2231 
   2232     fprintf(f, "static const uint8_t * const unicode_prop_table[] = {\n");
   2233     for(i = 0; i < PROP_TABLE_COUNT; i++) {
   2234         fprintf(f, "    unicode_prop_%s_table,\n", unicode_prop_name[i]);
   2235     }
   2236     fprintf(f, "};\n\n");
   2237 
   2238     fprintf(f, "static const uint16_t unicode_prop_len_table[] = {\n");
   2239     for(i = 0; i < PROP_TABLE_COUNT; i++) {
   2240         fprintf(f, "    countof(unicode_prop_%s_table),\n", unicode_prop_name[i]);
   2241     }
   2242     fprintf(f, "};\n\n");
   2243 }
   2244 
   2245 static BOOL is_emoji_hair_color(uint32_t c)
   2246 {
   2247     return (c >= 0x1F9B0 && c <= 0x1F9B3);
   2248 }
   2249 
   2250 #define EMOJI_MOD_NONE   0
   2251 #define EMOJI_MOD_TYPE1  1
   2252 #define EMOJI_MOD_TYPE2  2
   2253 #define EMOJI_MOD_TYPE2D 3
   2254 
   2255 static BOOL mark_zwj_string(REStringList *sl, uint32_t *buf, int len, int mod_type, int *mod_pos,
   2256                             int hc_pos, BOOL mark_flag)
   2257 {
   2258     REString *p;
   2259     int i, n_mod, i0, i1, hc_count, j;
   2260 
   2261 #if 0
   2262     if (mark_flag)
   2263         printf("mod_type=%d\n", mod_type);
   2264 #endif
   2265     
   2266     switch(mod_type) {
   2267     case EMOJI_MOD_NONE:
   2268         n_mod = 1;
   2269         break;
   2270     case EMOJI_MOD_TYPE1:
   2271         n_mod = 5;
   2272         break;
   2273     case EMOJI_MOD_TYPE2:
   2274         n_mod = 25;
   2275         break;
   2276     case EMOJI_MOD_TYPE2D:
   2277         n_mod = 20;
   2278         break;
   2279     default:
   2280         assert(0);
   2281     }
   2282     if (hc_pos >= 0)
   2283         hc_count = 4;
   2284     else
   2285         hc_count = 1;
   2286     /* check that all the related strings are present */
   2287     for(j = 0; j < hc_count; j++) {
   2288         for(i = 0; i < n_mod; i++) {
   2289             switch(mod_type) {
   2290             case EMOJI_MOD_NONE:
   2291                 break;
   2292             case EMOJI_MOD_TYPE1:
   2293                 buf[mod_pos[0]] = 0x1f3fb + i;
   2294                 break;
   2295             case EMOJI_MOD_TYPE2:
   2296             case EMOJI_MOD_TYPE2D:
   2297                 i0 = i / 5;
   2298                 i1 = i % 5;
   2299                 /* avoid identical values */
   2300                 if (mod_type == EMOJI_MOD_TYPE2D && i0 >= i1)
   2301                     i0++;
   2302                 buf[mod_pos[0]] = 0x1f3fb + i0;
   2303                 buf[mod_pos[1]] = 0x1f3fb + i1;
   2304                 break;
   2305             default:
   2306                 assert(0);
   2307             }
   2308 
   2309             if (hc_pos >= 0)
   2310                 buf[hc_pos] = 0x1F9B0 + j;
   2311             
   2312             p = re_string_find(sl, len, buf, FALSE);
   2313             if (!p)
   2314                 return FALSE;
   2315             if (mark_flag)
   2316                 p->flags |= 1;
   2317         }
   2318     }
   2319     return TRUE;
   2320 }
   2321 
   2322 static void zwj_encode_string(DynBuf *dbuf, const uint32_t *buf, int len, int mod_type, int *mod_pos,
   2323                               int hc_pos)
   2324 {
   2325     int i, j;
   2326     int c, code;
   2327     uint32_t buf1[SEQ_MAX_LEN];
   2328     
   2329     j = 0;
   2330     for(i = 0; i < len;) {
   2331         c = buf[i++];
   2332         if (c >= 0x2000 && c <= 0x2fff) {
   2333             code = c - 0x2000;
   2334         } else if (c >= 0x1f000 && c <= 0x1ffff) {
   2335             code = c - 0x1f000 + 0x1000;
   2336         } else {
   2337             assert(0);
   2338         }
   2339         if (i < len && is_emoji_modifier(buf[i])) {
   2340             /* modifier */
   2341             code |= (mod_type << 13);
   2342             i++;
   2343         }
   2344         if (i < len && buf[i] == 0xfe0f) {
   2345             /* presentation selector present */
   2346             code |= 0x8000;
   2347             i++;
   2348         }
   2349         if (i < len) {
   2350             /* zero width join */
   2351             assert(buf[i] == 0x200d);
   2352             i++;
   2353         }
   2354         buf1[j++] = code;
   2355     }
   2356     dbuf_putc(dbuf, j);
   2357     for(i = 0; i < j; i++) {
   2358         dbuf_putc(dbuf, buf1[i]);
   2359         dbuf_putc(dbuf, buf1[i] >> 8);
   2360     }
   2361 }
   2362 
   2363 static void build_rgi_emoji_zwj_sequence(FILE *f, REStringList *sl)
   2364 {
   2365     int mod_pos[2], mod_count, hair_color_pos, j, h;
   2366     REString *p;
   2367     uint32_t buf[SEQ_MAX_LEN];
   2368     DynBuf dbuf;
   2369 
   2370 #if 0
   2371     {
   2372         for(h = 0; h < sl->hash_size; h++) {
   2373             for(p = sl->hash_table[h]; p != NULL; p = p->next) {
   2374                 for(j = 0; j < p->len; j++)
   2375                     printf(" %04x", p->buf[j]);
   2376                 printf("\n");
   2377             }
   2378         }
   2379         exit(0);
   2380     }
   2381 #endif
   2382     //    printf("rgi_emoji_zwj_sequence: n=%d\n", sl->n_strings);
   2383 
   2384     dbuf_init(&dbuf);
   2385     
   2386     /* avoid duplicating strings with emoji modifiers or hair colors */
   2387     for(h = 0; h < sl->hash_size; h++) {
   2388         for(p = sl->hash_table[h]; p != NULL; p = p->next) {
   2389             if (p->flags) /* already examined */
   2390                 continue;
   2391             mod_count = 0;
   2392             hair_color_pos = -1;
   2393             for(j = 0; j < p->len; j++) {
   2394                 if (is_emoji_modifier(p->buf[j])) {
   2395                     assert(mod_count < 2);
   2396                     mod_pos[mod_count++] = j;
   2397                 } else if (is_emoji_hair_color(p->buf[j])) {
   2398                     hair_color_pos = j;
   2399                 }
   2400                 buf[j] = p->buf[j];
   2401             }
   2402             
   2403             if (mod_count != 0 || hair_color_pos >= 0) {
   2404                 int mod_type;
   2405                 if (mod_count == 0)
   2406                     mod_type = EMOJI_MOD_NONE;
   2407                 else if (mod_count == 1)
   2408                     mod_type = EMOJI_MOD_TYPE1;
   2409                 else
   2410                     mod_type = EMOJI_MOD_TYPE2;
   2411                 
   2412                 if (mark_zwj_string(sl, buf, p->len, mod_type, mod_pos, hair_color_pos, FALSE)) {
   2413                     mark_zwj_string(sl, buf, p->len, mod_type, mod_pos, hair_color_pos, TRUE);
   2414                 } else if (mod_type == EMOJI_MOD_TYPE2) {
   2415                     mod_type = EMOJI_MOD_TYPE2D;
   2416                     if (mark_zwj_string(sl, buf, p->len, mod_type, mod_pos, hair_color_pos, FALSE)) {
   2417                         mark_zwj_string(sl, buf, p->len, mod_type, mod_pos, hair_color_pos, TRUE);
   2418                     } else {
   2419                         dump_str("not_found", (int *)p->buf, p->len);
   2420                         goto keep;
   2421                     }
   2422                 }
   2423                 if (hair_color_pos >= 0)
   2424                     buf[hair_color_pos] = 0x1f9b0;
   2425                 /* encode the string */
   2426                 zwj_encode_string(&dbuf, buf, p->len, mod_type, mod_pos, hair_color_pos);
   2427             } else {
   2428             keep:
   2429                 zwj_encode_string(&dbuf, buf, p->len, EMOJI_MOD_NONE, NULL, -1);
   2430             }
   2431         }
   2432     }
   2433     
   2434     /* Encode */
   2435     dump_byte_table(f, "unicode_rgi_emoji_zwj_sequence", dbuf.buf, dbuf.size);
   2436 
   2437     dbuf_free(&dbuf);
   2438 }
   2439 
   2440 void build_sequence_prop_list_table(FILE *f)
   2441 {
   2442     int i;
   2443     fprintf(f, "typedef enum {\n");
   2444     for(i = 0; i < SEQUENCE_PROP_COUNT; i++)
   2445         fprintf(f, "    UNICODE_SEQUENCE_PROP_%s,\n", unicode_sequence_prop_name[i]);
   2446     fprintf(f, "    UNICODE_SEQUENCE_PROP_COUNT,\n");
   2447     fprintf(f, "} UnicodeSequencePropertyEnum;\n\n");
   2448 
   2449     dump_name_table(f, "unicode_sequence_prop_name_table",
   2450                     unicode_sequence_prop_name, SEQUENCE_PROP_COUNT, NULL);
   2451 
   2452     dump_byte_table(f, "unicode_rgi_emoji_tag_sequence", rgi_emoji_tag_sequence.buf, rgi_emoji_tag_sequence.size);
   2453 
   2454     build_rgi_emoji_zwj_sequence(f, &rgi_emoji_zwj_sequence);
   2455 }
   2456 
   2457 #ifdef USE_TEST
   2458 int check_conv(uint32_t *res, uint32_t c, int conv_type)
   2459 {
   2460     return lre_case_conv(res, c, conv_type);
   2461 }
   2462 
   2463 void check_case_conv(void)
   2464 {
   2465     CCInfo *tab = unicode_db;
   2466     uint32_t res[3];
   2467     int l, error;
   2468     CCInfo ci_s, *ci1, *ci = &ci_s;
   2469     int code;
   2470 
   2471     for(code = 0; code <= CHARCODE_MAX; code++) {
   2472         ci1 = &tab[code];
   2473         *ci = *ci1;
   2474         if (ci->l_len == 0) {
   2475             ci->l_len = 1;
   2476             ci->l_data[0] = code;
   2477         }
   2478         if (ci->u_len == 0) {
   2479             ci->u_len = 1;
   2480             ci->u_data[0] = code;
   2481         }
   2482         if (ci->f_len == 0) {
   2483             ci->f_len = 1;
   2484             ci->f_data[0] = code;
   2485         }
   2486 
   2487         error = 0;
   2488         l = check_conv(res, code, 0);
   2489         if (l != ci->u_len || tabcmp((int *)res, ci->u_data, l)) {
   2490             printf("ERROR: L\n");
   2491             error++;
   2492         }
   2493         l = check_conv(res, code, 1);
   2494         if (l != ci->l_len || tabcmp((int *)res, ci->l_data, l)) {
   2495             printf("ERROR: U\n");
   2496             error++;
   2497         }
   2498         l = check_conv(res, code, 2);
   2499         if (l != ci->f_len || tabcmp((int *)res, ci->f_data, l)) {
   2500             printf("ERROR: F\n");
   2501             error++;
   2502         }
   2503         if (error) {
   2504             dump_cc_info(ci, code);
   2505             exit(1);
   2506         }
   2507     }
   2508 }
   2509 
   2510 #ifdef PROFILE
   2511 static int64_t get_time_ns(void)
   2512 {
   2513     struct timespec ts;
   2514     clock_gettime(CLOCK_MONOTONIC, &ts);
   2515     return (int64_t)ts.tv_sec * 1000000000 + ts.tv_nsec;
   2516 }
   2517 #endif
   2518 
   2519 
   2520 void check_flags(void)
   2521 {
   2522     int c;
   2523     BOOL flag_ref, flag;
   2524     for(c = 0; c <= CHARCODE_MAX; c++) {
   2525         flag_ref = get_prop(c, PROP_Cased);
   2526         flag = !!lre_is_cased(c);
   2527         if (flag != flag_ref) {
   2528             printf("ERROR: c=%05x cased=%d ref=%d\n",
   2529                    c, flag, flag_ref);
   2530             exit(1);
   2531         }
   2532 
   2533         flag_ref = get_prop(c, PROP_Case_Ignorable);
   2534         flag = !!lre_is_case_ignorable(c);
   2535         if (flag != flag_ref) {
   2536             printf("ERROR: c=%05x case_ignorable=%d ref=%d\n",
   2537                    c, flag, flag_ref);
   2538             exit(1);
   2539         }
   2540 
   2541         flag_ref = get_prop(c, PROP_ID_Start);
   2542         flag = !!lre_is_id_start(c);
   2543         if (flag != flag_ref) {
   2544             printf("ERROR: c=%05x id_start=%d ref=%d\n",
   2545                    c, flag, flag_ref);
   2546             exit(1);
   2547         }
   2548 
   2549         flag_ref = get_prop(c, PROP_ID_Continue);
   2550         flag = !!lre_is_id_continue(c);
   2551         if (flag != flag_ref) {
   2552             printf("ERROR: c=%05x id_cont=%d ref=%d\n",
   2553                    c, flag, flag_ref);
   2554             exit(1);
   2555         }
   2556     }
   2557 #ifdef PROFILE
   2558     {
   2559         int64_t ti, count;
   2560         ti = get_time_ns();
   2561         count = 0;
   2562         for(c = 0x20; c <= 0xffff; c++) {
   2563             flag_ref = get_prop(c, PROP_ID_Start);
   2564             flag = !!lre_is_id_start(c);
   2565             assert(flag == flag_ref);
   2566             count++;
   2567         }
   2568         ti = get_time_ns() - ti;
   2569         printf("flags time=%0.1f ns/char\n",
   2570                (double)ti / count);
   2571     }
   2572 #endif
   2573 }
   2574 
   2575 #endif
   2576 
   2577 #define CC_BLOCK_LEN 32
   2578 
   2579 void build_cc_table(FILE *f)
   2580 {
   2581     // Compress combining class table
   2582     // see: https://www.unicode.org/reports/tr44/#Canonical_Combining_Class_Values
   2583     int i, cc, n, type, n1, block_end_pos;
   2584     DynBuf dbuf_s, *dbuf = &dbuf_s;
   2585     DynBuf dbuf1_s, *dbuf1 = &dbuf1_s;
   2586 #if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
   2587     int cw_len_tab[3], cw_start, cc_table_len;
   2588 #endif
   2589     uint32_t v;
   2590 
   2591     dbuf_init(dbuf);
   2592     dbuf_init(dbuf1);
   2593 #if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
   2594     cc_table_len = 0;
   2595     for(i = 0; i < countof(cw_len_tab); i++)
   2596         cw_len_tab[i] = 0;
   2597 #endif
   2598     block_end_pos = CC_BLOCK_LEN;
   2599     for(i = 0; i <= CHARCODE_MAX;) {
   2600         cc = unicode_db[i].combining_class;
   2601         assert(cc <= 255);
   2602         /* check increasing values */
   2603         n = 1;
   2604         while ((i + n) <= CHARCODE_MAX &&
   2605                unicode_db[i + n].combining_class == (cc + n))
   2606             n++;
   2607         if (n >= 2) {
   2608             type = 1;
   2609         } else {
   2610             type = 0;
   2611             n = 1;
   2612             while ((i + n) <= CHARCODE_MAX &&
   2613                    unicode_db[i + n].combining_class == cc)
   2614                 n++;
   2615         }
   2616         /* no need to encode the last run */
   2617         if (cc == 0 && (i + n - 1) == CHARCODE_MAX)
   2618             break;
   2619 #ifdef DUMP_CC_TABLE
   2620         printf("%05x %6d %d %d\n", i, n, type, cc);
   2621 #endif
   2622         if (type == 0) {
   2623             if (cc == 0)
   2624                 type = 2;
   2625             else if (cc == 230)
   2626                 type = 3;
   2627         }
   2628         n1 = n - 1;
   2629 
   2630         /* add an entry to the index if necessary */
   2631         if (dbuf->size >= block_end_pos) {
   2632             v = i | ((dbuf->size - block_end_pos) << 21);
   2633             dbuf_putc(dbuf1, v);
   2634             dbuf_putc(dbuf1, v >> 8);
   2635             dbuf_putc(dbuf1, v >> 16);
   2636             block_end_pos += CC_BLOCK_LEN;
   2637         }
   2638 #if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
   2639         cw_start = dbuf->size;
   2640 #endif
   2641         /* Compressed run length encoding:
   2642            - 2 high order bits are combining class type
   2643            -         0:0, 1:230, 2:extra byte linear progression, 3:extra byte
   2644            - 00..2F: range length (add 1)
   2645            - 30..37: 3-bit range-length + 1 extra byte
   2646            - 38..3F: 3-bit range-length + 2 extra byte
   2647          */
   2648         if (n1 < 48) {
   2649             dbuf_putc(dbuf, n1 | (type << 6));
   2650         } else if (n1 < 48 + (1 << 11)) {
   2651             n1 -= 48;
   2652             dbuf_putc(dbuf, ((n1 >> 8) + 48) | (type << 6));
   2653             dbuf_putc(dbuf, n1);
   2654         } else {
   2655             n1 -= 48 + (1 << 11);
   2656             assert(n1 < (1 << 20));
   2657             dbuf_putc(dbuf, ((n1 >> 16) + 56) | (type << 6));
   2658             dbuf_putc(dbuf, n1 >> 8);
   2659             dbuf_putc(dbuf, n1);
   2660         }
   2661 #if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
   2662         cw_len_tab[dbuf->size - cw_start - 1]++;
   2663         cc_table_len++;
   2664 #endif
   2665         if (type == 0 || type == 1)
   2666             dbuf_putc(dbuf, cc);
   2667         i += n;
   2668     }
   2669 
   2670     /* last index entry */
   2671     v = i;
   2672     dbuf_putc(dbuf1, v);
   2673     dbuf_putc(dbuf1, v >> 8);
   2674     dbuf_putc(dbuf1, v >> 16);
   2675 
   2676     dump_byte_table(f, "unicode_cc_table", dbuf->buf, dbuf->size);
   2677     dump_index_table(f, "unicode_cc_index", dbuf1->buf, dbuf1->size);
   2678 
   2679 #if defined(DUMP_CC_TABLE) || defined(DUMP_TABLE_SIZE)
   2680     printf("CC table: size=%d (%d entries) [",
   2681            (int)(dbuf->size + dbuf1->size),
   2682            cc_table_len);
   2683     for(i = 0; i < countof(cw_len_tab); i++)
   2684         printf(" %d", cw_len_tab[i]);
   2685     printf(" ]\n");
   2686 #endif
   2687     dbuf_free(dbuf);
   2688     dbuf_free(dbuf1);
   2689 }
   2690 
   2691 /* maximum length of decomposition: 18 chars (1), then 8 */
   2692 #ifndef USE_TEST
   2693 typedef enum {
   2694     DECOMP_TYPE_C1, /* 16 bit char */
   2695     DECOMP_TYPE_L1, /* 16 bit char table */
   2696     DECOMP_TYPE_L2,
   2697     DECOMP_TYPE_L3,
   2698     DECOMP_TYPE_L4,
   2699     DECOMP_TYPE_L5, /* XXX: not used */
   2700     DECOMP_TYPE_L6, /* XXX: could remove */
   2701     DECOMP_TYPE_L7, /* XXX: could remove */
   2702     DECOMP_TYPE_LL1, /* 18 bit char table */
   2703     DECOMP_TYPE_LL2,
   2704     DECOMP_TYPE_S1, /* 8 bit char table */
   2705     DECOMP_TYPE_S2,
   2706     DECOMP_TYPE_S3,
   2707     DECOMP_TYPE_S4,
   2708     DECOMP_TYPE_S5,
   2709     DECOMP_TYPE_I1, /* increment 16 bit char value */
   2710     DECOMP_TYPE_I2_0,
   2711     DECOMP_TYPE_I2_1,
   2712     DECOMP_TYPE_I3_1,
   2713     DECOMP_TYPE_I3_2,
   2714     DECOMP_TYPE_I4_1,
   2715     DECOMP_TYPE_I4_2,
   2716     DECOMP_TYPE_B1, /* 16 bit base + 8 bit offset */
   2717     DECOMP_TYPE_B2,
   2718     DECOMP_TYPE_B3,
   2719     DECOMP_TYPE_B4,
   2720     DECOMP_TYPE_B5,
   2721     DECOMP_TYPE_B6,
   2722     DECOMP_TYPE_B7,
   2723     DECOMP_TYPE_B8,
   2724     DECOMP_TYPE_B18,
   2725     DECOMP_TYPE_LS2,
   2726     DECOMP_TYPE_PAT3,
   2727     DECOMP_TYPE_S2_UL,
   2728     DECOMP_TYPE_LS2_UL,
   2729 } DecompTypeEnum;
   2730 #endif
   2731 
   2732 const char *decomp_type_str[] = {
   2733     "C1",
   2734     "L1",
   2735     "L2",
   2736     "L3",
   2737     "L4",
   2738     "L5",
   2739     "L6",
   2740     "L7",
   2741     "LL1",
   2742     "LL2",
   2743     "S1",
   2744     "S2",
   2745     "S3",
   2746     "S4",
   2747     "S5",
   2748     "I1",
   2749     "I2_0",
   2750     "I2_1",
   2751     "I3_1",
   2752     "I3_2",
   2753     "I4_1",
   2754     "I4_2",
   2755     "B1",
   2756     "B2",
   2757     "B3",
   2758     "B4",
   2759     "B5",
   2760     "B6",
   2761     "B7",
   2762     "B8",
   2763     "B18",
   2764     "LS2",
   2765     "PAT3",
   2766     "S2_UL",
   2767     "LS2_UL",
   2768 };
   2769 
   2770 const int decomp_incr_tab[4][4] = {
   2771     { DECOMP_TYPE_I1, 0, -1 },
   2772     { DECOMP_TYPE_I2_0, 0, 1, -1 },
   2773     { DECOMP_TYPE_I3_1, 1, 2, -1 },
   2774     { DECOMP_TYPE_I4_1, 1, 2, -1 },
   2775 };
   2776 
   2777 /*
   2778   entry size:
   2779   type   bits
   2780   code   18
   2781   len    7
   2782   compat 1
   2783   type   5
   2784   index  16
   2785   total  47
   2786 */
   2787 
   2788 typedef struct {
   2789     int code;
   2790     uint8_t len;
   2791     uint8_t type;
   2792     uint8_t c_len;
   2793     uint16_t c_min;
   2794     uint16_t data_index;
   2795     int cost; /* size in bytes from this entry to the end */
   2796 } DecompEntry;
   2797 
   2798 int get_decomp_run_size(const DecompEntry *de)
   2799 {
   2800     int s;
   2801     s = 6;
   2802     if (de->type <= DECOMP_TYPE_C1) {
   2803         /* nothing more */
   2804     } else if (de->type <= DECOMP_TYPE_L7) {
   2805         s += de->len * de->c_len * 2;
   2806     } else if (de->type <= DECOMP_TYPE_LL2) {
   2807         /* 18 bits per char */
   2808         s += (de->len * de->c_len * 18 + 7) / 8;
   2809     } else if (de->type <= DECOMP_TYPE_S5) {
   2810         s += de->len * de->c_len;
   2811     } else if (de->type <= DECOMP_TYPE_I4_2) {
   2812         s += de->c_len * 2;
   2813     } else if (de->type <= DECOMP_TYPE_B18) {
   2814         s += 2 + de->len * de->c_len;
   2815     } else if (de->type <= DECOMP_TYPE_LS2) {
   2816         s += de->len * 3;
   2817     } else if (de->type <= DECOMP_TYPE_PAT3) {
   2818         s += 4 + de->len * 2;
   2819     } else if (de->type <= DECOMP_TYPE_S2_UL) {
   2820         s += de->len;
   2821     } else if (de->type <= DECOMP_TYPE_LS2_UL) {
   2822         s += (de->len / 2) * 3;
   2823     } else {
   2824         abort();
   2825     }
   2826     return s;
   2827 }
   2828 
   2829 static const uint16_t unicode_short_table[2] = { 0x2044, 0x2215 };
   2830 
   2831 /* return -1 if not found */
   2832 int get_short_code(int c)
   2833 {
   2834     int i;
   2835     if (c < 0x80) {
   2836         return c;
   2837     } else if (c >= 0x300 && c < 0x350) {
   2838         return c - 0x300 + 0x80;
   2839     } else {
   2840         for(i = 0; i < countof(unicode_short_table); i++) {
   2841             if (c == unicode_short_table[i])
   2842                 return i + 0x80 + 0x50;
   2843         }
   2844         return -1;
   2845     }
   2846 }
   2847 
   2848 static BOOL is_short(int code)
   2849 {
   2850     return get_short_code(code) >= 0;
   2851 }
   2852 
   2853 static BOOL is_short_tab(const int *tab, int len)
   2854 {
   2855     int i;
   2856     for(i = 0; i < len; i++) {
   2857         if (!is_short(tab[i]))
   2858             return FALSE;
   2859     }
   2860     return TRUE;
   2861 }
   2862 
   2863 static BOOL is_16bit(const int *tab, int len)
   2864 {
   2865     int i;
   2866     for(i = 0; i < len; i++) {
   2867         if (tab[i] > 0xffff)
   2868             return FALSE;
   2869     }
   2870     return TRUE;
   2871 }
   2872 
   2873 static uint32_t to_lower_simple(uint32_t c)
   2874 {
   2875     /* Latin1 and Cyrillic */
   2876     if (c < 0x100 || (c >= 0x410 && c <= 0x42f))
   2877         c += 0x20;
   2878     else
   2879         c++;
   2880     return c;
   2881 }
   2882 
   2883 /* select best encoding with dynamic programming */
   2884 void find_decomp_run(DecompEntry *tab_de, int i)
   2885 {
   2886     DecompEntry de_s, *de = &de_s;
   2887     CCInfo *ci, *ci1, *ci2;
   2888     int l, j, n, len_max;
   2889 
   2890     ci = &unicode_db[i];
   2891     l = ci->decomp_len;
   2892     if (l == 0) {
   2893         tab_de[i].cost = tab_de[i + 1].cost;
   2894         return;
   2895     }
   2896 
   2897     /* the offset for the compose table has only 6 bits, so we must
   2898        limit if it can be used by the compose table */
   2899     if (!ci->is_compat && !ci->is_excluded && l == 2)
   2900         len_max = 64;
   2901     else
   2902         len_max = 127;
   2903 
   2904     tab_de[i].cost = 0x7fffffff;
   2905 
   2906     if (!is_16bit(ci->decomp_data, l)) {
   2907         assert(l <= 2);
   2908 
   2909         n = 1;
   2910         for(;;) {
   2911             de->code = i;
   2912             de->len = n;
   2913             de->type = DECOMP_TYPE_LL1 + l - 1;
   2914             de->c_len = l;
   2915             de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
   2916             if (de->cost < tab_de[i].cost) {
   2917                 tab_de[i] = *de;
   2918             }
   2919             if (!((i + n) <= CHARCODE_MAX && n < len_max))
   2920                 break;
   2921             ci1 = &unicode_db[i + n];
   2922             /* Note: we accept a hole */
   2923             if (!(ci1->decomp_len == 0 ||
   2924                   (ci1->decomp_len == l &&
   2925                    ci1->is_compat == ci->is_compat)))
   2926                 break;
   2927             n++;
   2928         }
   2929         return;
   2930     }
   2931 
   2932     if (l <= 7) {
   2933         n = 1;
   2934         for(;;) {
   2935             de->code = i;
   2936             de->len = n;
   2937             if (l == 1 && n == 1) {
   2938                 de->type = DECOMP_TYPE_C1;
   2939             } else {
   2940                 assert(l <= 8);
   2941                 de->type = DECOMP_TYPE_L1 + l - 1;
   2942             }
   2943             de->c_len = l;
   2944             de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
   2945             if (de->cost < tab_de[i].cost) {
   2946                 tab_de[i] = *de;
   2947             }
   2948 
   2949             if (!((i + n) <= CHARCODE_MAX && n < len_max))
   2950                 break;
   2951             ci1 = &unicode_db[i + n];
   2952             /* Note: we accept a hole */
   2953             if (!(ci1->decomp_len == 0 ||
   2954                   (ci1->decomp_len == l &&
   2955                    ci1->is_compat == ci->is_compat &&
   2956                    is_16bit(ci1->decomp_data, l))))
   2957                 break;
   2958             n++;
   2959         }
   2960     }
   2961 
   2962     if (l <= 8 || l == 18) {
   2963         int c_min, c_max, c;
   2964         c_min = c_max = -1;
   2965         n = 1;
   2966         for(;;) {
   2967             ci1 = &unicode_db[i + n - 1];
   2968             for(j = 0; j < l; j++) {
   2969                 c = ci1->decomp_data[j];
   2970                 if (c == 0x20) {
   2971                     /* we accept space for Arabic */
   2972                 } else if (c_min == -1) {
   2973                     c_min = c_max = c;
   2974                 } else {
   2975                     c_min = min_int(c_min, c);
   2976                     c_max = max_int(c_max, c);
   2977                 }
   2978             }
   2979             if ((c_max - c_min) > 254)
   2980                 break;
   2981             de->code = i;
   2982             de->len = n;
   2983             if (l == 18)
   2984                 de->type = DECOMP_TYPE_B18;
   2985             else
   2986                 de->type = DECOMP_TYPE_B1 + l - 1;
   2987             de->c_len = l;
   2988             de->c_min = c_min;
   2989             de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
   2990             if (de->cost < tab_de[i].cost) {
   2991                 tab_de[i] = *de;
   2992             }
   2993             if (!((i + n) <= CHARCODE_MAX && n < len_max))
   2994                 break;
   2995             ci1 = &unicode_db[i + n];
   2996             if (!(ci1->decomp_len == l &&
   2997                   ci1->is_compat == ci->is_compat))
   2998                 break;
   2999             n++;
   3000         }
   3001     }
   3002 
   3003     /* find an ascii run */
   3004     if (l <= 5 && is_short_tab(ci->decomp_data, l)) {
   3005         n = 1;
   3006         for(;;) {
   3007             de->code = i;
   3008             de->len = n;
   3009             de->type = DECOMP_TYPE_S1 + l - 1;
   3010             de->c_len = l;
   3011             de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
   3012             if (de->cost < tab_de[i].cost) {
   3013                 tab_de[i] = *de;
   3014             }
   3015 
   3016             if (!((i + n) <= CHARCODE_MAX && n < len_max))
   3017                 break;
   3018             ci1 = &unicode_db[i + n];
   3019             /* Note: we accept a hole */
   3020             if (!(ci1->decomp_len == 0 ||
   3021                   (ci1->decomp_len == l &&
   3022                    ci1->is_compat == ci->is_compat &&
   3023                    is_short_tab(ci1->decomp_data, l))))
   3024                 break;
   3025             n++;
   3026         }
   3027     }
   3028 
   3029     /* check if a single char is increasing */
   3030     if (l <= 4) {
   3031         int idx1, idx;
   3032 
   3033         for(idx1 = 1; (idx = decomp_incr_tab[l - 1][idx1]) >= 0; idx1++) {
   3034             n = 1;
   3035             for(;;) {
   3036                 de->code = i;
   3037                 de->len = n;
   3038                 de->type = decomp_incr_tab[l - 1][0] + idx1 - 1;
   3039                 de->c_len = l;
   3040                 de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
   3041                 if (de->cost < tab_de[i].cost) {
   3042                     tab_de[i] = *de;
   3043                 }
   3044 
   3045                 if (!((i + n) <= CHARCODE_MAX && n < len_max))
   3046                     break;
   3047                 ci1 = &unicode_db[i + n];
   3048                 if (!(ci1->decomp_len == l &&
   3049                       ci1->is_compat == ci->is_compat))
   3050                     goto next1;
   3051                 for(j = 0; j < l; j++) {
   3052                     if (j == idx) {
   3053                         if (ci1->decomp_data[j] != ci->decomp_data[j] + n)
   3054                             goto next1;
   3055                     } else {
   3056                         if (ci1->decomp_data[j] != ci->decomp_data[j])
   3057                             goto next1;
   3058                     }
   3059                 }
   3060                 n++;
   3061             }
   3062         next1: ;
   3063         }
   3064     }
   3065 
   3066     if (l == 3) {
   3067         n = 1;
   3068         for(;;) {
   3069             de->code = i;
   3070             de->len = n;
   3071             de->type = DECOMP_TYPE_PAT3;
   3072             de->c_len = l;
   3073             de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
   3074             if (de->cost < tab_de[i].cost) {
   3075                 tab_de[i] = *de;
   3076             }
   3077             if (!((i + n) <= CHARCODE_MAX && n < len_max))
   3078                 break;
   3079             ci1 = &unicode_db[i + n];
   3080             if (!(ci1->decomp_len == l &&
   3081                   ci1->is_compat == ci->is_compat &&
   3082                   ci1->decomp_data[1] <= 0xffff &&
   3083                   ci1->decomp_data[0] == ci->decomp_data[0] &&
   3084                   ci1->decomp_data[l - 1] == ci->decomp_data[l - 1]))
   3085                 break;
   3086             n++;
   3087         }
   3088     }
   3089 
   3090     if (l == 2 && is_short(ci->decomp_data[1])) {
   3091         n = 1;
   3092         for(;;) {
   3093             de->code = i;
   3094             de->len = n;
   3095             de->type = DECOMP_TYPE_LS2;
   3096             de->c_len = l;
   3097             de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
   3098             if (de->cost < tab_de[i].cost) {
   3099                 tab_de[i] = *de;
   3100             }
   3101             if (!((i + n) <= CHARCODE_MAX && n < len_max))
   3102                 break;
   3103             ci1 = &unicode_db[i + n];
   3104             if (!(ci1->decomp_len == 0 ||
   3105                   (ci1->decomp_len == l &&
   3106                    ci1->is_compat == ci->is_compat &&
   3107                    ci1->decomp_data[0] <= 0xffff &&
   3108                    is_short(ci1->decomp_data[1]))))
   3109                 break;
   3110             n++;
   3111         }
   3112     }
   3113 
   3114     if (l == 2) {
   3115         BOOL is_16bit;
   3116 
   3117         n = 0;
   3118         is_16bit = FALSE;
   3119         for(;;) {
   3120             if (!((i + n + 1) <= CHARCODE_MAX && n + 2 <= len_max))
   3121                 break;
   3122             ci1 = &unicode_db[i + n];
   3123             if (!(ci1->decomp_len == l &&
   3124                   ci1->is_compat == ci->is_compat &&
   3125                   is_short(ci1->decomp_data[1])))
   3126                 break;
   3127             if (!is_16bit && !is_short(ci1->decomp_data[0]))
   3128                 is_16bit = TRUE;
   3129             ci2 = &unicode_db[i + n + 1];
   3130             if (!(ci2->decomp_len == l &&
   3131                   ci2->is_compat == ci->is_compat &&
   3132                   ci2->decomp_data[0] == to_lower_simple(ci1->decomp_data[0])  &&
   3133                   ci2->decomp_data[1] == ci1->decomp_data[1]))
   3134                 break;
   3135             n += 2;
   3136             de->code = i;
   3137             de->len = n;
   3138             de->type = DECOMP_TYPE_S2_UL + is_16bit;
   3139             de->c_len = l;
   3140             de->cost = get_decomp_run_size(de) + tab_de[i + n].cost;
   3141             if (de->cost < tab_de[i].cost) {
   3142                 tab_de[i] = *de;
   3143             }
   3144         }
   3145     }
   3146 }
   3147 
   3148 void put16(uint8_t *data_buf, int *pidx, uint16_t c)
   3149 {
   3150     int idx;
   3151     idx = *pidx;
   3152     data_buf[idx++] = c;
   3153     data_buf[idx++] = c >> 8;
   3154     *pidx = idx;
   3155 }
   3156 
   3157 void add_decomp_data(uint8_t *data_buf, int *pidx, DecompEntry *de)
   3158 {
   3159     int i, j, idx, c;
   3160     CCInfo *ci;
   3161 
   3162     idx = *pidx;
   3163     de->data_index = idx;
   3164     if (de->type <= DECOMP_TYPE_C1) {
   3165         ci = &unicode_db[de->code];
   3166         assert(ci->decomp_len == 1);
   3167         de->data_index = ci->decomp_data[0];
   3168     } else if (de->type <= DECOMP_TYPE_L7) {
   3169         for(i = 0; i < de->len; i++) {
   3170             ci = &unicode_db[de->code + i];
   3171             for(j = 0; j < de->c_len; j++) {
   3172                 if (ci->decomp_len == 0)
   3173                     c = 0;
   3174                 else
   3175                     c = ci->decomp_data[j];
   3176                 put16(data_buf, &idx,  c);
   3177             }
   3178         }
   3179     } else if (de->type <= DECOMP_TYPE_LL2) {
   3180         int n, p, k;
   3181         n = (de->len * de->c_len * 18 + 7) / 8;
   3182         p = de->len * de->c_len * 2;
   3183         memset(data_buf + idx, 0, n);
   3184         k = 0;
   3185         for(i = 0; i < de->len; i++) {
   3186             ci = &unicode_db[de->code + i];
   3187             for(j = 0; j < de->c_len; j++) {
   3188                 if (ci->decomp_len == 0)
   3189                     c = 0;
   3190                 else
   3191                     c = ci->decomp_data[j];
   3192                 data_buf[idx + k * 2] = c;
   3193                 data_buf[idx + k * 2 + 1] = c >> 8;
   3194                 data_buf[idx + p + (k / 4)] |= (c >> 16) << ((k % 4) * 2);
   3195                 k++;
   3196             }
   3197         }
   3198         idx += n;
   3199     } else if (de->type <= DECOMP_TYPE_S5) {
   3200         for(i = 0; i < de->len; i++) {
   3201             ci = &unicode_db[de->code + i];
   3202             for(j = 0; j < de->c_len; j++) {
   3203                 if (ci->decomp_len == 0)
   3204                     c = 0;
   3205                 else
   3206                     c = ci->decomp_data[j];
   3207                 c = get_short_code(c);
   3208                 assert(c >= 0);
   3209                 data_buf[idx++] = c;
   3210             }
   3211         }
   3212     } else if (de->type <= DECOMP_TYPE_I4_2) {
   3213         ci = &unicode_db[de->code];
   3214         assert(ci->decomp_len == de->c_len);
   3215         for(j = 0; j < de->c_len; j++)
   3216             put16(data_buf, &idx, ci->decomp_data[j]);
   3217     } else if (de->type <= DECOMP_TYPE_B18) {
   3218         c = de->c_min;
   3219         data_buf[idx++] = c;
   3220         data_buf[idx++] = c >> 8;
   3221         for(i = 0; i < de->len; i++) {
   3222             ci = &unicode_db[de->code + i];
   3223             for(j = 0; j < de->c_len; j++) {
   3224                 assert(ci->decomp_len == de->c_len);
   3225                 c = ci->decomp_data[j];
   3226                 if (c == 0x20) {
   3227                     c = 0xff;
   3228                 } else {
   3229                     c -= de->c_min;
   3230                     assert((uint32_t)c <= 254);
   3231                 }
   3232                 data_buf[idx++] = c;
   3233             }
   3234         }
   3235     } else if (de->type <= DECOMP_TYPE_LS2) {
   3236         assert(de->c_len == 2);
   3237         for(i = 0; i < de->len; i++) {
   3238             ci = &unicode_db[de->code + i];
   3239             if (ci->decomp_len == 0)
   3240                 c = 0;
   3241             else
   3242                 c = ci->decomp_data[0];
   3243             put16(data_buf, &idx,  c);
   3244 
   3245             if (ci->decomp_len == 0)
   3246                 c = 0;
   3247             else
   3248                 c = ci->decomp_data[1];
   3249             c = get_short_code(c);
   3250             assert(c >= 0);
   3251             data_buf[idx++] = c;
   3252         }
   3253     } else if (de->type <= DECOMP_TYPE_PAT3) {
   3254         ci = &unicode_db[de->code];
   3255         assert(ci->decomp_len == 3);
   3256         put16(data_buf, &idx,  ci->decomp_data[0]);
   3257         put16(data_buf, &idx,  ci->decomp_data[2]);
   3258         for(i = 0; i < de->len; i++) {
   3259             ci = &unicode_db[de->code + i];
   3260             assert(ci->decomp_len == 3);
   3261             put16(data_buf, &idx,  ci->decomp_data[1]);
   3262         }
   3263     } else if (de->type <= DECOMP_TYPE_S2_UL) {
   3264         for(i = 0; i < de->len; i += 2) {
   3265             ci = &unicode_db[de->code + i];
   3266             c = ci->decomp_data[0];
   3267             c = get_short_code(c);
   3268             assert(c >= 0);
   3269             data_buf[idx++] = c;
   3270             c = ci->decomp_data[1];
   3271             c = get_short_code(c);
   3272             assert(c >= 0);
   3273             data_buf[idx++] = c;
   3274         }
   3275     } else if (de->type <= DECOMP_TYPE_LS2_UL) {
   3276         for(i = 0; i < de->len; i += 2) {
   3277             ci = &unicode_db[de->code + i];
   3278             c = ci->decomp_data[0];
   3279             put16(data_buf, &idx,  c);
   3280             c = ci->decomp_data[1];
   3281             c = get_short_code(c);
   3282             assert(c >= 0);
   3283             data_buf[idx++] = c;
   3284         }
   3285     } else {
   3286         abort();
   3287     }
   3288     *pidx = idx;
   3289 }
   3290 
   3291 #if 0
   3292 void dump_large_char(void)
   3293 {
   3294     int i, j;
   3295     for(i = 0; i <= CHARCODE_MAX; i++) {
   3296         CCInfo *ci = &unicode_db[i];
   3297         for(j = 0; j < ci->decomp_len; j++) {
   3298             if (ci->decomp_data[j] > 0xffff)
   3299                 printf("%05x\n", ci->decomp_data[j]);
   3300         }
   3301     }
   3302 }
   3303 #endif
   3304 
   3305 void build_compose_table(FILE *f, const DecompEntry *tab_de);
   3306 
   3307 void build_decompose_table(FILE *f)
   3308 {
   3309     int i, array_len, code_max, data_len, count;
   3310     DecompEntry *tab_de, de_s, *de = &de_s;
   3311     uint8_t *data_buf;
   3312 
   3313     code_max = CHARCODE_MAX;
   3314 
   3315     tab_de = mallocz((code_max + 2) * sizeof(*tab_de));
   3316 
   3317     for(i = code_max; i >= 0; i--) {
   3318         find_decomp_run(tab_de, i);
   3319     }
   3320 
   3321     /* build the data buffer */
   3322     data_buf = malloc(100000);
   3323     data_len = 0;
   3324     array_len = 0;
   3325     for(i = 0; i <= code_max; i++) {
   3326         de = &tab_de[i];
   3327         if (de->len != 0) {
   3328             add_decomp_data(data_buf, &data_len, de);
   3329             i += de->len - 1;
   3330             array_len++;
   3331         }
   3332     }
   3333 
   3334 #ifdef DUMP_DECOMP_TABLE
   3335     /* dump */
   3336     {
   3337         int size, size1;
   3338 
   3339         printf("START LEN   TYPE  L C SIZE\n");
   3340         size = 0;
   3341         for(i = 0; i <= code_max; i++) {
   3342             de = &tab_de[i];
   3343             if (de->len != 0) {
   3344                 size1 = get_decomp_run_size(de);
   3345                 printf("%05x %3d %6s %2d %1d %4d\n", i, de->len,
   3346                        decomp_type_str[de->type], de->c_len,
   3347                        unicode_db[i].is_compat, size1);
   3348                 i += de->len - 1;
   3349                 size += size1;
   3350             }
   3351         }
   3352 
   3353         printf("array_len=%d estimated size=%d bytes actual=%d bytes\n",
   3354                array_len, size, array_len * 6 + data_len);
   3355     }
   3356 #endif
   3357 
   3358     total_tables++;
   3359     total_table_bytes += array_len * sizeof(uint32_t);
   3360     fprintf(f, "static const uint32_t unicode_decomp_table1[%d] = {", array_len);
   3361     count = 0;
   3362     for(i = 0; i <= code_max; i++) {
   3363         de = &tab_de[i];
   3364         if (de->len != 0) {
   3365             uint32_t v;
   3366             if (count++ % 4 == 0)
   3367                 fprintf(f, "\n   ");
   3368             v = (de->code << (32 - 18)) |
   3369                 (de->len << (32 - 18 - 7)) |
   3370                 (de->type << (32 - 18 - 7 - 6)) |
   3371                 unicode_db[de->code].is_compat;
   3372             fprintf(f, " 0x%08x,", v);
   3373             i += de->len - 1;
   3374         }
   3375     }
   3376     fprintf(f, "\n};\n\n");
   3377 
   3378     total_tables++;
   3379     total_table_bytes += array_len * sizeof(uint16_t);
   3380     fprintf(f, "static const uint16_t unicode_decomp_table2[%d] = {", array_len);
   3381     count = 0;
   3382     for(i = 0; i <= code_max; i++) {
   3383         de = &tab_de[i];
   3384         if (de->len != 0) {
   3385             if (count++ % 8 == 0)
   3386                 fprintf(f, "\n   ");
   3387             fprintf(f, " 0x%04x,", de->data_index);
   3388             i += de->len - 1;
   3389         }
   3390     }
   3391     fprintf(f, "\n};\n\n");
   3392 
   3393     total_tables++;
   3394     total_table_bytes += data_len;
   3395     fprintf(f, "static const uint8_t unicode_decomp_data[%d] = {", data_len);
   3396     for(i = 0; i < data_len; i++) {
   3397         if (i % 8 == 0)
   3398             fprintf(f, "\n   ");
   3399         fprintf(f, " 0x%02x,", data_buf[i]);
   3400     }
   3401     fprintf(f, "\n};\n\n");
   3402 
   3403     build_compose_table(f, tab_de);
   3404 
   3405     free(data_buf);
   3406 
   3407     free(tab_de);
   3408 }
   3409 
   3410 typedef struct {
   3411     uint32_t c[2];
   3412     uint32_t p;
   3413 } ComposeEntry;
   3414 
   3415 #define COMPOSE_LEN_MAX 10000
   3416 
   3417 static int ce_cmp(const void *p1, const void *p2)
   3418 {
   3419     const ComposeEntry *ce1 = p1;
   3420     const ComposeEntry *ce2 = p2;
   3421     int i;
   3422 
   3423     for(i = 0; i < 2; i++) {
   3424         if (ce1->c[i] < ce2->c[i])
   3425             return -1;
   3426         else if (ce1->c[i] > ce2->c[i])
   3427             return 1;
   3428     }
   3429     return 0;
   3430 }
   3431 
   3432 
   3433 static int get_decomp_pos(const DecompEntry *tab_de, int c)
   3434 {
   3435     int i, v, k;
   3436     const DecompEntry *de;
   3437 
   3438     k = 0;
   3439     for(i = 0; i <= CHARCODE_MAX; i++) {
   3440         de = &tab_de[i];
   3441         if (de->len != 0) {
   3442             if (c >= de->code && c < de->code + de->len) {
   3443                 v = c - de->code;
   3444                 assert(v < 64);
   3445                 v |= k << 6;
   3446                 assert(v < 65536);
   3447                 return v;
   3448             }
   3449             i += de->len - 1;
   3450             k++;
   3451         }
   3452     }
   3453     return -1;
   3454 }
   3455 
   3456 void build_compose_table(FILE *f, const DecompEntry *tab_de)
   3457 {
   3458     int i, v, tab_ce_len;
   3459     ComposeEntry *ce, *tab_ce;
   3460 
   3461     tab_ce = malloc(sizeof(*tab_ce) * COMPOSE_LEN_MAX);
   3462     tab_ce_len = 0;
   3463     for(i = 0; i <= CHARCODE_MAX; i++) {
   3464         CCInfo *ci = &unicode_db[i];
   3465         if (ci->decomp_len == 2 && !ci->is_compat &&
   3466             !ci->is_excluded) {
   3467             assert(tab_ce_len < COMPOSE_LEN_MAX);
   3468             ce = &tab_ce[tab_ce_len++];
   3469             ce->c[0] = ci->decomp_data[0];
   3470             ce->c[1] = ci->decomp_data[1];
   3471             ce->p = i;
   3472         }
   3473     }
   3474     qsort(tab_ce, tab_ce_len, sizeof(*tab_ce), ce_cmp);
   3475 
   3476 #if 0
   3477     {
   3478         printf("tab_ce_len=%d\n", tab_ce_len);
   3479         for(i = 0; i < tab_ce_len; i++) {
   3480             ce = &tab_ce[i];
   3481             printf("%05x %05x %05x\n", ce->c[0], ce->c[1], ce->p);
   3482         }
   3483     }
   3484 #endif
   3485 
   3486     total_tables++;
   3487     total_table_bytes += tab_ce_len * sizeof(uint16_t);
   3488     fprintf(f, "static const uint16_t unicode_comp_table[%u] = {", tab_ce_len);
   3489     for(i = 0; i < tab_ce_len; i++) {
   3490         if (i % 8 == 0)
   3491             fprintf(f, "\n   ");
   3492         v = get_decomp_pos(tab_de, tab_ce[i].p);
   3493         if (v < 0) {
   3494             printf("ERROR: entry for c=%04x not found\n",
   3495                    tab_ce[i].p);
   3496             exit(1);
   3497         }
   3498         fprintf(f, " 0x%04x,", v);
   3499     }
   3500     fprintf(f, "\n};\n\n");
   3501 
   3502     free(tab_ce);
   3503 }
   3504 
   3505 #ifdef USE_TEST
   3506 void check_decompose_table(void)
   3507 {
   3508     int c;
   3509     CCInfo *ci;
   3510     int res[UNICODE_DECOMP_LEN_MAX], *ref;
   3511     int len, ref_len, is_compat;
   3512 
   3513     for(is_compat = 0; is_compat <= 1; is_compat++) {
   3514         for(c = 0; c < CHARCODE_MAX; c++) {
   3515             ci = &unicode_db[c];
   3516             ref_len = ci->decomp_len;
   3517             ref = ci->decomp_data;
   3518             if (!is_compat && ci->is_compat) {
   3519                 ref_len = 0;
   3520             }
   3521             len = unicode_decomp_char((uint32_t *)res, c, is_compat);
   3522             if (len != ref_len ||
   3523                 tabcmp(res, ref, ref_len) != 0) {
   3524                 printf("ERROR c=%05x compat=%d\n", c, is_compat);
   3525                 dump_str("res", res, len);
   3526                 dump_str("ref", ref, ref_len);
   3527                 exit(1);
   3528             }
   3529         }
   3530     }
   3531 }
   3532 
   3533 void check_compose_table(void)
   3534 {
   3535     int i, p;
   3536     /* XXX: we don't test all the cases */
   3537 
   3538     for(i = 0; i <= CHARCODE_MAX; i++) {
   3539         CCInfo *ci = &unicode_db[i];
   3540         if (ci->decomp_len == 2 && !ci->is_compat &&
   3541             !ci->is_excluded) {
   3542             p = unicode_compose_pair(ci->decomp_data[0], ci->decomp_data[1]);
   3543             if (p != i) {
   3544                 printf("ERROR compose: c=%05x %05x -> %05x ref=%05x\n",
   3545                        ci->decomp_data[0], ci->decomp_data[1], p, i);
   3546                 exit(1);
   3547             }
   3548         }
   3549     }
   3550 
   3551 
   3552 
   3553 }
   3554 
   3555 #endif
   3556 
   3557 
   3558 
   3559 #ifdef USE_TEST
   3560 
   3561 void check_str(const char *msg, int num, const int *in_buf, int in_len,
   3562                const int *buf1, int len1,
   3563                const int *buf2, int len2)
   3564 {
   3565     if (len1 != len2 || tabcmp(buf1, buf2, len1) != 0) {
   3566         printf("%d: ERROR %s:\n", num, msg);
   3567         dump_str(" in", in_buf, in_len);
   3568         dump_str("res", buf1, len1);
   3569         dump_str("ref", buf2, len2);
   3570         exit(1);
   3571     }
   3572 }
   3573 
   3574 void check_cc_table(void)
   3575 {
   3576     int cc, cc_ref, c;
   3577 
   3578     for(c = 0; c <= CHARCODE_MAX; c++) {
   3579         cc_ref = unicode_db[c].combining_class;
   3580         cc = unicode_get_cc(c);
   3581         if (cc != cc_ref) {
   3582             printf("ERROR: c=%04x cc=%d cc_ref=%d\n",
   3583                    c, cc, cc_ref);
   3584             exit(1);
   3585         }
   3586     }
   3587 #ifdef PROFILE
   3588     {
   3589         int64_t ti, count;
   3590 
   3591         ti = get_time_ns();
   3592         count = 0;
   3593         /* only do it on meaningful chars */
   3594         for(c = 0x20; c <= 0xffff; c++) {
   3595             cc_ref = unicode_db[c].combining_class;
   3596             cc = unicode_get_cc(c);
   3597             count++;
   3598         }
   3599         ti = get_time_ns() - ti;
   3600         printf("cc time=%0.1f ns/char\n",
   3601                (double)ti / count);
   3602     }
   3603 #endif
   3604 }
   3605 
   3606 void normalization_test(const char *filename)
   3607 {
   3608     FILE *f;
   3609     char line[4096], *p;
   3610     int *in_str, *nfc_str, *nfd_str, *nfkc_str, *nfkd_str;
   3611     int in_len, nfc_len, nfd_len, nfkc_len, nfkd_len;
   3612     int *buf, buf_len, pos;
   3613 
   3614     f = fopen(filename, "rb");
   3615     if (!f) {
   3616         perror(filename);
   3617         exit(1);
   3618     }
   3619     pos = 0;
   3620     for(;;) {
   3621         if (!get_line(line, sizeof(line), f))
   3622             break;
   3623         pos++;
   3624         p = line;
   3625         while (isspace(*p))
   3626             p++;
   3627         if (*p == '#' || *p == '@')
   3628             continue;
   3629         in_str = get_field_str(&in_len, p, 0);
   3630         nfc_str = get_field_str(&nfc_len, p, 1);
   3631         nfd_str = get_field_str(&nfd_len, p, 2);
   3632         nfkc_str = get_field_str(&nfkc_len, p, 3);
   3633         nfkd_str = get_field_str(&nfkd_len, p, 4);
   3634 
   3635         //        dump_str("in", in_str, in_len);
   3636 
   3637         buf_len = unicode_normalize((uint32_t **)&buf, (uint32_t *)in_str, in_len, UNICODE_NFD, NULL, NULL);
   3638         check_str("nfd", pos, in_str, in_len, buf, buf_len, nfd_str, nfd_len);
   3639         free(buf);
   3640 
   3641         buf_len = unicode_normalize((uint32_t **)&buf, (uint32_t *)in_str, in_len, UNICODE_NFKD, NULL, NULL);
   3642         check_str("nfkd", pos, in_str, in_len, buf, buf_len, nfkd_str, nfkd_len);
   3643         free(buf);
   3644 
   3645         buf_len = unicode_normalize((uint32_t **)&buf, (uint32_t *)in_str, in_len, UNICODE_NFC, NULL, NULL);
   3646         check_str("nfc", pos, in_str, in_len, buf, buf_len, nfc_str, nfc_len);
   3647         free(buf);
   3648 
   3649         buf_len = unicode_normalize((uint32_t **)&buf, (uint32_t *)in_str, in_len, UNICODE_NFKC, NULL, NULL);
   3650         check_str("nfkc", pos, in_str, in_len, buf, buf_len, nfkc_str, nfkc_len);
   3651         free(buf);
   3652 
   3653         free(in_str);
   3654         free(nfc_str);
   3655         free(nfd_str);
   3656         free(nfkc_str);
   3657         free(nfkd_str);
   3658     }
   3659     fclose(f);
   3660 }
   3661 #endif
   3662 
   3663 int main(int argc, char *argv[])
   3664 {
   3665     const char *unicode_db_path, *outfilename;
   3666     char filename[1024];
   3667     int arg = 1;
   3668 
   3669     if (arg >= argc || (!strcmp(argv[arg], "-h") || !strcmp(argv[arg], "--help"))) {
   3670         printf("usage: %s PATH [OUTPUT]\n"
   3671                "  PATH    path to the Unicode database directory\n"
   3672                "  OUTPUT  name of the output file.  If omitted, a self test is performed\n"
   3673                "          using the files from the Unicode library\n"
   3674                , argv[0]);
   3675         return 1;
   3676     }
   3677     unicode_db_path = argv[arg++];
   3678     outfilename = NULL;
   3679     if (arg < argc)
   3680         outfilename = argv[arg++];
   3681 
   3682     unicode_db = mallocz(sizeof(unicode_db[0]) * (CHARCODE_MAX + 1));
   3683     re_string_list_init(&rgi_emoji_zwj_sequence);
   3684     dbuf_init(&rgi_emoji_tag_sequence);
   3685 
   3686     snprintf(filename, sizeof(filename), "%s/UnicodeData.txt", unicode_db_path);
   3687 
   3688     parse_unicode_data(filename);
   3689 
   3690     snprintf(filename, sizeof(filename), "%s/SpecialCasing.txt", unicode_db_path);
   3691     parse_special_casing(unicode_db, filename);
   3692 
   3693     snprintf(filename, sizeof(filename), "%s/CaseFolding.txt", unicode_db_path);
   3694     parse_case_folding(unicode_db, filename);
   3695 
   3696     snprintf(filename, sizeof(filename), "%s/CompositionExclusions.txt", unicode_db_path);
   3697     parse_composition_exclusions(filename);
   3698 
   3699     snprintf(filename, sizeof(filename), "%s/DerivedCoreProperties.txt", unicode_db_path);
   3700     parse_derived_core_properties(filename);
   3701 
   3702     snprintf(filename, sizeof(filename), "%s/DerivedNormalizationProps.txt", unicode_db_path);
   3703     parse_derived_norm_properties(filename);
   3704 
   3705     snprintf(filename, sizeof(filename), "%s/PropList.txt", unicode_db_path);
   3706     parse_prop_list(filename);
   3707 
   3708     snprintf(filename, sizeof(filename), "%s/Scripts.txt", unicode_db_path);
   3709     parse_scripts(filename);
   3710 
   3711     snprintf(filename, sizeof(filename), "%s/ScriptExtensions.txt",
   3712              unicode_db_path);
   3713     parse_script_extensions(filename);
   3714 
   3715     snprintf(filename, sizeof(filename), "%s/emoji-data.txt",
   3716              unicode_db_path);
   3717     parse_prop_list(filename);
   3718 
   3719     snprintf(filename, sizeof(filename), "%s/emoji-sequences.txt",
   3720              unicode_db_path);
   3721     parse_sequence_prop_list(filename);
   3722 
   3723     snprintf(filename, sizeof(filename), "%s/emoji-zwj-sequences.txt",
   3724              unicode_db_path);
   3725     parse_sequence_prop_list(filename);
   3726 
   3727     //    dump_unicode_data(unicode_db);
   3728     build_conv_table(unicode_db);
   3729 
   3730 #ifdef DUMP_CASE_FOLDING_SPECIAL_CASES
   3731     dump_case_folding_special_cases(unicode_db);
   3732 #endif
   3733 
   3734     if (!outfilename) {
   3735 #ifdef USE_TEST
   3736         check_case_conv();
   3737         check_flags();
   3738         check_decompose_table();
   3739         check_compose_table();
   3740         check_cc_table();
   3741         snprintf(filename, sizeof(filename), "%s/NormalizationTest.txt", unicode_db_path);
   3742         normalization_test(filename);
   3743 #else
   3744         fprintf(stderr, "Tests are not compiled\n");
   3745         exit(1);
   3746 #endif
   3747     } else
   3748     {
   3749         FILE *fo = fopen(outfilename, "wb");
   3750 
   3751         if (!fo) {
   3752             perror(outfilename);
   3753             exit(1);
   3754         }
   3755         fprintf(fo,
   3756                 "/* Compressed unicode tables */\n"
   3757                 "/* Automatically generated file - do not edit */\n"
   3758                 "\n"
   3759                 "#include <stdint.h>\n"
   3760                 "\n");
   3761         dump_case_conv_table(fo);
   3762         compute_internal_props();
   3763         build_flags_tables(fo);
   3764         fprintf(fo, "#ifdef CONFIG_ALL_UNICODE\n\n");
   3765         build_cc_table(fo);
   3766         build_decompose_table(fo);
   3767         build_general_category_table(fo);
   3768         build_script_table(fo);
   3769         build_script_ext_table(fo);
   3770         build_prop_list_table(fo);
   3771         build_sequence_prop_list_table(fo);
   3772         fprintf(fo, "#endif /* CONFIG_ALL_UNICODE */\n");
   3773         fprintf(fo, "/* %u tables / %u bytes, %u index / %u bytes */\n",
   3774                 total_tables, total_table_bytes, total_index, total_index_bytes);
   3775         fclose(fo);
   3776     }
   3777     re_string_list_free(&rgi_emoji_zwj_sequence);
   3778     return 0;
   3779 }
	quickjs-tart quickjs-based runtime for wallet-core logic
	Log \| Files \| Refs \| README \| LICENSE