libunicode.c (63617B)
1 /* 2 * Unicode utilities 3 * 4 * Copyright (c) 2017-2018 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 #include <stdlib.h> 25 #include <stdio.h> 26 #include <stdarg.h> 27 #include <string.h> 28 #include <assert.h> 29 30 #include "cutils.h" 31 #include "libunicode.h" 32 #include "libunicode-table.h" 33 34 enum { 35 RUN_TYPE_U, 36 RUN_TYPE_L, 37 RUN_TYPE_UF, 38 RUN_TYPE_LF, 39 RUN_TYPE_UL, 40 RUN_TYPE_LSU, 41 RUN_TYPE_U2L_399_EXT2, 42 RUN_TYPE_UF_D20, 43 RUN_TYPE_UF_D1_EXT, 44 RUN_TYPE_U_EXT, 45 RUN_TYPE_LF_EXT, 46 RUN_TYPE_UF_EXT2, 47 RUN_TYPE_LF_EXT2, 48 RUN_TYPE_UF_EXT3, 49 }; 50 51 static int lre_case_conv1(uint32_t c, int conv_type) 52 { 53 uint32_t res[LRE_CC_RES_LEN_MAX]; 54 lre_case_conv(res, c, conv_type); 55 return res[0]; 56 } 57 58 /* case conversion using the table entry 'idx' with value 'v' */ 59 static int lre_case_conv_entry(uint32_t *res, uint32_t c, int conv_type, uint32_t idx, uint32_t v) 60 { 61 uint32_t code, data, type, a, is_lower; 62 is_lower = (conv_type != 0); 63 type = (v >> (32 - 17 - 7 - 4)) & 0xf; 64 data = ((v & 0xf) << 8) | case_conv_table2[idx]; 65 code = v >> (32 - 17); 66 switch(type) { 67 case RUN_TYPE_U: 68 case RUN_TYPE_L: 69 case RUN_TYPE_UF: 70 case RUN_TYPE_LF: 71 if (conv_type == (type & 1) || 72 (type >= RUN_TYPE_UF && conv_type == 2)) { 73 c = c - code + (case_conv_table1[data] >> (32 - 17)); 74 } 75 break; 76 case RUN_TYPE_UL: 77 a = c - code; 78 if ((a & 1) != (1 - is_lower)) 79 break; 80 c = (a ^ 1) + code; 81 break; 82 case RUN_TYPE_LSU: 83 a = c - code; 84 if (a == 1) { 85 c += 2 * is_lower - 1; 86 } else if (a == (1 - is_lower) * 2) { 87 c += (2 * is_lower - 1) * 2; 88 } 89 break; 90 case RUN_TYPE_U2L_399_EXT2: 91 if (!is_lower) { 92 res[0] = c - code + case_conv_ext[data >> 6]; 93 res[1] = 0x399; 94 return 2; 95 } else { 96 c = c - code + case_conv_ext[data & 0x3f]; 97 } 98 break; 99 case RUN_TYPE_UF_D20: 100 if (conv_type == 1) 101 break; 102 c = data + (conv_type == 2) * 0x20; 103 break; 104 case RUN_TYPE_UF_D1_EXT: 105 if (conv_type == 1) 106 break; 107 c = case_conv_ext[data] + (conv_type == 2); 108 break; 109 case RUN_TYPE_U_EXT: 110 case RUN_TYPE_LF_EXT: 111 if (is_lower != (type - RUN_TYPE_U_EXT)) 112 break; 113 c = case_conv_ext[data]; 114 break; 115 case RUN_TYPE_LF_EXT2: 116 if (!is_lower) 117 break; 118 res[0] = c - code + case_conv_ext[data >> 6]; 119 res[1] = case_conv_ext[data & 0x3f]; 120 return 2; 121 case RUN_TYPE_UF_EXT2: 122 if (conv_type == 1) 123 break; 124 res[0] = c - code + case_conv_ext[data >> 6]; 125 res[1] = case_conv_ext[data & 0x3f]; 126 if (conv_type == 2) { 127 /* convert to lower */ 128 res[0] = lre_case_conv1(res[0], 1); 129 res[1] = lre_case_conv1(res[1], 1); 130 } 131 return 2; 132 default: 133 case RUN_TYPE_UF_EXT3: 134 if (conv_type == 1) 135 break; 136 res[0] = case_conv_ext[data >> 8]; 137 res[1] = case_conv_ext[(data >> 4) & 0xf]; 138 res[2] = case_conv_ext[data & 0xf]; 139 if (conv_type == 2) { 140 /* convert to lower */ 141 res[0] = lre_case_conv1(res[0], 1); 142 res[1] = lre_case_conv1(res[1], 1); 143 res[2] = lre_case_conv1(res[2], 1); 144 } 145 return 3; 146 } 147 res[0] = c; 148 return 1; 149 } 150 151 /* conv_type: 152 0 = to upper 153 1 = to lower 154 2 = case folding (= to lower with modifications) 155 */ 156 int lre_case_conv(uint32_t *res, uint32_t c, int conv_type) 157 { 158 if (c < 128) { 159 if (conv_type) { 160 if (c >= 'A' && c <= 'Z') { 161 c = c - 'A' + 'a'; 162 } 163 } else { 164 if (c >= 'a' && c <= 'z') { 165 c = c - 'a' + 'A'; 166 } 167 } 168 } else { 169 uint32_t v, code, len; 170 int idx, idx_min, idx_max; 171 172 idx_min = 0; 173 idx_max = countof(case_conv_table1) - 1; 174 while (idx_min <= idx_max) { 175 idx = (unsigned)(idx_max + idx_min) / 2; 176 v = case_conv_table1[idx]; 177 code = v >> (32 - 17); 178 len = (v >> (32 - 17 - 7)) & 0x7f; 179 if (c < code) { 180 idx_max = idx - 1; 181 } else if (c >= code + len) { 182 idx_min = idx + 1; 183 } else { 184 return lre_case_conv_entry(res, c, conv_type, idx, v); 185 } 186 } 187 } 188 res[0] = c; 189 return 1; 190 } 191 192 static int lre_case_folding_entry(uint32_t c, uint32_t idx, uint32_t v, BOOL is_unicode) 193 { 194 uint32_t res[LRE_CC_RES_LEN_MAX]; 195 int len; 196 197 if (is_unicode) { 198 len = lre_case_conv_entry(res, c, 2, idx, v); 199 if (len == 1) { 200 c = res[0]; 201 } else { 202 /* handle the few specific multi-character cases (see 203 unicode_gen.c:dump_case_folding_special_cases()) */ 204 if (c == 0xfb06) { 205 c = 0xfb05; 206 } else if (c == 0x01fd3) { 207 c = 0x390; 208 } else if (c == 0x01fe3) { 209 c = 0x3b0; 210 } 211 } 212 } else { 213 if (likely(c < 128)) { 214 if (c >= 'a' && c <= 'z') 215 c = c - 'a' + 'A'; 216 } else { 217 /* legacy regexp: to upper case if single char >= 128 */ 218 len = lre_case_conv_entry(res, c, FALSE, idx, v); 219 if (len == 1 && res[0] >= 128) 220 c = res[0]; 221 } 222 } 223 return c; 224 } 225 226 /* JS regexp specific rules for case folding */ 227 int lre_canonicalize(uint32_t c, BOOL is_unicode) 228 { 229 if (c < 128) { 230 /* fast case */ 231 if (is_unicode) { 232 if (c >= 'A' && c <= 'Z') { 233 c = c - 'A' + 'a'; 234 } 235 } else { 236 if (c >= 'a' && c <= 'z') { 237 c = c - 'a' + 'A'; 238 } 239 } 240 } else { 241 uint32_t v, code, len; 242 int idx, idx_min, idx_max; 243 244 idx_min = 0; 245 idx_max = countof(case_conv_table1) - 1; 246 while (idx_min <= idx_max) { 247 idx = (unsigned)(idx_max + idx_min) / 2; 248 v = case_conv_table1[idx]; 249 code = v >> (32 - 17); 250 len = (v >> (32 - 17 - 7)) & 0x7f; 251 if (c < code) { 252 idx_max = idx - 1; 253 } else if (c >= code + len) { 254 idx_min = idx + 1; 255 } else { 256 return lre_case_folding_entry(c, idx, v, is_unicode); 257 } 258 } 259 } 260 return c; 261 } 262 263 static uint32_t get_le24(const uint8_t *ptr) 264 { 265 return ptr[0] | (ptr[1] << 8) | (ptr[2] << 16); 266 } 267 268 #define UNICODE_INDEX_BLOCK_LEN 32 269 270 /* return -1 if not in table, otherwise the offset in the block */ 271 static int get_index_pos(uint32_t *pcode, uint32_t c, 272 const uint8_t *index_table, int index_table_len) 273 { 274 uint32_t code, v; 275 int idx_min, idx_max, idx; 276 277 idx_min = 0; 278 v = get_le24(index_table); 279 code = v & ((1 << 21) - 1); 280 if (c < code) { 281 *pcode = 0; 282 return 0; 283 } 284 idx_max = index_table_len - 1; 285 code = get_le24(index_table + idx_max * 3); 286 if (c >= code) 287 return -1; 288 /* invariant: tab[idx_min] <= c < tab2[idx_max] */ 289 while ((idx_max - idx_min) > 1) { 290 idx = (idx_max + idx_min) / 2; 291 v = get_le24(index_table + idx * 3); 292 code = v & ((1 << 21) - 1); 293 if (c < code) { 294 idx_max = idx; 295 } else { 296 idx_min = idx; 297 } 298 } 299 v = get_le24(index_table + idx_min * 3); 300 *pcode = v & ((1 << 21) - 1); 301 return (idx_min + 1) * UNICODE_INDEX_BLOCK_LEN + (v >> 21); 302 } 303 304 static BOOL lre_is_in_table(uint32_t c, const uint8_t *table, 305 const uint8_t *index_table, int index_table_len) 306 { 307 uint32_t code, b, bit; 308 int pos; 309 const uint8_t *p; 310 311 pos = get_index_pos(&code, c, index_table, index_table_len); 312 if (pos < 0) 313 return FALSE; /* outside the table */ 314 p = table + pos; 315 bit = 0; 316 /* Compressed run length encoding: 317 00..3F: 2 packed lengths: 3-bit + 3-bit 318 40..5F: 5-bits plus extra byte for length 319 60..7F: 5-bits plus 2 extra bytes for length 320 80..FF: 7-bit length 321 lengths must be incremented to get character count 322 Ranges alternate between false and true return value. 323 */ 324 for(;;) { 325 b = *p++; 326 if (b < 64) { 327 code += (b >> 3) + 1; 328 if (c < code) 329 return bit; 330 bit ^= 1; 331 code += (b & 7) + 1; 332 } else if (b >= 0x80) { 333 code += b - 0x80 + 1; 334 } else if (b < 0x60) { 335 code += (((b - 0x40) << 8) | p[0]) + 1; 336 p++; 337 } else { 338 code += (((b - 0x60) << 16) | (p[0] << 8) | p[1]) + 1; 339 p += 2; 340 } 341 if (c < code) 342 return bit; 343 bit ^= 1; 344 } 345 } 346 347 BOOL lre_is_cased(uint32_t c) 348 { 349 uint32_t v, code, len; 350 int idx, idx_min, idx_max; 351 352 idx_min = 0; 353 idx_max = countof(case_conv_table1) - 1; 354 while (idx_min <= idx_max) { 355 idx = (unsigned)(idx_max + idx_min) / 2; 356 v = case_conv_table1[idx]; 357 code = v >> (32 - 17); 358 len = (v >> (32 - 17 - 7)) & 0x7f; 359 if (c < code) { 360 idx_max = idx - 1; 361 } else if (c >= code + len) { 362 idx_min = idx + 1; 363 } else { 364 return TRUE; 365 } 366 } 367 return lre_is_in_table(c, unicode_prop_Cased1_table, 368 unicode_prop_Cased1_index, 369 sizeof(unicode_prop_Cased1_index) / 3); 370 } 371 372 BOOL lre_is_case_ignorable(uint32_t c) 373 { 374 return lre_is_in_table(c, unicode_prop_Case_Ignorable_table, 375 unicode_prop_Case_Ignorable_index, 376 sizeof(unicode_prop_Case_Ignorable_index) / 3); 377 } 378 379 /* character range */ 380 381 static __maybe_unused void cr_dump(CharRange *cr) 382 { 383 int i; 384 for(i = 0; i < cr->len; i++) 385 printf("%d: 0x%04x\n", i, cr->points[i]); 386 } 387 388 static void *cr_default_realloc(void *opaque, void *ptr, size_t size) 389 { 390 return realloc(ptr, size); 391 } 392 393 void cr_init(CharRange *cr, void *mem_opaque, DynBufReallocFunc *realloc_func) 394 { 395 cr->len = cr->size = 0; 396 cr->points = NULL; 397 cr->mem_opaque = mem_opaque; 398 cr->realloc_func = realloc_func ? realloc_func : cr_default_realloc; 399 } 400 401 void cr_free(CharRange *cr) 402 { 403 cr->realloc_func(cr->mem_opaque, cr->points, 0); 404 } 405 406 int cr_realloc(CharRange *cr, int size) 407 { 408 int new_size; 409 uint32_t *new_buf; 410 411 if (size > cr->size) { 412 new_size = max_int(size, cr->size * 3 / 2); 413 new_buf = cr->realloc_func(cr->mem_opaque, cr->points, 414 new_size * sizeof(cr->points[0])); 415 if (!new_buf) 416 return -1; 417 cr->points = new_buf; 418 cr->size = new_size; 419 } 420 return 0; 421 } 422 423 int cr_copy(CharRange *cr, const CharRange *cr1) 424 { 425 if (cr_realloc(cr, cr1->len)) 426 return -1; 427 memcpy(cr->points, cr1->points, sizeof(cr->points[0]) * cr1->len); 428 cr->len = cr1->len; 429 return 0; 430 } 431 432 /* merge consecutive intervals and remove empty intervals */ 433 static void cr_compress(CharRange *cr) 434 { 435 int i, j, k, len; 436 uint32_t *pt; 437 438 pt = cr->points; 439 len = cr->len; 440 i = 0; 441 j = 0; 442 k = 0; 443 while ((i + 1) < len) { 444 if (pt[i] == pt[i + 1]) { 445 /* empty interval */ 446 i += 2; 447 } else { 448 j = i; 449 while ((j + 3) < len && pt[j + 1] == pt[j + 2]) 450 j += 2; 451 /* just copy */ 452 pt[k] = pt[i]; 453 pt[k + 1] = pt[j + 1]; 454 k += 2; 455 i = j + 2; 456 } 457 } 458 cr->len = k; 459 } 460 461 /* union or intersection */ 462 int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len, 463 const uint32_t *b_pt, int b_len, int op) 464 { 465 int a_idx, b_idx, is_in; 466 uint32_t v; 467 468 a_idx = 0; 469 b_idx = 0; 470 for(;;) { 471 /* get one more point from a or b in increasing order */ 472 if (a_idx < a_len && b_idx < b_len) { 473 if (a_pt[a_idx] < b_pt[b_idx]) { 474 goto a_add; 475 } else if (a_pt[a_idx] == b_pt[b_idx]) { 476 v = a_pt[a_idx]; 477 a_idx++; 478 b_idx++; 479 } else { 480 goto b_add; 481 } 482 } else if (a_idx < a_len) { 483 a_add: 484 v = a_pt[a_idx++]; 485 } else if (b_idx < b_len) { 486 b_add: 487 v = b_pt[b_idx++]; 488 } else { 489 break; 490 } 491 /* add the point if the in/out status changes */ 492 switch(op) { 493 case CR_OP_UNION: 494 is_in = (a_idx & 1) | (b_idx & 1); 495 break; 496 case CR_OP_INTER: 497 is_in = (a_idx & 1) & (b_idx & 1); 498 break; 499 case CR_OP_XOR: 500 is_in = (a_idx & 1) ^ (b_idx & 1); 501 break; 502 case CR_OP_SUB: 503 is_in = (a_idx & 1) & ((b_idx & 1) ^ 1); 504 break; 505 default: 506 abort(); 507 } 508 if (is_in != (cr->len & 1)) { 509 if (cr_add_point(cr, v)) 510 return -1; 511 } 512 } 513 cr_compress(cr); 514 return 0; 515 } 516 517 int cr_op1(CharRange *cr, const uint32_t *b_pt, int b_len, int op) 518 { 519 CharRange a = *cr; 520 int ret; 521 cr->len = 0; 522 cr->size = 0; 523 cr->points = NULL; 524 ret = cr_op(cr, a.points, a.len, b_pt, b_len, op); 525 cr_free(&a); 526 return ret; 527 } 528 529 int cr_invert(CharRange *cr) 530 { 531 int len; 532 len = cr->len; 533 if (cr_realloc(cr, len + 2)) 534 return -1; 535 memmove(cr->points + 1, cr->points, len * sizeof(cr->points[0])); 536 cr->points[0] = 0; 537 cr->points[len + 1] = UINT32_MAX; 538 cr->len = len + 2; 539 cr_compress(cr); 540 return 0; 541 } 542 543 #define CASE_U (1 << 0) 544 #define CASE_L (1 << 1) 545 #define CASE_F (1 << 2) 546 547 /* use the case conversion table to generate range of characters. 548 CASE_U: set char if modified by uppercasing, 549 CASE_L: set char if modified by lowercasing, 550 CASE_F: set char if modified by case folding, 551 */ 552 static int unicode_case1(CharRange *cr, int case_mask) 553 { 554 #define MR(x) (1 << RUN_TYPE_ ## x) 555 const uint32_t tab_run_mask[3] = { 556 MR(U) | MR(UF) | MR(UL) | MR(LSU) | MR(U2L_399_EXT2) | MR(UF_D20) | 557 MR(UF_D1_EXT) | MR(U_EXT) | MR(UF_EXT2) | MR(UF_EXT3), 558 559 MR(L) | MR(LF) | MR(UL) | MR(LSU) | MR(U2L_399_EXT2) | MR(LF_EXT) | MR(LF_EXT2), 560 561 MR(UF) | MR(LF) | MR(UL) | MR(LSU) | MR(U2L_399_EXT2) | MR(LF_EXT) | MR(LF_EXT2) | MR(UF_D20) | MR(UF_D1_EXT) | MR(LF_EXT) | MR(UF_EXT2) | MR(UF_EXT3), 562 }; 563 #undef MR 564 uint32_t mask, v, code, type, len, i, idx; 565 566 if (case_mask == 0) 567 return 0; 568 mask = 0; 569 for(i = 0; i < 3; i++) { 570 if ((case_mask >> i) & 1) 571 mask |= tab_run_mask[i]; 572 } 573 for(idx = 0; idx < countof(case_conv_table1); idx++) { 574 v = case_conv_table1[idx]; 575 type = (v >> (32 - 17 - 7 - 4)) & 0xf; 576 code = v >> (32 - 17); 577 len = (v >> (32 - 17 - 7)) & 0x7f; 578 if ((mask >> type) & 1) { 579 // printf("%d: type=%d %04x %04x\n", idx, type, code, code + len - 1); 580 switch(type) { 581 case RUN_TYPE_UL: 582 if ((case_mask & CASE_U) && (case_mask & (CASE_L | CASE_F))) 583 goto def_case; 584 code += ((case_mask & CASE_U) != 0); 585 for(i = 0; i < len; i += 2) { 586 if (cr_add_interval(cr, code + i, code + i + 1)) 587 return -1; 588 } 589 break; 590 case RUN_TYPE_LSU: 591 if ((case_mask & CASE_U) && (case_mask & (CASE_L | CASE_F))) 592 goto def_case; 593 if (!(case_mask & CASE_U)) { 594 if (cr_add_interval(cr, code, code + 1)) 595 return -1; 596 } 597 if (cr_add_interval(cr, code + 1, code + 2)) 598 return -1; 599 if (case_mask & CASE_U) { 600 if (cr_add_interval(cr, code + 2, code + 3)) 601 return -1; 602 } 603 break; 604 default: 605 def_case: 606 if (cr_add_interval(cr, code, code + len)) 607 return -1; 608 break; 609 } 610 } 611 } 612 return 0; 613 } 614 615 static int point_cmp(const void *p1, const void *p2, void *arg) 616 { 617 uint32_t v1 = *(uint32_t *)p1; 618 uint32_t v2 = *(uint32_t *)p2; 619 return (v1 > v2) - (v1 < v2); 620 } 621 622 static void cr_sort_and_remove_overlap(CharRange *cr) 623 { 624 uint32_t start, end, start1, end1, i, j; 625 626 /* the resulting ranges are not necessarily sorted and may overlap */ 627 rqsort(cr->points, cr->len / 2, sizeof(cr->points[0]) * 2, point_cmp, NULL); 628 j = 0; 629 for(i = 0; i < cr->len; ) { 630 start = cr->points[i]; 631 end = cr->points[i + 1]; 632 i += 2; 633 while (i < cr->len) { 634 start1 = cr->points[i]; 635 end1 = cr->points[i + 1]; 636 if (start1 > end) { 637 /* |------| 638 * |-------| */ 639 break; 640 } else if (end1 <= end) { 641 /* |------| 642 * |--| */ 643 i += 2; 644 } else { 645 /* |------| 646 * |-------| */ 647 end = end1; 648 i += 2; 649 } 650 } 651 cr->points[j] = start; 652 cr->points[j + 1] = end; 653 j += 2; 654 } 655 cr->len = j; 656 } 657 658 /* canonicalize a character set using the JS regex case folding rules 659 (see lre_canonicalize()) */ 660 int cr_regexp_canonicalize(CharRange *cr, BOOL is_unicode) 661 { 662 CharRange cr_inter, cr_mask, cr_result, cr_sub; 663 uint32_t v, code, len, i, idx, start, end, c, d_start, d_end, d; 664 665 cr_init(&cr_mask, cr->mem_opaque, cr->realloc_func); 666 cr_init(&cr_inter, cr->mem_opaque, cr->realloc_func); 667 cr_init(&cr_result, cr->mem_opaque, cr->realloc_func); 668 cr_init(&cr_sub, cr->mem_opaque, cr->realloc_func); 669 670 if (unicode_case1(&cr_mask, is_unicode ? CASE_F : CASE_U)) 671 goto fail; 672 if (cr_op(&cr_inter, cr_mask.points, cr_mask.len, cr->points, cr->len, CR_OP_INTER)) 673 goto fail; 674 675 if (cr_invert(&cr_mask)) 676 goto fail; 677 if (cr_op(&cr_sub, cr_mask.points, cr_mask.len, cr->points, cr->len, CR_OP_INTER)) 678 goto fail; 679 680 /* cr_inter = cr & cr_mask */ 681 /* cr_sub = cr & ~cr_mask */ 682 683 /* use the case conversion table to compute the result */ 684 d_start = -1; 685 d_end = -1; 686 idx = 0; 687 v = case_conv_table1[idx]; 688 code = v >> (32 - 17); 689 len = (v >> (32 - 17 - 7)) & 0x7f; 690 for(i = 0; i < cr_inter.len; i += 2) { 691 start = cr_inter.points[i]; 692 end = cr_inter.points[i + 1]; 693 694 for(c = start; c < end; c++) { 695 for(;;) { 696 if (c >= code && c < code + len) 697 break; 698 idx++; 699 assert(idx < countof(case_conv_table1)); 700 v = case_conv_table1[idx]; 701 code = v >> (32 - 17); 702 len = (v >> (32 - 17 - 7)) & 0x7f; 703 } 704 d = lre_case_folding_entry(c, idx, v, is_unicode); 705 /* try to merge with the current interval */ 706 if (d_start == -1) { 707 d_start = d; 708 d_end = d + 1; 709 } else if (d_end == d) { 710 d_end++; 711 } else { 712 cr_add_interval(&cr_result, d_start, d_end); 713 d_start = d; 714 d_end = d + 1; 715 } 716 } 717 } 718 if (d_start != -1) { 719 if (cr_add_interval(&cr_result, d_start, d_end)) 720 goto fail; 721 } 722 723 /* the resulting ranges are not necessarily sorted and may overlap */ 724 cr_sort_and_remove_overlap(&cr_result); 725 726 /* or with the character not affected by the case folding */ 727 cr->len = 0; 728 if (cr_op(cr, cr_result.points, cr_result.len, cr_sub.points, cr_sub.len, CR_OP_UNION)) 729 goto fail; 730 731 cr_free(&cr_inter); 732 cr_free(&cr_mask); 733 cr_free(&cr_result); 734 cr_free(&cr_sub); 735 return 0; 736 fail: 737 cr_free(&cr_inter); 738 cr_free(&cr_mask); 739 cr_free(&cr_result); 740 cr_free(&cr_sub); 741 return -1; 742 } 743 744 #ifdef CONFIG_ALL_UNICODE 745 746 BOOL lre_is_id_start(uint32_t c) 747 { 748 return lre_is_in_table(c, unicode_prop_ID_Start_table, 749 unicode_prop_ID_Start_index, 750 sizeof(unicode_prop_ID_Start_index) / 3); 751 } 752 753 BOOL lre_is_id_continue(uint32_t c) 754 { 755 return lre_is_id_start(c) || 756 lre_is_in_table(c, unicode_prop_ID_Continue1_table, 757 unicode_prop_ID_Continue1_index, 758 sizeof(unicode_prop_ID_Continue1_index) / 3); 759 } 760 761 #define UNICODE_DECOMP_LEN_MAX 18 762 763 typedef enum { 764 DECOMP_TYPE_C1, /* 16 bit char */ 765 DECOMP_TYPE_L1, /* 16 bit char table */ 766 DECOMP_TYPE_L2, 767 DECOMP_TYPE_L3, 768 DECOMP_TYPE_L4, 769 DECOMP_TYPE_L5, /* XXX: not used */ 770 DECOMP_TYPE_L6, /* XXX: could remove */ 771 DECOMP_TYPE_L7, /* XXX: could remove */ 772 DECOMP_TYPE_LL1, /* 18 bit char table */ 773 DECOMP_TYPE_LL2, 774 DECOMP_TYPE_S1, /* 8 bit char table */ 775 DECOMP_TYPE_S2, 776 DECOMP_TYPE_S3, 777 DECOMP_TYPE_S4, 778 DECOMP_TYPE_S5, 779 DECOMP_TYPE_I1, /* increment 16 bit char value */ 780 DECOMP_TYPE_I2_0, 781 DECOMP_TYPE_I2_1, 782 DECOMP_TYPE_I3_1, 783 DECOMP_TYPE_I3_2, 784 DECOMP_TYPE_I4_1, 785 DECOMP_TYPE_I4_2, 786 DECOMP_TYPE_B1, /* 16 bit base + 8 bit offset */ 787 DECOMP_TYPE_B2, 788 DECOMP_TYPE_B3, 789 DECOMP_TYPE_B4, 790 DECOMP_TYPE_B5, 791 DECOMP_TYPE_B6, 792 DECOMP_TYPE_B7, 793 DECOMP_TYPE_B8, 794 DECOMP_TYPE_B18, 795 DECOMP_TYPE_LS2, 796 DECOMP_TYPE_PAT3, 797 DECOMP_TYPE_S2_UL, 798 DECOMP_TYPE_LS2_UL, 799 } DecompTypeEnum; 800 801 static uint32_t unicode_get_short_code(uint32_t c) 802 { 803 static const uint16_t unicode_short_table[2] = { 0x2044, 0x2215 }; 804 805 if (c < 0x80) 806 return c; 807 else if (c < 0x80 + 0x50) 808 return c - 0x80 + 0x300; 809 else 810 return unicode_short_table[c - 0x80 - 0x50]; 811 } 812 813 static uint32_t unicode_get_lower_simple(uint32_t c) 814 { 815 if (c < 0x100 || (c >= 0x410 && c <= 0x42f)) 816 c += 0x20; 817 else 818 c++; 819 return c; 820 } 821 822 static uint16_t unicode_get16(const uint8_t *p) 823 { 824 return p[0] | (p[1] << 8); 825 } 826 827 static int unicode_decomp_entry(uint32_t *res, uint32_t c, 828 int idx, uint32_t code, uint32_t len, 829 uint32_t type) 830 { 831 uint32_t c1; 832 int l, i, p; 833 const uint8_t *d; 834 835 if (type == DECOMP_TYPE_C1) { 836 res[0] = unicode_decomp_table2[idx]; 837 return 1; 838 } else { 839 d = unicode_decomp_data + unicode_decomp_table2[idx]; 840 switch(type) { 841 case DECOMP_TYPE_L1: 842 case DECOMP_TYPE_L2: 843 case DECOMP_TYPE_L3: 844 case DECOMP_TYPE_L4: 845 case DECOMP_TYPE_L5: 846 case DECOMP_TYPE_L6: 847 case DECOMP_TYPE_L7: 848 l = type - DECOMP_TYPE_L1 + 1; 849 d += (c - code) * l * 2; 850 for(i = 0; i < l; i++) { 851 if ((res[i] = unicode_get16(d + 2 * i)) == 0) 852 return 0; 853 } 854 return l; 855 case DECOMP_TYPE_LL1: 856 case DECOMP_TYPE_LL2: 857 { 858 uint32_t k, p; 859 l = type - DECOMP_TYPE_LL1 + 1; 860 k = (c - code) * l; 861 p = len * l * 2; 862 for(i = 0; i < l; i++) { 863 c1 = unicode_get16(d + 2 * k) | 864 (((d[p + (k / 4)] >> ((k % 4) * 2)) & 3) << 16); 865 if (!c1) 866 return 0; 867 res[i] = c1; 868 k++; 869 } 870 } 871 return l; 872 case DECOMP_TYPE_S1: 873 case DECOMP_TYPE_S2: 874 case DECOMP_TYPE_S3: 875 case DECOMP_TYPE_S4: 876 case DECOMP_TYPE_S5: 877 l = type - DECOMP_TYPE_S1 + 1; 878 d += (c - code) * l; 879 for(i = 0; i < l; i++) { 880 if ((res[i] = unicode_get_short_code(d[i])) == 0) 881 return 0; 882 } 883 return l; 884 case DECOMP_TYPE_I1: 885 l = 1; 886 p = 0; 887 goto decomp_type_i; 888 case DECOMP_TYPE_I2_0: 889 case DECOMP_TYPE_I2_1: 890 case DECOMP_TYPE_I3_1: 891 case DECOMP_TYPE_I3_2: 892 case DECOMP_TYPE_I4_1: 893 case DECOMP_TYPE_I4_2: 894 l = 2 + ((type - DECOMP_TYPE_I2_0) >> 1); 895 p = ((type - DECOMP_TYPE_I2_0) & 1) + (l > 2); 896 decomp_type_i: 897 for(i = 0; i < l; i++) { 898 c1 = unicode_get16(d + 2 * i); 899 if (i == p) 900 c1 += c - code; 901 res[i] = c1; 902 } 903 return l; 904 case DECOMP_TYPE_B18: 905 l = 18; 906 goto decomp_type_b; 907 case DECOMP_TYPE_B1: 908 case DECOMP_TYPE_B2: 909 case DECOMP_TYPE_B3: 910 case DECOMP_TYPE_B4: 911 case DECOMP_TYPE_B5: 912 case DECOMP_TYPE_B6: 913 case DECOMP_TYPE_B7: 914 case DECOMP_TYPE_B8: 915 l = type - DECOMP_TYPE_B1 + 1; 916 decomp_type_b: 917 { 918 uint32_t c_min; 919 c_min = unicode_get16(d); 920 d += 2 + (c - code) * l; 921 for(i = 0; i < l; i++) { 922 c1 = d[i]; 923 if (c1 == 0xff) 924 c1 = 0x20; 925 else 926 c1 += c_min; 927 res[i] = c1; 928 } 929 } 930 return l; 931 case DECOMP_TYPE_LS2: 932 d += (c - code) * 3; 933 if (!(res[0] = unicode_get16(d))) 934 return 0; 935 res[1] = unicode_get_short_code(d[2]); 936 return 2; 937 case DECOMP_TYPE_PAT3: 938 res[0] = unicode_get16(d); 939 res[2] = unicode_get16(d + 2); 940 d += 4 + (c - code) * 2; 941 res[1] = unicode_get16(d); 942 return 3; 943 case DECOMP_TYPE_S2_UL: 944 case DECOMP_TYPE_LS2_UL: 945 c1 = c - code; 946 if (type == DECOMP_TYPE_S2_UL) { 947 d += c1 & ~1; 948 c = unicode_get_short_code(*d); 949 d++; 950 } else { 951 d += (c1 >> 1) * 3; 952 c = unicode_get16(d); 953 d += 2; 954 } 955 if (c1 & 1) 956 c = unicode_get_lower_simple(c); 957 res[0] = c; 958 res[1] = unicode_get_short_code(*d); 959 return 2; 960 } 961 } 962 return 0; 963 } 964 965 966 /* return the length of the decomposition (length <= 967 UNICODE_DECOMP_LEN_MAX) or 0 if no decomposition */ 968 static int unicode_decomp_char(uint32_t *res, uint32_t c, BOOL is_compat1) 969 { 970 uint32_t v, type, is_compat, code, len; 971 int idx_min, idx_max, idx; 972 973 idx_min = 0; 974 idx_max = countof(unicode_decomp_table1) - 1; 975 while (idx_min <= idx_max) { 976 idx = (idx_max + idx_min) / 2; 977 v = unicode_decomp_table1[idx]; 978 code = v >> (32 - 18); 979 len = (v >> (32 - 18 - 7)) & 0x7f; 980 // printf("idx=%d code=%05x len=%d\n", idx, code, len); 981 if (c < code) { 982 idx_max = idx - 1; 983 } else if (c >= code + len) { 984 idx_min = idx + 1; 985 } else { 986 is_compat = v & 1; 987 if (is_compat1 < is_compat) 988 break; 989 type = (v >> (32 - 18 - 7 - 6)) & 0x3f; 990 return unicode_decomp_entry(res, c, idx, code, len, type); 991 } 992 } 993 return 0; 994 } 995 996 /* return 0 if no pair found */ 997 static int unicode_compose_pair(uint32_t c0, uint32_t c1) 998 { 999 uint32_t code, len, type, v, idx1, d_idx, d_offset, ch; 1000 int idx_min, idx_max, idx, d; 1001 uint32_t pair[2]; 1002 1003 idx_min = 0; 1004 idx_max = countof(unicode_comp_table) - 1; 1005 while (idx_min <= idx_max) { 1006 idx = (idx_max + idx_min) / 2; 1007 idx1 = unicode_comp_table[idx]; 1008 1009 /* idx1 represent an entry of the decomposition table */ 1010 d_idx = idx1 >> 6; 1011 d_offset = idx1 & 0x3f; 1012 v = unicode_decomp_table1[d_idx]; 1013 code = v >> (32 - 18); 1014 len = (v >> (32 - 18 - 7)) & 0x7f; 1015 type = (v >> (32 - 18 - 7 - 6)) & 0x3f; 1016 ch = code + d_offset; 1017 unicode_decomp_entry(pair, ch, d_idx, code, len, type); 1018 d = c0 - pair[0]; 1019 if (d == 0) 1020 d = c1 - pair[1]; 1021 if (d < 0) { 1022 idx_max = idx - 1; 1023 } else if (d > 0) { 1024 idx_min = idx + 1; 1025 } else { 1026 return ch; 1027 } 1028 } 1029 return 0; 1030 } 1031 1032 /* return the combining class of character c (between 0 and 255) */ 1033 static int unicode_get_cc(uint32_t c) 1034 { 1035 uint32_t code, n, type, cc, c1, b; 1036 int pos; 1037 const uint8_t *p; 1038 1039 pos = get_index_pos(&code, c, 1040 unicode_cc_index, sizeof(unicode_cc_index) / 3); 1041 if (pos < 0) 1042 return 0; 1043 p = unicode_cc_table + pos; 1044 /* Compressed run length encoding: 1045 - 2 high order bits are combining class type 1046 - 0:0, 1:230, 2:extra byte linear progression, 3:extra byte 1047 - 00..2F: range length (add 1) 1048 - 30..37: 3-bit range-length + 1 extra byte 1049 - 38..3F: 3-bit range-length + 2 extra byte 1050 */ 1051 for(;;) { 1052 b = *p++; 1053 type = b >> 6; 1054 n = b & 0x3f; 1055 if (n < 48) { 1056 } else if (n < 56) { 1057 n = (n - 48) << 8; 1058 n |= *p++; 1059 n += 48; 1060 } else { 1061 n = (n - 56) << 8; 1062 n |= *p++ << 8; 1063 n |= *p++; 1064 n += 48 + (1 << 11); 1065 } 1066 if (type <= 1) 1067 p++; 1068 c1 = code + n + 1; 1069 if (c < c1) { 1070 switch(type) { 1071 case 0: 1072 cc = p[-1]; 1073 break; 1074 case 1: 1075 cc = p[-1] + c - code; 1076 break; 1077 case 2: 1078 cc = 0; 1079 break; 1080 default: 1081 case 3: 1082 cc = 230; 1083 break; 1084 } 1085 return cc; 1086 } 1087 code = c1; 1088 } 1089 } 1090 1091 static void sort_cc(int *buf, int len) 1092 { 1093 int i, j, k, cc, cc1, start, ch1; 1094 1095 for(i = 0; i < len; i++) { 1096 cc = unicode_get_cc(buf[i]); 1097 if (cc != 0) { 1098 start = i; 1099 j = i + 1; 1100 while (j < len) { 1101 ch1 = buf[j]; 1102 cc1 = unicode_get_cc(ch1); 1103 if (cc1 == 0) 1104 break; 1105 k = j - 1; 1106 while (k >= start) { 1107 if (unicode_get_cc(buf[k]) <= cc1) 1108 break; 1109 buf[k + 1] = buf[k]; 1110 k--; 1111 } 1112 buf[k + 1] = ch1; 1113 j++; 1114 } 1115 #if 0 1116 printf("cc:"); 1117 for(k = start; k < j; k++) { 1118 printf(" %3d", unicode_get_cc(buf[k])); 1119 } 1120 printf("\n"); 1121 #endif 1122 i = j; 1123 } 1124 } 1125 } 1126 1127 static void to_nfd_rec(DynBuf *dbuf, 1128 const int *src, int src_len, int is_compat) 1129 { 1130 uint32_t c, v; 1131 int i, l; 1132 uint32_t res[UNICODE_DECOMP_LEN_MAX]; 1133 1134 for(i = 0; i < src_len; i++) { 1135 c = src[i]; 1136 if (c >= 0xac00 && c < 0xd7a4) { 1137 /* Hangul decomposition */ 1138 c -= 0xac00; 1139 dbuf_put_u32(dbuf, 0x1100 + c / 588); 1140 dbuf_put_u32(dbuf, 0x1161 + (c % 588) / 28); 1141 v = c % 28; 1142 if (v != 0) 1143 dbuf_put_u32(dbuf, 0x11a7 + v); 1144 } else { 1145 l = unicode_decomp_char(res, c, is_compat); 1146 if (l) { 1147 to_nfd_rec(dbuf, (int *)res, l, is_compat); 1148 } else { 1149 dbuf_put_u32(dbuf, c); 1150 } 1151 } 1152 } 1153 } 1154 1155 /* return 0 if not found */ 1156 static int compose_pair(uint32_t c0, uint32_t c1) 1157 { 1158 /* Hangul composition */ 1159 if (c0 >= 0x1100 && c0 < 0x1100 + 19 && 1160 c1 >= 0x1161 && c1 < 0x1161 + 21) { 1161 return 0xac00 + (c0 - 0x1100) * 588 + (c1 - 0x1161) * 28; 1162 } else if (c0 >= 0xac00 && c0 < 0xac00 + 11172 && 1163 (c0 - 0xac00) % 28 == 0 && 1164 c1 >= 0x11a7 && c1 < 0x11a7 + 28) { 1165 return c0 + c1 - 0x11a7; 1166 } else { 1167 return unicode_compose_pair(c0, c1); 1168 } 1169 } 1170 1171 int unicode_normalize(uint32_t **pdst, const uint32_t *src, int src_len, 1172 UnicodeNormalizationEnum n_type, 1173 void *opaque, DynBufReallocFunc *realloc_func) 1174 { 1175 int *buf, buf_len, i, p, starter_pos, cc, last_cc, out_len; 1176 BOOL is_compat; 1177 DynBuf dbuf_s, *dbuf = &dbuf_s; 1178 1179 is_compat = n_type >> 1; 1180 1181 dbuf_init2(dbuf, opaque, realloc_func); 1182 if (dbuf_claim(dbuf, sizeof(int) * src_len)) 1183 goto fail; 1184 1185 /* common case: latin1 is unaffected by NFC */ 1186 if (n_type == UNICODE_NFC) { 1187 for(i = 0; i < src_len; i++) { 1188 if (src[i] >= 0x100) 1189 goto not_latin1; 1190 } 1191 buf = (int *)dbuf->buf; 1192 if (src_len != 0) 1193 memcpy(buf, src, src_len * sizeof(int)); 1194 *pdst = (uint32_t *)buf; 1195 return src_len; 1196 not_latin1: ; 1197 } 1198 1199 to_nfd_rec(dbuf, (const int *)src, src_len, is_compat); 1200 if (dbuf_error(dbuf)) { 1201 fail: 1202 *pdst = NULL; 1203 return -1; 1204 } 1205 buf = (int *)dbuf->buf; 1206 buf_len = dbuf->size / sizeof(int); 1207 1208 sort_cc(buf, buf_len); 1209 1210 if (buf_len <= 1 || (n_type & 1) != 0) { 1211 /* NFD / NFKD */ 1212 *pdst = (uint32_t *)buf; 1213 return buf_len; 1214 } 1215 1216 i = 1; 1217 out_len = 1; 1218 while (i < buf_len) { 1219 /* find the starter character and test if it is blocked from 1220 the character at 'i' */ 1221 last_cc = unicode_get_cc(buf[i]); 1222 starter_pos = out_len - 1; 1223 while (starter_pos >= 0) { 1224 cc = unicode_get_cc(buf[starter_pos]); 1225 if (cc == 0) 1226 break; 1227 if (cc >= last_cc) 1228 goto next; 1229 last_cc = 256; 1230 starter_pos--; 1231 } 1232 if (starter_pos >= 0 && 1233 (p = compose_pair(buf[starter_pos], buf[i])) != 0) { 1234 buf[starter_pos] = p; 1235 i++; 1236 } else { 1237 next: 1238 buf[out_len++] = buf[i++]; 1239 } 1240 } 1241 *pdst = (uint32_t *)buf; 1242 return out_len; 1243 } 1244 1245 /* char ranges for various unicode properties */ 1246 1247 static int unicode_find_name(const char *name_table, const char *name) 1248 { 1249 const char *p, *r; 1250 int pos; 1251 size_t name_len, len; 1252 1253 p = name_table; 1254 pos = 0; 1255 name_len = strlen(name); 1256 while (*p) { 1257 for(;;) { 1258 r = strchr(p, ','); 1259 if (!r) 1260 len = strlen(p); 1261 else 1262 len = r - p; 1263 if (len == name_len && !memcmp(p, name, name_len)) 1264 return pos; 1265 p += len + 1; 1266 if (!r) 1267 break; 1268 } 1269 pos++; 1270 } 1271 return -1; 1272 } 1273 1274 /* 'cr' must be initialized and empty. Return 0 if OK, -1 if error, -2 1275 if not found */ 1276 int unicode_script(CharRange *cr, 1277 const char *script_name, BOOL is_ext) 1278 { 1279 int script_idx; 1280 const uint8_t *p, *p_end; 1281 uint32_t c, c1, b, n, v, v_len, i, type; 1282 CharRange cr1_s, *cr1; 1283 CharRange cr2_s, *cr2 = &cr2_s; 1284 BOOL is_common; 1285 1286 script_idx = unicode_find_name(unicode_script_name_table, script_name); 1287 if (script_idx < 0) 1288 return -2; 1289 1290 is_common = (script_idx == UNICODE_SCRIPT_Common || 1291 script_idx == UNICODE_SCRIPT_Inherited); 1292 if (is_ext) { 1293 cr1 = &cr1_s; 1294 cr_init(cr1, cr->mem_opaque, cr->realloc_func); 1295 cr_init(cr2, cr->mem_opaque, cr->realloc_func); 1296 } else { 1297 cr1 = cr; 1298 } 1299 1300 p = unicode_script_table; 1301 p_end = unicode_script_table + countof(unicode_script_table); 1302 c = 0; 1303 while (p < p_end) { 1304 b = *p++; 1305 type = b >> 7; 1306 n = b & 0x7f; 1307 if (n < 96) { 1308 } else if (n < 112) { 1309 n = (n - 96) << 8; 1310 n |= *p++; 1311 n += 96; 1312 } else { 1313 n = (n - 112) << 16; 1314 n |= *p++ << 8; 1315 n |= *p++; 1316 n += 96 + (1 << 12); 1317 } 1318 c1 = c + n + 1; 1319 if (type != 0) { 1320 v = *p++; 1321 if (v == script_idx || script_idx == UNICODE_SCRIPT_Unknown) { 1322 if (cr_add_interval(cr1, c, c1)) 1323 goto fail; 1324 } 1325 } 1326 c = c1; 1327 } 1328 if (script_idx == UNICODE_SCRIPT_Unknown) { 1329 /* Unknown is all the characters outside scripts */ 1330 if (cr_invert(cr1)) 1331 goto fail; 1332 } 1333 1334 if (is_ext) { 1335 /* add the script extensions */ 1336 p = unicode_script_ext_table; 1337 p_end = unicode_script_ext_table + countof(unicode_script_ext_table); 1338 c = 0; 1339 while (p < p_end) { 1340 b = *p++; 1341 if (b < 128) { 1342 n = b; 1343 } else if (b < 128 + 64) { 1344 n = (b - 128) << 8; 1345 n |= *p++; 1346 n += 128; 1347 } else { 1348 n = (b - 128 - 64) << 16; 1349 n |= *p++ << 8; 1350 n |= *p++; 1351 n += 128 + (1 << 14); 1352 } 1353 c1 = c + n + 1; 1354 v_len = *p++; 1355 if (is_common) { 1356 if (v_len != 0) { 1357 if (cr_add_interval(cr2, c, c1)) 1358 goto fail; 1359 } 1360 } else { 1361 for(i = 0; i < v_len; i++) { 1362 if (p[i] == script_idx) { 1363 if (cr_add_interval(cr2, c, c1)) 1364 goto fail; 1365 break; 1366 } 1367 } 1368 } 1369 p += v_len; 1370 c = c1; 1371 } 1372 if (is_common) { 1373 /* remove all the characters with script extensions */ 1374 if (cr_invert(cr2)) 1375 goto fail; 1376 if (cr_op(cr, cr1->points, cr1->len, cr2->points, cr2->len, 1377 CR_OP_INTER)) 1378 goto fail; 1379 } else { 1380 if (cr_op(cr, cr1->points, cr1->len, cr2->points, cr2->len, 1381 CR_OP_UNION)) 1382 goto fail; 1383 } 1384 cr_free(cr1); 1385 cr_free(cr2); 1386 } 1387 return 0; 1388 fail: 1389 if (is_ext) { 1390 cr_free(cr1); 1391 cr_free(cr2); 1392 } 1393 goto fail; 1394 } 1395 1396 #define M(id) (1U << UNICODE_GC_ ## id) 1397 1398 static int unicode_general_category1(CharRange *cr, uint32_t gc_mask) 1399 { 1400 const uint8_t *p, *p_end; 1401 uint32_t c, c0, b, n, v; 1402 1403 p = unicode_gc_table; 1404 p_end = unicode_gc_table + countof(unicode_gc_table); 1405 c = 0; 1406 /* Compressed range encoding: 1407 initial byte: 1408 bits 0..4: category number (special case 31) 1409 bits 5..7: range length (add 1) 1410 special case bits 5..7 == 7: read an extra byte 1411 - 00..7F: range length (add 7 + 1) 1412 - 80..BF: 6-bits plus extra byte for range length (add 7 + 128) 1413 - C0..FF: 6-bits plus 2 extra bytes for range length (add 7 + 128 + 16384) 1414 */ 1415 while (p < p_end) { 1416 b = *p++; 1417 n = b >> 5; 1418 v = b & 0x1f; 1419 if (n == 7) { 1420 n = *p++; 1421 if (n < 128) { 1422 n += 7; 1423 } else if (n < 128 + 64) { 1424 n = (n - 128) << 8; 1425 n |= *p++; 1426 n += 7 + 128; 1427 } else { 1428 n = (n - 128 - 64) << 16; 1429 n |= *p++ << 8; 1430 n |= *p++; 1431 n += 7 + 128 + (1 << 14); 1432 } 1433 } 1434 c0 = c; 1435 c += n + 1; 1436 if (v == 31) { 1437 /* run of Lu / Ll */ 1438 b = gc_mask & (M(Lu) | M(Ll)); 1439 if (b != 0) { 1440 if (b == (M(Lu) | M(Ll))) { 1441 goto add_range; 1442 } else { 1443 c0 += ((gc_mask & M(Ll)) != 0); 1444 for(; c0 < c; c0 += 2) { 1445 if (cr_add_interval(cr, c0, c0 + 1)) 1446 return -1; 1447 } 1448 } 1449 } 1450 } else if ((gc_mask >> v) & 1) { 1451 add_range: 1452 if (cr_add_interval(cr, c0, c)) 1453 return -1; 1454 } 1455 } 1456 return 0; 1457 } 1458 1459 static int unicode_prop1(CharRange *cr, int prop_idx) 1460 { 1461 const uint8_t *p, *p_end; 1462 uint32_t c, c0, b, bit; 1463 1464 p = unicode_prop_table[prop_idx]; 1465 p_end = p + unicode_prop_len_table[prop_idx]; 1466 c = 0; 1467 bit = 0; 1468 /* Compressed range encoding: 1469 00..3F: 2 packed lengths: 3-bit + 3-bit 1470 40..5F: 5-bits plus extra byte for length 1471 60..7F: 5-bits plus 2 extra bytes for length 1472 80..FF: 7-bit length 1473 lengths must be incremented to get character count 1474 Ranges alternate between false and true return value. 1475 */ 1476 while (p < p_end) { 1477 c0 = c; 1478 b = *p++; 1479 if (b < 64) { 1480 c += (b >> 3) + 1; 1481 if (bit) { 1482 if (cr_add_interval(cr, c0, c)) 1483 return -1; 1484 } 1485 bit ^= 1; 1486 c0 = c; 1487 c += (b & 7) + 1; 1488 } else if (b >= 0x80) { 1489 c += b - 0x80 + 1; 1490 } else if (b < 0x60) { 1491 c += (((b - 0x40) << 8) | p[0]) + 1; 1492 p++; 1493 } else { 1494 c += (((b - 0x60) << 16) | (p[0] << 8) | p[1]) + 1; 1495 p += 2; 1496 } 1497 if (bit) { 1498 if (cr_add_interval(cr, c0, c)) 1499 return -1; 1500 } 1501 bit ^= 1; 1502 } 1503 return 0; 1504 } 1505 1506 typedef enum { 1507 POP_GC, 1508 POP_PROP, 1509 POP_CASE, 1510 POP_UNION, 1511 POP_INTER, 1512 POP_XOR, 1513 POP_INVERT, 1514 POP_END, 1515 } PropOPEnum; 1516 1517 #define POP_STACK_LEN_MAX 4 1518 1519 static int unicode_prop_ops(CharRange *cr, ...) 1520 { 1521 va_list ap; 1522 CharRange stack[POP_STACK_LEN_MAX]; 1523 int stack_len, op, ret, i; 1524 uint32_t a; 1525 1526 va_start(ap, cr); 1527 stack_len = 0; 1528 for(;;) { 1529 op = va_arg(ap, int); 1530 switch(op) { 1531 case POP_GC: 1532 assert(stack_len < POP_STACK_LEN_MAX); 1533 a = va_arg(ap, int); 1534 cr_init(&stack[stack_len++], cr->mem_opaque, cr->realloc_func); 1535 if (unicode_general_category1(&stack[stack_len - 1], a)) 1536 goto fail; 1537 break; 1538 case POP_PROP: 1539 assert(stack_len < POP_STACK_LEN_MAX); 1540 a = va_arg(ap, int); 1541 cr_init(&stack[stack_len++], cr->mem_opaque, cr->realloc_func); 1542 if (unicode_prop1(&stack[stack_len - 1], a)) 1543 goto fail; 1544 break; 1545 case POP_CASE: 1546 assert(stack_len < POP_STACK_LEN_MAX); 1547 a = va_arg(ap, int); 1548 cr_init(&stack[stack_len++], cr->mem_opaque, cr->realloc_func); 1549 if (unicode_case1(&stack[stack_len - 1], a)) 1550 goto fail; 1551 break; 1552 case POP_UNION: 1553 case POP_INTER: 1554 case POP_XOR: 1555 { 1556 CharRange *cr1, *cr2, *cr3; 1557 assert(stack_len >= 2); 1558 assert(stack_len < POP_STACK_LEN_MAX); 1559 cr1 = &stack[stack_len - 2]; 1560 cr2 = &stack[stack_len - 1]; 1561 cr3 = &stack[stack_len++]; 1562 cr_init(cr3, cr->mem_opaque, cr->realloc_func); 1563 /* CR_OP_XOR may be used here */ 1564 if (cr_op(cr3, cr1->points, cr1->len, 1565 cr2->points, cr2->len, op - POP_UNION + CR_OP_UNION)) 1566 goto fail; 1567 cr_free(cr1); 1568 cr_free(cr2); 1569 *cr1 = *cr3; 1570 stack_len -= 2; 1571 } 1572 break; 1573 case POP_INVERT: 1574 assert(stack_len >= 1); 1575 if (cr_invert(&stack[stack_len - 1])) 1576 goto fail; 1577 break; 1578 case POP_END: 1579 goto done; 1580 default: 1581 abort(); 1582 } 1583 } 1584 done: 1585 assert(stack_len == 1); 1586 ret = cr_copy(cr, &stack[0]); 1587 cr_free(&stack[0]); 1588 return ret; 1589 fail: 1590 for(i = 0; i < stack_len; i++) 1591 cr_free(&stack[i]); 1592 return -1; 1593 } 1594 1595 static const uint32_t unicode_gc_mask_table[] = { 1596 M(Lu) | M(Ll) | M(Lt), /* LC */ 1597 M(Lu) | M(Ll) | M(Lt) | M(Lm) | M(Lo), /* L */ 1598 M(Mn) | M(Mc) | M(Me), /* M */ 1599 M(Nd) | M(Nl) | M(No), /* N */ 1600 M(Sm) | M(Sc) | M(Sk) | M(So), /* S */ 1601 M(Pc) | M(Pd) | M(Ps) | M(Pe) | M(Pi) | M(Pf) | M(Po), /* P */ 1602 M(Zs) | M(Zl) | M(Zp), /* Z */ 1603 M(Cc) | M(Cf) | M(Cs) | M(Co) | M(Cn), /* C */ 1604 }; 1605 1606 /* 'cr' must be initialized and empty. Return 0 if OK, -1 if error, -2 1607 if not found */ 1608 int unicode_general_category(CharRange *cr, const char *gc_name) 1609 { 1610 int gc_idx; 1611 uint32_t gc_mask; 1612 1613 gc_idx = unicode_find_name(unicode_gc_name_table, gc_name); 1614 if (gc_idx < 0) 1615 return -2; 1616 if (gc_idx <= UNICODE_GC_Co) { 1617 gc_mask = (uint64_t)1 << gc_idx; 1618 } else { 1619 gc_mask = unicode_gc_mask_table[gc_idx - UNICODE_GC_LC]; 1620 } 1621 return unicode_general_category1(cr, gc_mask); 1622 } 1623 1624 1625 /* 'cr' must be initialized and empty. Return 0 if OK, -1 if error, -2 1626 if not found */ 1627 int unicode_prop(CharRange *cr, const char *prop_name) 1628 { 1629 int prop_idx, ret; 1630 1631 prop_idx = unicode_find_name(unicode_prop_name_table, prop_name); 1632 if (prop_idx < 0) 1633 return -2; 1634 prop_idx += UNICODE_PROP_ASCII_Hex_Digit; 1635 1636 ret = 0; 1637 switch(prop_idx) { 1638 case UNICODE_PROP_ASCII: 1639 if (cr_add_interval(cr, 0x00, 0x7f + 1)) 1640 return -1; 1641 break; 1642 case UNICODE_PROP_Any: 1643 if (cr_add_interval(cr, 0x00000, 0x10ffff + 1)) 1644 return -1; 1645 break; 1646 case UNICODE_PROP_Assigned: 1647 ret = unicode_prop_ops(cr, 1648 POP_GC, M(Cn), 1649 POP_INVERT, 1650 POP_END); 1651 break; 1652 case UNICODE_PROP_Math: 1653 ret = unicode_prop_ops(cr, 1654 POP_GC, M(Sm), 1655 POP_PROP, UNICODE_PROP_Other_Math, 1656 POP_UNION, 1657 POP_END); 1658 break; 1659 case UNICODE_PROP_Lowercase: 1660 ret = unicode_prop_ops(cr, 1661 POP_GC, M(Ll), 1662 POP_PROP, UNICODE_PROP_Other_Lowercase, 1663 POP_UNION, 1664 POP_END); 1665 break; 1666 case UNICODE_PROP_Uppercase: 1667 ret = unicode_prop_ops(cr, 1668 POP_GC, M(Lu), 1669 POP_PROP, UNICODE_PROP_Other_Uppercase, 1670 POP_UNION, 1671 POP_END); 1672 break; 1673 case UNICODE_PROP_Cased: 1674 ret = unicode_prop_ops(cr, 1675 POP_GC, M(Lu) | M(Ll) | M(Lt), 1676 POP_PROP, UNICODE_PROP_Other_Uppercase, 1677 POP_UNION, 1678 POP_PROP, UNICODE_PROP_Other_Lowercase, 1679 POP_UNION, 1680 POP_END); 1681 break; 1682 case UNICODE_PROP_Alphabetic: 1683 ret = unicode_prop_ops(cr, 1684 POP_GC, M(Lu) | M(Ll) | M(Lt) | M(Lm) | M(Lo) | M(Nl), 1685 POP_PROP, UNICODE_PROP_Other_Uppercase, 1686 POP_UNION, 1687 POP_PROP, UNICODE_PROP_Other_Lowercase, 1688 POP_UNION, 1689 POP_PROP, UNICODE_PROP_Other_Alphabetic, 1690 POP_UNION, 1691 POP_END); 1692 break; 1693 case UNICODE_PROP_Grapheme_Base: 1694 ret = unicode_prop_ops(cr, 1695 POP_GC, M(Cc) | M(Cf) | M(Cs) | M(Co) | M(Cn) | M(Zl) | M(Zp) | M(Me) | M(Mn), 1696 POP_PROP, UNICODE_PROP_Other_Grapheme_Extend, 1697 POP_UNION, 1698 POP_INVERT, 1699 POP_END); 1700 break; 1701 case UNICODE_PROP_Grapheme_Extend: 1702 ret = unicode_prop_ops(cr, 1703 POP_GC, M(Me) | M(Mn), 1704 POP_PROP, UNICODE_PROP_Other_Grapheme_Extend, 1705 POP_UNION, 1706 POP_END); 1707 break; 1708 case UNICODE_PROP_XID_Start: 1709 ret = unicode_prop_ops(cr, 1710 POP_GC, M(Lu) | M(Ll) | M(Lt) | M(Lm) | M(Lo) | M(Nl), 1711 POP_PROP, UNICODE_PROP_Other_ID_Start, 1712 POP_UNION, 1713 POP_PROP, UNICODE_PROP_Pattern_Syntax, 1714 POP_PROP, UNICODE_PROP_Pattern_White_Space, 1715 POP_UNION, 1716 POP_PROP, UNICODE_PROP_XID_Start1, 1717 POP_UNION, 1718 POP_INVERT, 1719 POP_INTER, 1720 POP_END); 1721 break; 1722 case UNICODE_PROP_XID_Continue: 1723 ret = unicode_prop_ops(cr, 1724 POP_GC, M(Lu) | M(Ll) | M(Lt) | M(Lm) | M(Lo) | M(Nl) | 1725 M(Mn) | M(Mc) | M(Nd) | M(Pc), 1726 POP_PROP, UNICODE_PROP_Other_ID_Start, 1727 POP_UNION, 1728 POP_PROP, UNICODE_PROP_Other_ID_Continue, 1729 POP_UNION, 1730 POP_PROP, UNICODE_PROP_Pattern_Syntax, 1731 POP_PROP, UNICODE_PROP_Pattern_White_Space, 1732 POP_UNION, 1733 POP_PROP, UNICODE_PROP_XID_Continue1, 1734 POP_UNION, 1735 POP_INVERT, 1736 POP_INTER, 1737 POP_END); 1738 break; 1739 case UNICODE_PROP_Changes_When_Uppercased: 1740 ret = unicode_case1(cr, CASE_U); 1741 break; 1742 case UNICODE_PROP_Changes_When_Lowercased: 1743 ret = unicode_case1(cr, CASE_L); 1744 break; 1745 case UNICODE_PROP_Changes_When_Casemapped: 1746 ret = unicode_case1(cr, CASE_U | CASE_L | CASE_F); 1747 break; 1748 case UNICODE_PROP_Changes_When_Titlecased: 1749 ret = unicode_prop_ops(cr, 1750 POP_CASE, CASE_U, 1751 POP_PROP, UNICODE_PROP_Changes_When_Titlecased1, 1752 POP_XOR, 1753 POP_END); 1754 break; 1755 case UNICODE_PROP_Changes_When_Casefolded: 1756 ret = unicode_prop_ops(cr, 1757 POP_CASE, CASE_F, 1758 POP_PROP, UNICODE_PROP_Changes_When_Casefolded1, 1759 POP_XOR, 1760 POP_END); 1761 break; 1762 case UNICODE_PROP_Changes_When_NFKC_Casefolded: 1763 ret = unicode_prop_ops(cr, 1764 POP_CASE, CASE_F, 1765 POP_PROP, UNICODE_PROP_Changes_When_NFKC_Casefolded1, 1766 POP_XOR, 1767 POP_END); 1768 break; 1769 #if 0 1770 case UNICODE_PROP_ID_Start: 1771 ret = unicode_prop_ops(cr, 1772 POP_GC, M(Lu) | M(Ll) | M(Lt) | M(Lm) | M(Lo) | M(Nl), 1773 POP_PROP, UNICODE_PROP_Other_ID_Start, 1774 POP_UNION, 1775 POP_PROP, UNICODE_PROP_Pattern_Syntax, 1776 POP_PROP, UNICODE_PROP_Pattern_White_Space, 1777 POP_UNION, 1778 POP_INVERT, 1779 POP_INTER, 1780 POP_END); 1781 break; 1782 case UNICODE_PROP_ID_Continue: 1783 ret = unicode_prop_ops(cr, 1784 POP_GC, M(Lu) | M(Ll) | M(Lt) | M(Lm) | M(Lo) | M(Nl) | 1785 M(Mn) | M(Mc) | M(Nd) | M(Pc), 1786 POP_PROP, UNICODE_PROP_Other_ID_Start, 1787 POP_UNION, 1788 POP_PROP, UNICODE_PROP_Other_ID_Continue, 1789 POP_UNION, 1790 POP_PROP, UNICODE_PROP_Pattern_Syntax, 1791 POP_PROP, UNICODE_PROP_Pattern_White_Space, 1792 POP_UNION, 1793 POP_INVERT, 1794 POP_INTER, 1795 POP_END); 1796 break; 1797 case UNICODE_PROP_Case_Ignorable: 1798 ret = unicode_prop_ops(cr, 1799 POP_GC, M(Mn) | M(Cf) | M(Lm) | M(Sk), 1800 POP_PROP, UNICODE_PROP_Case_Ignorable1, 1801 POP_XOR, 1802 POP_END); 1803 break; 1804 #else 1805 /* we use the existing tables */ 1806 case UNICODE_PROP_ID_Continue: 1807 ret = unicode_prop_ops(cr, 1808 POP_PROP, UNICODE_PROP_ID_Start, 1809 POP_PROP, UNICODE_PROP_ID_Continue1, 1810 POP_XOR, 1811 POP_END); 1812 break; 1813 #endif 1814 default: 1815 if (prop_idx >= countof(unicode_prop_table)) 1816 return -2; 1817 ret = unicode_prop1(cr, prop_idx); 1818 break; 1819 } 1820 return ret; 1821 } 1822 1823 #endif /* CONFIG_ALL_UNICODE */ 1824 1825 /*---- lre codepoint categorizing functions ----*/ 1826 1827 #define S UNICODE_C_SPACE 1828 #define D UNICODE_C_DIGIT 1829 #define X UNICODE_C_XDIGIT 1830 #define U UNICODE_C_UPPER 1831 #define L UNICODE_C_LOWER 1832 #define _ UNICODE_C_UNDER 1833 #define d UNICODE_C_DOLLAR 1834 1835 uint8_t const lre_ctype_bits[256] = { 1836 0, 0, 0, 0, 0, 0, 0, 0, 1837 0, S, S, S, S, S, 0, 0, 1838 0, 0, 0, 0, 0, 0, 0, 0, 1839 0, 0, 0, 0, 0, 0, 0, 0, 1840 1841 S, 0, 0, 0, d, 0, 0, 0, 1842 0, 0, 0, 0, 0, 0, 0, 0, 1843 X|D, X|D, X|D, X|D, X|D, X|D, X|D, X|D, 1844 X|D, X|D, 0, 0, 0, 0, 0, 0, 1845 1846 0, X|U, X|U, X|U, X|U, X|U, X|U, U, 1847 U, U, U, U, U, U, U, U, 1848 U, U, U, U, U, U, U, U, 1849 U, U, U, 0, 0, 0, 0, _, 1850 1851 0, X|L, X|L, X|L, X|L, X|L, X|L, L, 1852 L, L, L, L, L, L, L, L, 1853 L, L, L, L, L, L, L, L, 1854 L, L, L, 0, 0, 0, 0, 0, 1855 1856 0, 0, 0, 0, 0, 0, 0, 0, 1857 0, 0, 0, 0, 0, 0, 0, 0, 1858 0, 0, 0, 0, 0, 0, 0, 0, 1859 0, 0, 0, 0, 0, 0, 0, 0, 1860 1861 S, 0, 0, 0, 0, 0, 0, 0, 1862 0, 0, 0, 0, 0, 0, 0, 0, 1863 0, 0, 0, 0, 0, 0, 0, 0, 1864 0, 0, 0, 0, 0, 0, 0, 0, 1865 1866 0, 0, 0, 0, 0, 0, 0, 0, 1867 0, 0, 0, 0, 0, 0, 0, 0, 1868 0, 0, 0, 0, 0, 0, 0, 0, 1869 0, 0, 0, 0, 0, 0, 0, 0, 1870 1871 0, 0, 0, 0, 0, 0, 0, 0, 1872 0, 0, 0, 0, 0, 0, 0, 0, 1873 0, 0, 0, 0, 0, 0, 0, 0, 1874 0, 0, 0, 0, 0, 0, 0, 0, 1875 }; 1876 1877 #undef S 1878 #undef D 1879 #undef X 1880 #undef U 1881 #undef L 1882 #undef _ 1883 #undef d 1884 1885 /* code point ranges for Zs,Zl or Zp property */ 1886 static const uint16_t char_range_s[] = { 1887 10, 1888 0x0009, 0x000D + 1, 1889 0x0020, 0x0020 + 1, 1890 0x00A0, 0x00A0 + 1, 1891 0x1680, 0x1680 + 1, 1892 0x2000, 0x200A + 1, 1893 /* 2028;LINE SEPARATOR;Zl;0;WS;;;;;N;;;;; */ 1894 /* 2029;PARAGRAPH SEPARATOR;Zp;0;B;;;;;N;;;;; */ 1895 0x2028, 0x2029 + 1, 1896 0x202F, 0x202F + 1, 1897 0x205F, 0x205F + 1, 1898 0x3000, 0x3000 + 1, 1899 /* FEFF;ZERO WIDTH NO-BREAK SPACE;Cf;0;BN;;;;;N;BYTE ORDER MARK;;;; */ 1900 0xFEFF, 0xFEFF + 1, 1901 }; 1902 1903 BOOL lre_is_space_non_ascii(uint32_t c) 1904 { 1905 size_t i, n; 1906 1907 n = countof(char_range_s); 1908 for(i = 5; i < n; i += 2) { 1909 uint32_t low = char_range_s[i]; 1910 uint32_t high = char_range_s[i + 1]; 1911 if (c < low) 1912 return FALSE; 1913 if (c < high) 1914 return TRUE; 1915 } 1916 return FALSE; 1917 } 1918 1919 #define SEQ_MAX_LEN 16 1920 1921 static int unicode_sequence_prop1(int seq_prop_idx, UnicodeSequencePropCB *cb, void *opaque, 1922 CharRange *cr) 1923 { 1924 int i, c, j; 1925 uint32_t seq[SEQ_MAX_LEN]; 1926 1927 switch(seq_prop_idx) { 1928 case UNICODE_SEQUENCE_PROP_Basic_Emoji: 1929 if (unicode_prop1(cr, UNICODE_PROP_Basic_Emoji1) < 0) 1930 return -1; 1931 for(i = 0; i < cr->len; i += 2) { 1932 for(c = cr->points[i]; c < cr->points[i + 1]; c++) { 1933 seq[0] = c; 1934 cb(opaque, seq, 1); 1935 } 1936 } 1937 1938 cr->len = 0; 1939 1940 if (unicode_prop1(cr, UNICODE_PROP_Basic_Emoji2) < 0) 1941 return -1; 1942 for(i = 0; i < cr->len; i += 2) { 1943 for(c = cr->points[i]; c < cr->points[i + 1]; c++) { 1944 seq[0] = c; 1945 seq[1] = 0xfe0f; 1946 cb(opaque, seq, 2); 1947 } 1948 } 1949 1950 break; 1951 case UNICODE_SEQUENCE_PROP_RGI_Emoji_Modifier_Sequence: 1952 if (unicode_prop1(cr, UNICODE_PROP_Emoji_Modifier_Base) < 0) 1953 return -1; 1954 for(i = 0; i < cr->len; i += 2) { 1955 for(c = cr->points[i]; c < cr->points[i + 1]; c++) { 1956 for(j = 0; j < 5; j++) { 1957 seq[0] = c; 1958 seq[1] = 0x1f3fb + j; 1959 cb(opaque, seq, 2); 1960 } 1961 } 1962 } 1963 break; 1964 case UNICODE_SEQUENCE_PROP_RGI_Emoji_Flag_Sequence: 1965 if (unicode_prop1(cr, UNICODE_PROP_RGI_Emoji_Flag_Sequence) < 0) 1966 return -1; 1967 for(i = 0; i < cr->len; i += 2) { 1968 for(c = cr->points[i]; c < cr->points[i + 1]; c++) { 1969 int c0, c1; 1970 c0 = c / 26; 1971 c1 = c % 26; 1972 seq[0] = 0x1F1E6 + c0; 1973 seq[1] = 0x1F1E6 + c1; 1974 cb(opaque, seq, 2); 1975 } 1976 } 1977 break; 1978 case UNICODE_SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence: 1979 { 1980 int len, code, pres, k, mod, mod_count, mod_pos[2], hc_pos, n_mod, n_hc, mod1; 1981 int mod_idx, hc_idx, i0, i1; 1982 const uint8_t *tab = unicode_rgi_emoji_zwj_sequence; 1983 1984 for(i = 0; i < countof(unicode_rgi_emoji_zwj_sequence);) { 1985 len = tab[i++]; 1986 k = 0; 1987 mod = 0; 1988 mod_count = 0; 1989 hc_pos = -1; 1990 for(j = 0; j < len; j++) { 1991 code = tab[i++]; 1992 code |= tab[i++] << 8; 1993 pres = code >> 15; 1994 mod1 = (code >> 13) & 3; 1995 code &= 0x1fff; 1996 if (code < 0x1000) { 1997 c = code + 0x2000; 1998 } else { 1999 c = 0x1f000 + (code - 0x1000); 2000 } 2001 if (c == 0x1f9b0) 2002 hc_pos = k; 2003 seq[k++] = c; 2004 if (mod1 != 0) { 2005 assert(mod_count < 2); 2006 mod = mod1; 2007 mod_pos[mod_count++] = k; 2008 seq[k++] = 0; /* will be filled later */ 2009 } 2010 if (pres) { 2011 seq[k++] = 0xfe0f; 2012 } 2013 if (j < len - 1) { 2014 seq[k++] = 0x200d; 2015 } 2016 } 2017 2018 /* genrate all the variants */ 2019 switch(mod) { 2020 case 1: 2021 n_mod = 5; 2022 break; 2023 case 2: 2024 n_mod = 25; 2025 break; 2026 case 3: 2027 n_mod = 20; 2028 break; 2029 default: 2030 n_mod = 1; 2031 break; 2032 } 2033 if (hc_pos >= 0) 2034 n_hc = 4; 2035 else 2036 n_hc = 1; 2037 for(hc_idx = 0; hc_idx < n_hc; hc_idx++) { 2038 for(mod_idx = 0; mod_idx < n_mod; mod_idx++) { 2039 if (hc_pos >= 0) 2040 seq[hc_pos] = 0x1f9b0 + hc_idx; 2041 2042 switch(mod) { 2043 case 1: 2044 seq[mod_pos[0]] = 0x1f3fb + mod_idx; 2045 break; 2046 case 2: 2047 case 3: 2048 i0 = mod_idx / 5; 2049 i1 = mod_idx % 5; 2050 /* avoid identical values */ 2051 if (mod == 3 && i0 >= i1) 2052 i0++; 2053 seq[mod_pos[0]] = 0x1f3fb + i0; 2054 seq[mod_pos[1]] = 0x1f3fb + i1; 2055 break; 2056 default: 2057 break; 2058 } 2059 #if 0 2060 for(j = 0; j < k; j++) 2061 printf(" %04x", seq[j]); 2062 printf("\n"); 2063 #endif 2064 cb(opaque, seq, k); 2065 } 2066 } 2067 } 2068 } 2069 break; 2070 case UNICODE_SEQUENCE_PROP_RGI_Emoji_Tag_Sequence: 2071 { 2072 for(i = 0; i < countof(unicode_rgi_emoji_tag_sequence);) { 2073 j = 0; 2074 seq[j++] = 0x1F3F4; 2075 for(;;) { 2076 c = unicode_rgi_emoji_tag_sequence[i++]; 2077 if (c == 0x00) 2078 break; 2079 seq[j++] = 0xe0000 + c; 2080 } 2081 seq[j++] = 0xe007f; 2082 cb(opaque, seq, j); 2083 } 2084 } 2085 break; 2086 case UNICODE_SEQUENCE_PROP_Emoji_Keycap_Sequence: 2087 if (unicode_prop1(cr, UNICODE_PROP_Emoji_Keycap_Sequence) < 0) 2088 return -1; 2089 for(i = 0; i < cr->len; i += 2) { 2090 for(c = cr->points[i]; c < cr->points[i + 1]; c++) { 2091 seq[0] = c; 2092 seq[1] = 0xfe0f; 2093 seq[2] = 0x20e3; 2094 cb(opaque, seq, 3); 2095 } 2096 } 2097 break; 2098 case UNICODE_SEQUENCE_PROP_RGI_Emoji: 2099 /* all prevous sequences */ 2100 for(i = UNICODE_SEQUENCE_PROP_Basic_Emoji; i <= UNICODE_SEQUENCE_PROP_RGI_Emoji_ZWJ_Sequence; i++) { 2101 int ret; 2102 ret = unicode_sequence_prop1(i, cb, opaque, cr); 2103 if (ret < 0) 2104 return ret; 2105 cr->len = 0; 2106 } 2107 break; 2108 default: 2109 return -2; 2110 } 2111 return 0; 2112 } 2113 2114 /* build a unicode sequence property */ 2115 /* return -2 if not found, -1 if other error. 'cr' is used as temporary memory. */ 2116 int unicode_sequence_prop(const char *prop_name, UnicodeSequencePropCB *cb, void *opaque, 2117 CharRange *cr) 2118 { 2119 int seq_prop_idx; 2120 seq_prop_idx = unicode_find_name(unicode_sequence_prop_name_table, prop_name); 2121 if (seq_prop_idx < 0) 2122 return -2; 2123 return unicode_sequence_prop1(seq_prop_idx, cb, opaque, cr); 2124 }