1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 27 28/* 29 * UTF-8 text preparation functions (PSARC/2007/149, PSARC/2007/458). 30 * 31 * Man pages: u8_textprep_open(9F), u8_textprep_buf(9F), u8_textprep_close(9F), 32 * u8_textprep_str(9F), u8_strcmp(9F), and u8_validate(9F). See also 33 * the section 3C man pages. 34 * Interface stability: Committed. 35 */ 36 37#include <sys/types.h> 38#ifdef _KERNEL 39#include <sys/param.h> 40#include <sys/sysmacros.h> 41#include <sys/systm.h> 42#include <sys/debug.h> 43#include <sys/kmem.h> 44#include <sys/sunddi.h> 45#else 46#include <strings.h> 47#endif /* _KERNEL */ 48#include <sys/byteorder.h> 49#include <sys/errno.h> 50#include <sys/u8_textprep.h> 51#include <sys/u8_textprep_data.h> 52 53 54/* The maximum possible number of bytes in a UTF-8 character. */ 55#define U8_MB_CUR_MAX (4) 56 57/* 58 * The maximum number of bytes needed for a UTF-8 character to cover 59 * U+0000 - U+FFFF, i.e., the coding space of now deprecated UCS-2. 60 */ 61#define U8_MAX_BYTES_UCS2 (3) 62 63/* The maximum possible number of bytes in a Stream-Safe Text. */ 64#define U8_STREAM_SAFE_TEXT_MAX (128) 65 66/* 67 * The maximum number of characters in a combining/conjoining sequence and 68 * the actual upperbound limit of a combining/conjoining sequence. 69 */ 70#define U8_MAX_CHARS_A_SEQ (32) 71#define U8_UPPER_LIMIT_IN_A_SEQ (31) 72 73/* The combining class value for Starter. */ 74#define U8_COMBINING_CLASS_STARTER (0) 75 76/* 77 * Some Hangul related macros at below. 78 * 79 * The first and the last of Hangul syllables, Hangul Jamo Leading consonants, 80 * Vowels, and optional Trailing consonants in Unicode scalar values. 81 * 82 * Please be noted that the U8_HANGUL_JAMO_T_FIRST is 0x11A7 at below not 83 * the actual U+11A8. This is due to that the trailing consonant is optional 84 * and thus we are doing a pre-calculation of subtracting one. 85 * 86 * Each of 19 modern leading consonants has total 588 possible syllables since 87 * Hangul has 21 modern vowels and 27 modern trailing consonants plus 1 for 88 * no trailing consonant case, i.e., 21 x 28 = 588. 89 * 90 * We also have bunch of Hangul related macros at below. Please bear in mind 91 * that the U8_HANGUL_JAMO_1ST_BYTE can be used to check whether it is 92 * a Hangul Jamo or not but the value does not guarantee that it is a Hangul 93 * Jamo; it just guarantee that it will be most likely. 94 */ 95#define U8_HANGUL_SYL_FIRST (0xAC00U) 96#define U8_HANGUL_SYL_LAST (0xD7A3U) 97 98#define U8_HANGUL_JAMO_L_FIRST (0x1100U) 99#define U8_HANGUL_JAMO_L_LAST (0x1112U) 100#define U8_HANGUL_JAMO_V_FIRST (0x1161U) 101#define U8_HANGUL_JAMO_V_LAST (0x1175U) 102#define U8_HANGUL_JAMO_T_FIRST (0x11A7U) 103#define U8_HANGUL_JAMO_T_LAST (0x11C2U) 104 105#define U8_HANGUL_V_COUNT (21) 106#define U8_HANGUL_VT_COUNT (588) 107#define U8_HANGUL_T_COUNT (28) 108 109#define U8_HANGUL_JAMO_1ST_BYTE (0xE1U) 110 111#define U8_SAVE_HANGUL_AS_UTF8(s, i, j, k, b) \ 112 (s)[(i)] = (uchar_t)(0xE0U | ((uint32_t)(b) & 0xF000U) >> 12); \ 113 (s)[(j)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x0FC0U) >> 6); \ 114 (s)[(k)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x003FU)); 115 116#define U8_HANGUL_JAMO_L(u) \ 117 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_L_LAST) 118 119#define U8_HANGUL_JAMO_V(u) \ 120 ((u) >= U8_HANGUL_JAMO_V_FIRST && (u) <= U8_HANGUL_JAMO_V_LAST) 121 122#define U8_HANGUL_JAMO_T(u) \ 123 ((u) > U8_HANGUL_JAMO_T_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST) 124 125#define U8_HANGUL_JAMO(u) \ 126 ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST) 127 128#define U8_HANGUL_SYLLABLE(u) \ 129 ((u) >= U8_HANGUL_SYL_FIRST && (u) <= U8_HANGUL_SYL_LAST) 130 131#define U8_HANGUL_COMPOSABLE_L_V(s, u) \ 132 ((s) == U8_STATE_HANGUL_L && U8_HANGUL_JAMO_V((u))) 133 134#define U8_HANGUL_COMPOSABLE_LV_T(s, u) \ 135 ((s) == U8_STATE_HANGUL_LV && U8_HANGUL_JAMO_T((u))) 136 137/* The types of decomposition mappings. */ 138#define U8_DECOMP_BOTH (0xF5U) 139#define U8_DECOMP_CANONICAL (0xF6U) 140 141/* The indicator for 16-bit table. */ 142#define U8_16BIT_TABLE_INDICATOR (0x8000U) 143 144/* The following are some convenience macros. */ 145#define U8_PUT_3BYTES_INTO_UTF32(u, b1, b2, b3) \ 146 (u) = ((((uint32_t)(b1) & 0x0F) << 12) | \ 147 (((uint32_t)(b2) & 0x3F) << 6) | \ 148 ((uint32_t)(b3) & 0x3F)); 149#define U8_SIMPLE_SWAP(a, b, t) \ 150 (t) = (a); \ 151 (a) = (b); \ 152 (b) = (t); 153 154#define U8_ASCII_TOUPPER(c) \ 155 (((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 'A' : (c)) 156 157#define U8_ASCII_TOLOWER(c) \ 158 (((c) >= 'A' && (c) <= 'Z') ? (c) - 'A' + 'a' : (c)) 159 160#define U8_ISASCII(c) (((uchar_t)(c)) < 0x80U) 161/* 162 * The following macro assumes that the two characters that are to be 163 * swapped are adjacent to each other and 'a' comes before 'b'. 164 * 165 * If the assumptions are not met, then, the macro will fail. 166 */ 167#define U8_SWAP_COMB_MARKS(a, b) \ 168 for (k = 0; k < disp[(a)]; k++) \ 169 u8t[k] = u8s[start[(a)] + k]; \ 170 for (k = 0; k < disp[(b)]; k++) \ 171 u8s[start[(a)] + k] = u8s[start[(b)] + k]; \ 172 start[(b)] = start[(a)] + disp[(b)]; \ 173 for (k = 0; k < disp[(a)]; k++) \ 174 u8s[start[(b)] + k] = u8t[k]; \ 175 U8_SIMPLE_SWAP(comb_class[(a)], comb_class[(b)], tc); \ 176 U8_SIMPLE_SWAP(disp[(a)], disp[(b)], tc); 177 178/* The possible states during normalization. */ 179typedef enum { 180 U8_STATE_START = 0, 181 U8_STATE_HANGUL_L = 1, 182 U8_STATE_HANGUL_LV = 2, 183 U8_STATE_HANGUL_LVT = 3, 184 U8_STATE_HANGUL_V = 4, 185 U8_STATE_HANGUL_T = 5, 186 U8_STATE_COMBINING_MARK = 6 187} u8_normalization_states_t; 188 189/* 190 * The three vectors at below are used to check bytes of a given UTF-8 191 * character are valid and not containing any malformed byte values. 192 * 193 * We used to have a quite relaxed UTF-8 binary representation but then there 194 * was some security related issues and so the Unicode Consortium defined 195 * and announced the UTF-8 Corrigendum at Unicode 3.1 and then refined it 196 * one more time at the Unicode 3.2. The following three tables are based on 197 * that. 198 */ 199 200#define U8_ILLEGAL_NEXT_BYTE_COMMON(c) ((c) < 0x80 || (c) > 0xBF) 201 202#define I_ U8_ILLEGAL_CHAR 203#define O_ U8_OUT_OF_RANGE_CHAR 204 205const int8_t u8_number_of_bytes[0x100] = { 206 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 207 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 208 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 209 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 210 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 211 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 212 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 213 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 214 215/* 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F */ 216 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 217 218/* 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F */ 219 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 220 221/* A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF */ 222 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 223 224/* B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF */ 225 I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 226 227/* C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF */ 228 I_, I_, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 229 230/* D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF */ 231 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 232 233/* E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF */ 234 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 235 236/* F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF */ 237 4, 4, 4, 4, 4, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_, 238}; 239 240#undef I_ 241#undef O_ 242 243const uint8_t u8_valid_min_2nd_byte[0x100] = { 244 0, 0, 0, 0, 0, 0, 0, 0, 245 0, 0, 0, 0, 0, 0, 0, 0, 246 0, 0, 0, 0, 0, 0, 0, 0, 247 0, 0, 0, 0, 0, 0, 0, 0, 248 0, 0, 0, 0, 0, 0, 0, 0, 249 0, 0, 0, 0, 0, 0, 0, 0, 250 0, 0, 0, 0, 0, 0, 0, 0, 251 0, 0, 0, 0, 0, 0, 0, 0, 252 0, 0, 0, 0, 0, 0, 0, 0, 253 0, 0, 0, 0, 0, 0, 0, 0, 254 0, 0, 0, 0, 0, 0, 0, 0, 255 0, 0, 0, 0, 0, 0, 0, 0, 256 0, 0, 0, 0, 0, 0, 0, 0, 257 0, 0, 0, 0, 0, 0, 0, 0, 258 0, 0, 0, 0, 0, 0, 0, 0, 259 0, 0, 0, 0, 0, 0, 0, 0, 260 0, 0, 0, 0, 0, 0, 0, 0, 261 0, 0, 0, 0, 0, 0, 0, 0, 262 0, 0, 0, 0, 0, 0, 0, 0, 263 0, 0, 0, 0, 0, 0, 0, 0, 264 0, 0, 0, 0, 0, 0, 0, 0, 265 0, 0, 0, 0, 0, 0, 0, 0, 266 0, 0, 0, 0, 0, 0, 0, 0, 267 0, 0, 0, 0, 0, 0, 0, 0, 268/* C0 C1 C2 C3 C4 C5 C6 C7 */ 269 0, 0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 270/* C8 C9 CA CB CC CD CE CF */ 271 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 272/* D0 D1 D2 D3 D4 D5 D6 D7 */ 273 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 274/* D8 D9 DA DB DC DD DE DF */ 275 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 276/* E0 E1 E2 E3 E4 E5 E6 E7 */ 277 0xa0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 278/* E8 E9 EA EB EC ED EE EF */ 279 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 280/* F0 F1 F2 F3 F4 F5 F6 F7 */ 281 0x90, 0x80, 0x80, 0x80, 0x80, 0, 0, 0, 282 0, 0, 0, 0, 0, 0, 0, 0, 283}; 284 285const uint8_t u8_valid_max_2nd_byte[0x100] = { 286 0, 0, 0, 0, 0, 0, 0, 0, 287 0, 0, 0, 0, 0, 0, 0, 0, 288 0, 0, 0, 0, 0, 0, 0, 0, 289 0, 0, 0, 0, 0, 0, 0, 0, 290 0, 0, 0, 0, 0, 0, 0, 0, 291 0, 0, 0, 0, 0, 0, 0, 0, 292 0, 0, 0, 0, 0, 0, 0, 0, 293 0, 0, 0, 0, 0, 0, 0, 0, 294 0, 0, 0, 0, 0, 0, 0, 0, 295 0, 0, 0, 0, 0, 0, 0, 0, 296 0, 0, 0, 0, 0, 0, 0, 0, 297 0, 0, 0, 0, 0, 0, 0, 0, 298 0, 0, 0, 0, 0, 0, 0, 0, 299 0, 0, 0, 0, 0, 0, 0, 0, 300 0, 0, 0, 0, 0, 0, 0, 0, 301 0, 0, 0, 0, 0, 0, 0, 0, 302 0, 0, 0, 0, 0, 0, 0, 0, 303 0, 0, 0, 0, 0, 0, 0, 0, 304 0, 0, 0, 0, 0, 0, 0, 0, 305 0, 0, 0, 0, 0, 0, 0, 0, 306 0, 0, 0, 0, 0, 0, 0, 0, 307 0, 0, 0, 0, 0, 0, 0, 0, 308 0, 0, 0, 0, 0, 0, 0, 0, 309 0, 0, 0, 0, 0, 0, 0, 0, 310/* C0 C1 C2 C3 C4 C5 C6 C7 */ 311 0, 0, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 312/* C8 C9 CA CB CC CD CE CF */ 313 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 314/* D0 D1 D2 D3 D4 D5 D6 D7 */ 315 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 316/* D8 D9 DA DB DC DD DE DF */ 317 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 318/* E0 E1 E2 E3 E4 E5 E6 E7 */ 319 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 320/* E8 E9 EA EB EC ED EE EF */ 321 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0x9f, 0xbf, 0xbf, 322/* F0 F1 F2 F3 F4 F5 F6 F7 */ 323 0xbf, 0xbf, 0xbf, 0xbf, 0x8f, 0, 0, 0, 324 0, 0, 0, 0, 0, 0, 0, 0, 325}; 326 327 328/* 329 * The u8_validate() validates on the given UTF-8 character string and 330 * calculate the byte length. It is quite similar to mblen(3C) except that 331 * this will validate against the list of characters if required and 332 * specific to UTF-8 and Unicode. 333 */ 334int 335u8_validate(char *u8str, size_t n, char **list, int flag, int *errnum) 336{ 337 uchar_t *ib; 338 uchar_t *ibtail; 339 uchar_t **p; 340 uchar_t *s1; 341 uchar_t *s2; 342 uchar_t f; 343 int sz; 344 size_t i; 345 int ret_val; 346 boolean_t second; 347 boolean_t no_need_to_validate_entire; 348 boolean_t check_additional; 349 boolean_t validate_ucs2_range_only; 350 351 if (! u8str) 352 return (0); 353 354 ib = (uchar_t *)u8str; 355 ibtail = ib + n; 356 357 ret_val = 0; 358 359 no_need_to_validate_entire = ! (flag & U8_VALIDATE_ENTIRE); 360 check_additional = flag & U8_VALIDATE_CHECK_ADDITIONAL; 361 validate_ucs2_range_only = flag & U8_VALIDATE_UCS2_RANGE; 362 363 while (ib < ibtail) { 364 /* 365 * The first byte of a UTF-8 character tells how many 366 * bytes will follow for the character. If the first byte 367 * is an illegal byte value or out of range value, we just 368 * return -1 with an appropriate error number. 369 */ 370 sz = u8_number_of_bytes[*ib]; 371 if (sz == U8_ILLEGAL_CHAR) { 372 *errnum = EILSEQ; 373 return (-1); 374 } 375 376 if (sz == U8_OUT_OF_RANGE_CHAR || 377 (validate_ucs2_range_only && sz > U8_MAX_BYTES_UCS2)) { 378 *errnum = ERANGE; 379 return (-1); 380 } 381 382 /* 383 * If we don't have enough bytes to check on, that's also 384 * an error. As you can see, we give illegal byte sequence 385 * checking higher priority then EINVAL cases. 386 */ 387 if ((ibtail - ib) < sz) { 388 *errnum = EINVAL; 389 return (-1); 390 } 391 392 if (sz == 1) { 393 ib++; 394 ret_val++; 395 } else { 396 /* 397 * Check on the multi-byte UTF-8 character. For more 398 * details on this, see comment added for the used 399 * data structures at the beginning of the file. 400 */ 401 f = *ib++; 402 ret_val++; 403 second = B_TRUE; 404 for (i = 1; i < sz; i++) { 405 if (second) { 406 if (*ib < u8_valid_min_2nd_byte[f] || 407 *ib > u8_valid_max_2nd_byte[f]) { 408 *errnum = EILSEQ; 409 return (-1); 410 } 411 second = B_FALSE; 412 } else if (U8_ILLEGAL_NEXT_BYTE_COMMON(*ib)) { 413 *errnum = EILSEQ; 414 return (-1); 415 } 416 ib++; 417 ret_val++; 418 } 419 } 420 421 if (check_additional) { 422 for (p = (uchar_t **)list, i = 0; p[i]; i++) { 423 s1 = ib - sz; 424 s2 = p[i]; 425 while (s1 < ib) { 426 if (*s1 != *s2 || *s2 == '\0') 427 break; 428 s1++; 429 s2++; 430 } 431 432 if (s1 >= ib && *s2 == '\0') { 433 *errnum = EBADF; 434 return (-1); 435 } 436 } 437 } 438 439 if (no_need_to_validate_entire) 440 break; 441 } 442 443 return (ret_val); 444} 445 446/* 447 * The do_case_conv() looks at the mapping tables and returns found 448 * bytes if any. If not found, the input bytes are returned. The function 449 * always terminate the return bytes with a null character assuming that 450 * there are plenty of room to do so. 451 * 452 * The case conversions are simple case conversions mapping a character to 453 * another character as specified in the Unicode data. The byte size of 454 * the mapped character could be different from that of the input character. 455 * 456 * The return value is the byte length of the returned character excluding 457 * the terminating null byte. 458 */ 459static size_t 460do_case_conv(int uv, uchar_t *u8s, uchar_t *s, int sz, boolean_t is_it_toupper) 461{ 462 size_t i; 463 uint16_t b1 = 0; 464 uint16_t b2 = 0; 465 uint16_t b3 = 0; 466 uint16_t b3_tbl; 467 uint16_t b3_base; 468 uint16_t b4 = 0; 469 size_t start_id; 470 size_t end_id; 471 472 /* 473 * At this point, the only possible values for sz are 2, 3, and 4. 474 * The u8s should point to a vector that is well beyond the size of 475 * 5 bytes. 476 */ 477 if (sz == 2) { 478 b3 = u8s[0] = s[0]; 479 b4 = u8s[1] = s[1]; 480 } else if (sz == 3) { 481 b2 = u8s[0] = s[0]; 482 b3 = u8s[1] = s[1]; 483 b4 = u8s[2] = s[2]; 484 } else if (sz == 4) { 485 b1 = u8s[0] = s[0]; 486 b2 = u8s[1] = s[1]; 487 b3 = u8s[2] = s[2]; 488 b4 = u8s[3] = s[3]; 489 } else { 490 /* This is not possible but just in case as a fallback. */ 491 if (is_it_toupper) 492 *u8s = U8_ASCII_TOUPPER(*s); 493 else 494 *u8s = U8_ASCII_TOLOWER(*s); 495 u8s[1] = '\0'; 496 497 return (1); 498 } 499 u8s[sz] = '\0'; 500 501 /* 502 * Let's find out if we have a corresponding character. 503 */ 504 b1 = u8_common_b1_tbl[uv][b1]; 505 if (b1 == U8_TBL_ELEMENT_NOT_DEF) 506 return ((size_t)sz); 507 508 b2 = u8_case_common_b2_tbl[uv][b1][b2]; 509 if (b2 == U8_TBL_ELEMENT_NOT_DEF) 510 return ((size_t)sz); 511 512 if (is_it_toupper) { 513 b3_tbl = u8_toupper_b3_tbl[uv][b2][b3].tbl_id; 514 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 515 return ((size_t)sz); 516 517 start_id = u8_toupper_b4_tbl[uv][b3_tbl][b4]; 518 end_id = u8_toupper_b4_tbl[uv][b3_tbl][b4 + 1]; 519 520 /* Either there is no match or an error at the table. */ 521 if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX) 522 return ((size_t)sz); 523 524 b3_base = u8_toupper_b3_tbl[uv][b2][b3].base; 525 526 for (i = 0; start_id < end_id; start_id++) 527 u8s[i++] = u8_toupper_final_tbl[uv][b3_base + start_id]; 528 } else { 529 b3_tbl = u8_tolower_b3_tbl[uv][b2][b3].tbl_id; 530 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 531 return ((size_t)sz); 532 533 start_id = u8_tolower_b4_tbl[uv][b3_tbl][b4]; 534 end_id = u8_tolower_b4_tbl[uv][b3_tbl][b4 + 1]; 535 536 if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX) 537 return ((size_t)sz); 538 539 b3_base = u8_tolower_b3_tbl[uv][b2][b3].base; 540 541 for (i = 0; start_id < end_id; start_id++) 542 u8s[i++] = u8_tolower_final_tbl[uv][b3_base + start_id]; 543 } 544 545 /* 546 * If i is still zero, that means there is no corresponding character. 547 */ 548 if (i == 0) 549 return ((size_t)sz); 550 551 u8s[i] = '\0'; 552 553 return (i); 554} 555 556/* 557 * The do_case_compare() function compares the two input strings, s1 and s2, 558 * one character at a time doing case conversions if applicable and return 559 * the comparison result as like strcmp(). 560 * 561 * Since, in empirical sense, most of text data are 7-bit ASCII characters, 562 * we treat the 7-bit ASCII characters as a special case trying to yield 563 * faster processing time. 564 */ 565static int 566do_case_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, 567 size_t n2, boolean_t is_it_toupper, int *errnum) 568{ 569 int f; 570 int sz1; 571 int sz2; 572 size_t j; 573 size_t i1; 574 size_t i2; 575 uchar_t u8s1[U8_MB_CUR_MAX + 1]; 576 uchar_t u8s2[U8_MB_CUR_MAX + 1]; 577 578 i1 = i2 = 0; 579 while (i1 < n1 && i2 < n2) { 580 /* 581 * Find out what would be the byte length for this UTF-8 582 * character at string s1 and also find out if this is 583 * an illegal start byte or not and if so, issue a proper 584 * error number and yet treat this byte as a character. 585 */ 586 sz1 = u8_number_of_bytes[*s1]; 587 if (sz1 < 0) { 588 *errnum = EILSEQ; 589 sz1 = 1; 590 } 591 592 /* 593 * For 7-bit ASCII characters mainly, we do a quick case 594 * conversion right at here. 595 * 596 * If we don't have enough bytes for this character, issue 597 * an EINVAL error and use what are available. 598 * 599 * If we have enough bytes, find out if there is 600 * a corresponding uppercase character and if so, copy over 601 * the bytes for a comparison later. If there is no 602 * corresponding uppercase character, then, use what we have 603 * for the comparison. 604 */ 605 if (sz1 == 1) { 606 if (is_it_toupper) 607 u8s1[0] = U8_ASCII_TOUPPER(*s1); 608 else 609 u8s1[0] = U8_ASCII_TOLOWER(*s1); 610 s1++; 611 u8s1[1] = '\0'; 612 } else if ((i1 + sz1) > n1) { 613 *errnum = EINVAL; 614 for (j = 0; (i1 + j) < n1; ) 615 u8s1[j++] = *s1++; 616 u8s1[j] = '\0'; 617 } else { 618 (void) do_case_conv(uv, u8s1, s1, sz1, is_it_toupper); 619 s1 += sz1; 620 } 621 622 /* Do the same for the string s2. */ 623 sz2 = u8_number_of_bytes[*s2]; 624 if (sz2 < 0) { 625 *errnum = EILSEQ; 626 sz2 = 1; 627 } 628 629 if (sz2 == 1) { 630 if (is_it_toupper) 631 u8s2[0] = U8_ASCII_TOUPPER(*s2); 632 else 633 u8s2[0] = U8_ASCII_TOLOWER(*s2); 634 s2++; 635 u8s2[1] = '\0'; 636 } else if ((i2 + sz2) > n2) { 637 *errnum = EINVAL; 638 for (j = 0; (i2 + j) < n2; ) 639 u8s2[j++] = *s2++; 640 u8s2[j] = '\0'; 641 } else { 642 (void) do_case_conv(uv, u8s2, s2, sz2, is_it_toupper); 643 s2 += sz2; 644 } 645 646 /* Now compare the two characters. */ 647 if (sz1 == 1 && sz2 == 1) { 648 if (*u8s1 > *u8s2) 649 return (1); 650 if (*u8s1 < *u8s2) 651 return (-1); 652 } else { 653 f = strcmp((const char *)u8s1, (const char *)u8s2); 654 if (f != 0) 655 return (f); 656 } 657 658 /* 659 * They were the same. Let's move on to the next 660 * characters then. 661 */ 662 i1 += sz1; 663 i2 += sz2; 664 } 665 666 /* 667 * We compared until the end of either or both strings. 668 * 669 * If we reached to or went over the ends for the both, that means 670 * they are the same. 671 * 672 * If we reached only one of the two ends, that means the other string 673 * has something which then the fact can be used to determine 674 * the return value. 675 */ 676 if (i1 >= n1) { 677 if (i2 >= n2) 678 return (0); 679 return (-1); 680 } 681 return (1); 682} 683 684/* 685 * The combining_class() function checks on the given bytes and find out 686 * the corresponding Unicode combining class value. The return value 0 means 687 * it is a Starter. Any illegal UTF-8 character will also be treated as 688 * a Starter. 689 */ 690static uchar_t 691combining_class(size_t uv, uchar_t *s, size_t sz) 692{ 693 uint16_t b1 = 0; 694 uint16_t b2 = 0; 695 uint16_t b3 = 0; 696 uint16_t b4 = 0; 697 698 if (sz == 1 || sz > 4) 699 return (0); 700 701 if (sz == 2) { 702 b3 = s[0]; 703 b4 = s[1]; 704 } else if (sz == 3) { 705 b2 = s[0]; 706 b3 = s[1]; 707 b4 = s[2]; 708 } else if (sz == 4) { 709 b1 = s[0]; 710 b2 = s[1]; 711 b3 = s[2]; 712 b4 = s[3]; 713 } 714 715 b1 = u8_common_b1_tbl[uv][b1]; 716 if (b1 == U8_TBL_ELEMENT_NOT_DEF) 717 return (0); 718 719 b2 = u8_combining_class_b2_tbl[uv][b1][b2]; 720 if (b2 == U8_TBL_ELEMENT_NOT_DEF) 721 return (0); 722 723 b3 = u8_combining_class_b3_tbl[uv][b2][b3]; 724 if (b3 == U8_TBL_ELEMENT_NOT_DEF) 725 return (0); 726 727 return (u8_combining_class_b4_tbl[uv][b3][b4]); 728} 729 730/* 731 * The do_decomp() function finds out a matching decomposition if any 732 * and return. If there is no match, the input bytes are copied and returned. 733 * The function also checks if there is a Hangul, decomposes it if necessary 734 * and returns. 735 * 736 * To save time, a single byte 7-bit ASCII character should be handled by 737 * the caller. 738 * 739 * The function returns the number of bytes returned sans always terminating 740 * the null byte. It will also return a state that will tell if there was 741 * a Hangul character decomposed which then will be used by the caller. 742 */ 743static size_t 744do_decomp(size_t uv, uchar_t *u8s, uchar_t *s, int sz, 745 boolean_t canonical_decomposition, u8_normalization_states_t *state) 746{ 747 uint16_t b1 = 0; 748 uint16_t b2 = 0; 749 uint16_t b3 = 0; 750 uint16_t b3_tbl; 751 uint16_t b3_base; 752 uint16_t b4 = 0; 753 size_t start_id; 754 size_t end_id; 755 size_t i; 756 uint32_t u1; 757 758 if (sz == 2) { 759 b3 = u8s[0] = s[0]; 760 b4 = u8s[1] = s[1]; 761 u8s[2] = '\0'; 762 } else if (sz == 3) { 763 /* Convert it to a Unicode scalar value. */ 764 U8_PUT_3BYTES_INTO_UTF32(u1, s[0], s[1], s[2]); 765 766 /* 767 * If this is a Hangul syllable, we decompose it into 768 * a leading consonant, a vowel, and an optional trailing 769 * consonant and then return. 770 */ 771 if (U8_HANGUL_SYLLABLE(u1)) { 772 u1 -= U8_HANGUL_SYL_FIRST; 773 774 b1 = U8_HANGUL_JAMO_L_FIRST + u1 / U8_HANGUL_VT_COUNT; 775 b2 = U8_HANGUL_JAMO_V_FIRST + (u1 % U8_HANGUL_VT_COUNT) 776 / U8_HANGUL_T_COUNT; 777 b3 = u1 % U8_HANGUL_T_COUNT; 778 779 U8_SAVE_HANGUL_AS_UTF8(u8s, 0, 1, 2, b1); 780 U8_SAVE_HANGUL_AS_UTF8(u8s, 3, 4, 5, b2); 781 if (b3) { 782 b3 += U8_HANGUL_JAMO_T_FIRST; 783 U8_SAVE_HANGUL_AS_UTF8(u8s, 6, 7, 8, b3); 784 785 u8s[9] = '\0'; 786 *state = U8_STATE_HANGUL_LVT; 787 return (9); 788 } 789 790 u8s[6] = '\0'; 791 *state = U8_STATE_HANGUL_LV; 792 return (6); 793 } 794 795 b2 = u8s[0] = s[0]; 796 b3 = u8s[1] = s[1]; 797 b4 = u8s[2] = s[2]; 798 u8s[3] = '\0'; 799 800 /* 801 * If this is a Hangul Jamo, we know there is nothing 802 * further that we can decompose. 803 */ 804 if (U8_HANGUL_JAMO_L(u1)) { 805 *state = U8_STATE_HANGUL_L; 806 return (3); 807 } 808 809 if (U8_HANGUL_JAMO_V(u1)) { 810 if (*state == U8_STATE_HANGUL_L) 811 *state = U8_STATE_HANGUL_LV; 812 else 813 *state = U8_STATE_HANGUL_V; 814 return (3); 815 } 816 817 if (U8_HANGUL_JAMO_T(u1)) { 818 if (*state == U8_STATE_HANGUL_LV) 819 *state = U8_STATE_HANGUL_LVT; 820 else 821 *state = U8_STATE_HANGUL_T; 822 return (3); 823 } 824 } else if (sz == 4) { 825 b1 = u8s[0] = s[0]; 826 b2 = u8s[1] = s[1]; 827 b3 = u8s[2] = s[2]; 828 b4 = u8s[3] = s[3]; 829 u8s[4] = '\0'; 830 } else { 831 /* 832 * This is a fallback and should not happen if the function 833 * was called properly. 834 */ 835 u8s[0] = s[0]; 836 u8s[1] = '\0'; 837 *state = U8_STATE_START; 838 return (1); 839 } 840 841 /* 842 * At this point, this rountine does not know what it would get. 843 * The caller should sort it out if the state isn't a Hangul one. 844 */ 845 *state = U8_STATE_START; 846 847 /* Try to find matching decomposition mapping byte sequence. */ 848 b1 = u8_common_b1_tbl[uv][b1]; 849 if (b1 == U8_TBL_ELEMENT_NOT_DEF) 850 return ((size_t)sz); 851 852 b2 = u8_decomp_b2_tbl[uv][b1][b2]; 853 if (b2 == U8_TBL_ELEMENT_NOT_DEF) 854 return ((size_t)sz); 855 856 b3_tbl = u8_decomp_b3_tbl[uv][b2][b3].tbl_id; 857 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 858 return ((size_t)sz); 859 860 /* 861 * If b3_tbl is bigger than or equal to U8_16BIT_TABLE_INDICATOR 862 * which is 0x8000, this means we couldn't fit the mappings into 863 * the cardinality of a unsigned byte. 864 */ 865 if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) { 866 b3_tbl -= U8_16BIT_TABLE_INDICATOR; 867 start_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4]; 868 end_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4 + 1]; 869 } else { 870 start_id = u8_decomp_b4_tbl[uv][b3_tbl][b4]; 871 end_id = u8_decomp_b4_tbl[uv][b3_tbl][b4 + 1]; 872 } 873 874 /* This also means there wasn't any matching decomposition. */ 875 if (start_id >= end_id) 876 return ((size_t)sz); 877 878 /* 879 * The final table for decomposition mappings has three types of 880 * byte sequences depending on whether a mapping is for compatibility 881 * decomposition, canonical decomposition, or both like the following: 882 * 883 * (1) Compatibility decomposition mappings: 884 * 885 * +---+---+-...-+---+ 886 * | B0| B1| ... | Bm| 887 * +---+---+-...-+---+ 888 * 889 * The first byte, B0, is always less then 0xF5 (U8_DECOMP_BOTH). 890 * 891 * (2) Canonical decomposition mappings: 892 * 893 * +---+---+---+-...-+---+ 894 * | T | b0| b1| ... | bn| 895 * +---+---+---+-...-+---+ 896 * 897 * where the first byte, T, is 0xF6 (U8_DECOMP_CANONICAL). 898 * 899 * (3) Both mappings: 900 * 901 * +---+---+---+---+-...-+---+---+---+-...-+---+ 902 * | T | D | b0| b1| ... | bn| B0| B1| ... | Bm| 903 * +---+---+---+---+-...-+---+---+---+-...-+---+ 904 * 905 * where T is 0xF5 (U8_DECOMP_BOTH) and D is a displacement 906 * byte, b0 to bn are canonical mapping bytes and B0 to Bm are 907 * compatibility mapping bytes. 908 * 909 * Note that compatibility decomposition means doing recursive 910 * decompositions using both compatibility decomposition mappings and 911 * canonical decomposition mappings. On the other hand, canonical 912 * decomposition means doing recursive decompositions using only 913 * canonical decomposition mappings. Since the table we have has gone 914 * through the recursions already, we do not need to do so during 915 * runtime, i.e., the table has been completely flattened out 916 * already. 917 */ 918 919 b3_base = u8_decomp_b3_tbl[uv][b2][b3].base; 920 921 /* Get the type, T, of the byte sequence. */ 922 b1 = u8_decomp_final_tbl[uv][b3_base + start_id]; 923 924 /* 925 * If necessary, adjust start_id, end_id, or both. Note that if 926 * this is compatibility decomposition mapping, there is no 927 * adjustment. 928 */ 929 if (canonical_decomposition) { 930 /* Is the mapping only for compatibility decomposition? */ 931 if (b1 < U8_DECOMP_BOTH) 932 return ((size_t)sz); 933 934 start_id++; 935 936 if (b1 == U8_DECOMP_BOTH) { 937 end_id = start_id + 938 u8_decomp_final_tbl[uv][b3_base + start_id]; 939 start_id++; 940 } 941 } else { 942 /* 943 * Unless this is a compatibility decomposition mapping, 944 * we adjust the start_id. 945 */ 946 if (b1 == U8_DECOMP_BOTH) { 947 start_id++; 948 start_id += u8_decomp_final_tbl[uv][b3_base + start_id]; 949 } else if (b1 == U8_DECOMP_CANONICAL) { 950 start_id++; 951 } 952 } 953 954 for (i = 0; start_id < end_id; start_id++) 955 u8s[i++] = u8_decomp_final_tbl[uv][b3_base + start_id]; 956 u8s[i] = '\0'; 957 958 return (i); 959} 960 961/* 962 * The find_composition_start() function uses the character bytes given and 963 * find out the matching composition mappings if any and return the address 964 * to the composition mappings as explained in the do_composition(). 965 */ 966static uchar_t * 967find_composition_start(size_t uv, uchar_t *s, size_t sz) 968{ 969 uint16_t b1 = 0; 970 uint16_t b2 = 0; 971 uint16_t b3 = 0; 972 uint16_t b3_tbl; 973 uint16_t b3_base; 974 uint16_t b4 = 0; 975 size_t start_id; 976 size_t end_id; 977 978 if (sz == 1) { 979 b4 = s[0]; 980 } else if (sz == 2) { 981 b3 = s[0]; 982 b4 = s[1]; 983 } else if (sz == 3) { 984 b2 = s[0]; 985 b3 = s[1]; 986 b4 = s[2]; 987 } else if (sz == 4) { 988 b1 = s[0]; 989 b2 = s[1]; 990 b3 = s[2]; 991 b4 = s[3]; 992 } else { 993 /* 994 * This is a fallback and should not happen if the function 995 * was called properly. 996 */ 997 return (NULL); 998 } 999 1000 b1 = u8_composition_b1_tbl[uv][b1]; 1001 if (b1 == U8_TBL_ELEMENT_NOT_DEF) 1002 return (NULL); 1003 1004 b2 = u8_composition_b2_tbl[uv][b1][b2]; 1005 if (b2 == U8_TBL_ELEMENT_NOT_DEF) 1006 return (NULL); 1007 1008 b3_tbl = u8_composition_b3_tbl[uv][b2][b3].tbl_id; 1009 if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 1010 return (NULL); 1011 1012 if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) { 1013 b3_tbl -= U8_16BIT_TABLE_INDICATOR; 1014 start_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4]; 1015 end_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4 + 1]; 1016 } else { 1017 start_id = u8_composition_b4_tbl[uv][b3_tbl][b4]; 1018 end_id = u8_composition_b4_tbl[uv][b3_tbl][b4 + 1]; 1019 } 1020 1021 if (start_id >= end_id) 1022 return (NULL); 1023 1024 b3_base = u8_composition_b3_tbl[uv][b2][b3].base; 1025 1026 return ((uchar_t *)&(u8_composition_final_tbl[uv][b3_base + start_id])); 1027} 1028 1029/* 1030 * The blocked() function checks on the combining class values of previous 1031 * characters in this sequence and return whether it is blocked or not. 1032 */ 1033static boolean_t 1034blocked(uchar_t *comb_class, size_t last) 1035{ 1036 uchar_t my_comb_class; 1037 size_t i; 1038 1039 my_comb_class = comb_class[last]; 1040 for (i = 1; i < last; i++) 1041 if (comb_class[i] >= my_comb_class || 1042 comb_class[i] == U8_COMBINING_CLASS_STARTER) 1043 return (B_TRUE); 1044 1045 return (B_FALSE); 1046} 1047 1048/* 1049 * The do_composition() reads the character string pointed by 's' and 1050 * do necessary canonical composition and then copy over the result back to 1051 * the 's'. 1052 * 1053 * The input argument 's' cannot contain more than 32 characters. 1054 */ 1055static size_t 1056do_composition(size_t uv, uchar_t *s, uchar_t *comb_class, uchar_t *start, 1057 uchar_t *disp, size_t last, uchar_t **os, uchar_t *oslast) 1058{ 1059 uchar_t t[U8_STREAM_SAFE_TEXT_MAX + 1]; 1060 uchar_t tc[U8_MB_CUR_MAX]; 1061 uint8_t saved_marks[U8_MAX_CHARS_A_SEQ]; 1062 size_t saved_marks_count; 1063 uchar_t *p; 1064 uchar_t *saved_p; 1065 uchar_t *q; 1066 size_t i; 1067 size_t saved_i; 1068 size_t j; 1069 size_t k; 1070 size_t l; 1071 size_t C; 1072 size_t saved_l; 1073 size_t size; 1074 uint32_t u1; 1075 uint32_t u2; 1076 boolean_t match_not_found = B_TRUE; 1077 1078 /* 1079 * This should never happen unless the callers are doing some strange 1080 * and unexpected things. 1081 * 1082 * The "last" is the index pointing to the last character not last + 1. 1083 */ 1084 if (last >= U8_MAX_CHARS_A_SEQ) 1085 last = U8_UPPER_LIMIT_IN_A_SEQ; 1086 1087 for (i = l = 0; i <= last; i++) { 1088 /* 1089 * The last or any non-Starters at the beginning, we don't 1090 * have any chance to do composition and so we just copy them 1091 * to the temporary buffer. 1092 */ 1093 if (i >= last || comb_class[i] != U8_COMBINING_CLASS_STARTER) { 1094SAVE_THE_CHAR: 1095 p = s + start[i]; 1096 size = disp[i]; 1097 for (k = 0; k < size; k++) 1098 t[l++] = *p++; 1099 continue; 1100 } 1101 1102 /* 1103 * If this could be a start of Hangul Jamos, then, we try to 1104 * conjoin them. 1105 */ 1106 if (s[start[i]] == U8_HANGUL_JAMO_1ST_BYTE) { 1107 U8_PUT_3BYTES_INTO_UTF32(u1, s[start[i]], 1108 s[start[i] + 1], s[start[i] + 2]); 1109 U8_PUT_3BYTES_INTO_UTF32(u2, s[start[i] + 3], 1110 s[start[i] + 4], s[start[i] + 5]); 1111 1112 if (U8_HANGUL_JAMO_L(u1) && U8_HANGUL_JAMO_V(u2)) { 1113 u1 -= U8_HANGUL_JAMO_L_FIRST; 1114 u2 -= U8_HANGUL_JAMO_V_FIRST; 1115 u1 = U8_HANGUL_SYL_FIRST + 1116 (u1 * U8_HANGUL_V_COUNT + u2) * 1117 U8_HANGUL_T_COUNT; 1118 1119 i += 2; 1120 if (i <= last) { 1121 U8_PUT_3BYTES_INTO_UTF32(u2, 1122 s[start[i]], s[start[i] + 1], 1123 s[start[i] + 2]); 1124 1125 if (U8_HANGUL_JAMO_T(u2)) { 1126 u1 += u2 - 1127 U8_HANGUL_JAMO_T_FIRST; 1128 i++; 1129 } 1130 } 1131 1132 U8_SAVE_HANGUL_AS_UTF8(t + l, 0, 1, 2, u1); 1133 i--; 1134 l += 3; 1135 continue; 1136 } 1137 } 1138 1139 /* 1140 * Let's then find out if this Starter has composition 1141 * mapping. 1142 */ 1143 p = find_composition_start(uv, s + start[i], disp[i]); 1144 if (p == NULL) 1145 goto SAVE_THE_CHAR; 1146 1147 /* 1148 * We have a Starter with composition mapping and the next 1149 * character is a non-Starter. Let's try to find out if 1150 * we can do composition. 1151 */ 1152 1153 saved_p = p; 1154 saved_i = i; 1155 saved_l = l; 1156 saved_marks_count = 0; 1157 1158TRY_THE_NEXT_MARK: 1159 q = s + start[++i]; 1160 size = disp[i]; 1161 1162 /* 1163 * The next for() loop compares the non-Starter pointed by 1164 * 'q' with the possible (joinable) characters pointed by 'p'. 1165 * 1166 * The composition final table entry pointed by the 'p' 1167 * looks like the following: 1168 * 1169 * +---+---+---+-...-+---+---+---+---+-...-+---+---+ 1170 * | C | b0| b2| ... | bn| F | B0| B1| ... | Bm| F | 1171 * +---+---+---+-...-+---+---+---+---+-...-+---+---+ 1172 * 1173 * where C is the count byte indicating the number of 1174 * mapping pairs where each pair would be look like 1175 * (b0-bn F, B0-Bm F). The b0-bn are the bytes of the second 1176 * character of a canonical decomposition and the B0-Bm are 1177 * the bytes of a matching composite character. The F is 1178 * a filler byte after each character as the separator. 1179 */ 1180 1181 match_not_found = B_TRUE; 1182 1183 for (C = *p++; C > 0; C--) { 1184 for (k = 0; k < size; p++, k++) 1185 if (*p != q[k]) 1186 break; 1187 1188 /* Have we found it? */ 1189 if (k >= size && *p == U8_TBL_ELEMENT_FILLER) { 1190 match_not_found = B_FALSE; 1191 1192 l = saved_l; 1193 1194 while (*++p != U8_TBL_ELEMENT_FILLER) 1195 t[l++] = *p; 1196 1197 break; 1198 } 1199 1200 /* We didn't find; skip to the next pair. */ 1201 if (*p != U8_TBL_ELEMENT_FILLER) 1202 while (*++p != U8_TBL_ELEMENT_FILLER) 1203 ; 1204 while (*++p != U8_TBL_ELEMENT_FILLER) 1205 ; 1206 p++; 1207 } 1208 1209 /* 1210 * If there was no match, we will need to save the combining 1211 * mark for later appending. After that, if the next one 1212 * is a non-Starter and not blocked, then, we try once 1213 * again to do composition with the next non-Starter. 1214 * 1215 * If there was no match and this was a Starter, then, 1216 * this is a new start. 1217 * 1218 * If there was a match and a composition done and we have 1219 * more to check on, then, we retrieve a new composition final 1220 * table entry for the composite and then try to do the 1221 * composition again. 1222 */ 1223 1224 if (match_not_found) { 1225 if (comb_class[i] == U8_COMBINING_CLASS_STARTER) { 1226 i--; 1227 goto SAVE_THE_CHAR; 1228 } 1229 1230 saved_marks[saved_marks_count++] = i; 1231 } 1232 1233 if (saved_l == l) { 1234 while (i < last) { 1235 if (blocked(comb_class, i + 1)) 1236 saved_marks[saved_marks_count++] = ++i; 1237 else 1238 break; 1239 } 1240 if (i < last) { 1241 p = saved_p; 1242 goto TRY_THE_NEXT_MARK; 1243 } 1244 } else if (i < last) { 1245 p = find_composition_start(uv, t + saved_l, 1246 l - saved_l); 1247 if (p != NULL) { 1248 saved_p = p; 1249 goto TRY_THE_NEXT_MARK; 1250 } 1251 } 1252 1253 /* 1254 * There is no more composition possible. 1255 * 1256 * If there was no composition what so ever then we copy 1257 * over the original Starter and then append any non-Starters 1258 * remaining at the target string sequentially after that. 1259 */ 1260 1261 if (saved_l == l) { 1262 p = s + start[saved_i]; 1263 size = disp[saved_i]; 1264 for (j = 0; j < size; j++) 1265 t[l++] = *p++; 1266 } 1267 1268 for (k = 0; k < saved_marks_count; k++) { 1269 p = s + start[saved_marks[k]]; 1270 size = disp[saved_marks[k]]; 1271 for (j = 0; j < size; j++) 1272 t[l++] = *p++; 1273 } 1274 } 1275 1276 /* 1277 * If the last character is a Starter and if we have a character 1278 * (possibly another Starter) that can be turned into a composite, 1279 * we do so and we do so until there is no more of composition 1280 * possible. 1281 */ 1282 if (comb_class[last] == U8_COMBINING_CLASS_STARTER) { 1283 p = *os; 1284 saved_l = l - disp[last]; 1285 1286 while (p < oslast) { 1287 size = u8_number_of_bytes[*p]; 1288 if (size <= 1 || (p + size) > oslast) 1289 break; 1290 1291 saved_p = p; 1292 1293 for (i = 0; i < size; i++) 1294 tc[i] = *p++; 1295 1296 q = find_composition_start(uv, t + saved_l, 1297 l - saved_l); 1298 if (q == NULL) { 1299 p = saved_p; 1300 break; 1301 } 1302 1303 match_not_found = B_TRUE; 1304 1305 for (C = *q++; C > 0; C--) { 1306 for (k = 0; k < size; q++, k++) 1307 if (*q != tc[k]) 1308 break; 1309 1310 if (k >= size && *q == U8_TBL_ELEMENT_FILLER) { 1311 match_not_found = B_FALSE; 1312 1313 l = saved_l; 1314 1315 while (*++q != U8_TBL_ELEMENT_FILLER) { 1316 /* 1317 * This is practically 1318 * impossible but we don't 1319 * want to take any chances. 1320 */ 1321 if (l >= 1322 U8_STREAM_SAFE_TEXT_MAX) { 1323 p = saved_p; 1324 goto SAFE_RETURN; 1325 } 1326 t[l++] = *q; 1327 } 1328 1329 break; 1330 } 1331 1332 if (*q != U8_TBL_ELEMENT_FILLER) 1333 while (*++q != U8_TBL_ELEMENT_FILLER) 1334 ; 1335 while (*++q != U8_TBL_ELEMENT_FILLER) 1336 ; 1337 q++; 1338 } 1339 1340 if (match_not_found) { 1341 p = saved_p; 1342 break; 1343 } 1344 } 1345SAFE_RETURN: 1346 *os = p; 1347 } 1348 1349 /* 1350 * Now we copy over the temporary string to the target string. 1351 * Since composition always reduces the number of characters or 1352 * the number of characters stay, we don't need to worry about 1353 * the buffer overflow here. 1354 */ 1355 for (i = 0; i < l; i++) 1356 s[i] = t[i]; 1357 s[l] = '\0'; 1358 1359 return (l); 1360} 1361 1362/* 1363 * The collect_a_seq() function checks on the given string s, collect 1364 * a sequence of characters at u8s, and return the sequence. While it collects 1365 * a sequence, it also applies case conversion, canonical or compatibility 1366 * decomposition, canonical decomposition, or some or all of them and 1367 * in that order. 1368 * 1369 * The collected sequence cannot be bigger than 32 characters since if 1370 * it is having more than 31 characters, the sequence will be terminated 1371 * with a U+034F COMBINING GRAPHEME JOINER (CGJ) character and turned into 1372 * a Stream-Safe Text. The collected sequence is always terminated with 1373 * a null byte and the return value is the byte length of the sequence 1374 * including 0. The return value does not include the terminating 1375 * null byte. 1376 */ 1377static size_t 1378collect_a_seq(size_t uv, uchar_t *u8s, uchar_t **source, uchar_t *slast, 1379 boolean_t is_it_toupper, 1380 boolean_t is_it_tolower, 1381 boolean_t canonical_decomposition, 1382 boolean_t compatibility_decomposition, 1383 boolean_t canonical_composition, 1384 int *errnum, u8_normalization_states_t *state) 1385{ 1386 uchar_t *s; 1387 int sz; 1388 int saved_sz; 1389 size_t i; 1390 size_t j; 1391 size_t k; 1392 size_t l; 1393 uchar_t comb_class[U8_MAX_CHARS_A_SEQ]; 1394 uchar_t disp[U8_MAX_CHARS_A_SEQ]; 1395 uchar_t start[U8_MAX_CHARS_A_SEQ]; 1396 uchar_t u8t[U8_MB_CUR_MAX]; 1397 uchar_t uts[U8_STREAM_SAFE_TEXT_MAX + 1]; 1398 uchar_t tc; 1399 size_t last; 1400 size_t saved_last; 1401 uint32_t u1; 1402 1403 /* 1404 * Save the source string pointer which we will return a changed 1405 * pointer if we do processing. 1406 */ 1407 s = *source; 1408 1409 /* 1410 * The following is a fallback for just in case callers are not 1411 * checking the string boundaries before the calling. 1412 */ 1413 if (s >= slast) { 1414 u8s[0] = '\0'; 1415 1416 return (0); 1417 } 1418 1419 /* 1420 * As the first thing, let's collect a character and do case 1421 * conversion if necessary. 1422 */ 1423 1424 sz = u8_number_of_bytes[*s]; 1425 1426 if (sz < 0) { 1427 *errnum = EILSEQ; 1428 1429 u8s[0] = *s++; 1430 u8s[1] = '\0'; 1431 1432 *source = s; 1433 1434 return (1); 1435 } 1436 1437 if (sz == 1) { 1438 if (is_it_toupper) 1439 u8s[0] = U8_ASCII_TOUPPER(*s); 1440 else if (is_it_tolower) 1441 u8s[0] = U8_ASCII_TOLOWER(*s); 1442 else 1443 u8s[0] = *s; 1444 s++; 1445 u8s[1] = '\0'; 1446 } else if ((s + sz) > slast) { 1447 *errnum = EINVAL; 1448 1449 for (i = 0; s < slast; ) 1450 u8s[i++] = *s++; 1451 u8s[i] = '\0'; 1452 1453 *source = s; 1454 1455 return (i); 1456 } else { 1457 if (is_it_toupper || is_it_tolower) { 1458 i = do_case_conv(uv, u8s, s, sz, is_it_toupper); 1459 s += sz; 1460 sz = i; 1461 } else { 1462 for (i = 0; i < sz; ) 1463 u8s[i++] = *s++; 1464 u8s[i] = '\0'; 1465 } 1466 } 1467 1468 /* 1469 * And then canonical/compatibility decomposition followed by 1470 * an optional canonical composition. Please be noted that 1471 * canonical composition is done only when a decomposition is 1472 * done. 1473 */ 1474 if (canonical_decomposition || compatibility_decomposition) { 1475 if (sz == 1) { 1476 *state = U8_STATE_START; 1477 1478 saved_sz = 1; 1479 1480 comb_class[0] = 0; 1481 start[0] = 0; 1482 disp[0] = 1; 1483 1484 last = 1; 1485 } else { 1486 saved_sz = do_decomp(uv, u8s, u8s, sz, 1487 canonical_decomposition, state); 1488 1489 last = 0; 1490 1491 for (i = 0; i < saved_sz; ) { 1492 sz = u8_number_of_bytes[u8s[i]]; 1493 1494 comb_class[last] = combining_class(uv, 1495 u8s + i, sz); 1496 start[last] = i; 1497 disp[last] = sz; 1498 1499 last++; 1500 i += sz; 1501 } 1502 1503 /* 1504 * Decomposition yields various Hangul related 1505 * states but not on combining marks. We need to 1506 * find out at here by checking on the last 1507 * character. 1508 */ 1509 if (*state == U8_STATE_START) { 1510 if (comb_class[last - 1]) 1511 *state = U8_STATE_COMBINING_MARK; 1512 } 1513 } 1514 1515 saved_last = last; 1516 1517 while (s < slast) { 1518 sz = u8_number_of_bytes[*s]; 1519 1520 /* 1521 * If this is an illegal character, an incomplete 1522 * character, or an 7-bit ASCII Starter character, 1523 * then we have collected a sequence; break and let 1524 * the next call deal with the two cases. 1525 * 1526 * Note that this is okay only if you are using this 1527 * function with a fixed length string, not on 1528 * a buffer with multiple calls of one chunk at a time. 1529 */ 1530 if (sz <= 1) { 1531 break; 1532 } else if ((s + sz) > slast) { 1533 break; 1534 } else { 1535 /* 1536 * If the previous character was a Hangul Jamo 1537 * and this character is a Hangul Jamo that 1538 * can be conjoined, we collect the Jamo. 1539 */ 1540 if (*s == U8_HANGUL_JAMO_1ST_BYTE) { 1541 U8_PUT_3BYTES_INTO_UTF32(u1, 1542 *s, *(s + 1), *(s + 2)); 1543 1544 if (U8_HANGUL_COMPOSABLE_L_V(*state, 1545 u1)) { 1546 i = 0; 1547 *state = U8_STATE_HANGUL_LV; 1548 goto COLLECT_A_HANGUL; 1549 } 1550 1551 if (U8_HANGUL_COMPOSABLE_LV_T(*state, 1552 u1)) { 1553 i = 0; 1554 *state = U8_STATE_HANGUL_LVT; 1555 goto COLLECT_A_HANGUL; 1556 } 1557 } 1558 1559 /* 1560 * Regardless of whatever it was, if this is 1561 * a Starter, we don't collect the character 1562 * since that's a new start and we will deal 1563 * with it at the next time. 1564 */ 1565 i = combining_class(uv, s, sz); 1566 if (i == U8_COMBINING_CLASS_STARTER) 1567 break; 1568 1569 /* 1570 * We know the current character is a combining 1571 * mark. If the previous character wasn't 1572 * a Starter (not Hangul) or a combining mark, 1573 * then, we don't collect this combining mark. 1574 */ 1575 if (*state != U8_STATE_START && 1576 *state != U8_STATE_COMBINING_MARK) 1577 break; 1578 1579 *state = U8_STATE_COMBINING_MARK; 1580COLLECT_A_HANGUL: 1581 /* 1582 * If we collected a Starter and combining 1583 * marks up to 30, i.e., total 31 characters, 1584 * then, we terminate this degenerately long 1585 * combining sequence with a U+034F COMBINING 1586 * GRAPHEME JOINER (CGJ) which is 0xCD 0x8F in 1587 * UTF-8 and turn this into a Stream-Safe 1588 * Text. This will be extremely rare but 1589 * possible. 1590 * 1591 * The following will also guarantee that 1592 * we are not writing more than 32 characters 1593 * plus a NULL at u8s[]. 1594 */ 1595 if (last >= U8_UPPER_LIMIT_IN_A_SEQ) { 1596TURN_STREAM_SAFE: 1597 *state = U8_STATE_START; 1598 comb_class[last] = 0; 1599 start[last] = saved_sz; 1600 disp[last] = 2; 1601 last++; 1602 1603 u8s[saved_sz++] = 0xCD; 1604 u8s[saved_sz++] = 0x8F; 1605 1606 break; 1607 } 1608 1609 /* 1610 * Some combining marks also do decompose into 1611 * another combining mark or marks. 1612 */ 1613 if (*state == U8_STATE_COMBINING_MARK) { 1614 k = last; 1615 l = sz; 1616 i = do_decomp(uv, uts, s, sz, 1617 canonical_decomposition, state); 1618 for (j = 0; j < i; ) { 1619 sz = u8_number_of_bytes[uts[j]]; 1620 1621 comb_class[last] = 1622 combining_class(uv, 1623 uts + j, sz); 1624 start[last] = saved_sz + j; 1625 disp[last] = sz; 1626 1627 last++; 1628 if (last >= 1629 U8_UPPER_LIMIT_IN_A_SEQ) { 1630 last = k; 1631 goto TURN_STREAM_SAFE; 1632 } 1633 j += sz; 1634 } 1635 1636 *state = U8_STATE_COMBINING_MARK; 1637 sz = i; 1638 s += l; 1639 1640 for (i = 0; i < sz; i++) 1641 u8s[saved_sz++] = uts[i]; 1642 } else { 1643 comb_class[last] = i; 1644 start[last] = saved_sz; 1645 disp[last] = sz; 1646 last++; 1647 1648 for (i = 0; i < sz; i++) 1649 u8s[saved_sz++] = *s++; 1650 } 1651 1652 /* 1653 * If this is U+0345 COMBINING GREEK 1654 * YPOGEGRAMMENI (0xCD 0x85 in UTF-8), a.k.a., 1655 * iota subscript, and need to be converted to 1656 * uppercase letter, convert it to U+0399 GREEK 1657 * CAPITAL LETTER IOTA (0xCE 0x99 in UTF-8), 1658 * i.e., convert to capital adscript form as 1659 * specified in the Unicode standard. 1660 * 1661 * This is the only special case of (ambiguous) 1662 * case conversion at combining marks and 1663 * probably the standard will never have 1664 * anything similar like this in future. 1665 */ 1666 if (is_it_toupper && sz >= 2 && 1667 u8s[saved_sz - 2] == 0xCD && 1668 u8s[saved_sz - 1] == 0x85) { 1669 u8s[saved_sz - 2] = 0xCE; 1670 u8s[saved_sz - 1] = 0x99; 1671 } 1672 } 1673 } 1674 1675 /* 1676 * Let's try to ensure a canonical ordering for the collected 1677 * combining marks. We do this only if we have collected 1678 * at least one more non-Starter. (The decomposition mapping 1679 * data tables have fully (and recursively) expanded and 1680 * canonically ordered decompositions.) 1681 * 1682 * The U8_SWAP_COMB_MARKS() convenience macro has some 1683 * assumptions and we are meeting the assumptions. 1684 */ 1685 last--; 1686 if (last >= saved_last) { 1687 for (i = 0; i < last; i++) 1688 for (j = last; j > i; j--) 1689 if (comb_class[j] && 1690 comb_class[j - 1] > comb_class[j]) { 1691 U8_SWAP_COMB_MARKS(j - 1, j); 1692 } 1693 } 1694 1695 *source = s; 1696 1697 if (! canonical_composition) { 1698 u8s[saved_sz] = '\0'; 1699 return (saved_sz); 1700 } 1701 1702 /* 1703 * Now do the canonical composition. Note that we do this 1704 * only after a canonical or compatibility decomposition to 1705 * finish up NFC or NFKC. 1706 */ 1707 sz = do_composition(uv, u8s, comb_class, start, disp, last, 1708 &s, slast); 1709 } 1710 1711 *source = s; 1712 1713 return ((size_t)sz); 1714} 1715 1716/* 1717 * The do_norm_compare() function does string comparion based on Unicode 1718 * simple case mappings and Unicode Normalization definitions. 1719 * 1720 * It does so by collecting a sequence of character at a time and comparing 1721 * the collected sequences from the strings. 1722 * 1723 * The meanings on the return values are the same as the usual strcmp(). 1724 */ 1725static int 1726do_norm_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, size_t n2, 1727 int flag, int *errnum) 1728{ 1729 int result; 1730 size_t sz1; 1731 size_t sz2; 1732 uchar_t u8s1[U8_STREAM_SAFE_TEXT_MAX + 1]; 1733 uchar_t u8s2[U8_STREAM_SAFE_TEXT_MAX + 1]; 1734 uchar_t *s1last; 1735 uchar_t *s2last; 1736 boolean_t is_it_toupper; 1737 boolean_t is_it_tolower; 1738 boolean_t canonical_decomposition; 1739 boolean_t compatibility_decomposition; 1740 boolean_t canonical_composition; 1741 u8_normalization_states_t state; 1742 1743 s1last = s1 + n1; 1744 s2last = s2 + n2; 1745 1746 is_it_toupper = flag & U8_TEXTPREP_TOUPPER; 1747 is_it_tolower = flag & U8_TEXTPREP_TOLOWER; 1748 canonical_decomposition = flag & U8_CANON_DECOMP; 1749 compatibility_decomposition = flag & U8_COMPAT_DECOMP; 1750 canonical_composition = flag & U8_CANON_COMP; 1751 1752 while (s1 < s1last && s2 < s2last) { 1753 /* 1754 * If the current character is a 7-bit ASCII and the last 1755 * character, or, if the current character and the next 1756 * character are both some 7-bit ASCII characters then 1757 * we treat the current character as a sequence. 1758 * 1759 * In any other cases, we need to call collect_a_seq(). 1760 */ 1761 1762 if (U8_ISASCII(*s1) && ((s1 + 1) >= s1last || 1763 ((s1 + 1) < s1last && U8_ISASCII(*(s1 + 1))))) { 1764 if (is_it_toupper) 1765 u8s1[0] = U8_ASCII_TOUPPER(*s1); 1766 else if (is_it_tolower) 1767 u8s1[0] = U8_ASCII_TOLOWER(*s1); 1768 else 1769 u8s1[0] = *s1; 1770 u8s1[1] = '\0'; 1771 sz1 = 1; 1772 s1++; 1773 } else { 1774 state = U8_STATE_START; 1775 sz1 = collect_a_seq(uv, u8s1, &s1, s1last, 1776 is_it_toupper, is_it_tolower, 1777 canonical_decomposition, 1778 compatibility_decomposition, 1779 canonical_composition, errnum, &state); 1780 } 1781 1782 if (U8_ISASCII(*s2) && ((s2 + 1) >= s2last || 1783 ((s2 + 1) < s2last && U8_ISASCII(*(s2 + 1))))) { 1784 if (is_it_toupper) 1785 u8s2[0] = U8_ASCII_TOUPPER(*s2); 1786 else if (is_it_tolower) 1787 u8s2[0] = U8_ASCII_TOLOWER(*s2); 1788 else 1789 u8s2[0] = *s2; 1790 u8s2[1] = '\0'; 1791 sz2 = 1; 1792 s2++; 1793 } else { 1794 state = U8_STATE_START; 1795 sz2 = collect_a_seq(uv, u8s2, &s2, s2last, 1796 is_it_toupper, is_it_tolower, 1797 canonical_decomposition, 1798 compatibility_decomposition, 1799 canonical_composition, errnum, &state); 1800 } 1801 1802 /* 1803 * Now compare the two characters. If they are the same, 1804 * we move on to the next character sequences. 1805 */ 1806 if (sz1 == 1 && sz2 == 1) { 1807 if (*u8s1 > *u8s2) 1808 return (1); 1809 if (*u8s1 < *u8s2) 1810 return (-1); 1811 } else { 1812 result = strcmp((const char *)u8s1, (const char *)u8s2); 1813 if (result != 0) 1814 return (result); 1815 } 1816 } 1817 1818 /* 1819 * We compared until the end of either or both strings. 1820 * 1821 * If we reached to or went over the ends for the both, that means 1822 * they are the same. 1823 * 1824 * If we reached only one end, that means the other string has 1825 * something which then can be used to determine the return value. 1826 */ 1827 if (s1 >= s1last) { 1828 if (s2 >= s2last) 1829 return (0); 1830 return (-1); 1831 } 1832 return (1); 1833} 1834 1835/* 1836 * The u8_strcmp() function compares two UTF-8 strings quite similar to 1837 * the strcmp(). For the comparison, however, Unicode Normalization specific 1838 * equivalency and Unicode simple case conversion mappings based equivalency 1839 * can be requested and checked against. 1840 */ 1841int 1842u8_strcmp(const char *s1, const char *s2, size_t n, int flag, size_t uv, 1843 int *errnum) 1844{ 1845 int f; 1846 size_t n1; 1847 size_t n2; 1848 1849 *errnum = 0; 1850 1851 /* 1852 * Check on the requested Unicode version, case conversion, and 1853 * normalization flag values. 1854 */ 1855 1856 if (uv > U8_UNICODE_LATEST) { 1857 *errnum = ERANGE; 1858 uv = U8_UNICODE_LATEST; 1859 } 1860 1861 if (flag == 0) { 1862 flag = U8_STRCMP_CS; 1863 } else { 1864 f = flag & (U8_STRCMP_CS | U8_STRCMP_CI_UPPER | 1865 U8_STRCMP_CI_LOWER); 1866 if (f == 0) { 1867 flag |= U8_STRCMP_CS; 1868 } else if (f != U8_STRCMP_CS && f != U8_STRCMP_CI_UPPER && 1869 f != U8_STRCMP_CI_LOWER) { 1870 *errnum = EBADF; 1871 flag = U8_STRCMP_CS; 1872 } 1873 1874 f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP); 1875 if (f && f != U8_STRCMP_NFD && f != U8_STRCMP_NFC && 1876 f != U8_STRCMP_NFKD && f != U8_STRCMP_NFKC) { 1877 *errnum = EBADF; 1878 flag = U8_STRCMP_CS; 1879 } 1880 } 1881 1882 if (flag == U8_STRCMP_CS) { 1883 return (n == 0 ? strcmp(s1, s2) : strncmp(s1, s2, n)); 1884 } 1885 1886 n1 = strlen(s1); 1887 n2 = strlen(s2); 1888 if (n != 0) { 1889 if (n < n1) 1890 n1 = n; 1891 if (n < n2) 1892 n2 = n; 1893 } 1894 1895 /* 1896 * Simple case conversion can be done much faster and so we do 1897 * them separately here. 1898 */ 1899 if (flag == U8_STRCMP_CI_UPPER) { 1900 return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2, 1901 n1, n2, B_TRUE, errnum)); 1902 } else if (flag == U8_STRCMP_CI_LOWER) { 1903 return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2, 1904 n1, n2, B_FALSE, errnum)); 1905 } 1906 1907 return (do_norm_compare(uv, (uchar_t *)s1, (uchar_t *)s2, n1, n2, 1908 flag, errnum)); 1909} 1910 1911size_t 1912u8_textprep_str(char *inarray, size_t *inlen, char *outarray, size_t *outlen, 1913 int flag, size_t unicode_version, int *errnum) 1914{ 1915 int f; 1916 int sz; 1917 uchar_t *ib; 1918 uchar_t *ibtail; 1919 uchar_t *ob; 1920 uchar_t *obtail; 1921 boolean_t do_not_ignore_null; 1922 boolean_t do_not_ignore_invalid; 1923 boolean_t is_it_toupper; 1924 boolean_t is_it_tolower; 1925 boolean_t canonical_decomposition; 1926 boolean_t compatibility_decomposition; 1927 boolean_t canonical_composition; 1928 size_t ret_val; 1929 size_t i; 1930 size_t j; 1931 uchar_t u8s[U8_STREAM_SAFE_TEXT_MAX + 1]; 1932 u8_normalization_states_t state; 1933 1934 if (unicode_version > U8_UNICODE_LATEST) { 1935 *errnum = ERANGE; 1936 return ((size_t)-1); 1937 } 1938 1939 f = flag & (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER); 1940 if (f == (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER)) { 1941 *errnum = EBADF; 1942 return ((size_t)-1); 1943 } 1944 1945 f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP); 1946 if (f && f != U8_TEXTPREP_NFD && f != U8_TEXTPREP_NFC && 1947 f != U8_TEXTPREP_NFKD && f != U8_TEXTPREP_NFKC) { 1948 *errnum = EBADF; 1949 return ((size_t)-1); 1950 } 1951 1952 if (inarray == NULL || *inlen == 0) 1953 return (0); 1954 1955 if (outarray == NULL) { 1956 *errnum = E2BIG; 1957 return ((size_t)-1); 1958 } 1959 1960 ib = (uchar_t *)inarray; 1961 ob = (uchar_t *)outarray; 1962 ibtail = ib + *inlen; 1963 obtail = ob + *outlen; 1964 1965 do_not_ignore_null = !(flag & U8_TEXTPREP_IGNORE_NULL); 1966 do_not_ignore_invalid = !(flag & U8_TEXTPREP_IGNORE_INVALID); 1967 is_it_toupper = flag & U8_TEXTPREP_TOUPPER; 1968 is_it_tolower = flag & U8_TEXTPREP_TOLOWER; 1969 1970 ret_val = 0; 1971 1972 /* 1973 * If we don't have a normalization flag set, we do the simple case 1974 * conversion based text preparation separately below. Text 1975 * preparation involving Normalization will be done in the false task 1976 * block, again, separately since it will take much more time and 1977 * resource than doing simple case conversions. 1978 */ 1979 if (f == 0) { 1980 while (ib < ibtail) { 1981 if (*ib == '\0' && do_not_ignore_null) 1982 break; 1983 1984 sz = u8_number_of_bytes[*ib]; 1985 1986 if (sz < 0) { 1987 if (do_not_ignore_invalid) { 1988 *errnum = EILSEQ; 1989 ret_val = (size_t)-1; 1990 break; 1991 } 1992 1993 sz = 1; 1994 ret_val++; 1995 } 1996 1997 if (sz == 1) { 1998 if (ob >= obtail) { 1999 *errnum = E2BIG; 2000 ret_val = (size_t)-1; 2001 break; 2002 } 2003 2004 if (is_it_toupper) 2005 *ob = U8_ASCII_TOUPPER(*ib); 2006 else if (is_it_tolower) 2007 *ob = U8_ASCII_TOLOWER(*ib); 2008 else 2009 *ob = *ib; 2010 ib++; 2011 ob++; 2012 } else if ((ib + sz) > ibtail) { 2013 if (do_not_ignore_invalid) { 2014 *errnum = EINVAL; 2015 ret_val = (size_t)-1; 2016 break; 2017 } 2018 2019 if ((obtail - ob) < (ibtail - ib)) { 2020 *errnum = E2BIG; 2021 ret_val = (size_t)-1; 2022 break; 2023 } 2024 2025 /* 2026 * We treat the remaining incomplete character 2027 * bytes as a character. 2028 */ 2029 ret_val++; 2030 2031 while (ib < ibtail) 2032 *ob++ = *ib++; 2033 } else { 2034 if (is_it_toupper || is_it_tolower) { 2035 i = do_case_conv(unicode_version, u8s, 2036 ib, sz, is_it_toupper); 2037 2038 if ((obtail - ob) < i) { 2039 *errnum = E2BIG; 2040 ret_val = (size_t)-1; 2041 break; 2042 } 2043 2044 ib += sz; 2045 2046 for (sz = 0; sz < i; sz++) 2047 *ob++ = u8s[sz]; 2048 } else { 2049 if ((obtail - ob) < sz) { 2050 *errnum = E2BIG; 2051 ret_val = (size_t)-1; 2052 break; 2053 } 2054 2055 for (i = 0; i < sz; i++) 2056 *ob++ = *ib++; 2057 } 2058 } 2059 } 2060 } else { 2061 canonical_decomposition = flag & U8_CANON_DECOMP; 2062 compatibility_decomposition = flag & U8_COMPAT_DECOMP; 2063 canonical_composition = flag & U8_CANON_COMP; 2064 2065 while (ib < ibtail) { 2066 if (*ib == '\0' && do_not_ignore_null) 2067 break; 2068 2069 /* 2070 * If the current character is a 7-bit ASCII 2071 * character and it is the last character, or, 2072 * if the current character is a 7-bit ASCII 2073 * character and the next character is also a 7-bit 2074 * ASCII character, then, we copy over this 2075 * character without going through collect_a_seq(). 2076 * 2077 * In any other cases, we need to look further with 2078 * the collect_a_seq() function. 2079 */ 2080 if (U8_ISASCII(*ib) && ((ib + 1) >= ibtail || 2081 ((ib + 1) < ibtail && U8_ISASCII(*(ib + 1))))) { 2082 if (ob >= obtail) { 2083 *errnum = E2BIG; 2084 ret_val = (size_t)-1; 2085 break; 2086 } 2087 2088 if (is_it_toupper) 2089 *ob = U8_ASCII_TOUPPER(*ib); 2090 else if (is_it_tolower) 2091 *ob = U8_ASCII_TOLOWER(*ib); 2092 else 2093 *ob = *ib; 2094 ib++; 2095 ob++; 2096 } else { 2097 *errnum = 0; 2098 state = U8_STATE_START; 2099 2100 j = collect_a_seq(unicode_version, u8s, 2101 &ib, ibtail, 2102 is_it_toupper, 2103 is_it_tolower, 2104 canonical_decomposition, 2105 compatibility_decomposition, 2106 canonical_composition, 2107 errnum, &state); 2108 2109 if (*errnum && do_not_ignore_invalid) { 2110 ret_val = (size_t)-1; 2111 break; 2112 } 2113 2114 if ((obtail - ob) < j) { 2115 *errnum = E2BIG; 2116 ret_val = (size_t)-1; 2117 break; 2118 } 2119 2120 for (i = 0; i < j; i++) 2121 *ob++ = u8s[i]; 2122 } 2123 } 2124 } 2125 2126 *inlen = ibtail - ib; 2127 *outlen = obtail - ob; 2128 2129 return (ret_val); 2130} 2131