1185029Spjd/* 2185029Spjd * CDDL HEADER START 3185029Spjd * 4185029Spjd * The contents of this file are subject to the terms of the 5185029Spjd * Common Development and Distribution License (the "License"). 6185029Spjd * You may not use this file except in compliance with the License. 7185029Spjd * 8185029Spjd * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9185029Spjd * or http://www.opensolaris.org/os/licensing. 10185029Spjd * See the License for the specific language governing permissions 11185029Spjd * and limitations under the License. 12185029Spjd * 13185029Spjd * When distributing Covered Code, include this CDDL HEADER in each 14185029Spjd * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15185029Spjd * If applicable, add the following below this CDDL HEADER, with the 16185029Spjd * fields enclosed by brackets "[]" replaced with your own identifying 17185029Spjd * information: Portions Copyright [yyyy] [name of copyright owner] 18185029Spjd * 19185029Spjd * CDDL HEADER END 20185029Spjd */ 21185029Spjd/* 22185029Spjd * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23185029Spjd * Use is subject to license terms. 24185029Spjd */ 25185029Spjd 26185029Spjd 27185029Spjd 28185029Spjd/* 29185029Spjd * UTF-8 text preparation functions (PSARC/2007/149, PSARC/2007/458). 30185029Spjd * 31185029Spjd * Man pages: u8_textprep_open(9F), u8_textprep_buf(9F), u8_textprep_close(9F), 32185029Spjd * u8_textprep_str(9F), u8_strcmp(9F), and u8_validate(9F). See also 33185029Spjd * the section 3C man pages. 34185029Spjd * Interface stability: Committed. 35185029Spjd */ 36185029Spjd 37185029Spjd#include <sys/types.h> 38185029Spjd#ifdef _KERNEL 39185029Spjd#include <sys/param.h> 40185029Spjd#include <sys/sysmacros.h> 41185029Spjd#include <sys/systm.h> 42185029Spjd#include <sys/debug.h> 43185029Spjd#include <sys/kmem.h> 44219089Spjd#include <sys/sunddi.h> 45185029Spjd#else 46185029Spjd#include <strings.h> 47185029Spjd#endif /* _KERNEL */ 48185029Spjd#include <sys/byteorder.h> 49185029Spjd#include <sys/errno.h> 50185029Spjd#include <sys/u8_textprep.h> 51185029Spjd#include <sys/u8_textprep_data.h> 52185029Spjd 53185029Spjd 54185029Spjd/* The maximum possible number of bytes in a UTF-8 character. */ 55185029Spjd#define U8_MB_CUR_MAX (4) 56185029Spjd 57185029Spjd/* 58185029Spjd * The maximum number of bytes needed for a UTF-8 character to cover 59185029Spjd * U+0000 - U+FFFF, i.e., the coding space of now deprecated UCS-2. 60185029Spjd */ 61185029Spjd#define U8_MAX_BYTES_UCS2 (3) 62185029Spjd 63185029Spjd/* The maximum possible number of bytes in a Stream-Safe Text. */ 64185029Spjd#define U8_STREAM_SAFE_TEXT_MAX (128) 65185029Spjd 66185029Spjd/* 67185029Spjd * The maximum number of characters in a combining/conjoining sequence and 68185029Spjd * the actual upperbound limit of a combining/conjoining sequence. 69185029Spjd */ 70185029Spjd#define U8_MAX_CHARS_A_SEQ (32) 71185029Spjd#define U8_UPPER_LIMIT_IN_A_SEQ (31) 72185029Spjd 73185029Spjd/* The combining class value for Starter. */ 74185029Spjd#define U8_COMBINING_CLASS_STARTER (0) 75185029Spjd 76185029Spjd/* 77185029Spjd * Some Hangul related macros at below. 78185029Spjd * 79185029Spjd * The first and the last of Hangul syllables, Hangul Jamo Leading consonants, 80185029Spjd * Vowels, and optional Trailing consonants in Unicode scalar values. 81185029Spjd * 82185029Spjd * Please be noted that the U8_HANGUL_JAMO_T_FIRST is 0x11A7 at below not 83185029Spjd * the actual U+11A8. This is due to that the trailing consonant is optional 84185029Spjd * and thus we are doing a pre-calculation of subtracting one. 85185029Spjd * 86185029Spjd * Each of 19 modern leading consonants has total 588 possible syllables since 87185029Spjd * Hangul has 21 modern vowels and 27 modern trailing consonants plus 1 for 88185029Spjd * no trailing consonant case, i.e., 21 x 28 = 588. 89185029Spjd * 90185029Spjd * We also have bunch of Hangul related macros at below. Please bear in mind 91185029Spjd * that the U8_HANGUL_JAMO_1ST_BYTE can be used to check whether it is 92185029Spjd * a Hangul Jamo or not but the value does not guarantee that it is a Hangul 93185029Spjd * Jamo; it just guarantee that it will be most likely. 94185029Spjd */ 95185029Spjd#define U8_HANGUL_SYL_FIRST (0xAC00U) 96185029Spjd#define U8_HANGUL_SYL_LAST (0xD7A3U) 97185029Spjd 98185029Spjd#define U8_HANGUL_JAMO_L_FIRST (0x1100U) 99185029Spjd#define U8_HANGUL_JAMO_L_LAST (0x1112U) 100185029Spjd#define U8_HANGUL_JAMO_V_FIRST (0x1161U) 101185029Spjd#define U8_HANGUL_JAMO_V_LAST (0x1175U) 102185029Spjd#define U8_HANGUL_JAMO_T_FIRST (0x11A7U) 103185029Spjd#define U8_HANGUL_JAMO_T_LAST (0x11C2U) 104185029Spjd 105185029Spjd#define U8_HANGUL_V_COUNT (21) 106185029Spjd#define U8_HANGUL_VT_COUNT (588) 107185029Spjd#define U8_HANGUL_T_COUNT (28) 108185029Spjd 109185029Spjd#define U8_HANGUL_JAMO_1ST_BYTE (0xE1U) 110185029Spjd 111185029Spjd#define U8_SAVE_HANGUL_AS_UTF8(s, i, j, k, b) \ 112185029Spjd (s)[(i)] = (uchar_t)(0xE0U | ((uint32_t)(b) & 0xF000U) >> 12); \ 113185029Spjd (s)[(j)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x0FC0U) >> 6); \ 114185029Spjd (s)[(k)] = (uchar_t)(0x80U | ((uint32_t)(b) & 0x003FU)); 115185029Spjd 116185029Spjd#define U8_HANGUL_JAMO_L(u) \ 117185029Spjd ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_L_LAST) 118185029Spjd 119185029Spjd#define U8_HANGUL_JAMO_V(u) \ 120185029Spjd ((u) >= U8_HANGUL_JAMO_V_FIRST && (u) <= U8_HANGUL_JAMO_V_LAST) 121185029Spjd 122185029Spjd#define U8_HANGUL_JAMO_T(u) \ 123185029Spjd ((u) > U8_HANGUL_JAMO_T_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST) 124185029Spjd 125185029Spjd#define U8_HANGUL_JAMO(u) \ 126185029Spjd ((u) >= U8_HANGUL_JAMO_L_FIRST && (u) <= U8_HANGUL_JAMO_T_LAST) 127185029Spjd 128185029Spjd#define U8_HANGUL_SYLLABLE(u) \ 129185029Spjd ((u) >= U8_HANGUL_SYL_FIRST && (u) <= U8_HANGUL_SYL_LAST) 130185029Spjd 131185029Spjd#define U8_HANGUL_COMPOSABLE_L_V(s, u) \ 132185029Spjd ((s) == U8_STATE_HANGUL_L && U8_HANGUL_JAMO_V((u))) 133185029Spjd 134185029Spjd#define U8_HANGUL_COMPOSABLE_LV_T(s, u) \ 135185029Spjd ((s) == U8_STATE_HANGUL_LV && U8_HANGUL_JAMO_T((u))) 136185029Spjd 137185029Spjd/* The types of decomposition mappings. */ 138185029Spjd#define U8_DECOMP_BOTH (0xF5U) 139185029Spjd#define U8_DECOMP_CANONICAL (0xF6U) 140185029Spjd 141185029Spjd/* The indicator for 16-bit table. */ 142185029Spjd#define U8_16BIT_TABLE_INDICATOR (0x8000U) 143185029Spjd 144185029Spjd/* The following are some convenience macros. */ 145268014Spfg#define U8_PUT_3BYTES_INTO_UTF32(u, b1, b2, b3) \ 146268014Spfg (u) = ((((uint32_t)(b1) & 0x0F) << 12) | \ 147268014Spfg (((uint32_t)(b2) & 0x3F) << 6) | \ 148268014Spfg ((uint32_t)(b3) & 0x3F)); 149185029Spjd#define U8_SIMPLE_SWAP(a, b, t) \ 150185029Spjd (t) = (a); \ 151185029Spjd (a) = (b); \ 152185029Spjd (b) = (t); 153185029Spjd 154185029Spjd#define U8_ASCII_TOUPPER(c) \ 155185029Spjd (((c) >= 'a' && (c) <= 'z') ? (c) - 'a' + 'A' : (c)) 156185029Spjd 157185029Spjd#define U8_ASCII_TOLOWER(c) \ 158185029Spjd (((c) >= 'A' && (c) <= 'Z') ? (c) - 'A' + 'a' : (c)) 159185029Spjd 160185029Spjd#define U8_ISASCII(c) (((uchar_t)(c)) < 0x80U) 161185029Spjd/* 162185029Spjd * The following macro assumes that the two characters that are to be 163185029Spjd * swapped are adjacent to each other and 'a' comes before 'b'. 164185029Spjd * 165185029Spjd * If the assumptions are not met, then, the macro will fail. 166185029Spjd */ 167185029Spjd#define U8_SWAP_COMB_MARKS(a, b) \ 168185029Spjd for (k = 0; k < disp[(a)]; k++) \ 169185029Spjd u8t[k] = u8s[start[(a)] + k]; \ 170185029Spjd for (k = 0; k < disp[(b)]; k++) \ 171185029Spjd u8s[start[(a)] + k] = u8s[start[(b)] + k]; \ 172185029Spjd start[(b)] = start[(a)] + disp[(b)]; \ 173185029Spjd for (k = 0; k < disp[(a)]; k++) \ 174185029Spjd u8s[start[(b)] + k] = u8t[k]; \ 175185029Spjd U8_SIMPLE_SWAP(comb_class[(a)], comb_class[(b)], tc); \ 176185029Spjd U8_SIMPLE_SWAP(disp[(a)], disp[(b)], tc); 177185029Spjd 178185029Spjd/* The possible states during normalization. */ 179185029Spjdtypedef enum { 180185029Spjd U8_STATE_START = 0, 181185029Spjd U8_STATE_HANGUL_L = 1, 182185029Spjd U8_STATE_HANGUL_LV = 2, 183185029Spjd U8_STATE_HANGUL_LVT = 3, 184185029Spjd U8_STATE_HANGUL_V = 4, 185185029Spjd U8_STATE_HANGUL_T = 5, 186185029Spjd U8_STATE_COMBINING_MARK = 6 187185029Spjd} u8_normalization_states_t; 188185029Spjd 189185029Spjd/* 190185029Spjd * The three vectors at below are used to check bytes of a given UTF-8 191185029Spjd * character are valid and not containing any malformed byte values. 192185029Spjd * 193185029Spjd * We used to have a quite relaxed UTF-8 binary representation but then there 194185029Spjd * was some security related issues and so the Unicode Consortium defined 195185029Spjd * and announced the UTF-8 Corrigendum at Unicode 3.1 and then refined it 196185029Spjd * one more time at the Unicode 3.2. The following three tables are based on 197185029Spjd * that. 198185029Spjd */ 199185029Spjd 200185029Spjd#define U8_ILLEGAL_NEXT_BYTE_COMMON(c) ((c) < 0x80 || (c) > 0xBF) 201185029Spjd 202185029Spjd#define I_ U8_ILLEGAL_CHAR 203185029Spjd#define O_ U8_OUT_OF_RANGE_CHAR 204185029Spjd 205185029Spjdconst int8_t u8_number_of_bytes[0x100] = { 206185029Spjd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 207185029Spjd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 208185029Spjd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 209185029Spjd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 210185029Spjd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 211185029Spjd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 212185029Spjd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 213185029Spjd 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 214185029Spjd 215185029Spjd/* 80 81 82 83 84 85 86 87 88 89 8A 8B 8C 8D 8E 8F */ 216185029Spjd I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 217185029Spjd 218268014Spfg/* 90 91 92 93 94 95 96 97 98 99 9A 9B 9C 9D 9E 9F */ 219185029Spjd I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 220185029Spjd 221268014Spfg/* A0 A1 A2 A3 A4 A5 A6 A7 A8 A9 AA AB AC AD AE AF */ 222185029Spjd I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 223185029Spjd 224185029Spjd/* B0 B1 B2 B3 B4 B5 B6 B7 B8 B9 BA BB BC BD BE BF */ 225185029Spjd I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, I_, 226185029Spjd 227185029Spjd/* C0 C1 C2 C3 C4 C5 C6 C7 C8 C9 CA CB CC CD CE CF */ 228185029Spjd I_, I_, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 229185029Spjd 230185029Spjd/* D0 D1 D2 D3 D4 D5 D6 D7 D8 D9 DA DB DC DD DE DF */ 231185029Spjd 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 232185029Spjd 233185029Spjd/* E0 E1 E2 E3 E4 E5 E6 E7 E8 E9 EA EB EC ED EE EF */ 234185029Spjd 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 235185029Spjd 236185029Spjd/* F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 FA FB FC FD FE FF */ 237185029Spjd 4, 4, 4, 4, 4, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_, O_, 238185029Spjd}; 239185029Spjd 240185029Spjd#undef I_ 241185029Spjd#undef O_ 242185029Spjd 243185029Spjdconst uint8_t u8_valid_min_2nd_byte[0x100] = { 244185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 245185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 246185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 247185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 248185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 249185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 250185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 251185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 252185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 253185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 254185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 255185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 256185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 257185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 258185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 259185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 260185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 261185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 262185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 263185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 264185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 265185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 266185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 267185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 268185029Spjd/* C0 C1 C2 C3 C4 C5 C6 C7 */ 269185029Spjd 0, 0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 270185029Spjd/* C8 C9 CA CB CC CD CE CF */ 271185029Spjd 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 272185029Spjd/* D0 D1 D2 D3 D4 D5 D6 D7 */ 273185029Spjd 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 274185029Spjd/* D8 D9 DA DB DC DD DE DF */ 275185029Spjd 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 276185029Spjd/* E0 E1 E2 E3 E4 E5 E6 E7 */ 277185029Spjd 0xa0, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 278185029Spjd/* E8 E9 EA EB EC ED EE EF */ 279185029Spjd 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 280185029Spjd/* F0 F1 F2 F3 F4 F5 F6 F7 */ 281185029Spjd 0x90, 0x80, 0x80, 0x80, 0x80, 0, 0, 0, 282185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 283185029Spjd}; 284185029Spjd 285185029Spjdconst uint8_t u8_valid_max_2nd_byte[0x100] = { 286185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 287185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 288185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 289185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 290185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 291185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 292185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 293185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 294185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 295185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 296185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 297185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 298185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 299185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 300185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 301185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 302185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 303185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 304185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 305185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 306185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 307185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 308185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 309185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 310185029Spjd/* C0 C1 C2 C3 C4 C5 C6 C7 */ 311185029Spjd 0, 0, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 312185029Spjd/* C8 C9 CA CB CC CD CE CF */ 313185029Spjd 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 314185029Spjd/* D0 D1 D2 D3 D4 D5 D6 D7 */ 315185029Spjd 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 316185029Spjd/* D8 D9 DA DB DC DD DE DF */ 317185029Spjd 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 318185029Spjd/* E0 E1 E2 E3 E4 E5 E6 E7 */ 319185029Spjd 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 320185029Spjd/* E8 E9 EA EB EC ED EE EF */ 321185029Spjd 0xbf, 0xbf, 0xbf, 0xbf, 0xbf, 0x9f, 0xbf, 0xbf, 322185029Spjd/* F0 F1 F2 F3 F4 F5 F6 F7 */ 323185029Spjd 0xbf, 0xbf, 0xbf, 0xbf, 0x8f, 0, 0, 0, 324185029Spjd 0, 0, 0, 0, 0, 0, 0, 0, 325185029Spjd}; 326185029Spjd 327185029Spjd 328185029Spjd/* 329185029Spjd * The u8_validate() validates on the given UTF-8 character string and 330185029Spjd * calculate the byte length. It is quite similar to mblen(3C) except that 331185029Spjd * this will validate against the list of characters if required and 332185029Spjd * specific to UTF-8 and Unicode. 333185029Spjd */ 334185029Spjdint 335185029Spjdu8_validate(char *u8str, size_t n, char **list, int flag, int *errnum) 336185029Spjd{ 337185029Spjd uchar_t *ib; 338185029Spjd uchar_t *ibtail; 339185029Spjd uchar_t **p; 340185029Spjd uchar_t *s1; 341185029Spjd uchar_t *s2; 342185029Spjd uchar_t f; 343185029Spjd int sz; 344185029Spjd size_t i; 345185029Spjd int ret_val; 346185029Spjd boolean_t second; 347185029Spjd boolean_t no_need_to_validate_entire; 348185029Spjd boolean_t check_additional; 349185029Spjd boolean_t validate_ucs2_range_only; 350185029Spjd 351185029Spjd if (! u8str) 352185029Spjd return (0); 353185029Spjd 354185029Spjd ib = (uchar_t *)u8str; 355185029Spjd ibtail = ib + n; 356185029Spjd 357185029Spjd ret_val = 0; 358185029Spjd 359185029Spjd no_need_to_validate_entire = ! (flag & U8_VALIDATE_ENTIRE); 360185029Spjd check_additional = flag & U8_VALIDATE_CHECK_ADDITIONAL; 361185029Spjd validate_ucs2_range_only = flag & U8_VALIDATE_UCS2_RANGE; 362185029Spjd 363185029Spjd while (ib < ibtail) { 364185029Spjd /* 365185029Spjd * The first byte of a UTF-8 character tells how many 366185029Spjd * bytes will follow for the character. If the first byte 367185029Spjd * is an illegal byte value or out of range value, we just 368185029Spjd * return -1 with an appropriate error number. 369185029Spjd */ 370185029Spjd sz = u8_number_of_bytes[*ib]; 371185029Spjd if (sz == U8_ILLEGAL_CHAR) { 372185029Spjd *errnum = EILSEQ; 373185029Spjd return (-1); 374185029Spjd } 375185029Spjd 376185029Spjd if (sz == U8_OUT_OF_RANGE_CHAR || 377185029Spjd (validate_ucs2_range_only && sz > U8_MAX_BYTES_UCS2)) { 378185029Spjd *errnum = ERANGE; 379185029Spjd return (-1); 380185029Spjd } 381185029Spjd 382185029Spjd /* 383185029Spjd * If we don't have enough bytes to check on, that's also 384185029Spjd * an error. As you can see, we give illegal byte sequence 385185029Spjd * checking higher priority then EINVAL cases. 386185029Spjd */ 387185029Spjd if ((ibtail - ib) < sz) { 388185029Spjd *errnum = EINVAL; 389185029Spjd return (-1); 390185029Spjd } 391185029Spjd 392185029Spjd if (sz == 1) { 393185029Spjd ib++; 394185029Spjd ret_val++; 395185029Spjd } else { 396185029Spjd /* 397185029Spjd * Check on the multi-byte UTF-8 character. For more 398185029Spjd * details on this, see comment added for the used 399185029Spjd * data structures at the beginning of the file. 400185029Spjd */ 401185029Spjd f = *ib++; 402185029Spjd ret_val++; 403185029Spjd second = B_TRUE; 404185029Spjd for (i = 1; i < sz; i++) { 405185029Spjd if (second) { 406185029Spjd if (*ib < u8_valid_min_2nd_byte[f] || 407185029Spjd *ib > u8_valid_max_2nd_byte[f]) { 408185029Spjd *errnum = EILSEQ; 409185029Spjd return (-1); 410185029Spjd } 411185029Spjd second = B_FALSE; 412185029Spjd } else if (U8_ILLEGAL_NEXT_BYTE_COMMON(*ib)) { 413185029Spjd *errnum = EILSEQ; 414185029Spjd return (-1); 415185029Spjd } 416185029Spjd ib++; 417185029Spjd ret_val++; 418185029Spjd } 419185029Spjd } 420185029Spjd 421185029Spjd if (check_additional) { 422185029Spjd for (p = (uchar_t **)list, i = 0; p[i]; i++) { 423185029Spjd s1 = ib - sz; 424185029Spjd s2 = p[i]; 425185029Spjd while (s1 < ib) { 426185029Spjd if (*s1 != *s2 || *s2 == '\0') 427185029Spjd break; 428185029Spjd s1++; 429185029Spjd s2++; 430185029Spjd } 431185029Spjd 432185029Spjd if (s1 >= ib && *s2 == '\0') { 433185029Spjd *errnum = EBADF; 434185029Spjd return (-1); 435185029Spjd } 436185029Spjd } 437185029Spjd } 438185029Spjd 439185029Spjd if (no_need_to_validate_entire) 440185029Spjd break; 441185029Spjd } 442185029Spjd 443185029Spjd return (ret_val); 444185029Spjd} 445185029Spjd 446185029Spjd/* 447185029Spjd * The do_case_conv() looks at the mapping tables and returns found 448185029Spjd * bytes if any. If not found, the input bytes are returned. The function 449185029Spjd * always terminate the return bytes with a null character assuming that 450185029Spjd * there are plenty of room to do so. 451185029Spjd * 452185029Spjd * The case conversions are simple case conversions mapping a character to 453185029Spjd * another character as specified in the Unicode data. The byte size of 454185029Spjd * the mapped character could be different from that of the input character. 455185029Spjd * 456185029Spjd * The return value is the byte length of the returned character excluding 457185029Spjd * the terminating null byte. 458185029Spjd */ 459185029Spjdstatic size_t 460185029Spjddo_case_conv(int uv, uchar_t *u8s, uchar_t *s, int sz, boolean_t is_it_toupper) 461185029Spjd{ 462185029Spjd size_t i; 463185029Spjd uint16_t b1 = 0; 464185029Spjd uint16_t b2 = 0; 465185029Spjd uint16_t b3 = 0; 466185029Spjd uint16_t b3_tbl; 467185029Spjd uint16_t b3_base; 468185029Spjd uint16_t b4 = 0; 469185029Spjd size_t start_id; 470185029Spjd size_t end_id; 471185029Spjd 472185029Spjd /* 473185029Spjd * At this point, the only possible values for sz are 2, 3, and 4. 474185029Spjd * The u8s should point to a vector that is well beyond the size of 475185029Spjd * 5 bytes. 476185029Spjd */ 477185029Spjd if (sz == 2) { 478185029Spjd b3 = u8s[0] = s[0]; 479185029Spjd b4 = u8s[1] = s[1]; 480185029Spjd } else if (sz == 3) { 481185029Spjd b2 = u8s[0] = s[0]; 482185029Spjd b3 = u8s[1] = s[1]; 483185029Spjd b4 = u8s[2] = s[2]; 484185029Spjd } else if (sz == 4) { 485185029Spjd b1 = u8s[0] = s[0]; 486185029Spjd b2 = u8s[1] = s[1]; 487185029Spjd b3 = u8s[2] = s[2]; 488185029Spjd b4 = u8s[3] = s[3]; 489185029Spjd } else { 490185029Spjd /* This is not possible but just in case as a fallback. */ 491185029Spjd if (is_it_toupper) 492185029Spjd *u8s = U8_ASCII_TOUPPER(*s); 493185029Spjd else 494185029Spjd *u8s = U8_ASCII_TOLOWER(*s); 495185029Spjd u8s[1] = '\0'; 496185029Spjd 497185029Spjd return (1); 498185029Spjd } 499185029Spjd u8s[sz] = '\0'; 500185029Spjd 501185029Spjd /* 502185029Spjd * Let's find out if we have a corresponding character. 503185029Spjd */ 504185029Spjd b1 = u8_common_b1_tbl[uv][b1]; 505185029Spjd if (b1 == U8_TBL_ELEMENT_NOT_DEF) 506185029Spjd return ((size_t)sz); 507185029Spjd 508185029Spjd b2 = u8_case_common_b2_tbl[uv][b1][b2]; 509185029Spjd if (b2 == U8_TBL_ELEMENT_NOT_DEF) 510185029Spjd return ((size_t)sz); 511185029Spjd 512185029Spjd if (is_it_toupper) { 513185029Spjd b3_tbl = u8_toupper_b3_tbl[uv][b2][b3].tbl_id; 514185029Spjd if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 515185029Spjd return ((size_t)sz); 516185029Spjd 517185029Spjd start_id = u8_toupper_b4_tbl[uv][b3_tbl][b4]; 518185029Spjd end_id = u8_toupper_b4_tbl[uv][b3_tbl][b4 + 1]; 519185029Spjd 520185029Spjd /* Either there is no match or an error at the table. */ 521185029Spjd if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX) 522185029Spjd return ((size_t)sz); 523185029Spjd 524185029Spjd b3_base = u8_toupper_b3_tbl[uv][b2][b3].base; 525185029Spjd 526185029Spjd for (i = 0; start_id < end_id; start_id++) 527185029Spjd u8s[i++] = u8_toupper_final_tbl[uv][b3_base + start_id]; 528185029Spjd } else { 529185029Spjd b3_tbl = u8_tolower_b3_tbl[uv][b2][b3].tbl_id; 530185029Spjd if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 531185029Spjd return ((size_t)sz); 532185029Spjd 533185029Spjd start_id = u8_tolower_b4_tbl[uv][b3_tbl][b4]; 534185029Spjd end_id = u8_tolower_b4_tbl[uv][b3_tbl][b4 + 1]; 535185029Spjd 536185029Spjd if (start_id >= end_id || (end_id - start_id) > U8_MB_CUR_MAX) 537185029Spjd return ((size_t)sz); 538185029Spjd 539185029Spjd b3_base = u8_tolower_b3_tbl[uv][b2][b3].base; 540185029Spjd 541185029Spjd for (i = 0; start_id < end_id; start_id++) 542185029Spjd u8s[i++] = u8_tolower_final_tbl[uv][b3_base + start_id]; 543185029Spjd } 544185029Spjd 545185029Spjd /* 546185029Spjd * If i is still zero, that means there is no corresponding character. 547185029Spjd */ 548185029Spjd if (i == 0) 549185029Spjd return ((size_t)sz); 550185029Spjd 551185029Spjd u8s[i] = '\0'; 552185029Spjd 553185029Spjd return (i); 554185029Spjd} 555185029Spjd 556185029Spjd/* 557185029Spjd * The do_case_compare() function compares the two input strings, s1 and s2, 558185029Spjd * one character at a time doing case conversions if applicable and return 559185029Spjd * the comparison result as like strcmp(). 560185029Spjd * 561185029Spjd * Since, in empirical sense, most of text data are 7-bit ASCII characters, 562185029Spjd * we treat the 7-bit ASCII characters as a special case trying to yield 563185029Spjd * faster processing time. 564185029Spjd */ 565185029Spjdstatic int 566185029Spjddo_case_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, 567185029Spjd size_t n2, boolean_t is_it_toupper, int *errnum) 568185029Spjd{ 569185029Spjd int f; 570185029Spjd int sz1; 571185029Spjd int sz2; 572185029Spjd size_t j; 573185029Spjd size_t i1; 574185029Spjd size_t i2; 575185029Spjd uchar_t u8s1[U8_MB_CUR_MAX + 1]; 576185029Spjd uchar_t u8s2[U8_MB_CUR_MAX + 1]; 577185029Spjd 578185029Spjd i1 = i2 = 0; 579185029Spjd while (i1 < n1 && i2 < n2) { 580185029Spjd /* 581185029Spjd * Find out what would be the byte length for this UTF-8 582185029Spjd * character at string s1 and also find out if this is 583185029Spjd * an illegal start byte or not and if so, issue a proper 584185029Spjd * error number and yet treat this byte as a character. 585185029Spjd */ 586185029Spjd sz1 = u8_number_of_bytes[*s1]; 587185029Spjd if (sz1 < 0) { 588185029Spjd *errnum = EILSEQ; 589185029Spjd sz1 = 1; 590185029Spjd } 591185029Spjd 592185029Spjd /* 593185029Spjd * For 7-bit ASCII characters mainly, we do a quick case 594185029Spjd * conversion right at here. 595185029Spjd * 596185029Spjd * If we don't have enough bytes for this character, issue 597185029Spjd * an EINVAL error and use what are available. 598185029Spjd * 599185029Spjd * If we have enough bytes, find out if there is 600185029Spjd * a corresponding uppercase character and if so, copy over 601185029Spjd * the bytes for a comparison later. If there is no 602185029Spjd * corresponding uppercase character, then, use what we have 603185029Spjd * for the comparison. 604185029Spjd */ 605185029Spjd if (sz1 == 1) { 606185029Spjd if (is_it_toupper) 607185029Spjd u8s1[0] = U8_ASCII_TOUPPER(*s1); 608185029Spjd else 609185029Spjd u8s1[0] = U8_ASCII_TOLOWER(*s1); 610185029Spjd s1++; 611185029Spjd u8s1[1] = '\0'; 612185029Spjd } else if ((i1 + sz1) > n1) { 613185029Spjd *errnum = EINVAL; 614185029Spjd for (j = 0; (i1 + j) < n1; ) 615185029Spjd u8s1[j++] = *s1++; 616185029Spjd u8s1[j] = '\0'; 617185029Spjd } else { 618185029Spjd (void) do_case_conv(uv, u8s1, s1, sz1, is_it_toupper); 619185029Spjd s1 += sz1; 620185029Spjd } 621185029Spjd 622185029Spjd /* Do the same for the string s2. */ 623185029Spjd sz2 = u8_number_of_bytes[*s2]; 624185029Spjd if (sz2 < 0) { 625185029Spjd *errnum = EILSEQ; 626185029Spjd sz2 = 1; 627185029Spjd } 628185029Spjd 629185029Spjd if (sz2 == 1) { 630185029Spjd if (is_it_toupper) 631185029Spjd u8s2[0] = U8_ASCII_TOUPPER(*s2); 632185029Spjd else 633185029Spjd u8s2[0] = U8_ASCII_TOLOWER(*s2); 634185029Spjd s2++; 635185029Spjd u8s2[1] = '\0'; 636185029Spjd } else if ((i2 + sz2) > n2) { 637185029Spjd *errnum = EINVAL; 638185029Spjd for (j = 0; (i2 + j) < n2; ) 639185029Spjd u8s2[j++] = *s2++; 640185029Spjd u8s2[j] = '\0'; 641185029Spjd } else { 642185029Spjd (void) do_case_conv(uv, u8s2, s2, sz2, is_it_toupper); 643185029Spjd s2 += sz2; 644185029Spjd } 645185029Spjd 646185029Spjd /* Now compare the two characters. */ 647185029Spjd if (sz1 == 1 && sz2 == 1) { 648185029Spjd if (*u8s1 > *u8s2) 649185029Spjd return (1); 650185029Spjd if (*u8s1 < *u8s2) 651185029Spjd return (-1); 652185029Spjd } else { 653185029Spjd f = strcmp((const char *)u8s1, (const char *)u8s2); 654185029Spjd if (f != 0) 655185029Spjd return (f); 656185029Spjd } 657185029Spjd 658185029Spjd /* 659185029Spjd * They were the same. Let's move on to the next 660185029Spjd * characters then. 661185029Spjd */ 662185029Spjd i1 += sz1; 663185029Spjd i2 += sz2; 664185029Spjd } 665185029Spjd 666185029Spjd /* 667185029Spjd * We compared until the end of either or both strings. 668185029Spjd * 669185029Spjd * If we reached to or went over the ends for the both, that means 670185029Spjd * they are the same. 671185029Spjd * 672185029Spjd * If we reached only one of the two ends, that means the other string 673185029Spjd * has something which then the fact can be used to determine 674185029Spjd * the return value. 675185029Spjd */ 676185029Spjd if (i1 >= n1) { 677185029Spjd if (i2 >= n2) 678185029Spjd return (0); 679185029Spjd return (-1); 680185029Spjd } 681185029Spjd return (1); 682185029Spjd} 683185029Spjd 684185029Spjd/* 685185029Spjd * The combining_class() function checks on the given bytes and find out 686185029Spjd * the corresponding Unicode combining class value. The return value 0 means 687185029Spjd * it is a Starter. Any illegal UTF-8 character will also be treated as 688185029Spjd * a Starter. 689185029Spjd */ 690185029Spjdstatic uchar_t 691185029Spjdcombining_class(size_t uv, uchar_t *s, size_t sz) 692185029Spjd{ 693185029Spjd uint16_t b1 = 0; 694185029Spjd uint16_t b2 = 0; 695185029Spjd uint16_t b3 = 0; 696185029Spjd uint16_t b4 = 0; 697185029Spjd 698185029Spjd if (sz == 1 || sz > 4) 699185029Spjd return (0); 700185029Spjd 701185029Spjd if (sz == 2) { 702185029Spjd b3 = s[0]; 703185029Spjd b4 = s[1]; 704185029Spjd } else if (sz == 3) { 705185029Spjd b2 = s[0]; 706185029Spjd b3 = s[1]; 707185029Spjd b4 = s[2]; 708185029Spjd } else if (sz == 4) { 709185029Spjd b1 = s[0]; 710185029Spjd b2 = s[1]; 711185029Spjd b3 = s[2]; 712185029Spjd b4 = s[3]; 713185029Spjd } 714185029Spjd 715185029Spjd b1 = u8_common_b1_tbl[uv][b1]; 716185029Spjd if (b1 == U8_TBL_ELEMENT_NOT_DEF) 717185029Spjd return (0); 718185029Spjd 719185029Spjd b2 = u8_combining_class_b2_tbl[uv][b1][b2]; 720185029Spjd if (b2 == U8_TBL_ELEMENT_NOT_DEF) 721185029Spjd return (0); 722185029Spjd 723185029Spjd b3 = u8_combining_class_b3_tbl[uv][b2][b3]; 724185029Spjd if (b3 == U8_TBL_ELEMENT_NOT_DEF) 725185029Spjd return (0); 726185029Spjd 727185029Spjd return (u8_combining_class_b4_tbl[uv][b3][b4]); 728185029Spjd} 729185029Spjd 730185029Spjd/* 731185029Spjd * The do_decomp() function finds out a matching decomposition if any 732185029Spjd * and return. If there is no match, the input bytes are copied and returned. 733185029Spjd * The function also checks if there is a Hangul, decomposes it if necessary 734185029Spjd * and returns. 735185029Spjd * 736185029Spjd * To save time, a single byte 7-bit ASCII character should be handled by 737185029Spjd * the caller. 738185029Spjd * 739185029Spjd * The function returns the number of bytes returned sans always terminating 740185029Spjd * the null byte. It will also return a state that will tell if there was 741185029Spjd * a Hangul character decomposed which then will be used by the caller. 742185029Spjd */ 743185029Spjdstatic size_t 744185029Spjddo_decomp(size_t uv, uchar_t *u8s, uchar_t *s, int sz, 745185029Spjd boolean_t canonical_decomposition, u8_normalization_states_t *state) 746185029Spjd{ 747185029Spjd uint16_t b1 = 0; 748185029Spjd uint16_t b2 = 0; 749185029Spjd uint16_t b3 = 0; 750185029Spjd uint16_t b3_tbl; 751185029Spjd uint16_t b3_base; 752185029Spjd uint16_t b4 = 0; 753185029Spjd size_t start_id; 754185029Spjd size_t end_id; 755185029Spjd size_t i; 756185029Spjd uint32_t u1; 757185029Spjd 758185029Spjd if (sz == 2) { 759185029Spjd b3 = u8s[0] = s[0]; 760185029Spjd b4 = u8s[1] = s[1]; 761185029Spjd u8s[2] = '\0'; 762185029Spjd } else if (sz == 3) { 763185029Spjd /* Convert it to a Unicode scalar value. */ 764185029Spjd U8_PUT_3BYTES_INTO_UTF32(u1, s[0], s[1], s[2]); 765185029Spjd 766185029Spjd /* 767185029Spjd * If this is a Hangul syllable, we decompose it into 768185029Spjd * a leading consonant, a vowel, and an optional trailing 769185029Spjd * consonant and then return. 770185029Spjd */ 771185029Spjd if (U8_HANGUL_SYLLABLE(u1)) { 772185029Spjd u1 -= U8_HANGUL_SYL_FIRST; 773185029Spjd 774185029Spjd b1 = U8_HANGUL_JAMO_L_FIRST + u1 / U8_HANGUL_VT_COUNT; 775185029Spjd b2 = U8_HANGUL_JAMO_V_FIRST + (u1 % U8_HANGUL_VT_COUNT) 776185029Spjd / U8_HANGUL_T_COUNT; 777185029Spjd b3 = u1 % U8_HANGUL_T_COUNT; 778185029Spjd 779185029Spjd U8_SAVE_HANGUL_AS_UTF8(u8s, 0, 1, 2, b1); 780185029Spjd U8_SAVE_HANGUL_AS_UTF8(u8s, 3, 4, 5, b2); 781185029Spjd if (b3) { 782185029Spjd b3 += U8_HANGUL_JAMO_T_FIRST; 783185029Spjd U8_SAVE_HANGUL_AS_UTF8(u8s, 6, 7, 8, b3); 784185029Spjd 785185029Spjd u8s[9] = '\0'; 786185029Spjd *state = U8_STATE_HANGUL_LVT; 787185029Spjd return (9); 788185029Spjd } 789185029Spjd 790185029Spjd u8s[6] = '\0'; 791185029Spjd *state = U8_STATE_HANGUL_LV; 792185029Spjd return (6); 793185029Spjd } 794185029Spjd 795185029Spjd b2 = u8s[0] = s[0]; 796185029Spjd b3 = u8s[1] = s[1]; 797185029Spjd b4 = u8s[2] = s[2]; 798185029Spjd u8s[3] = '\0'; 799185029Spjd 800185029Spjd /* 801185029Spjd * If this is a Hangul Jamo, we know there is nothing 802185029Spjd * further that we can decompose. 803185029Spjd */ 804185029Spjd if (U8_HANGUL_JAMO_L(u1)) { 805185029Spjd *state = U8_STATE_HANGUL_L; 806185029Spjd return (3); 807185029Spjd } 808185029Spjd 809185029Spjd if (U8_HANGUL_JAMO_V(u1)) { 810185029Spjd if (*state == U8_STATE_HANGUL_L) 811185029Spjd *state = U8_STATE_HANGUL_LV; 812185029Spjd else 813185029Spjd *state = U8_STATE_HANGUL_V; 814185029Spjd return (3); 815185029Spjd } 816185029Spjd 817185029Spjd if (U8_HANGUL_JAMO_T(u1)) { 818185029Spjd if (*state == U8_STATE_HANGUL_LV) 819185029Spjd *state = U8_STATE_HANGUL_LVT; 820185029Spjd else 821185029Spjd *state = U8_STATE_HANGUL_T; 822185029Spjd return (3); 823185029Spjd } 824185029Spjd } else if (sz == 4) { 825185029Spjd b1 = u8s[0] = s[0]; 826185029Spjd b2 = u8s[1] = s[1]; 827185029Spjd b3 = u8s[2] = s[2]; 828185029Spjd b4 = u8s[3] = s[3]; 829185029Spjd u8s[4] = '\0'; 830185029Spjd } else { 831185029Spjd /* 832185029Spjd * This is a fallback and should not happen if the function 833185029Spjd * was called properly. 834185029Spjd */ 835185029Spjd u8s[0] = s[0]; 836185029Spjd u8s[1] = '\0'; 837185029Spjd *state = U8_STATE_START; 838185029Spjd return (1); 839185029Spjd } 840185029Spjd 841185029Spjd /* 842185029Spjd * At this point, this rountine does not know what it would get. 843185029Spjd * The caller should sort it out if the state isn't a Hangul one. 844185029Spjd */ 845185029Spjd *state = U8_STATE_START; 846185029Spjd 847185029Spjd /* Try to find matching decomposition mapping byte sequence. */ 848185029Spjd b1 = u8_common_b1_tbl[uv][b1]; 849185029Spjd if (b1 == U8_TBL_ELEMENT_NOT_DEF) 850185029Spjd return ((size_t)sz); 851185029Spjd 852185029Spjd b2 = u8_decomp_b2_tbl[uv][b1][b2]; 853185029Spjd if (b2 == U8_TBL_ELEMENT_NOT_DEF) 854185029Spjd return ((size_t)sz); 855185029Spjd 856185029Spjd b3_tbl = u8_decomp_b3_tbl[uv][b2][b3].tbl_id; 857185029Spjd if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 858185029Spjd return ((size_t)sz); 859185029Spjd 860185029Spjd /* 861185029Spjd * If b3_tbl is bigger than or equal to U8_16BIT_TABLE_INDICATOR 862185029Spjd * which is 0x8000, this means we couldn't fit the mappings into 863185029Spjd * the cardinality of a unsigned byte. 864185029Spjd */ 865185029Spjd if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) { 866185029Spjd b3_tbl -= U8_16BIT_TABLE_INDICATOR; 867185029Spjd start_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4]; 868185029Spjd end_id = u8_decomp_b4_16bit_tbl[uv][b3_tbl][b4 + 1]; 869185029Spjd } else { 870185029Spjd start_id = u8_decomp_b4_tbl[uv][b3_tbl][b4]; 871185029Spjd end_id = u8_decomp_b4_tbl[uv][b3_tbl][b4 + 1]; 872185029Spjd } 873185029Spjd 874185029Spjd /* This also means there wasn't any matching decomposition. */ 875185029Spjd if (start_id >= end_id) 876185029Spjd return ((size_t)sz); 877185029Spjd 878185029Spjd /* 879185029Spjd * The final table for decomposition mappings has three types of 880185029Spjd * byte sequences depending on whether a mapping is for compatibility 881185029Spjd * decomposition, canonical decomposition, or both like the following: 882185029Spjd * 883185029Spjd * (1) Compatibility decomposition mappings: 884185029Spjd * 885185029Spjd * +---+---+-...-+---+ 886185029Spjd * | B0| B1| ... | Bm| 887185029Spjd * +---+---+-...-+---+ 888185029Spjd * 889185029Spjd * The first byte, B0, is always less then 0xF5 (U8_DECOMP_BOTH). 890185029Spjd * 891185029Spjd * (2) Canonical decomposition mappings: 892185029Spjd * 893185029Spjd * +---+---+---+-...-+---+ 894185029Spjd * | T | b0| b1| ... | bn| 895185029Spjd * +---+---+---+-...-+---+ 896185029Spjd * 897185029Spjd * where the first byte, T, is 0xF6 (U8_DECOMP_CANONICAL). 898185029Spjd * 899185029Spjd * (3) Both mappings: 900185029Spjd * 901185029Spjd * +---+---+---+---+-...-+---+---+---+-...-+---+ 902185029Spjd * | T | D | b0| b1| ... | bn| B0| B1| ... | Bm| 903185029Spjd * +---+---+---+---+-...-+---+---+---+-...-+---+ 904185029Spjd * 905185029Spjd * where T is 0xF5 (U8_DECOMP_BOTH) and D is a displacement 906185029Spjd * byte, b0 to bn are canonical mapping bytes and B0 to Bm are 907185029Spjd * compatibility mapping bytes. 908185029Spjd * 909185029Spjd * Note that compatibility decomposition means doing recursive 910185029Spjd * decompositions using both compatibility decomposition mappings and 911185029Spjd * canonical decomposition mappings. On the other hand, canonical 912185029Spjd * decomposition means doing recursive decompositions using only 913185029Spjd * canonical decomposition mappings. Since the table we have has gone 914185029Spjd * through the recursions already, we do not need to do so during 915185029Spjd * runtime, i.e., the table has been completely flattened out 916185029Spjd * already. 917185029Spjd */ 918185029Spjd 919185029Spjd b3_base = u8_decomp_b3_tbl[uv][b2][b3].base; 920185029Spjd 921185029Spjd /* Get the type, T, of the byte sequence. */ 922185029Spjd b1 = u8_decomp_final_tbl[uv][b3_base + start_id]; 923185029Spjd 924185029Spjd /* 925185029Spjd * If necessary, adjust start_id, end_id, or both. Note that if 926185029Spjd * this is compatibility decomposition mapping, there is no 927185029Spjd * adjustment. 928185029Spjd */ 929185029Spjd if (canonical_decomposition) { 930185029Spjd /* Is the mapping only for compatibility decomposition? */ 931185029Spjd if (b1 < U8_DECOMP_BOTH) 932185029Spjd return ((size_t)sz); 933185029Spjd 934185029Spjd start_id++; 935185029Spjd 936185029Spjd if (b1 == U8_DECOMP_BOTH) { 937185029Spjd end_id = start_id + 938185029Spjd u8_decomp_final_tbl[uv][b3_base + start_id]; 939185029Spjd start_id++; 940185029Spjd } 941185029Spjd } else { 942185029Spjd /* 943185029Spjd * Unless this is a compatibility decomposition mapping, 944185029Spjd * we adjust the start_id. 945185029Spjd */ 946185029Spjd if (b1 == U8_DECOMP_BOTH) { 947185029Spjd start_id++; 948185029Spjd start_id += u8_decomp_final_tbl[uv][b3_base + start_id]; 949185029Spjd } else if (b1 == U8_DECOMP_CANONICAL) { 950185029Spjd start_id++; 951185029Spjd } 952185029Spjd } 953185029Spjd 954185029Spjd for (i = 0; start_id < end_id; start_id++) 955185029Spjd u8s[i++] = u8_decomp_final_tbl[uv][b3_base + start_id]; 956185029Spjd u8s[i] = '\0'; 957185029Spjd 958185029Spjd return (i); 959185029Spjd} 960185029Spjd 961185029Spjd/* 962185029Spjd * The find_composition_start() function uses the character bytes given and 963185029Spjd * find out the matching composition mappings if any and return the address 964185029Spjd * to the composition mappings as explained in the do_composition(). 965185029Spjd */ 966185029Spjdstatic uchar_t * 967185029Spjdfind_composition_start(size_t uv, uchar_t *s, size_t sz) 968185029Spjd{ 969185029Spjd uint16_t b1 = 0; 970185029Spjd uint16_t b2 = 0; 971185029Spjd uint16_t b3 = 0; 972185029Spjd uint16_t b3_tbl; 973185029Spjd uint16_t b3_base; 974185029Spjd uint16_t b4 = 0; 975185029Spjd size_t start_id; 976185029Spjd size_t end_id; 977185029Spjd 978185029Spjd if (sz == 1) { 979185029Spjd b4 = s[0]; 980185029Spjd } else if (sz == 2) { 981185029Spjd b3 = s[0]; 982185029Spjd b4 = s[1]; 983185029Spjd } else if (sz == 3) { 984185029Spjd b2 = s[0]; 985185029Spjd b3 = s[1]; 986185029Spjd b4 = s[2]; 987185029Spjd } else if (sz == 4) { 988185029Spjd b1 = s[0]; 989185029Spjd b2 = s[1]; 990185029Spjd b3 = s[2]; 991185029Spjd b4 = s[3]; 992185029Spjd } else { 993185029Spjd /* 994185029Spjd * This is a fallback and should not happen if the function 995185029Spjd * was called properly. 996185029Spjd */ 997185029Spjd return (NULL); 998185029Spjd } 999185029Spjd 1000185029Spjd b1 = u8_composition_b1_tbl[uv][b1]; 1001185029Spjd if (b1 == U8_TBL_ELEMENT_NOT_DEF) 1002185029Spjd return (NULL); 1003185029Spjd 1004185029Spjd b2 = u8_composition_b2_tbl[uv][b1][b2]; 1005185029Spjd if (b2 == U8_TBL_ELEMENT_NOT_DEF) 1006185029Spjd return (NULL); 1007185029Spjd 1008185029Spjd b3_tbl = u8_composition_b3_tbl[uv][b2][b3].tbl_id; 1009185029Spjd if (b3_tbl == U8_TBL_ELEMENT_NOT_DEF) 1010185029Spjd return (NULL); 1011185029Spjd 1012185029Spjd if (b3_tbl >= U8_16BIT_TABLE_INDICATOR) { 1013185029Spjd b3_tbl -= U8_16BIT_TABLE_INDICATOR; 1014185029Spjd start_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4]; 1015185029Spjd end_id = u8_composition_b4_16bit_tbl[uv][b3_tbl][b4 + 1]; 1016185029Spjd } else { 1017185029Spjd start_id = u8_composition_b4_tbl[uv][b3_tbl][b4]; 1018185029Spjd end_id = u8_composition_b4_tbl[uv][b3_tbl][b4 + 1]; 1019185029Spjd } 1020185029Spjd 1021185029Spjd if (start_id >= end_id) 1022185029Spjd return (NULL); 1023185029Spjd 1024185029Spjd b3_base = u8_composition_b3_tbl[uv][b2][b3].base; 1025185029Spjd 1026185029Spjd return ((uchar_t *)&(u8_composition_final_tbl[uv][b3_base + start_id])); 1027185029Spjd} 1028185029Spjd 1029185029Spjd/* 1030185029Spjd * The blocked() function checks on the combining class values of previous 1031185029Spjd * characters in this sequence and return whether it is blocked or not. 1032185029Spjd */ 1033185029Spjdstatic boolean_t 1034185029Spjdblocked(uchar_t *comb_class, size_t last) 1035185029Spjd{ 1036185029Spjd uchar_t my_comb_class; 1037185029Spjd size_t i; 1038185029Spjd 1039185029Spjd my_comb_class = comb_class[last]; 1040185029Spjd for (i = 1; i < last; i++) 1041185029Spjd if (comb_class[i] >= my_comb_class || 1042185029Spjd comb_class[i] == U8_COMBINING_CLASS_STARTER) 1043185029Spjd return (B_TRUE); 1044185029Spjd 1045185029Spjd return (B_FALSE); 1046185029Spjd} 1047185029Spjd 1048185029Spjd/* 1049185029Spjd * The do_composition() reads the character string pointed by 's' and 1050185029Spjd * do necessary canonical composition and then copy over the result back to 1051185029Spjd * the 's'. 1052185029Spjd * 1053185029Spjd * The input argument 's' cannot contain more than 32 characters. 1054185029Spjd */ 1055185029Spjdstatic size_t 1056185029Spjddo_composition(size_t uv, uchar_t *s, uchar_t *comb_class, uchar_t *start, 1057185029Spjd uchar_t *disp, size_t last, uchar_t **os, uchar_t *oslast) 1058185029Spjd{ 1059185029Spjd uchar_t t[U8_STREAM_SAFE_TEXT_MAX + 1]; 1060185029Spjd uchar_t tc[U8_MB_CUR_MAX]; 1061185029Spjd uint8_t saved_marks[U8_MAX_CHARS_A_SEQ]; 1062185029Spjd size_t saved_marks_count; 1063185029Spjd uchar_t *p; 1064185029Spjd uchar_t *saved_p; 1065185029Spjd uchar_t *q; 1066185029Spjd size_t i; 1067185029Spjd size_t saved_i; 1068185029Spjd size_t j; 1069185029Spjd size_t k; 1070185029Spjd size_t l; 1071185029Spjd size_t C; 1072185029Spjd size_t saved_l; 1073185029Spjd size_t size; 1074185029Spjd uint32_t u1; 1075185029Spjd uint32_t u2; 1076185029Spjd boolean_t match_not_found = B_TRUE; 1077185029Spjd 1078185029Spjd /* 1079185029Spjd * This should never happen unless the callers are doing some strange 1080185029Spjd * and unexpected things. 1081185029Spjd * 1082185029Spjd * The "last" is the index pointing to the last character not last + 1. 1083185029Spjd */ 1084185029Spjd if (last >= U8_MAX_CHARS_A_SEQ) 1085185029Spjd last = U8_UPPER_LIMIT_IN_A_SEQ; 1086185029Spjd 1087185029Spjd for (i = l = 0; i <= last; i++) { 1088185029Spjd /* 1089185029Spjd * The last or any non-Starters at the beginning, we don't 1090185029Spjd * have any chance to do composition and so we just copy them 1091185029Spjd * to the temporary buffer. 1092185029Spjd */ 1093185029Spjd if (i >= last || comb_class[i] != U8_COMBINING_CLASS_STARTER) { 1094185029SpjdSAVE_THE_CHAR: 1095185029Spjd p = s + start[i]; 1096185029Spjd size = disp[i]; 1097185029Spjd for (k = 0; k < size; k++) 1098185029Spjd t[l++] = *p++; 1099185029Spjd continue; 1100185029Spjd } 1101185029Spjd 1102185029Spjd /* 1103185029Spjd * If this could be a start of Hangul Jamos, then, we try to 1104185029Spjd * conjoin them. 1105185029Spjd */ 1106185029Spjd if (s[start[i]] == U8_HANGUL_JAMO_1ST_BYTE) { 1107185029Spjd U8_PUT_3BYTES_INTO_UTF32(u1, s[start[i]], 1108185029Spjd s[start[i] + 1], s[start[i] + 2]); 1109185029Spjd U8_PUT_3BYTES_INTO_UTF32(u2, s[start[i] + 3], 1110185029Spjd s[start[i] + 4], s[start[i] + 5]); 1111185029Spjd 1112185029Spjd if (U8_HANGUL_JAMO_L(u1) && U8_HANGUL_JAMO_V(u2)) { 1113185029Spjd u1 -= U8_HANGUL_JAMO_L_FIRST; 1114185029Spjd u2 -= U8_HANGUL_JAMO_V_FIRST; 1115185029Spjd u1 = U8_HANGUL_SYL_FIRST + 1116185029Spjd (u1 * U8_HANGUL_V_COUNT + u2) * 1117185029Spjd U8_HANGUL_T_COUNT; 1118185029Spjd 1119185029Spjd i += 2; 1120185029Spjd if (i <= last) { 1121185029Spjd U8_PUT_3BYTES_INTO_UTF32(u2, 1122185029Spjd s[start[i]], s[start[i] + 1], 1123185029Spjd s[start[i] + 2]); 1124185029Spjd 1125185029Spjd if (U8_HANGUL_JAMO_T(u2)) { 1126185029Spjd u1 += u2 - 1127185029Spjd U8_HANGUL_JAMO_T_FIRST; 1128185029Spjd i++; 1129185029Spjd } 1130185029Spjd } 1131185029Spjd 1132185029Spjd U8_SAVE_HANGUL_AS_UTF8(t + l, 0, 1, 2, u1); 1133185029Spjd i--; 1134185029Spjd l += 3; 1135185029Spjd continue; 1136185029Spjd } 1137185029Spjd } 1138185029Spjd 1139185029Spjd /* 1140185029Spjd * Let's then find out if this Starter has composition 1141185029Spjd * mapping. 1142185029Spjd */ 1143185029Spjd p = find_composition_start(uv, s + start[i], disp[i]); 1144185029Spjd if (p == NULL) 1145185029Spjd goto SAVE_THE_CHAR; 1146185029Spjd 1147185029Spjd /* 1148185029Spjd * We have a Starter with composition mapping and the next 1149185029Spjd * character is a non-Starter. Let's try to find out if 1150185029Spjd * we can do composition. 1151185029Spjd */ 1152185029Spjd 1153185029Spjd saved_p = p; 1154185029Spjd saved_i = i; 1155185029Spjd saved_l = l; 1156185029Spjd saved_marks_count = 0; 1157185029Spjd 1158185029SpjdTRY_THE_NEXT_MARK: 1159185029Spjd q = s + start[++i]; 1160185029Spjd size = disp[i]; 1161185029Spjd 1162185029Spjd /* 1163185029Spjd * The next for() loop compares the non-Starter pointed by 1164185029Spjd * 'q' with the possible (joinable) characters pointed by 'p'. 1165185029Spjd * 1166185029Spjd * The composition final table entry pointed by the 'p' 1167185029Spjd * looks like the following: 1168185029Spjd * 1169185029Spjd * +---+---+---+-...-+---+---+---+---+-...-+---+---+ 1170185029Spjd * | C | b0| b2| ... | bn| F | B0| B1| ... | Bm| F | 1171185029Spjd * +---+---+---+-...-+---+---+---+---+-...-+---+---+ 1172185029Spjd * 1173185029Spjd * where C is the count byte indicating the number of 1174185029Spjd * mapping pairs where each pair would be look like 1175185029Spjd * (b0-bn F, B0-Bm F). The b0-bn are the bytes of the second 1176185029Spjd * character of a canonical decomposition and the B0-Bm are 1177185029Spjd * the bytes of a matching composite character. The F is 1178185029Spjd * a filler byte after each character as the separator. 1179185029Spjd */ 1180185029Spjd 1181185029Spjd match_not_found = B_TRUE; 1182185029Spjd 1183185029Spjd for (C = *p++; C > 0; C--) { 1184185029Spjd for (k = 0; k < size; p++, k++) 1185185029Spjd if (*p != q[k]) 1186185029Spjd break; 1187185029Spjd 1188185029Spjd /* Have we found it? */ 1189185029Spjd if (k >= size && *p == U8_TBL_ELEMENT_FILLER) { 1190185029Spjd match_not_found = B_FALSE; 1191185029Spjd 1192185029Spjd l = saved_l; 1193185029Spjd 1194185029Spjd while (*++p != U8_TBL_ELEMENT_FILLER) 1195185029Spjd t[l++] = *p; 1196185029Spjd 1197185029Spjd break; 1198185029Spjd } 1199185029Spjd 1200185029Spjd /* We didn't find; skip to the next pair. */ 1201185029Spjd if (*p != U8_TBL_ELEMENT_FILLER) 1202185029Spjd while (*++p != U8_TBL_ELEMENT_FILLER) 1203185029Spjd ; 1204185029Spjd while (*++p != U8_TBL_ELEMENT_FILLER) 1205185029Spjd ; 1206185029Spjd p++; 1207185029Spjd } 1208185029Spjd 1209185029Spjd /* 1210185029Spjd * If there was no match, we will need to save the combining 1211185029Spjd * mark for later appending. After that, if the next one 1212185029Spjd * is a non-Starter and not blocked, then, we try once 1213185029Spjd * again to do composition with the next non-Starter. 1214185029Spjd * 1215185029Spjd * If there was no match and this was a Starter, then, 1216185029Spjd * this is a new start. 1217185029Spjd * 1218185029Spjd * If there was a match and a composition done and we have 1219185029Spjd * more to check on, then, we retrieve a new composition final 1220185029Spjd * table entry for the composite and then try to do the 1221185029Spjd * composition again. 1222185029Spjd */ 1223185029Spjd 1224185029Spjd if (match_not_found) { 1225185029Spjd if (comb_class[i] == U8_COMBINING_CLASS_STARTER) { 1226185029Spjd i--; 1227185029Spjd goto SAVE_THE_CHAR; 1228185029Spjd } 1229185029Spjd 1230185029Spjd saved_marks[saved_marks_count++] = i; 1231185029Spjd } 1232185029Spjd 1233185029Spjd if (saved_l == l) { 1234185029Spjd while (i < last) { 1235185029Spjd if (blocked(comb_class, i + 1)) 1236185029Spjd saved_marks[saved_marks_count++] = ++i; 1237185029Spjd else 1238185029Spjd break; 1239185029Spjd } 1240185029Spjd if (i < last) { 1241185029Spjd p = saved_p; 1242185029Spjd goto TRY_THE_NEXT_MARK; 1243185029Spjd } 1244185029Spjd } else if (i < last) { 1245185029Spjd p = find_composition_start(uv, t + saved_l, 1246185029Spjd l - saved_l); 1247185029Spjd if (p != NULL) { 1248185029Spjd saved_p = p; 1249185029Spjd goto TRY_THE_NEXT_MARK; 1250185029Spjd } 1251185029Spjd } 1252185029Spjd 1253185029Spjd /* 1254185029Spjd * There is no more composition possible. 1255185029Spjd * 1256185029Spjd * If there was no composition what so ever then we copy 1257185029Spjd * over the original Starter and then append any non-Starters 1258185029Spjd * remaining at the target string sequentially after that. 1259185029Spjd */ 1260185029Spjd 1261185029Spjd if (saved_l == l) { 1262185029Spjd p = s + start[saved_i]; 1263185029Spjd size = disp[saved_i]; 1264185029Spjd for (j = 0; j < size; j++) 1265185029Spjd t[l++] = *p++; 1266185029Spjd } 1267185029Spjd 1268185029Spjd for (k = 0; k < saved_marks_count; k++) { 1269185029Spjd p = s + start[saved_marks[k]]; 1270185029Spjd size = disp[saved_marks[k]]; 1271185029Spjd for (j = 0; j < size; j++) 1272185029Spjd t[l++] = *p++; 1273185029Spjd } 1274185029Spjd } 1275185029Spjd 1276185029Spjd /* 1277185029Spjd * If the last character is a Starter and if we have a character 1278185029Spjd * (possibly another Starter) that can be turned into a composite, 1279185029Spjd * we do so and we do so until there is no more of composition 1280185029Spjd * possible. 1281185029Spjd */ 1282185029Spjd if (comb_class[last] == U8_COMBINING_CLASS_STARTER) { 1283185029Spjd p = *os; 1284185029Spjd saved_l = l - disp[last]; 1285185029Spjd 1286185029Spjd while (p < oslast) { 1287185029Spjd size = u8_number_of_bytes[*p]; 1288185029Spjd if (size <= 1 || (p + size) > oslast) 1289185029Spjd break; 1290185029Spjd 1291185029Spjd saved_p = p; 1292185029Spjd 1293185029Spjd for (i = 0; i < size; i++) 1294185029Spjd tc[i] = *p++; 1295185029Spjd 1296185029Spjd q = find_composition_start(uv, t + saved_l, 1297185029Spjd l - saved_l); 1298185029Spjd if (q == NULL) { 1299185029Spjd p = saved_p; 1300185029Spjd break; 1301185029Spjd } 1302185029Spjd 1303185029Spjd match_not_found = B_TRUE; 1304185029Spjd 1305185029Spjd for (C = *q++; C > 0; C--) { 1306185029Spjd for (k = 0; k < size; q++, k++) 1307185029Spjd if (*q != tc[k]) 1308185029Spjd break; 1309185029Spjd 1310185029Spjd if (k >= size && *q == U8_TBL_ELEMENT_FILLER) { 1311185029Spjd match_not_found = B_FALSE; 1312185029Spjd 1313185029Spjd l = saved_l; 1314185029Spjd 1315185029Spjd while (*++q != U8_TBL_ELEMENT_FILLER) { 1316185029Spjd /* 1317185029Spjd * This is practically 1318185029Spjd * impossible but we don't 1319185029Spjd * want to take any chances. 1320185029Spjd */ 1321185029Spjd if (l >= 1322185029Spjd U8_STREAM_SAFE_TEXT_MAX) { 1323185029Spjd p = saved_p; 1324185029Spjd goto SAFE_RETURN; 1325185029Spjd } 1326185029Spjd t[l++] = *q; 1327185029Spjd } 1328185029Spjd 1329185029Spjd break; 1330185029Spjd } 1331185029Spjd 1332185029Spjd if (*q != U8_TBL_ELEMENT_FILLER) 1333185029Spjd while (*++q != U8_TBL_ELEMENT_FILLER) 1334185029Spjd ; 1335185029Spjd while (*++q != U8_TBL_ELEMENT_FILLER) 1336185029Spjd ; 1337185029Spjd q++; 1338185029Spjd } 1339185029Spjd 1340185029Spjd if (match_not_found) { 1341185029Spjd p = saved_p; 1342185029Spjd break; 1343185029Spjd } 1344185029Spjd } 1345185029SpjdSAFE_RETURN: 1346185029Spjd *os = p; 1347185029Spjd } 1348185029Spjd 1349185029Spjd /* 1350185029Spjd * Now we copy over the temporary string to the target string. 1351185029Spjd * Since composition always reduces the number of characters or 1352185029Spjd * the number of characters stay, we don't need to worry about 1353185029Spjd * the buffer overflow here. 1354185029Spjd */ 1355185029Spjd for (i = 0; i < l; i++) 1356185029Spjd s[i] = t[i]; 1357185029Spjd s[l] = '\0'; 1358185029Spjd 1359185029Spjd return (l); 1360185029Spjd} 1361185029Spjd 1362185029Spjd/* 1363185029Spjd * The collect_a_seq() function checks on the given string s, collect 1364185029Spjd * a sequence of characters at u8s, and return the sequence. While it collects 1365185029Spjd * a sequence, it also applies case conversion, canonical or compatibility 1366185029Spjd * decomposition, canonical decomposition, or some or all of them and 1367185029Spjd * in that order. 1368185029Spjd * 1369185029Spjd * The collected sequence cannot be bigger than 32 characters since if 1370185029Spjd * it is having more than 31 characters, the sequence will be terminated 1371185029Spjd * with a U+034F COMBINING GRAPHEME JOINER (CGJ) character and turned into 1372185029Spjd * a Stream-Safe Text. The collected sequence is always terminated with 1373185029Spjd * a null byte and the return value is the byte length of the sequence 1374185029Spjd * including 0. The return value does not include the terminating 1375185029Spjd * null byte. 1376185029Spjd */ 1377185029Spjdstatic size_t 1378185029Spjdcollect_a_seq(size_t uv, uchar_t *u8s, uchar_t **source, uchar_t *slast, 1379185029Spjd boolean_t is_it_toupper, 1380185029Spjd boolean_t is_it_tolower, 1381185029Spjd boolean_t canonical_decomposition, 1382185029Spjd boolean_t compatibility_decomposition, 1383185029Spjd boolean_t canonical_composition, 1384185029Spjd int *errnum, u8_normalization_states_t *state) 1385185029Spjd{ 1386185029Spjd uchar_t *s; 1387185029Spjd int sz; 1388185029Spjd int saved_sz; 1389185029Spjd size_t i; 1390185029Spjd size_t j; 1391185029Spjd size_t k; 1392185029Spjd size_t l; 1393185029Spjd uchar_t comb_class[U8_MAX_CHARS_A_SEQ]; 1394185029Spjd uchar_t disp[U8_MAX_CHARS_A_SEQ]; 1395185029Spjd uchar_t start[U8_MAX_CHARS_A_SEQ]; 1396185029Spjd uchar_t u8t[U8_MB_CUR_MAX]; 1397185029Spjd uchar_t uts[U8_STREAM_SAFE_TEXT_MAX + 1]; 1398185029Spjd uchar_t tc; 1399185029Spjd size_t last; 1400185029Spjd size_t saved_last; 1401185029Spjd uint32_t u1; 1402185029Spjd 1403185029Spjd /* 1404185029Spjd * Save the source string pointer which we will return a changed 1405185029Spjd * pointer if we do processing. 1406185029Spjd */ 1407185029Spjd s = *source; 1408185029Spjd 1409185029Spjd /* 1410185029Spjd * The following is a fallback for just in case callers are not 1411185029Spjd * checking the string boundaries before the calling. 1412185029Spjd */ 1413185029Spjd if (s >= slast) { 1414185029Spjd u8s[0] = '\0'; 1415185029Spjd 1416185029Spjd return (0); 1417185029Spjd } 1418185029Spjd 1419185029Spjd /* 1420185029Spjd * As the first thing, let's collect a character and do case 1421185029Spjd * conversion if necessary. 1422185029Spjd */ 1423185029Spjd 1424185029Spjd sz = u8_number_of_bytes[*s]; 1425185029Spjd 1426185029Spjd if (sz < 0) { 1427185029Spjd *errnum = EILSEQ; 1428185029Spjd 1429185029Spjd u8s[0] = *s++; 1430185029Spjd u8s[1] = '\0'; 1431185029Spjd 1432185029Spjd *source = s; 1433185029Spjd 1434185029Spjd return (1); 1435185029Spjd } 1436185029Spjd 1437185029Spjd if (sz == 1) { 1438185029Spjd if (is_it_toupper) 1439185029Spjd u8s[0] = U8_ASCII_TOUPPER(*s); 1440185029Spjd else if (is_it_tolower) 1441185029Spjd u8s[0] = U8_ASCII_TOLOWER(*s); 1442185029Spjd else 1443185029Spjd u8s[0] = *s; 1444185029Spjd s++; 1445185029Spjd u8s[1] = '\0'; 1446185029Spjd } else if ((s + sz) > slast) { 1447185029Spjd *errnum = EINVAL; 1448185029Spjd 1449185029Spjd for (i = 0; s < slast; ) 1450185029Spjd u8s[i++] = *s++; 1451185029Spjd u8s[i] = '\0'; 1452185029Spjd 1453185029Spjd *source = s; 1454185029Spjd 1455185029Spjd return (i); 1456185029Spjd } else { 1457185029Spjd if (is_it_toupper || is_it_tolower) { 1458185029Spjd i = do_case_conv(uv, u8s, s, sz, is_it_toupper); 1459185029Spjd s += sz; 1460185029Spjd sz = i; 1461185029Spjd } else { 1462185029Spjd for (i = 0; i < sz; ) 1463185029Spjd u8s[i++] = *s++; 1464185029Spjd u8s[i] = '\0'; 1465185029Spjd } 1466185029Spjd } 1467185029Spjd 1468185029Spjd /* 1469185029Spjd * And then canonical/compatibility decomposition followed by 1470185029Spjd * an optional canonical composition. Please be noted that 1471185029Spjd * canonical composition is done only when a decomposition is 1472185029Spjd * done. 1473185029Spjd */ 1474185029Spjd if (canonical_decomposition || compatibility_decomposition) { 1475185029Spjd if (sz == 1) { 1476185029Spjd *state = U8_STATE_START; 1477185029Spjd 1478185029Spjd saved_sz = 1; 1479185029Spjd 1480185029Spjd comb_class[0] = 0; 1481185029Spjd start[0] = 0; 1482185029Spjd disp[0] = 1; 1483185029Spjd 1484185029Spjd last = 1; 1485185029Spjd } else { 1486185029Spjd saved_sz = do_decomp(uv, u8s, u8s, sz, 1487185029Spjd canonical_decomposition, state); 1488185029Spjd 1489185029Spjd last = 0; 1490185029Spjd 1491185029Spjd for (i = 0; i < saved_sz; ) { 1492185029Spjd sz = u8_number_of_bytes[u8s[i]]; 1493185029Spjd 1494185029Spjd comb_class[last] = combining_class(uv, 1495185029Spjd u8s + i, sz); 1496185029Spjd start[last] = i; 1497185029Spjd disp[last] = sz; 1498185029Spjd 1499185029Spjd last++; 1500185029Spjd i += sz; 1501185029Spjd } 1502185029Spjd 1503185029Spjd /* 1504185029Spjd * Decomposition yields various Hangul related 1505185029Spjd * states but not on combining marks. We need to 1506185029Spjd * find out at here by checking on the last 1507185029Spjd * character. 1508185029Spjd */ 1509185029Spjd if (*state == U8_STATE_START) { 1510185029Spjd if (comb_class[last - 1]) 1511185029Spjd *state = U8_STATE_COMBINING_MARK; 1512185029Spjd } 1513185029Spjd } 1514185029Spjd 1515185029Spjd saved_last = last; 1516185029Spjd 1517185029Spjd while (s < slast) { 1518185029Spjd sz = u8_number_of_bytes[*s]; 1519185029Spjd 1520185029Spjd /* 1521185029Spjd * If this is an illegal character, an incomplete 1522185029Spjd * character, or an 7-bit ASCII Starter character, 1523185029Spjd * then we have collected a sequence; break and let 1524185029Spjd * the next call deal with the two cases. 1525185029Spjd * 1526185029Spjd * Note that this is okay only if you are using this 1527185029Spjd * function with a fixed length string, not on 1528185029Spjd * a buffer with multiple calls of one chunk at a time. 1529185029Spjd */ 1530185029Spjd if (sz <= 1) { 1531185029Spjd break; 1532185029Spjd } else if ((s + sz) > slast) { 1533185029Spjd break; 1534185029Spjd } else { 1535185029Spjd /* 1536185029Spjd * If the previous character was a Hangul Jamo 1537185029Spjd * and this character is a Hangul Jamo that 1538185029Spjd * can be conjoined, we collect the Jamo. 1539185029Spjd */ 1540185029Spjd if (*s == U8_HANGUL_JAMO_1ST_BYTE) { 1541185029Spjd U8_PUT_3BYTES_INTO_UTF32(u1, 1542185029Spjd *s, *(s + 1), *(s + 2)); 1543185029Spjd 1544185029Spjd if (U8_HANGUL_COMPOSABLE_L_V(*state, 1545185029Spjd u1)) { 1546185029Spjd i = 0; 1547185029Spjd *state = U8_STATE_HANGUL_LV; 1548185029Spjd goto COLLECT_A_HANGUL; 1549185029Spjd } 1550185029Spjd 1551185029Spjd if (U8_HANGUL_COMPOSABLE_LV_T(*state, 1552185029Spjd u1)) { 1553185029Spjd i = 0; 1554185029Spjd *state = U8_STATE_HANGUL_LVT; 1555185029Spjd goto COLLECT_A_HANGUL; 1556185029Spjd } 1557185029Spjd } 1558185029Spjd 1559185029Spjd /* 1560185029Spjd * Regardless of whatever it was, if this is 1561185029Spjd * a Starter, we don't collect the character 1562185029Spjd * since that's a new start and we will deal 1563185029Spjd * with it at the next time. 1564185029Spjd */ 1565185029Spjd i = combining_class(uv, s, sz); 1566185029Spjd if (i == U8_COMBINING_CLASS_STARTER) 1567185029Spjd break; 1568185029Spjd 1569185029Spjd /* 1570185029Spjd * We know the current character is a combining 1571185029Spjd * mark. If the previous character wasn't 1572185029Spjd * a Starter (not Hangul) or a combining mark, 1573185029Spjd * then, we don't collect this combining mark. 1574185029Spjd */ 1575185029Spjd if (*state != U8_STATE_START && 1576185029Spjd *state != U8_STATE_COMBINING_MARK) 1577185029Spjd break; 1578185029Spjd 1579185029Spjd *state = U8_STATE_COMBINING_MARK; 1580185029SpjdCOLLECT_A_HANGUL: 1581185029Spjd /* 1582185029Spjd * If we collected a Starter and combining 1583185029Spjd * marks up to 30, i.e., total 31 characters, 1584185029Spjd * then, we terminate this degenerately long 1585185029Spjd * combining sequence with a U+034F COMBINING 1586185029Spjd * GRAPHEME JOINER (CGJ) which is 0xCD 0x8F in 1587185029Spjd * UTF-8 and turn this into a Stream-Safe 1588185029Spjd * Text. This will be extremely rare but 1589185029Spjd * possible. 1590185029Spjd * 1591185029Spjd * The following will also guarantee that 1592185029Spjd * we are not writing more than 32 characters 1593185029Spjd * plus a NULL at u8s[]. 1594185029Spjd */ 1595185029Spjd if (last >= U8_UPPER_LIMIT_IN_A_SEQ) { 1596185029SpjdTURN_STREAM_SAFE: 1597185029Spjd *state = U8_STATE_START; 1598185029Spjd comb_class[last] = 0; 1599185029Spjd start[last] = saved_sz; 1600185029Spjd disp[last] = 2; 1601185029Spjd last++; 1602185029Spjd 1603185029Spjd u8s[saved_sz++] = 0xCD; 1604185029Spjd u8s[saved_sz++] = 0x8F; 1605185029Spjd 1606185029Spjd break; 1607185029Spjd } 1608185029Spjd 1609185029Spjd /* 1610185029Spjd * Some combining marks also do decompose into 1611185029Spjd * another combining mark or marks. 1612185029Spjd */ 1613185029Spjd if (*state == U8_STATE_COMBINING_MARK) { 1614185029Spjd k = last; 1615185029Spjd l = sz; 1616185029Spjd i = do_decomp(uv, uts, s, sz, 1617185029Spjd canonical_decomposition, state); 1618185029Spjd for (j = 0; j < i; ) { 1619185029Spjd sz = u8_number_of_bytes[uts[j]]; 1620185029Spjd 1621185029Spjd comb_class[last] = 1622185029Spjd combining_class(uv, 1623185029Spjd uts + j, sz); 1624185029Spjd start[last] = saved_sz + j; 1625185029Spjd disp[last] = sz; 1626185029Spjd 1627185029Spjd last++; 1628185029Spjd if (last >= 1629185029Spjd U8_UPPER_LIMIT_IN_A_SEQ) { 1630185029Spjd last = k; 1631185029Spjd goto TURN_STREAM_SAFE; 1632185029Spjd } 1633185029Spjd j += sz; 1634185029Spjd } 1635185029Spjd 1636185029Spjd *state = U8_STATE_COMBINING_MARK; 1637185029Spjd sz = i; 1638185029Spjd s += l; 1639185029Spjd 1640185029Spjd for (i = 0; i < sz; i++) 1641185029Spjd u8s[saved_sz++] = uts[i]; 1642185029Spjd } else { 1643185029Spjd comb_class[last] = i; 1644185029Spjd start[last] = saved_sz; 1645185029Spjd disp[last] = sz; 1646185029Spjd last++; 1647185029Spjd 1648185029Spjd for (i = 0; i < sz; i++) 1649185029Spjd u8s[saved_sz++] = *s++; 1650185029Spjd } 1651185029Spjd 1652185029Spjd /* 1653185029Spjd * If this is U+0345 COMBINING GREEK 1654185029Spjd * YPOGEGRAMMENI (0xCD 0x85 in UTF-8), a.k.a., 1655185029Spjd * iota subscript, and need to be converted to 1656185029Spjd * uppercase letter, convert it to U+0399 GREEK 1657185029Spjd * CAPITAL LETTER IOTA (0xCE 0x99 in UTF-8), 1658185029Spjd * i.e., convert to capital adscript form as 1659185029Spjd * specified in the Unicode standard. 1660185029Spjd * 1661185029Spjd * This is the only special case of (ambiguous) 1662185029Spjd * case conversion at combining marks and 1663185029Spjd * probably the standard will never have 1664185029Spjd * anything similar like this in future. 1665185029Spjd */ 1666185029Spjd if (is_it_toupper && sz >= 2 && 1667185029Spjd u8s[saved_sz - 2] == 0xCD && 1668185029Spjd u8s[saved_sz - 1] == 0x85) { 1669185029Spjd u8s[saved_sz - 2] = 0xCE; 1670185029Spjd u8s[saved_sz - 1] = 0x99; 1671185029Spjd } 1672185029Spjd } 1673185029Spjd } 1674185029Spjd 1675185029Spjd /* 1676185029Spjd * Let's try to ensure a canonical ordering for the collected 1677185029Spjd * combining marks. We do this only if we have collected 1678185029Spjd * at least one more non-Starter. (The decomposition mapping 1679185029Spjd * data tables have fully (and recursively) expanded and 1680185029Spjd * canonically ordered decompositions.) 1681185029Spjd * 1682185029Spjd * The U8_SWAP_COMB_MARKS() convenience macro has some 1683185029Spjd * assumptions and we are meeting the assumptions. 1684185029Spjd */ 1685185029Spjd last--; 1686185029Spjd if (last >= saved_last) { 1687185029Spjd for (i = 0; i < last; i++) 1688185029Spjd for (j = last; j > i; j--) 1689185029Spjd if (comb_class[j] && 1690185029Spjd comb_class[j - 1] > comb_class[j]) { 1691185029Spjd U8_SWAP_COMB_MARKS(j - 1, j); 1692185029Spjd } 1693185029Spjd } 1694185029Spjd 1695185029Spjd *source = s; 1696185029Spjd 1697185029Spjd if (! canonical_composition) { 1698185029Spjd u8s[saved_sz] = '\0'; 1699185029Spjd return (saved_sz); 1700185029Spjd } 1701185029Spjd 1702185029Spjd /* 1703185029Spjd * Now do the canonical composition. Note that we do this 1704185029Spjd * only after a canonical or compatibility decomposition to 1705185029Spjd * finish up NFC or NFKC. 1706185029Spjd */ 1707185029Spjd sz = do_composition(uv, u8s, comb_class, start, disp, last, 1708185029Spjd &s, slast); 1709185029Spjd } 1710185029Spjd 1711185029Spjd *source = s; 1712185029Spjd 1713185029Spjd return ((size_t)sz); 1714185029Spjd} 1715185029Spjd 1716185029Spjd/* 1717185029Spjd * The do_norm_compare() function does string comparion based on Unicode 1718185029Spjd * simple case mappings and Unicode Normalization definitions. 1719185029Spjd * 1720185029Spjd * It does so by collecting a sequence of character at a time and comparing 1721185029Spjd * the collected sequences from the strings. 1722185029Spjd * 1723185029Spjd * The meanings on the return values are the same as the usual strcmp(). 1724185029Spjd */ 1725185029Spjdstatic int 1726185029Spjddo_norm_compare(size_t uv, uchar_t *s1, uchar_t *s2, size_t n1, size_t n2, 1727185029Spjd int flag, int *errnum) 1728185029Spjd{ 1729185029Spjd int result; 1730185029Spjd size_t sz1; 1731185029Spjd size_t sz2; 1732185029Spjd uchar_t u8s1[U8_STREAM_SAFE_TEXT_MAX + 1]; 1733185029Spjd uchar_t u8s2[U8_STREAM_SAFE_TEXT_MAX + 1]; 1734185029Spjd uchar_t *s1last; 1735185029Spjd uchar_t *s2last; 1736185029Spjd boolean_t is_it_toupper; 1737185029Spjd boolean_t is_it_tolower; 1738185029Spjd boolean_t canonical_decomposition; 1739185029Spjd boolean_t compatibility_decomposition; 1740185029Spjd boolean_t canonical_composition; 1741185029Spjd u8_normalization_states_t state; 1742185029Spjd 1743185029Spjd s1last = s1 + n1; 1744185029Spjd s2last = s2 + n2; 1745185029Spjd 1746185029Spjd is_it_toupper = flag & U8_TEXTPREP_TOUPPER; 1747185029Spjd is_it_tolower = flag & U8_TEXTPREP_TOLOWER; 1748185029Spjd canonical_decomposition = flag & U8_CANON_DECOMP; 1749185029Spjd compatibility_decomposition = flag & U8_COMPAT_DECOMP; 1750185029Spjd canonical_composition = flag & U8_CANON_COMP; 1751185029Spjd 1752185029Spjd while (s1 < s1last && s2 < s2last) { 1753185029Spjd /* 1754185029Spjd * If the current character is a 7-bit ASCII and the last 1755185029Spjd * character, or, if the current character and the next 1756185029Spjd * character are both some 7-bit ASCII characters then 1757185029Spjd * we treat the current character as a sequence. 1758185029Spjd * 1759185029Spjd * In any other cases, we need to call collect_a_seq(). 1760185029Spjd */ 1761185029Spjd 1762185029Spjd if (U8_ISASCII(*s1) && ((s1 + 1) >= s1last || 1763185029Spjd ((s1 + 1) < s1last && U8_ISASCII(*(s1 + 1))))) { 1764185029Spjd if (is_it_toupper) 1765185029Spjd u8s1[0] = U8_ASCII_TOUPPER(*s1); 1766185029Spjd else if (is_it_tolower) 1767185029Spjd u8s1[0] = U8_ASCII_TOLOWER(*s1); 1768185029Spjd else 1769185029Spjd u8s1[0] = *s1; 1770185029Spjd u8s1[1] = '\0'; 1771185029Spjd sz1 = 1; 1772185029Spjd s1++; 1773185029Spjd } else { 1774185029Spjd state = U8_STATE_START; 1775185029Spjd sz1 = collect_a_seq(uv, u8s1, &s1, s1last, 1776185029Spjd is_it_toupper, is_it_tolower, 1777185029Spjd canonical_decomposition, 1778185029Spjd compatibility_decomposition, 1779185029Spjd canonical_composition, errnum, &state); 1780185029Spjd } 1781185029Spjd 1782185029Spjd if (U8_ISASCII(*s2) && ((s2 + 1) >= s2last || 1783185029Spjd ((s2 + 1) < s2last && U8_ISASCII(*(s2 + 1))))) { 1784185029Spjd if (is_it_toupper) 1785185029Spjd u8s2[0] = U8_ASCII_TOUPPER(*s2); 1786185029Spjd else if (is_it_tolower) 1787185029Spjd u8s2[0] = U8_ASCII_TOLOWER(*s2); 1788185029Spjd else 1789185029Spjd u8s2[0] = *s2; 1790185029Spjd u8s2[1] = '\0'; 1791185029Spjd sz2 = 1; 1792185029Spjd s2++; 1793185029Spjd } else { 1794185029Spjd state = U8_STATE_START; 1795185029Spjd sz2 = collect_a_seq(uv, u8s2, &s2, s2last, 1796185029Spjd is_it_toupper, is_it_tolower, 1797185029Spjd canonical_decomposition, 1798185029Spjd compatibility_decomposition, 1799185029Spjd canonical_composition, errnum, &state); 1800185029Spjd } 1801185029Spjd 1802185029Spjd /* 1803185029Spjd * Now compare the two characters. If they are the same, 1804185029Spjd * we move on to the next character sequences. 1805185029Spjd */ 1806185029Spjd if (sz1 == 1 && sz2 == 1) { 1807185029Spjd if (*u8s1 > *u8s2) 1808185029Spjd return (1); 1809185029Spjd if (*u8s1 < *u8s2) 1810185029Spjd return (-1); 1811185029Spjd } else { 1812185029Spjd result = strcmp((const char *)u8s1, (const char *)u8s2); 1813185029Spjd if (result != 0) 1814185029Spjd return (result); 1815185029Spjd } 1816185029Spjd } 1817185029Spjd 1818185029Spjd /* 1819185029Spjd * We compared until the end of either or both strings. 1820185029Spjd * 1821185029Spjd * If we reached to or went over the ends for the both, that means 1822185029Spjd * they are the same. 1823185029Spjd * 1824185029Spjd * If we reached only one end, that means the other string has 1825185029Spjd * something which then can be used to determine the return value. 1826185029Spjd */ 1827185029Spjd if (s1 >= s1last) { 1828185029Spjd if (s2 >= s2last) 1829185029Spjd return (0); 1830185029Spjd return (-1); 1831185029Spjd } 1832185029Spjd return (1); 1833185029Spjd} 1834185029Spjd 1835185029Spjd/* 1836185029Spjd * The u8_strcmp() function compares two UTF-8 strings quite similar to 1837185029Spjd * the strcmp(). For the comparison, however, Unicode Normalization specific 1838185029Spjd * equivalency and Unicode simple case conversion mappings based equivalency 1839185029Spjd * can be requested and checked against. 1840185029Spjd */ 1841185029Spjdint 1842185029Spjdu8_strcmp(const char *s1, const char *s2, size_t n, int flag, size_t uv, 1843185029Spjd int *errnum) 1844185029Spjd{ 1845185029Spjd int f; 1846185029Spjd size_t n1; 1847185029Spjd size_t n2; 1848185029Spjd 1849185029Spjd *errnum = 0; 1850185029Spjd 1851185029Spjd /* 1852185029Spjd * Check on the requested Unicode version, case conversion, and 1853185029Spjd * normalization flag values. 1854185029Spjd */ 1855185029Spjd 1856185029Spjd if (uv > U8_UNICODE_LATEST) { 1857185029Spjd *errnum = ERANGE; 1858185029Spjd uv = U8_UNICODE_LATEST; 1859185029Spjd } 1860185029Spjd 1861185029Spjd if (flag == 0) { 1862185029Spjd flag = U8_STRCMP_CS; 1863185029Spjd } else { 1864185029Spjd f = flag & (U8_STRCMP_CS | U8_STRCMP_CI_UPPER | 1865185029Spjd U8_STRCMP_CI_LOWER); 1866185029Spjd if (f == 0) { 1867185029Spjd flag |= U8_STRCMP_CS; 1868185029Spjd } else if (f != U8_STRCMP_CS && f != U8_STRCMP_CI_UPPER && 1869185029Spjd f != U8_STRCMP_CI_LOWER) { 1870185029Spjd *errnum = EBADF; 1871185029Spjd flag = U8_STRCMP_CS; 1872185029Spjd } 1873185029Spjd 1874185029Spjd f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP); 1875185029Spjd if (f && f != U8_STRCMP_NFD && f != U8_STRCMP_NFC && 1876185029Spjd f != U8_STRCMP_NFKD && f != U8_STRCMP_NFKC) { 1877185029Spjd *errnum = EBADF; 1878185029Spjd flag = U8_STRCMP_CS; 1879185029Spjd } 1880185029Spjd } 1881185029Spjd 1882185029Spjd if (flag == U8_STRCMP_CS) { 1883185029Spjd return (n == 0 ? strcmp(s1, s2) : strncmp(s1, s2, n)); 1884185029Spjd } 1885185029Spjd 1886185029Spjd n1 = strlen(s1); 1887185029Spjd n2 = strlen(s2); 1888185029Spjd if (n != 0) { 1889185029Spjd if (n < n1) 1890185029Spjd n1 = n; 1891185029Spjd if (n < n2) 1892185029Spjd n2 = n; 1893185029Spjd } 1894185029Spjd 1895185029Spjd /* 1896185029Spjd * Simple case conversion can be done much faster and so we do 1897185029Spjd * them separately here. 1898185029Spjd */ 1899185029Spjd if (flag == U8_STRCMP_CI_UPPER) { 1900185029Spjd return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2, 1901185029Spjd n1, n2, B_TRUE, errnum)); 1902185029Spjd } else if (flag == U8_STRCMP_CI_LOWER) { 1903185029Spjd return (do_case_compare(uv, (uchar_t *)s1, (uchar_t *)s2, 1904185029Spjd n1, n2, B_FALSE, errnum)); 1905185029Spjd } 1906185029Spjd 1907185029Spjd return (do_norm_compare(uv, (uchar_t *)s1, (uchar_t *)s2, n1, n2, 1908185029Spjd flag, errnum)); 1909185029Spjd} 1910185029Spjd 1911185029Spjdsize_t 1912185029Spjdu8_textprep_str(char *inarray, size_t *inlen, char *outarray, size_t *outlen, 1913185029Spjd int flag, size_t unicode_version, int *errnum) 1914185029Spjd{ 1915185029Spjd int f; 1916185029Spjd int sz; 1917185029Spjd uchar_t *ib; 1918185029Spjd uchar_t *ibtail; 1919185029Spjd uchar_t *ob; 1920185029Spjd uchar_t *obtail; 1921185029Spjd boolean_t do_not_ignore_null; 1922185029Spjd boolean_t do_not_ignore_invalid; 1923185029Spjd boolean_t is_it_toupper; 1924185029Spjd boolean_t is_it_tolower; 1925185029Spjd boolean_t canonical_decomposition; 1926185029Spjd boolean_t compatibility_decomposition; 1927185029Spjd boolean_t canonical_composition; 1928185029Spjd size_t ret_val; 1929185029Spjd size_t i; 1930185029Spjd size_t j; 1931185029Spjd uchar_t u8s[U8_STREAM_SAFE_TEXT_MAX + 1]; 1932185029Spjd u8_normalization_states_t state; 1933185029Spjd 1934185029Spjd if (unicode_version > U8_UNICODE_LATEST) { 1935185029Spjd *errnum = ERANGE; 1936185029Spjd return ((size_t)-1); 1937185029Spjd } 1938185029Spjd 1939185029Spjd f = flag & (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER); 1940185029Spjd if (f == (U8_TEXTPREP_TOUPPER | U8_TEXTPREP_TOLOWER)) { 1941185029Spjd *errnum = EBADF; 1942185029Spjd return ((size_t)-1); 1943185029Spjd } 1944185029Spjd 1945185029Spjd f = flag & (U8_CANON_DECOMP | U8_COMPAT_DECOMP | U8_CANON_COMP); 1946185029Spjd if (f && f != U8_TEXTPREP_NFD && f != U8_TEXTPREP_NFC && 1947185029Spjd f != U8_TEXTPREP_NFKD && f != U8_TEXTPREP_NFKC) { 1948185029Spjd *errnum = EBADF; 1949185029Spjd return ((size_t)-1); 1950185029Spjd } 1951185029Spjd 1952185029Spjd if (inarray == NULL || *inlen == 0) 1953185029Spjd return (0); 1954185029Spjd 1955185029Spjd if (outarray == NULL) { 1956185029Spjd *errnum = E2BIG; 1957185029Spjd return ((size_t)-1); 1958185029Spjd } 1959185029Spjd 1960185029Spjd ib = (uchar_t *)inarray; 1961185029Spjd ob = (uchar_t *)outarray; 1962185029Spjd ibtail = ib + *inlen; 1963185029Spjd obtail = ob + *outlen; 1964185029Spjd 1965185029Spjd do_not_ignore_null = !(flag & U8_TEXTPREP_IGNORE_NULL); 1966185029Spjd do_not_ignore_invalid = !(flag & U8_TEXTPREP_IGNORE_INVALID); 1967185029Spjd is_it_toupper = flag & U8_TEXTPREP_TOUPPER; 1968185029Spjd is_it_tolower = flag & U8_TEXTPREP_TOLOWER; 1969185029Spjd 1970185029Spjd ret_val = 0; 1971185029Spjd 1972185029Spjd /* 1973185029Spjd * If we don't have a normalization flag set, we do the simple case 1974185029Spjd * conversion based text preparation separately below. Text 1975185029Spjd * preparation involving Normalization will be done in the false task 1976185029Spjd * block, again, separately since it will take much more time and 1977185029Spjd * resource than doing simple case conversions. 1978185029Spjd */ 1979185029Spjd if (f == 0) { 1980185029Spjd while (ib < ibtail) { 1981185029Spjd if (*ib == '\0' && do_not_ignore_null) 1982185029Spjd break; 1983185029Spjd 1984185029Spjd sz = u8_number_of_bytes[*ib]; 1985185029Spjd 1986185029Spjd if (sz < 0) { 1987185029Spjd if (do_not_ignore_invalid) { 1988185029Spjd *errnum = EILSEQ; 1989185029Spjd ret_val = (size_t)-1; 1990185029Spjd break; 1991185029Spjd } 1992185029Spjd 1993185029Spjd sz = 1; 1994185029Spjd ret_val++; 1995185029Spjd } 1996185029Spjd 1997185029Spjd if (sz == 1) { 1998185029Spjd if (ob >= obtail) { 1999185029Spjd *errnum = E2BIG; 2000185029Spjd ret_val = (size_t)-1; 2001185029Spjd break; 2002185029Spjd } 2003185029Spjd 2004185029Spjd if (is_it_toupper) 2005185029Spjd *ob = U8_ASCII_TOUPPER(*ib); 2006185029Spjd else if (is_it_tolower) 2007185029Spjd *ob = U8_ASCII_TOLOWER(*ib); 2008185029Spjd else 2009185029Spjd *ob = *ib; 2010185029Spjd ib++; 2011185029Spjd ob++; 2012185029Spjd } else if ((ib + sz) > ibtail) { 2013185029Spjd if (do_not_ignore_invalid) { 2014185029Spjd *errnum = EINVAL; 2015185029Spjd ret_val = (size_t)-1; 2016185029Spjd break; 2017185029Spjd } 2018185029Spjd 2019185029Spjd if ((obtail - ob) < (ibtail - ib)) { 2020185029Spjd *errnum = E2BIG; 2021185029Spjd ret_val = (size_t)-1; 2022185029Spjd break; 2023185029Spjd } 2024185029Spjd 2025185029Spjd /* 2026185029Spjd * We treat the remaining incomplete character 2027185029Spjd * bytes as a character. 2028185029Spjd */ 2029185029Spjd ret_val++; 2030185029Spjd 2031185029Spjd while (ib < ibtail) 2032185029Spjd *ob++ = *ib++; 2033185029Spjd } else { 2034185029Spjd if (is_it_toupper || is_it_tolower) { 2035185029Spjd i = do_case_conv(unicode_version, u8s, 2036185029Spjd ib, sz, is_it_toupper); 2037185029Spjd 2038185029Spjd if ((obtail - ob) < i) { 2039185029Spjd *errnum = E2BIG; 2040185029Spjd ret_val = (size_t)-1; 2041185029Spjd break; 2042185029Spjd } 2043185029Spjd 2044185029Spjd ib += sz; 2045185029Spjd 2046185029Spjd for (sz = 0; sz < i; sz++) 2047185029Spjd *ob++ = u8s[sz]; 2048185029Spjd } else { 2049185029Spjd if ((obtail - ob) < sz) { 2050185029Spjd *errnum = E2BIG; 2051185029Spjd ret_val = (size_t)-1; 2052185029Spjd break; 2053185029Spjd } 2054185029Spjd 2055185029Spjd for (i = 0; i < sz; i++) 2056185029Spjd *ob++ = *ib++; 2057185029Spjd } 2058185029Spjd } 2059185029Spjd } 2060185029Spjd } else { 2061185029Spjd canonical_decomposition = flag & U8_CANON_DECOMP; 2062185029Spjd compatibility_decomposition = flag & U8_COMPAT_DECOMP; 2063185029Spjd canonical_composition = flag & U8_CANON_COMP; 2064185029Spjd 2065185029Spjd while (ib < ibtail) { 2066185029Spjd if (*ib == '\0' && do_not_ignore_null) 2067185029Spjd break; 2068185029Spjd 2069185029Spjd /* 2070185029Spjd * If the current character is a 7-bit ASCII 2071185029Spjd * character and it is the last character, or, 2072185029Spjd * if the current character is a 7-bit ASCII 2073185029Spjd * character and the next character is also a 7-bit 2074185029Spjd * ASCII character, then, we copy over this 2075185029Spjd * character without going through collect_a_seq(). 2076185029Spjd * 2077185029Spjd * In any other cases, we need to look further with 2078185029Spjd * the collect_a_seq() function. 2079185029Spjd */ 2080185029Spjd if (U8_ISASCII(*ib) && ((ib + 1) >= ibtail || 2081185029Spjd ((ib + 1) < ibtail && U8_ISASCII(*(ib + 1))))) { 2082185029Spjd if (ob >= obtail) { 2083185029Spjd *errnum = E2BIG; 2084185029Spjd ret_val = (size_t)-1; 2085185029Spjd break; 2086185029Spjd } 2087185029Spjd 2088185029Spjd if (is_it_toupper) 2089185029Spjd *ob = U8_ASCII_TOUPPER(*ib); 2090185029Spjd else if (is_it_tolower) 2091185029Spjd *ob = U8_ASCII_TOLOWER(*ib); 2092185029Spjd else 2093185029Spjd *ob = *ib; 2094185029Spjd ib++; 2095185029Spjd ob++; 2096185029Spjd } else { 2097185029Spjd *errnum = 0; 2098185029Spjd state = U8_STATE_START; 2099185029Spjd 2100185029Spjd j = collect_a_seq(unicode_version, u8s, 2101185029Spjd &ib, ibtail, 2102185029Spjd is_it_toupper, 2103185029Spjd is_it_tolower, 2104185029Spjd canonical_decomposition, 2105185029Spjd compatibility_decomposition, 2106185029Spjd canonical_composition, 2107185029Spjd errnum, &state); 2108185029Spjd 2109185029Spjd if (*errnum && do_not_ignore_invalid) { 2110185029Spjd ret_val = (size_t)-1; 2111185029Spjd break; 2112185029Spjd } 2113185029Spjd 2114185029Spjd if ((obtail - ob) < j) { 2115185029Spjd *errnum = E2BIG; 2116185029Spjd ret_val = (size_t)-1; 2117185029Spjd break; 2118185029Spjd } 2119185029Spjd 2120185029Spjd for (i = 0; i < j; i++) 2121185029Spjd *ob++ = u8s[i]; 2122185029Spjd } 2123185029Spjd } 2124185029Spjd } 2125185029Spjd 2126185029Spjd *inlen = ibtail - ib; 2127185029Spjd *outlen = obtail - ob; 2128185029Spjd 2129185029Spjd return (ret_val); 2130185029Spjd} 2131