1/*
2 * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
3 * Copyright 2012 Garrett D'Amore <garrett@damore.org>  All rights reserved.
4 * Copyright 2015 John Marino <draco@marino.st>
5 *
6 * This source code is derived from the illumos localedef command, and
7 * provided under BSD-style license terms by Nexenta Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 *
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * The functions in this file convert from the standard multibyte forms
34 * to the wide character forms used internally by libc.  Unfortunately,
35 * this approach means that we need a method for each and every encoding.
36 */
37#include <sys/cdefs.h>
38__FBSDID("$FreeBSD: stable/11/usr.bin/localedef/wide.c 321121 2017-07-18 08:35:22Z ngie $");
39
40#include <ctype.h>
41#include <stdlib.h>
42#include <wchar.h>
43#include <string.h>
44#include <sys/types.h>
45#include "localedef.h"
46
47static int towide_none(wchar_t *, const char *, unsigned);
48static int towide_utf8(wchar_t *, const char *, unsigned);
49static int towide_big5(wchar_t *, const char *, unsigned);
50static int towide_gbk(wchar_t *, const char *, unsigned);
51static int towide_gb2312(wchar_t *, const char *, unsigned);
52static int towide_gb18030(wchar_t *, const char *, unsigned);
53static int towide_mskanji(wchar_t *, const char *, unsigned);
54static int towide_euccn(wchar_t *, const char *, unsigned);
55static int towide_eucjp(wchar_t *, const char *, unsigned);
56static int towide_euckr(wchar_t *, const char *, unsigned);
57static int towide_euctw(wchar_t *, const char *, unsigned);
58
59static int tomb_none(char *, wchar_t);
60static int tomb_utf8(char *, wchar_t);
61static int tomb_mbs(char *, wchar_t);
62
63static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none;
64static int (*_tomb)(char *, wchar_t) = tomb_none;
65static char _encoding_buffer[20] = {'N','O','N','E'};
66static const char *_encoding = _encoding_buffer;
67static int _nbits = 7;
68
69/*
70 * Table of supported encodings.  We only bother to list the multibyte
71 * encodings here, because single byte locales are handed by "NONE".
72 */
73static struct {
74	const char *name;
75	/* the name that the underlying libc implemenation uses */
76	const char *cname;
77	/* the maximum number of bits required for priorities */
78	int nbits;
79	int (*towide)(wchar_t *, const char *, unsigned);
80	int (*tomb)(char *, wchar_t);
81} mb_encodings[] = {
82	/*
83	 * UTF8 values max out at 0x1fffff (although in theory there could
84	 * be later extensions, but it won't happen.)  This means we only need
85	 * 21 bits to be able to encode the entire range of priorities.
86	 */
87	{ "UTF-8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
88	{ "UTF8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
89	{ "utf8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
90	{ "utf-8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
91
92	{ "EUC-CN",	"EUC-CN",	16, towide_euccn, tomb_mbs },
93	{ "eucCN",	"EUC-CN",	16, towide_euccn, tomb_mbs },
94	/*
95	 * Because the 3-byte form of EUC-JP use the same leading byte,
96	 * only 17 bits required to provide unique priorities.  (The low
97	 * bit of that first byte is set.)  By setting this value low,
98	 * we can get by with only 3 bytes in the strxfrm expansion.
99	 */
100	{ "EUC-JP",	"EUC-JP",	17, towide_eucjp, tomb_mbs },
101	{ "eucJP",	"EUC-JP",	17, towide_eucjp, tomb_mbs },
102
103	{ "EUC-KR",	"EUC-KR",	16, towide_euckr, tomb_mbs },
104	{ "eucKR",	"EUC-KR",	16, towide_euckr, tomb_mbs },
105	/*
106	 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
107	 * high order byte is 0x8E.  However, with 4 byte encodings,
108	 * the third byte will be A0-B0.  So we only need to consider
109	 * the lower order 24 bits for collation.
110	 */
111	{ "EUC-TW",	"EUC-TW",	24, towide_euctw, tomb_mbs },
112	{ "eucTW",	"EUC-TW",	24, towide_euctw, tomb_mbs },
113
114	{ "MS_Kanji",	"MSKanji",	16, towide_mskanji, tomb_mbs },
115	{ "MSKanji",	"MSKanji",	16, towide_mskanji, tomb_mbs },
116	{ "PCK",	"MSKanji",	16, towide_mskanji, tomb_mbs },
117	{ "SJIS",	"MSKanji",	16, towide_mskanji, tomb_mbs },
118	{ "Shift_JIS",	"MSKanji",	16, towide_mskanji, tomb_mbs },
119
120	{ "BIG5",	"BIG5",		16, towide_big5, tomb_mbs },
121	{ "big5",	"BIG5",		16, towide_big5, tomb_mbs },
122	{ "Big5",	"BIG5",		16, towide_big5, tomb_mbs },
123
124	{ "GBK",	"GBK",		16, towide_gbk,	tomb_mbs },
125
126	/*
127	 * GB18030 can get away with just 31 bits.  This is because the
128	 * high order bit is always set for 4 byte values, and the
129	 * at least one of the other bits in that 4 byte value will
130	 * be non-zero.
131	 */
132	{ "GB18030",	"GB18030",	31, towide_gb18030, tomb_mbs },
133
134	/*
135	 * This should probably be an aliase for euc-cn, or vice versa.
136	 */
137	{ "GB2312",	"GB2312",	16, towide_gb2312, tomb_mbs },
138
139	{ NULL, NULL, 0, 0, 0 },
140};
141
142static char *
143show_mb(const char *mb)
144{
145	static char buf[64];
146
147	/* ASCII stuff we just print */
148	if (isascii(*mb) && isgraph(*mb)) {
149		buf[0] = *mb;
150		buf[1] = 0;
151		return (buf);
152	}
153	buf[0] = 0;
154	while (*mb != 0) {
155		char scr[8];
156		(void) snprintf(scr, sizeof (scr), "\\x%02x", *mb);
157		(void) strlcat(buf, scr, sizeof (buf));
158		mb++;
159	}
160	return (buf);
161}
162
163static char	*widemsg;
164
165void
166werr(const char *fmt, ...)
167{
168	char	*msg;
169
170	va_list	va;
171	va_start(va, fmt);
172	(void) vasprintf(&msg, fmt, va);
173	va_end(va);
174
175	free(widemsg);
176	widemsg = msg;
177}
178
179/*
180 * This is used for 8-bit encodings.
181 */
182int
183towide_none(wchar_t *c, const char *mb, unsigned n __unused)
184{
185	if (mb_cur_max != 1) {
186		werr("invalid or unsupported multibyte locale");
187		return (-1);
188	}
189	*c = (uint8_t)*mb;
190	return (1);
191}
192
193int
194tomb_none(char *mb, wchar_t wc)
195{
196	if (mb_cur_max != 1) {
197		werr("invalid or unsupported multibyte locale");
198		return (-1);
199	}
200	*(uint8_t *)mb = (wc & 0xff);
201	mb[1] = 0;
202	return (1);
203}
204
205/*
206 * UTF-8 stores wide characters in UTF-32 form.
207 */
208int
209towide_utf8(wchar_t *wc, const char *mb, unsigned n)
210{
211	wchar_t	c;
212	int	nb;
213	wchar_t	lv;	/* lowest legal value */
214	int	i;
215	const uint8_t *s = (const uint8_t *)mb;
216
217	c = *s;
218
219	if ((c & 0x80) == 0) {
220		/* 7-bit ASCII */
221		*wc = c;
222		return (1);
223	} else if ((c & 0xe0) == 0xc0) {
224		/* u80-u7ff - two bytes encoded */
225		nb = 2;
226		lv = 0x80;
227		c &= ~0xe0;
228	} else if ((c & 0xf0) == 0xe0) {
229		/* u800-uffff - three bytes encoded */
230		nb = 3;
231		lv = 0x800;
232		c &= ~0xf0;
233	} else if ((c & 0xf8) == 0xf0) {
234		/* u1000-u1fffff - four bytes encoded */
235		nb = 4;
236		lv = 0x1000;
237		c &= ~0xf8;
238	} else {
239		/* 5 and 6 byte encodings are not legal unicode */
240		werr("utf8 encoding too large (%s)", show_mb(mb));
241		return (-1);
242	}
243	if (nb > (int)n) {
244		werr("incomplete utf8 sequence (%s)", show_mb(mb));
245		return (-1);
246	}
247
248	for (i = 1; i < nb; i++) {
249		if (((s[i]) & 0xc0) != 0x80) {
250			werr("illegal utf8 byte (%x)", s[i]);
251			return (-1);
252		}
253		c <<= 6;
254		c |= (s[i] & 0x3f);
255	}
256
257	if (c < lv) {
258		werr("illegal redundant utf8 encoding (%s)", show_mb(mb));
259		return (-1);
260	}
261	*wc = c;
262	return (nb);
263}
264
265int
266tomb_utf8(char *mb, wchar_t wc)
267{
268	uint8_t *s = (uint8_t *)mb;
269	uint8_t msk;
270	int cnt;
271	int i;
272
273	if (wc <= 0x7f) {
274		s[0] = wc & 0x7f;
275		s[1] = 0;
276		return (1);
277	}
278	if (wc <= 0x7ff) {
279		cnt = 2;
280		msk = 0xc0;
281	} else if (wc <= 0xffff) {
282		cnt = 3;
283		msk = 0xe0;
284	} else if (wc <= 0x1fffff) {
285		cnt = 4;
286		msk = 0xf0;
287	} else {
288		werr("illegal uf8 char (%x)", wc);
289		return (-1);
290	}
291	for (i = cnt - 1; i; i--) {
292		s[i] = (wc & 0x3f) | 0x80;
293		wc >>= 6;
294	}
295	s[0] = (msk) | wc;
296	s[cnt] = 0;
297	return (cnt);
298}
299
300/*
301 * Several encodings share a simplistic dual byte encoding.  In these
302 * forms, they all indicate that a two byte sequence is to be used if
303 * the first byte has its high bit set.  They all store this simple
304 * encoding as a 16-bit value, although a great many of the possible
305 * code points are not used in most character sets.  This gives a possible
306 * set of just over 32,000 valid code points.
307 *
308 * 0x00 - 0x7f		- 1 byte encoding
309 * 0x80 - 0x7fff	- illegal
310 * 0x8000 - 0xffff	- 2 byte encoding
311 */
312
313static int
314towide_dbcs(wchar_t *wc, const char *mb, unsigned n)
315{
316	wchar_t	c;
317
318	c = *(const uint8_t *)mb;
319
320	if ((c & 0x80) == 0) {
321		/* 7-bit */
322		*wc = c;
323		return (1);
324	}
325	if (n < 2) {
326		werr("incomplete character sequence (%s)", show_mb(mb));
327		return (-1);
328	}
329
330	/* Store both bytes as a single 16-bit wide. */
331	c <<= 8;
332	c |= (uint8_t)(mb[1]);
333	*wc = c;
334	return (2);
335}
336
337/*
338 * Most multibyte locales just convert the wide character to the multibyte
339 * form by stripping leading null bytes, and writing the 32-bit quantity
340 * in big-endian order.
341 */
342int
343tomb_mbs(char *mb, wchar_t wc)
344{
345	uint8_t *s = (uint8_t *)mb;
346	int 	n = 0, c;
347
348	if ((wc & 0xff000000U) != 0) {
349		n = 4;
350	} else if ((wc & 0x00ff0000U) != 0) {
351		n = 3;
352	} else if ((wc & 0x0000ff00U) != 0) {
353		n = 2;
354	} else {
355		n = 1;
356	}
357	c = n;
358	while (n) {
359		n--;
360		s[n] = wc & 0xff;
361		wc >>= 8;
362	}
363	/* ensure null termination */
364	s[c] = 0;
365	return (c);
366}
367
368
369/*
370 * big5 is a simple dual byte character set.
371 */
372int
373towide_big5(wchar_t *wc, const char *mb, unsigned n)
374{
375	return (towide_dbcs(wc, mb, n));
376}
377
378/*
379 * GBK encodes wides in the same way that big5 does, the high order
380 * bit of the first byte indicates a double byte character.
381 */
382int
383towide_gbk(wchar_t *wc, const char *mb, unsigned n)
384{
385	return (towide_dbcs(wc, mb, n));
386}
387
388/*
389 * GB2312 is another DBCS.  Its cleaner than others in that the second
390 * byte does not encode ASCII, but it supports characters.
391 */
392int
393towide_gb2312(wchar_t *wc, const char *mb, unsigned n)
394{
395	return (towide_dbcs(wc, mb, n));
396}
397
398/*
399 * GB18030.  This encodes as 8, 16, or 32-bits.
400 * 7-bit values are in 1 byte,  4 byte sequences are used when
401 * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
402 */
403int
404towide_gb18030(wchar_t *wc, const char *mb, unsigned n)
405{
406	wchar_t	c;
407
408	c = *(const uint8_t *)mb;
409
410	if ((c & 0x80) == 0) {
411		/* 7-bit */
412		*wc = c;
413		return (1);
414	}
415	if (n < 2) {
416		werr("incomplete character sequence (%s)", show_mb(mb));
417		return (-1);
418	}
419
420	/* pull in the second byte */
421	c <<= 8;
422	c |= (uint8_t)(mb[1]);
423
424	if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) {
425		if (n < 4) {
426			werr("incomplete 4-byte character sequence (%s)",
427			    show_mb(mb));
428			return (-1);
429		}
430		c <<= 8;
431		c |= (uint8_t)(mb[2]);
432		c <<= 8;
433		c |= (uint8_t)(mb[3]);
434		*wc = c;
435		return (4);
436	}
437
438	*wc = c;
439	return (2);
440}
441
442/*
443 * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
444 * also has a range of single byte characters above 0x80.  (0xa1-0xdf).
445 */
446int
447towide_mskanji(wchar_t *wc, const char *mb, unsigned n)
448{
449	wchar_t	c;
450
451	c = *(const uint8_t *)mb;
452
453	if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) {
454		/* 7-bit */
455		*wc = c;
456		return (1);
457	}
458
459	if (n < 2) {
460		werr("incomplete character sequence (%s)", show_mb(mb));
461		return (-1);
462	}
463
464	/* Store both bytes as a single 16-bit wide. */
465	c <<= 8;
466	c |= (uint8_t)(mb[1]);
467	*wc = c;
468	return (2);
469}
470
471/*
472 * EUC forms.  EUC encodings are "variable".  FreeBSD carries some additional
473 * variable data to encode these, but we're going to treat each as independent
474 * instead.  Its the only way we can sensibly move forward.
475 *
476 * Note that the way in which the different EUC forms vary is how wide
477 * CS2 and CS3 are and what the first byte of them is.
478 */
479static int
480towide_euc_impl(wchar_t *wc, const char *mb, unsigned n,
481    uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width)
482{
483	int i;
484	int width = 2;
485	wchar_t	c;
486
487	c = *(const uint8_t *)mb;
488
489	/*
490	 * All variations of EUC encode 7-bit ASCII as one byte, and use
491	 * additional bytes for more than that.
492	 */
493	if ((c & 0x80) == 0) {
494		/* 7-bit */
495		*wc = c;
496		return (1);
497	}
498
499	/*
500	 * All EUC variants reserve 0xa1-0xff to identify CS1, which
501	 * is always two bytes wide.  Note that unused CS will be zero,
502	 * and that cannot be true because we know that the high order
503	 * bit must be set.
504	 */
505	if (c >= 0xa1) {
506		width = 2;
507	} else if (c == cs2) {
508		width = cs2width;
509	} else if (c == cs3) {
510		width = cs3width;
511	}
512
513	if ((int)n < width) {
514		werr("incomplete character sequence (%s)", show_mb(mb));
515		return (-1);
516	}
517
518	for (i = 1; i < width; i++) {
519		/* pull in the next byte */
520		c <<= 8;
521		c |= (uint8_t)(mb[i]);
522	}
523
524	*wc = c;
525	return (width);
526}
527
528/*
529 * EUC-CN encodes as follows:
530 *
531 * Code set 0 (ASCII):				0x21-0x7E
532 * Code set 1 (CNS 11643-1992 Plane 1):		0xA1A1-0xFEFE
533 * Code set 2:					unused
534 * Code set 3:					unused
535 */
536int
537towide_euccn(wchar_t *wc, const char *mb, unsigned n)
538{
539	return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
540}
541
542/*
543 * EUC-JP encodes as follows:
544 *
545 * Code set 0 (ASCII or JIS X 0201-1976 Roman):	0x21-0x7E
546 * Code set 1 (JIS X 0208):			0xA1A1-0xFEFE
547 * Code set 2 (half-width katakana):		0x8EA1-0x8EDF
548 * Code set 3 (JIS X 0212-1990):		0x8FA1A1-0x8FFEFE
549 */
550int
551towide_eucjp(wchar_t *wc, const char *mb, unsigned n)
552{
553	return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3));
554}
555
556/*
557 * EUC-KR encodes as follows:
558 *
559 * Code set 0 (ASCII or KS C 5636-1993):	0x21-0x7E
560 * Code set 1 (KS C 5601-1992):			0xA1A1-0xFEFE
561 * Code set 2:					unused
562 * Code set 3:					unused
563 */
564int
565towide_euckr(wchar_t *wc, const char *mb, unsigned n)
566{
567	return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0));
568}
569
570/*
571 * EUC-TW encodes as follows:
572 *
573 * Code set 0 (ASCII):				0x21-0x7E
574 * Code set 1 (CNS 11643-1992 Plane 1):		0xA1A1-0xFEFE
575 * Code set 2 (CNS 11643-1992 Planes 1-16):	0x8EA1A1A1-0x8EB0FEFE
576 * Code set 3:					unused
577 */
578int
579towide_euctw(wchar_t *wc, const char *mb, unsigned n)
580{
581	return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
582}
583
584/*
585 * Public entry points.
586 */
587
588int
589to_wide(wchar_t *wc, const char *mb)
590{
591	/* this won't fail hard */
592	return (_towide(wc, mb, strlen(mb)));
593}
594
595int
596to_mb(char *mb, wchar_t wc)
597{
598	int	rv;
599
600	if ((rv = _tomb(mb, wc)) < 0) {
601		warn("%s", widemsg);
602		free(widemsg);
603		widemsg = NULL;
604	}
605	return (rv);
606}
607
608char *
609to_mb_string(const wchar_t *wcs)
610{
611	char	*mbs;
612	char	*ptr;
613	int	len;
614
615	mbs = malloc((wcslen(wcs) * mb_cur_max) + 1);
616	if (mbs == NULL) {
617		warn("out of memory");
618		return (NULL);
619	}
620	ptr = mbs;
621	while (*wcs) {
622		if ((len = to_mb(ptr, *wcs)) < 0) {
623			INTERR;
624			free(mbs);
625			return (NULL);
626		}
627		wcs++;
628		ptr += len;
629	}
630	*ptr = 0;
631	return (mbs);
632}
633
634void
635set_wide_encoding(const char *encoding)
636{
637	int i;
638
639	_towide = towide_none;
640	_tomb = tomb_none;
641	_nbits = 8;
642
643	snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s",
644	    encoding);
645	for (i = 0; mb_encodings[i].name; i++) {
646		if (strcasecmp(encoding, mb_encodings[i].name) == 0) {
647			_towide = mb_encodings[i].towide;
648			_tomb = mb_encodings[i].tomb;
649			_encoding = mb_encodings[i].cname;
650			_nbits = mb_encodings[i].nbits;
651			break;
652		}
653	}
654}
655
656const char *
657get_wide_encoding(void)
658{
659	return (_encoding);
660}
661
662int
663max_wide(void)
664{
665	return ((int)((1U << _nbits) - 1));
666}
667