5.8.4/distrib/utf8.c

32 =for apidoc A|U8 *|uvuni_to_utf8_flags|U8 *d|UV uv|UV flags
56 U8 *
57 Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags)
78 	*d++ = (U8)UTF_TO_NATIVE(uv);
84 	U8 *p = d+len-1;
86 	    *p-- = (U8)UTF_TO_NATIVE((uv & UTF_CONTINUATION_MASK) | UTF_CONTINUATION_MARK);
89 	*p = (U8)UTF_TO_NATIVE((uv & UTF_START_MASK(len)) | UTF_START_MARK(len));
94 	*d++ = (U8)(( uv >>  6)         | 0xc0);
95 	*d++ = (U8)(( uv        & 0x3f) | 0x80);
99 	*d++ = (U8)(( uv >> 12)         | 0xe0);
100 	*d++ = (U8)(((uv >>  6) & 0x3f) | 0x80);
101 	*d++ = (U8)(( uv        & 0x3f) | 0x80);
105 	*d++ = (U8)(( uv >> 18)         | 0xf0);
106 	*d++ = (U8)(((uv >> 12) & 0x3f) | 0x80);
107 	*d++ = (U8)(((uv >>  6) & 0x3f) | 0x80);
108 	*d++ = (U8)(( uv        & 0x3f) | 0x80);
112 	*d++ = (U8)(( uv >> 24)         | 0xf8);
113 	*d++ = (U8)(((uv >> 18) & 0x3f) | 0x80);
114 	*d++ = (U8)(((uv >> 12) & 0x3f) | 0x80);
115 	*d++ = (U8)(((uv >>  6) & 0x3f) | 0x80);
116 	*d++ = (U8)(( uv        & 0x3f) | 0x80);
120 	*d++ = (U8)(( uv >> 30)         | 0xfc);
121 	*d++ = (U8)(((uv >> 24) & 0x3f) | 0x80);
122 	*d++ = (U8)(((uv >> 18) & 0x3f) | 0x80);
123 	*d++ = (U8)(((uv >> 12) & 0x3f) | 0x80);
124 	*d++ = (U8)(((uv >>  6) & 0x3f) | 0x80);
125 	*d++ = (U8)(( uv        & 0x3f) | 0x80);
133 	*d++ = (U8)(((uv >> 30) & 0x3f) | 0x80);
134 	*d++ = (U8)(((uv >> 24) & 0x3f) | 0x80);
135 	*d++ = (U8)(((uv >> 18) & 0x3f) | 0x80);
136 	*d++ = (U8)(((uv >> 12) & 0x3f) | 0x80);
137 	*d++ = (U8)(((uv >>  6) & 0x3f) | 0x80);
138 	*d++ = (U8)(( uv        & 0x3f) | 0x80);
145 	*d++ = (U8)(((uv >> 60) & 0x0f) | 0x80);	/* 2 Reserved bits */
146 	*d++ = (U8)(((uv >> 54) & 0x3f) | 0x80);
147 	*d++ = (U8)(((uv >> 48) & 0x3f) | 0x80);
148 	*d++ = (U8)(((uv >> 42) & 0x3f) | 0x80);
149 	*d++ = (U8)(((uv >> 36) & 0x3f) | 0x80);
150 	*d++ = (U8)(((uv >> 30) & 0x3f) | 0x80);
151 	*d++ = (U8)(((uv >> 24) & 0x3f) | 0x80);
152 	*d++ = (U8)(((uv >> 18) & 0x3f) | 0x80);
153 	*d++ = (U8)(((uv >> 12) & 0x3f) | 0x80);
154 	*d++ = (U8)(((uv >>  6) & 0x3f) | 0x80);
155 	*d++ = (U8)(( uv        & 0x3f) | 0x80);
162 U8 *
163 Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv)
170 =for apidoc A|STRLEN|is_utf8_char|U8 *s
179 Perl_is_utf8_char(pTHX_ U8 *s)
181     U8 u = *s;
218 =for apidoc A|bool|is_utf8_string|U8 *s|STRLEN len
229 Perl_is_utf8_string(pTHX_ U8 *s, STRLEN len)
231     U8* x = s;
232     U8* send;
260 =for apidoc A|bool|is_utf8_string_loc|U8 *s|STRLEN len|U8 **p
269 Perl_is_utf8_string_loc(pTHX_ U8 *s, STRLEN len, U8 **p)
271     U8* x = s;
272     U8* send;
309 =for apidoc A|UV|utf8n_to_uvuni|U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags
333 Perl_utf8n_to_uvuni(pTHX_ U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags)
335     U8 *s0 = s;
533 =for apidoc A|UV|utf8_to_uvchr|U8 *s|STRLEN *retlen
546 Perl_utf8_to_uvchr(pTHX_ U8 *s, STRLEN *retlen)
553 =for apidoc A|UV|utf8_to_uvuni|U8 *s|STRLEN *retlen
569 Perl_utf8_to_uvuni(pTHX_ U8 *s, STRLEN *retlen)
577 =for apidoc A|STRLEN|utf8_length|U8 *s|U8 *e
587 Perl_utf8_length(pTHX_ U8 *s, U8 *e)
606 	U8 t = UTF8SKIP(s);
626 =for apidoc A|IV|utf8_distance|U8 *a|U8 *b
638 Perl_utf8_distance(pTHX_ U8 *a, U8 *b)
648 	    U8 c = UTF8SKIP(a);
666 	    U8 c = UTF8SKIP(b);
687 =for apidoc A|U8 *|utf8_hop|U8 *s|I32 off
699 U8 *
700 Perl_utf8_hop(pTHX_ U8 *s, I32 off)
721 =for apidoc A|U8 *|utf8_to_bytes|U8 *s|STRLEN *len
731 U8 *
732 Perl_utf8_to_bytes(pTHX_ U8 *s, STRLEN *len)
734     U8 *send;
735     U8 *d;
736     U8 *save = s;
740         U8 c = *s++;
753         *d++ = (U8)utf8_to_uvchr(s, &ulen);
762 =for apidoc A|U8 *|bytes_from_utf8|U8 *s|STRLEN *len|bool *is_utf8
774 U8 *
775 Perl_bytes_from_utf8(pTHX_ U8 *s, STRLEN *len, bool *is_utf8)
777     U8 *d;
778     U8 *start = s;
779     U8 *send;
787 	U8 c = *s++;
799     Newz(801, d, (*len) - count + 1, U8);
802 	U8 c = *s++;
816 =for apidoc A|U8 *|bytes_to_utf8|U8 *s|STRLEN *len
828 U8*
829 Perl_bytes_to_utf8(pTHX_ U8 *s, STRLEN *len)
831     U8 *send;
832     U8 *d;
833     U8 *dst;
836     Newz(801, d, (*len) * 2 + 1, U8);
842             *d++ = (U8)UTF_TO_NATIVE(uv);
844             *d++ = (U8)UTF8_EIGHT_BIT_HI(uv);
845             *d++ = (U8)UTF8_EIGHT_BIT_LO(uv);
859 U8*
860 Perl_utf16_to_utf8(pTHX_ U8* p, U8* d, I32 bytelen, I32 *newlen)
862     U8* pend;
863     U8* dstart = d;
874 	    *d++ = (U8)uv;
878 	    *d++ = (U8)(( uv >>  6)         | 0xc0);
879 	    *d++ = (U8)(( uv        & 0x3f) | 0x80);
890 	    *d++ = (U8)(( uv >> 12)         | 0xe0);
891 	    *d++ = (U8)(((uv >>  6) & 0x3f) | 0x80);
892 	    *d++ = (U8)(( uv        & 0x3f) | 0x80);
896 	    *d++ = (U8)(( uv >> 18)         | 0xf0);
897 	    *d++ = (U8)(((uv >> 12) & 0x3f) | 0x80);
898 	    *d++ = (U8)(((uv >>  6) & 0x3f) | 0x80);
899 	    *d++ = (U8)(( uv        & 0x3f) | 0x80);
909 U8*
910 Perl_utf16_to_utf8_reversed(pTHX_ U8* p, U8* d, I32 bytelen, I32 *newlen)
912     U8* s = (U8*)p;
913     U8* send = s + bytelen;
915 	U8 tmp = s[0];
928     U8 tmpbuf[UTF8_MAXLEN+1];
936     U8 tmpbuf[UTF8_MAXLEN+1];
944     U8 tmpbuf[UTF8_MAXLEN+1];
952     U8 tmpbuf[UTF8_MAXLEN+1];
960     U8 tmpbuf[UTF8_MAXLEN+1];
968     U8 tmpbuf[UTF8_MAXLEN+1];
976     U8 tmpbuf[UTF8_MAXLEN+1];
984     U8 tmpbuf[UTF8_MAXLEN+1];
992     U8 tmpbuf[UTF8_MAXLEN+1];
1000     U8 tmpbuf[UTF8_MAXLEN+1];
1008     U8 tmpbuf[UTF8_MAXLEN+1];
1016     U8 tmpbuf[UTF8_MAXLEN+1];
1024     U8 tmpbuf[UTF8_MAXLEN+1];
1032     U8 tmpbuf[UTF8_MAXLEN_UCLC+1];
1038 Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp)
1045 Perl_to_uni_title(pTHX_ UV c, U8* p, STRLEN *lenp)
1052 Perl_to_uni_lower(pTHX_ UV c, U8* p, STRLEN *lenp)
1059 Perl_to_uni_fold(pTHX_ UV c, U8* p, STRLEN *lenp)
1157     U8 tmpbuf[UTF8_MAXLEN_UCLC+1];
1167     U8 tmpbuf[UTF8_MAXLEN_UCLC+1];
1177     U8 tmpbuf[UTF8_MAXLEN_UCLC+1];
1182 Perl_is_utf8_alnum(pTHX_ U8 *p)
1202 Perl_is_utf8_alnumc(pTHX_ U8 *p)
1219 Perl_is_utf8_idfirst(pTHX_ U8 *p) /* The naming is historical. */
1231 Perl_is_utf8_idcont(pTHX_ U8 *p)
1243 Perl_is_utf8_alpha(pTHX_ U8 *p)
1253 Perl_is_utf8_ascii(pTHX_ U8 *p)
1263 Perl_is_utf8_space(pTHX_ U8 *p)
1273 Perl_is_utf8_digit(pTHX_ U8 *p)
1283 Perl_is_utf8_upper(pTHX_ U8 *p)
1293 Perl_is_utf8_lower(pTHX_ U8 *p)
1303 Perl_is_utf8_cntrl(pTHX_ U8 *p)
1313 Perl_is_utf8_graph(pTHX_ U8 *p)
1323 Perl_is_utf8_print(pTHX_ U8 *p)
1333 Perl_is_utf8_punct(pTHX_ U8 *p)
1343 Perl_is_utf8_xdigit(pTHX_ U8 *p)
1353 Perl_is_utf8_mark(pTHX_ U8 *p)
1363 =for apidoc A|UV|to_utf8_case|U8 *p|U8* ustrp|STRLEN *lenp|SV **swash|char *normal|char *special
1388 Perl_to_utf8_case(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp, SV **swashp, char *normal, char *special)
1391     U8 tmpbuf[UTF8_MAXLEN_FOLD+1];
1418 		   len = uvuni_to_utf8(ustrp, NATIVE_TO_UNI(*(U8*)s)) - ustrp;
1424 		   U8 *t = (U8*)s, *tend = t + len, *d;
1447 		   Copy(tmpbuf, ustrp, len, U8);
1449 		   Copy(s, ustrp, len, U8);
1476 =for apidoc A|UV|to_utf8_upper|U8 *p|U8 *ustrp|STRLEN *lenp
1490 Perl_to_utf8_upper(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp)
1497 =for apidoc A|UV|to_utf8_title|U8 *p|U8 *ustrp|STRLEN *lenp
1511 Perl_to_utf8_title(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp)
1518 =for apidoc A|UV|to_utf8_lower|U8 *p|U8 *ustrp|STRLEN *lenp
1532 Perl_to_utf8_lower(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp)
1539 =for apidoc A|UV|to_utf8_fold|U8 *p|U8 *ustrp|STRLEN *lenp
1553 Perl_to_utf8_fold(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp)
1614 	PL_curcop->op_private = (U8)(PL_hints & HINT_PRIVATE_MASK);
1632 Perl_swash_fetch(pTHX_ SV *sv, U8 *ptr, bool do_utf8)
1639     U8 *tmps = NULL;
1642     U8 tmputf8[2];
1646         tmputf8[0] = (U8)UTF8_EIGHT_BIT_HI(c);
1647         tmputf8[1] = (U8)UTF8_EIGHT_BIT_LO(c);
1694 	if (!svp || !SvPOK(*svp) || !(tmps = (U8*)SvPV(*svp, slen))) {
1727 		PL_curcop->op_private = (U8)(PL_hints & HINT_PRIVATE_MASK);
1731 	    if (!svp || !(tmps = (U8*)SvPV(*svp, slen)) || (slen << 3) < needents)
1740 	    Copy(ptr, PL_last_swash_key, klen, U8);
1763 =for apidoc A|U8 *|uvchr_to_utf8|U8 *d|UV uv
1783 U8 *
1784 Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv)
1789 U8 *
1790 Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags)
1796 =for apidoc A|UV|utf8n_to_uvchr|U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags
1811 Perl_utf8n_to_uvchr(pTHX_ U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags)
1818 =for apidoc A|char *|pv_uni_display|SV *dsv|U8 *spv|STRLEN len|STRLEN pvlim|UV flags
1835 Perl_pv_uni_display(pTHX_ SV *dsv, U8 *spv, STRLEN len, STRLEN pvlim, UV flags)
1849 	 u = utf8_to_uvchr((U8*)s, 0);
1898      return Perl_pv_uni_display(aTHX_ dsv, (U8*)SvPVX(ssv), SvCUR(ssv),
1930      register U8 *p1  = (U8*)s1;
1931      register U8 *p2  = (U8*)s2;
1932      register U8 *e1 = 0, *f1 = 0, *q1 = 0;
1933      register U8 *e2 = 0, *f2 = 0, *q2 = 0;
1935      U8 foldbuf1[UTF8_MAXLEN_FOLD+1];
1936      U8 foldbuf2[UTF8_MAXLEN_FOLD+1];
1937      U8 natbuf[1+1];
1942 	  e1 = *(U8**)pe1;
1943      if (e1 == 0 || (l1 && l1 < (UV)(e1 - (U8*)s1)))
1944 	  f1 = (U8*)s1 + l1;
1946 	  e2 = *(U8**)pe2;
1947      if (e2 == 0 || (l2 && l2 < (UV)(e2 - (U8*)s2)))
1948 	  f2 = (U8*)s2 + l2;