5.8.4/distrib/toke.c

788 	if (UTF && !IN_BYTES && is_utf8_string((U8*)PL_tokenbuf, len))
839 	    n = utf8n_to_uvchr((U8*)start, len, &skip, 0);
841 	    n = *(U8*)start;
1250 		    char *c = (char*)utf8_hop((U8*)d, -1);
1266 		min = (U8)*d;			/* first char in range */
1267 		max = (U8)d[1];			/* last char in range  */
1481 			U8 *c;
1482 			for (c = (U8 *) SvPVX(sv); c < (U8 *)d; c++) {
1489 			    U8 *src, *dst;
1491 			    src = (U8 *)d - 1;
1494 			    while (src >= (U8 *)SvPVX(sv)) {
1496 				    U8 ch = NATIVE_TO_ASCII(*src);
1497 				    *dst-- = (U8)UTF8_EIGHT_BIT_LO(ch);
1498 				    *dst-- = (U8)UTF8_EIGHT_BIT_HI(ch);
1509 		        d = (char*)uvchr_to_utf8((U8*)d, uv);
1566 			 UV uv = utf8_to_uvchr((U8*)str, 0);
1569 			      U8 tmpbuf[UTF8_MAXLEN+1], *d;
1608 		    U8 c = *s++;
1651 	if ((has_utf8 || this_utf8) && !NATIVE_IS_INVARIANT((U8)(*s))) {
1653 	    UV uv       = (this_utf8) ? utf8n_to_uvchr((U8*)s, send - s, &len, 0) : (UV) ((U8) *s);
1661 	    d = (char*)uvchr_to_utf8((U8*)d, uv);
2537 		      *(U8*)s == 0xEF ||
2538 		      *(U8*)s >= 0xFE ||
2565 		    s = swallow_bom((U8*)s);
3716 	    if (*d == '$' || *d == '@' || *d == '\\' || !UTF8_IS_INVARIANT((U8)*d)) {
3844 	    if (UTF && !IN_BYTES && is_utf8_string((U8*)PL_tokenbuf, len))
3967 		    is_utf8_string((U8*)SvPVX(sv), SvCUR(sv)))
4015 		    if (UTF && !IN_BYTES && is_utf8_string((U8*)PL_tokenbuf, len))
6169 	else if (UTF && UTF8_IS_START(*s) && isALNUM_utf8((U8*)s)) {
6171 	    while (UTF8_IS_CONTINUED(*t) && is_utf8_mark((U8*)t))
6221 	    else if (UTF && UTF8_IS_START(*s) && isALNUM_utf8((U8*)s)) {
6223 		while (UTF8_IS_CONTINUED(*t) && is_utf8_mark((U8*)t))
6276 		    while (e < send && UTF8_IS_CONTINUED(*e) && is_utf8_mark((U8*)e))
6709 	if (UTF && is_utf8_string((U8*)SvPVX(tmpstr), SvCUR(tmpstr)))
6933     U8 termstr[UTF8_MAXLEN];		/* terminating string */
6951 	termcode = utf8_to_uvchr((U8*)s, &termlen);
6952 	Copy(s, termstr, termlen, U8);
7076 		else if (!has_utf8 && !UTF8_IS_INVARIANT((U8)*s) && UTF)
7105 		else if (!has_utf8 && !UTF8_IS_INVARIANT((U8)*s) && UTF)
7669 	    if (UTF && is_utf8_string((U8*)SvPVX(stuff), SvCUR(stuff)))
7841 S_swallow_bom(pTHX_ U8 *s)
7856 		U8 *news;
7860 		New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8);
7868 		s = (U8*)SvPVX(PL_linestr);
7883 		U8 *news;
7887 		New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8);
7895 		s = (U8*)SvPVX(PL_linestr);
7960 	U8* tmps;
7961 	U8* tend;
7963 	New(898, tmps, SvCUR(sv) * 3 / 2 + 1, U8);
7968 	tend = utf16_to_utf8((U8*)SvPVX(sv), tmps, SvCUR(sv), &newlen);
7979 	U8* tmps;
7980 	U8* tend;
7986 	New(898, tmps, SvCUR(sv) * 3 / 2 + 1, U8);
7987 	tend = utf16_to_utf8_reversed((U8*)SvPVX(sv), tmps, SvCUR(sv), &newlen);
8030 	U8 tmpbuf[UTF8_MAXLEN+1];
8031 	U8 *tmpend;