1/* 2 * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26package java.lang; 27 28import java.io.UnsupportedEncodingException; 29import java.lang.ref.SoftReference; 30import java.nio.ByteBuffer; 31import java.nio.CharBuffer; 32import java.nio.charset.Charset; 33import java.nio.charset.CharsetDecoder; 34import java.nio.charset.CharsetEncoder; 35import java.nio.charset.CharacterCodingException; 36import java.nio.charset.CoderResult; 37import java.nio.charset.CodingErrorAction; 38import java.nio.charset.IllegalCharsetNameException; 39import java.nio.charset.UnsupportedCharsetException; 40import java.util.Arrays; 41import jdk.internal.HotSpotIntrinsicCandidate; 42import sun.nio.cs.HistoricallyNamedCharset; 43import sun.nio.cs.ArrayDecoder; 44import sun.nio.cs.ArrayEncoder; 45 46import static java.lang.String.LATIN1; 47import static java.lang.String.UTF16; 48import static java.lang.String.COMPACT_STRINGS; 49 50/** 51 * Utility class for string encoding and decoding. 52 */ 53 54class StringCoding { 55 56 private StringCoding() { } 57 58 /** The cached coders for each thread */ 59 private static final ThreadLocal<SoftReference<StringDecoder>> decoder = 60 new ThreadLocal<>(); 61 private static final ThreadLocal<SoftReference<StringEncoder>> encoder = 62 new ThreadLocal<>(); 63 64 private static final Charset ISO_8859_1 = Charset.forName("iso-8859-1"); 65 private static final Charset US_ASCII = Charset.forName("us-ascii"); 66 private static final Charset UTF_8 = Charset.forName("utf-8"); 67 68 private static boolean warnUnsupportedCharset = true; 69 70 private static <T> T deref(ThreadLocal<SoftReference<T>> tl) { 71 SoftReference<T> sr = tl.get(); 72 if (sr == null) 73 return null; 74 return sr.get(); 75 } 76 77 private static <T> void set(ThreadLocal<SoftReference<T>> tl, T ob) { 78 tl.set(new SoftReference<>(ob)); 79 } 80 81 // Trim the given byte array to the given length 82 // 83 private static byte[] safeTrim(byte[] ba, int len, boolean isTrusted) { 84 if (len == ba.length && (isTrusted || System.getSecurityManager() == null)) 85 return ba; 86 else 87 return Arrays.copyOf(ba, len); 88 } 89 90 private static int scale(int len, float expansionFactor) { 91 // We need to perform double, not float, arithmetic; otherwise 92 // we lose low order bits when len is larger than 2**24. 93 return (int)(len * (double)expansionFactor); 94 } 95 96 private static Charset lookupCharset(String csn) { 97 if (Charset.isSupported(csn)) { 98 try { 99 return Charset.forName(csn); 100 } catch (UnsupportedCharsetException x) { 101 throw new Error(x); 102 } 103 } 104 return null; 105 } 106 107 private static void warnUnsupportedCharset(String csn) { 108 if (warnUnsupportedCharset) { 109 // Use err(String) rather than the Logging API or System.err 110 // since this method may be called during VM initialization 111 // before either is available. 112 err("WARNING: Default charset " + csn + 113 " not supported, using ISO-8859-1 instead\n"); 114 warnUnsupportedCharset = false; 115 } 116 } 117 118 static class Result { 119 byte[] value; 120 byte coder; 121 122 Result with() { 123 coder = COMPACT_STRINGS ? LATIN1 : UTF16; 124 value = new byte[0]; 125 return this; 126 } 127 128 Result with(char[] val, int off, int len) { 129 if (String.COMPACT_STRINGS) { 130 byte[] bs = StringUTF16.compress(val, off, len); 131 if (bs != null) { 132 value = bs; 133 coder = LATIN1; 134 return this; 135 } 136 } 137 coder = UTF16; 138 value = StringUTF16.toBytes(val, off, len); 139 return this; 140 } 141 142 Result with(byte[] val, byte coder) { 143 this.coder = coder; 144 value = val; 145 return this; 146 } 147 } 148 149 @HotSpotIntrinsicCandidate 150 public static boolean hasNegatives(byte[] ba, int off, int len) { 151 for (int i = off; i < off + len; i++) { 152 if (ba[i] < 0) { 153 return true; 154 } 155 } 156 return false; 157 } 158 159 // -- Decoding -- 160 static class StringDecoder { 161 private final String requestedCharsetName; 162 private final Charset cs; 163 private final boolean isASCIICompatible; 164 private final CharsetDecoder cd; 165 protected final Result result; 166 167 StringDecoder(Charset cs, String rcn) { 168 this.requestedCharsetName = rcn; 169 this.cs = cs; 170 this.cd = cs.newDecoder() 171 .onMalformedInput(CodingErrorAction.REPLACE) 172 .onUnmappableCharacter(CodingErrorAction.REPLACE); 173 this.result = new Result(); 174 this.isASCIICompatible = (cd instanceof ArrayDecoder) && 175 ((ArrayDecoder)cd).isASCIICompatible(); 176 } 177 178 String charsetName() { 179 if (cs instanceof HistoricallyNamedCharset) 180 return ((HistoricallyNamedCharset)cs).historicalName(); 181 return cs.name(); 182 } 183 184 final String requestedCharsetName() { 185 return requestedCharsetName; 186 } 187 188 Result decode(byte[] ba, int off, int len) { 189 if (len == 0) { 190 return result.with(); 191 } 192 // fastpath for ascii compatible 193 if (isASCIICompatible && !hasNegatives(ba, off, len)) { 194 if (COMPACT_STRINGS) { 195 return result.with(Arrays.copyOfRange(ba, off, off + len), 196 LATIN1); 197 } else { 198 return result.with(StringLatin1.inflate(ba, off, len), UTF16); 199 } 200 } 201 int en = scale(len, cd.maxCharsPerByte()); 202 char[] ca = new char[en]; 203 if (cd instanceof ArrayDecoder) { 204 int clen = ((ArrayDecoder)cd).decode(ba, off, len, ca); 205 return result.with(ca, 0, clen); 206 } 207 cd.reset(); 208 ByteBuffer bb = ByteBuffer.wrap(ba, off, len); 209 CharBuffer cb = CharBuffer.wrap(ca); 210 try { 211 CoderResult cr = cd.decode(bb, cb, true); 212 if (!cr.isUnderflow()) 213 cr.throwException(); 214 cr = cd.flush(cb); 215 if (!cr.isUnderflow()) 216 cr.throwException(); 217 } catch (CharacterCodingException x) { 218 // Substitution is always enabled, 219 // so this shouldn't happen 220 throw new Error(x); 221 } 222 return result.with(ca, 0, cb.position()); 223 } 224 } 225 226 private static class StringDecoder8859_1 extends StringDecoder { 227 StringDecoder8859_1(Charset cs, String rcn) { 228 super(cs, rcn); 229 } 230 Result decode(byte[] ba, int off, int len) { 231 if (COMPACT_STRINGS) { 232 return result.with(Arrays.copyOfRange(ba, off, off + len), LATIN1); 233 } else { 234 return result.with(StringLatin1.inflate(ba, off, len), UTF16); 235 } 236 } 237 } 238 239 static Result decode(String charsetName, byte[] ba, int off, int len) 240 throws UnsupportedEncodingException 241 { 242 StringDecoder sd = deref(decoder); 243 String csn = (charsetName == null) ? "ISO-8859-1" : charsetName; 244 if ((sd == null) || !(csn.equals(sd.requestedCharsetName()) 245 || csn.equals(sd.charsetName()))) { 246 sd = null; 247 try { 248 Charset cs = lookupCharset(csn); 249 if (cs != null) { 250 if (cs == UTF_8) { 251 sd = new StringDecoderUTF8(cs, csn); 252 } else if (cs == ISO_8859_1) { 253 sd = new StringDecoder8859_1(cs, csn); 254 } else { 255 sd = new StringDecoder(cs, csn); 256 } 257 } 258 } catch (IllegalCharsetNameException x) {} 259 if (sd == null) 260 throw new UnsupportedEncodingException(csn); 261 set(decoder, sd); 262 } 263 return sd.decode(ba, off, len); 264 } 265 266 static Result decode(Charset cs, byte[] ba, int off, int len) { 267 // (1)We never cache the "external" cs, the only benefit of creating 268 // an additional StringDe/Encoder object to wrap it is to share the 269 // de/encode() method. These SD/E objects are short-lived, the young-gen 270 // gc should be able to take care of them well. But the best approach 271 // is still not to generate them if not really necessary. 272 // (2)The defensive copy of the input byte/char[] has a big performance 273 // impact, as well as the outgoing result byte/char[]. Need to do the 274 // optimization check of (sm==null && classLoader0==null) for both. 275 // (3)getClass().getClassLoader0() is expensive 276 // (4)There might be a timing gap in isTrusted setting. getClassLoader0() 277 // is only checked (and then isTrusted gets set) when (SM==null). It is 278 // possible that the SM==null for now but then SM is NOT null later 279 // when safeTrim() is invoked...the "safe" way to do is to redundant 280 // check (... && (isTrusted || SM == null || getClassLoader0())) in trim 281 // but it then can be argued that the SM is null when the operation 282 // is started... 283 if (cs == UTF_8) { 284 return StringDecoderUTF8.decode(ba, off, len, new Result()); 285 } 286 CharsetDecoder cd = cs.newDecoder(); 287 // ascii fastpath 288 if (cs == ISO_8859_1 || ((cd instanceof ArrayDecoder) && 289 ((ArrayDecoder)cd).isASCIICompatible() && 290 !hasNegatives(ba, off, len))) { 291 if (COMPACT_STRINGS) { 292 return new Result().with(Arrays.copyOfRange(ba, off, off + len), 293 LATIN1); 294 } else { 295 return new Result().with(StringLatin1.inflate(ba, off, len), UTF16); 296 } 297 } 298 int en = scale(len, cd.maxCharsPerByte()); 299 if (len == 0) { 300 return new Result().with(); 301 } 302 if (System.getSecurityManager() != null && 303 cs.getClass().getClassLoader0() != null) { 304 ba = Arrays.copyOfRange(ba, off, off + len); 305 off = 0; 306 } 307 cd.onMalformedInput(CodingErrorAction.REPLACE) 308 .onUnmappableCharacter(CodingErrorAction.REPLACE) 309 .reset(); 310 311 char[] ca = new char[en]; 312 if (cd instanceof ArrayDecoder) { 313 int clen = ((ArrayDecoder)cd).decode(ba, off, len, ca); 314 return new Result().with(ca, 0, clen); 315 } 316 ByteBuffer bb = ByteBuffer.wrap(ba, off, len); 317 CharBuffer cb = CharBuffer.wrap(ca); 318 try { 319 CoderResult cr = cd.decode(bb, cb, true); 320 if (!cr.isUnderflow()) 321 cr.throwException(); 322 cr = cd.flush(cb); 323 if (!cr.isUnderflow()) 324 cr.throwException(); 325 } catch (CharacterCodingException x) { 326 // Substitution is always enabled, 327 // so this shouldn't happen 328 throw new Error(x); 329 } 330 return new Result().with(ca, 0, cb.position()); 331 } 332 333 static Result decode(byte[] ba, int off, int len) { 334 String csn = Charset.defaultCharset().name(); 335 try { 336 // use charset name decode() variant which provides caching. 337 return decode(csn, ba, off, len); 338 } catch (UnsupportedEncodingException x) { 339 warnUnsupportedCharset(csn); 340 } 341 try { 342 return decode("ISO-8859-1", ba, off, len); 343 } catch (UnsupportedEncodingException x) { 344 // If this code is hit during VM initialization, err(String) is 345 // the only way we will be able to get any kind of error message. 346 err("ISO-8859-1 charset not available: " + x.toString() + "\n"); 347 // If we can not find ISO-8859-1 (a required encoding) then things 348 // are seriously wrong with the installation. 349 System.exit(1); 350 return null; 351 } 352 } 353 354 // -- Encoding -- 355 private static class StringEncoder { 356 private Charset cs; 357 private CharsetEncoder ce; 358 private final boolean isASCIICompatible; 359 private final String requestedCharsetName; 360 private final boolean isTrusted; 361 362 private StringEncoder(Charset cs, String rcn) { 363 this.requestedCharsetName = rcn; 364 this.cs = cs; 365 this.ce = cs.newEncoder() 366 .onMalformedInput(CodingErrorAction.REPLACE) 367 .onUnmappableCharacter(CodingErrorAction.REPLACE); 368 this.isTrusted = (cs.getClass().getClassLoader0() == null); 369 this.isASCIICompatible = (ce instanceof ArrayEncoder) && 370 ((ArrayEncoder)ce).isASCIICompatible(); 371 } 372 373 String charsetName() { 374 if (cs instanceof HistoricallyNamedCharset) 375 return ((HistoricallyNamedCharset)cs).historicalName(); 376 return cs.name(); 377 } 378 379 final String requestedCharsetName() { 380 return requestedCharsetName; 381 } 382 383 byte[] encode(byte coder, byte[] val) { 384 // fastpath for ascii compatible 385 if (coder == LATIN1 && isASCIICompatible && 386 !hasNegatives(val, 0, val.length)) { 387 return Arrays.copyOf(val, val.length); 388 } 389 int len = val.length >> coder; // assume LATIN1=0/UTF16=1; 390 int en = scale(len, ce.maxBytesPerChar()); 391 byte[] ba = new byte[en]; 392 if (len == 0) { 393 return ba; 394 } 395 if (ce instanceof ArrayEncoder) { 396 if (!isTrusted) { 397 val = Arrays.copyOf(val, val.length); 398 } 399 int blen = (coder == LATIN1 ) ? ((ArrayEncoder)ce).encodeFromLatin1(val, 0, len, ba) 400 : ((ArrayEncoder)ce).encodeFromUTF16(val, 0, len, ba); 401 if (blen != -1) { 402 return safeTrim(ba, blen, isTrusted); 403 } 404 } 405 char[] ca = (coder == LATIN1 ) ? StringLatin1.toChars(val) 406 : StringUTF16.toChars(val); 407 ce.reset(); 408 ByteBuffer bb = ByteBuffer.wrap(ba); 409 CharBuffer cb = CharBuffer.wrap(ca, 0, len); 410 try { 411 CoderResult cr = ce.encode(cb, bb, true); 412 if (!cr.isUnderflow()) 413 cr.throwException(); 414 cr = ce.flush(bb); 415 if (!cr.isUnderflow()) 416 cr.throwException(); 417 } catch (CharacterCodingException x) { 418 // Substitution is always enabled, 419 // so this shouldn't happen 420 throw new Error(x); 421 } 422 return safeTrim(ba, bb.position(), isTrusted); 423 } 424 } 425 426 @HotSpotIntrinsicCandidate 427 private static int implEncodeISOArray(byte[] sa, int sp, 428 byte[] da, int dp, int len) { 429 int i = 0; 430 for (; i < len; i++) { 431 char c = StringUTF16.getChar(sa, sp++); 432 if (c > '\u00FF') 433 break; 434 da[dp++] = (byte)c; 435 } 436 return i; 437 } 438 439 static byte[] encode8859_1(byte coder, byte[] val) { 440 if (coder == LATIN1) { 441 return Arrays.copyOf(val, val.length); 442 } 443 int len = val.length >> 1; 444 byte[] dst = new byte[len]; 445 int dp = 0; 446 int sp = 0; 447 int sl = len; 448 while (sp < sl) { 449 int ret = implEncodeISOArray(val, sp, dst, dp, len); 450 sp = sp + ret; 451 dp = dp + ret; 452 if (ret != len) { 453 char c = StringUTF16.getChar(val, sp++); 454 if (Character.isHighSurrogate(c) && sp < sl && 455 Character.isLowSurrogate(StringUTF16.getChar(val, sp))) { 456 sp++; 457 } 458 dst[dp++] = '?'; 459 len = sl - sp; 460 } 461 } 462 if (dp == dst.length) { 463 return dst; 464 } 465 return Arrays.copyOf(dst, dp); 466 } 467 468 static byte[] encodeASCII(byte coder, byte[] val) { 469 if (coder == LATIN1) { 470 byte[] dst = new byte[val.length]; 471 for (int i = 0; i < val.length; i++) { 472 if (val[i] < 0) { 473 dst[i] = '?'; 474 } else { 475 dst[i] = val[i]; 476 } 477 } 478 return dst; 479 } 480 int len = val.length >> 1; 481 byte[] dst = new byte[len]; 482 int dp = 0; 483 for (int i = 0; i < len; i++) { 484 char c = StringUTF16.getChar(val, i); 485 if (c < 0x80) { 486 dst[dp++] = (byte)c; 487 continue; 488 } 489 if (Character.isHighSurrogate(c) && i + 1 < len && 490 Character.isLowSurrogate(StringUTF16.getChar(val, i + 1))) { 491 i++; 492 } 493 dst[dp++] = '?'; 494 } 495 if (len == dp) { 496 return dst; 497 } 498 return Arrays.copyOf(dst, dp); 499 } 500 501 static byte[] encodeUTF8(byte coder, byte[] val) { 502 int dp = 0; 503 byte[] dst; 504 if (coder == LATIN1) { 505 dst = new byte[val.length << 1]; 506 for (int sp = 0; sp < val.length; sp++) { 507 byte c = val[sp]; 508 if (c < 0) { 509 dst[dp++] = (byte)(0xc0 | ((c & 0xff) >> 6)); 510 dst[dp++] = (byte)(0x80 | (c & 0x3f)); 511 } else { 512 dst[dp++] = c; 513 } 514 } 515 } else { 516 int sp = 0; 517 int sl = val.length >> 1; 518 dst = new byte[sl * 3]; 519 char c; 520 while (sp < sl && (c = StringUTF16.getChar(val, sp)) < '\u0080') { 521 // ascii fast loop; 522 dst[dp++] = (byte)c; 523 sp++; 524 } 525 while (sp < sl) { 526 c = StringUTF16.getChar(val, sp++); 527 if (c < 0x80) { 528 dst[dp++] = (byte)c; 529 } else if (c < 0x800) { 530 dst[dp++] = (byte)(0xc0 | (c >> 6)); 531 dst[dp++] = (byte)(0x80 | (c & 0x3f)); 532 } else if (Character.isSurrogate(c)) { 533 int uc = -1; 534 char c2; 535 if (Character.isHighSurrogate(c) && sp < sl && 536 Character.isLowSurrogate(c2 = StringUTF16.getChar(val, sp))) { 537 uc = Character.toCodePoint(c, c2); 538 } 539 if (uc < 0) { 540 dst[dp++] = '?'; 541 } else { 542 dst[dp++] = (byte)(0xf0 | ((uc >> 18))); 543 dst[dp++] = (byte)(0x80 | ((uc >> 12) & 0x3f)); 544 dst[dp++] = (byte)(0x80 | ((uc >> 6) & 0x3f)); 545 dst[dp++] = (byte)(0x80 | (uc & 0x3f)); 546 sp++; // 2 chars 547 } 548 } else { 549 // 3 bytes, 16 bits 550 dst[dp++] = (byte)(0xe0 | ((c >> 12))); 551 dst[dp++] = (byte)(0x80 | ((c >> 6) & 0x3f)); 552 dst[dp++] = (byte)(0x80 | (c & 0x3f)); 553 } 554 } 555 } 556 if (dp == dst.length) { 557 return dst; 558 } 559 return Arrays.copyOf(dst, dp); 560 } 561 562 static byte[] encode(String charsetName, byte coder, byte[] val) 563 throws UnsupportedEncodingException 564 { 565 StringEncoder se = deref(encoder); 566 String csn = (charsetName == null) ? "ISO-8859-1" : charsetName; 567 if ((se == null) || !(csn.equals(se.requestedCharsetName()) 568 || csn.equals(se.charsetName()))) { 569 se = null; 570 try { 571 Charset cs = lookupCharset(csn); 572 if (cs != null) { 573 if (cs == UTF_8) { 574 return encodeUTF8(coder, val); 575 } else if (cs == ISO_8859_1) { 576 return encode8859_1(coder, val); 577 } else if (cs == US_ASCII) { 578 return encodeASCII(coder, val); 579 } 580 se = new StringEncoder(cs, csn); 581 } 582 } catch (IllegalCharsetNameException x) {} 583 if (se == null) { 584 throw new UnsupportedEncodingException (csn); 585 } 586 set(encoder, se); 587 } 588 return se.encode(coder, val); 589 } 590 591 static byte[] encode(Charset cs, byte coder, byte[] val) { 592 if (cs == UTF_8) { 593 return encodeUTF8(coder, val); 594 } else if (cs == ISO_8859_1) { 595 return encode8859_1(coder, val); 596 } else if (cs == US_ASCII) { 597 return encodeASCII(coder, val); 598 } 599 CharsetEncoder ce = cs.newEncoder(); 600 // fastpath for ascii compatible 601 if (coder == LATIN1 && (((ce instanceof ArrayEncoder) && 602 ((ArrayEncoder)ce).isASCIICompatible() && 603 !hasNegatives(val, 0, val.length)))) { 604 return Arrays.copyOf(val, val.length); 605 } 606 int len = val.length >> coder; // assume LATIN1=0/UTF16=1; 607 int en = scale(len, ce.maxBytesPerChar()); 608 byte[] ba = new byte[en]; 609 if (len == 0) { 610 return ba; 611 } 612 boolean isTrusted = System.getSecurityManager() == null || 613 cs.getClass().getClassLoader0() == null; 614 ce.onMalformedInput(CodingErrorAction.REPLACE) 615 .onUnmappableCharacter(CodingErrorAction.REPLACE) 616 .reset(); 617 if (ce instanceof ArrayEncoder) { 618 if (!isTrusted) { 619 val = Arrays.copyOf(val, val.length); 620 } 621 int blen = (coder == LATIN1 ) ? ((ArrayEncoder)ce).encodeFromLatin1(val, 0, len, ba) 622 : ((ArrayEncoder)ce).encodeFromUTF16(val, 0, len, ba); 623 if (blen != -1) { 624 return safeTrim(ba, blen, isTrusted); 625 } 626 } 627 char[] ca = (coder == LATIN1 ) ? StringLatin1.toChars(val) 628 : StringUTF16.toChars(val); 629 ByteBuffer bb = ByteBuffer.wrap(ba); 630 CharBuffer cb = CharBuffer.wrap(ca, 0, len); 631 try { 632 CoderResult cr = ce.encode(cb, bb, true); 633 if (!cr.isUnderflow()) 634 cr.throwException(); 635 cr = ce.flush(bb); 636 if (!cr.isUnderflow()) 637 cr.throwException(); 638 } catch (CharacterCodingException x) { 639 throw new Error(x); 640 } 641 return safeTrim(ba, bb.position(), isTrusted); 642 } 643 644 static byte[] encode(byte coder, byte[] val) { 645 String csn = Charset.defaultCharset().name(); 646 try { 647 // use charset name encode() variant which provides caching. 648 return encode(csn, coder, val); 649 } catch (UnsupportedEncodingException x) { 650 warnUnsupportedCharset(csn); 651 } 652 try { 653 return encode("ISO-8859-1", coder, val); 654 } catch (UnsupportedEncodingException x) { 655 // If this code is hit during VM initialization, err(String) is 656 // the only way we will be able to get any kind of error message. 657 err("ISO-8859-1 charset not available: " + x.toString() + "\n"); 658 // If we can not find ISO-8859-1 (a required encoding) then things 659 // are seriously wrong with the installation. 660 System.exit(1); 661 return null; 662 } 663 } 664 665 /** 666 * Print a message directly to stderr, bypassing all character conversion 667 * methods. 668 * @param msg message to print 669 */ 670 private static native void err(String msg); 671} 672