1/* 2 * Copyright (C) 2011, 2012 Apple Inc. All rights reserved. 3 * Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). 4 * 5 * This library is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU Library General Public 7 * License as published by the Free Software Foundation; either 8 * version 2 of the License, or (at your option) any later version. 9 * 10 * This library is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 * Library General Public License for more details. 14 * 15 * You should have received a copy of the GNU Library General Public License 16 * along with this library; see the file COPYING.LIB. If not, write to 17 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 18 * Boston, MA 02110-1301, USA. 19 * 20 */ 21 22#ifndef ASCIIFastPath_h 23#define ASCIIFastPath_h 24 25#include <stdint.h> 26#include <unicode/utypes.h> 27#include <wtf/StdLibExtras.h> 28#include <wtf/text/LChar.h> 29 30#if OS(DARWIN) && (CPU(X86) || CPU(X86_64)) 31#include <emmintrin.h> 32#endif 33 34namespace WTF { 35 36template <uintptr_t mask> 37inline bool isAlignedTo(const void* pointer) 38{ 39 return !(reinterpret_cast<uintptr_t>(pointer) & mask); 40} 41 42// Assuming that a pointer is the size of a "machine word", then 43// uintptr_t is an integer type that is also a machine word. 44typedef uintptr_t MachineWord; 45const uintptr_t machineWordAlignmentMask = sizeof(MachineWord) - 1; 46 47inline bool isAlignedToMachineWord(const void* pointer) 48{ 49 return isAlignedTo<machineWordAlignmentMask>(pointer); 50} 51 52template<typename T> inline T* alignToMachineWord(T* pointer) 53{ 54 return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(pointer) & ~machineWordAlignmentMask); 55} 56 57template<size_t size, typename CharacterType> struct NonASCIIMask; 58template<> struct NonASCIIMask<4, UChar> { 59 static inline uint32_t value() { return 0xFF80FF80U; } 60}; 61template<> struct NonASCIIMask<4, LChar> { 62 static inline uint32_t value() { return 0x80808080U; } 63}; 64template<> struct NonASCIIMask<8, UChar> { 65 static inline uint64_t value() { return 0xFF80FF80FF80FF80ULL; } 66}; 67template<> struct NonASCIIMask<8, LChar> { 68 static inline uint64_t value() { return 0x8080808080808080ULL; } 69}; 70 71 72template<typename CharacterType> 73inline bool isAllASCII(MachineWord word) 74{ 75 return !(word & NonASCIIMask<sizeof(MachineWord), CharacterType>::value()); 76} 77 78// Note: This function assume the input is likely all ASCII, and 79// does not leave early if it is not the case. 80template<typename CharacterType> 81inline bool charactersAreAllASCII(const CharacterType* characters, size_t length) 82{ 83 MachineWord allCharBits = 0; 84 const CharacterType* end = characters + length; 85 86 // Prologue: align the input. 87 while (!isAlignedToMachineWord(characters) && characters != end) { 88 allCharBits |= *characters; 89 ++characters; 90 } 91 92 // Compare the values of CPU word size. 93 const CharacterType* wordEnd = alignToMachineWord(end); 94 const size_t loopIncrement = sizeof(MachineWord) / sizeof(CharacterType); 95 while (characters < wordEnd) { 96 allCharBits |= *(reinterpret_cast_ptr<const MachineWord*>(characters)); 97 characters += loopIncrement; 98 } 99 100 // Process the remaining bytes. 101 while (characters != end) { 102 allCharBits |= *characters; 103 ++characters; 104 } 105 106 MachineWord nonASCIIBitMask = NonASCIIMask<sizeof(MachineWord), CharacterType>::value(); 107 return !(allCharBits & nonASCIIBitMask); 108} 109 110inline void copyLCharsFromUCharSource(LChar* destination, const UChar* source, size_t length) 111{ 112#if OS(DARWIN) && (CPU(X86) || CPU(X86_64)) 113 const uintptr_t memoryAccessSize = 16; // Memory accesses on 16 byte (128 bit) alignment 114 const uintptr_t memoryAccessMask = memoryAccessSize - 1; 115 116 size_t i = 0; 117 for (;i < length && !isAlignedTo<memoryAccessMask>(&source[i]); ++i) { 118 ASSERT(!(source[i] & 0xff00)); 119 destination[i] = static_cast<LChar>(source[i]); 120 } 121 122 const uintptr_t sourceLoadSize = 32; // Process 32 bytes (16 UChars) each iteration 123 const size_t ucharsPerLoop = sourceLoadSize / sizeof(UChar); 124 if (length > ucharsPerLoop) { 125 const size_t endLength = length - ucharsPerLoop + 1; 126 for (; i < endLength; i += ucharsPerLoop) { 127#ifndef NDEBUG 128 for (unsigned checkIndex = 0; checkIndex < ucharsPerLoop; ++checkIndex) 129 ASSERT(!(source[i+checkIndex] & 0xff00)); 130#endif 131 __m128i first8UChars = _mm_load_si128(reinterpret_cast<const __m128i*>(&source[i])); 132 __m128i second8UChars = _mm_load_si128(reinterpret_cast<const __m128i*>(&source[i+8])); 133 __m128i packedChars = _mm_packus_epi16(first8UChars, second8UChars); 134 _mm_storeu_si128(reinterpret_cast<__m128i*>(&destination[i]), packedChars); 135 } 136 } 137 138 for (; i < length; ++i) { 139 ASSERT(!(source[i] & 0xff00)); 140 destination[i] = static_cast<LChar>(source[i]); 141 } 142#elif COMPILER(GCC) && CPU(ARM64) && defined(NDEBUG) 143 const LChar* const end = destination + length; 144 const uintptr_t memoryAccessSize = 16; 145 146 if (length >= memoryAccessSize) { 147 const uintptr_t memoryAccessMask = memoryAccessSize - 1; 148 149 // Vector interleaved unpack, we only store the lower 8 bits. 150 const uintptr_t lengthLeft = end - destination; 151 const LChar* const simdEnd = destination + (lengthLeft & ~memoryAccessMask); 152 do { 153 asm("ld2 { v0.16B, v1.16B }, [%[SOURCE]], #32\n\t" 154 "st1 { v0.16B }, [%[DESTINATION]], #16\n\t" 155 : [SOURCE]"+r" (source), [DESTINATION]"+r" (destination) 156 : 157 : "memory", "v0", "v1"); 158 } while (destination != simdEnd); 159 } 160 161 while (destination != end) 162 *destination++ = static_cast<LChar>(*source++); 163#elif COMPILER(GCC) && CPU(ARM_NEON) && !(CPU(BIG_ENDIAN) || CPU(MIDDLE_ENDIAN)) && defined(NDEBUG) 164 const LChar* const end = destination + length; 165 const uintptr_t memoryAccessSize = 8; 166 167 if (length >= (2 * memoryAccessSize) - 1) { 168 // Prefix: align dst on 64 bits. 169 const uintptr_t memoryAccessMask = memoryAccessSize - 1; 170 while (!isAlignedTo<memoryAccessMask>(destination)) 171 *destination++ = static_cast<LChar>(*source++); 172 173 // Vector interleaved unpack, we only store the lower 8 bits. 174 const uintptr_t lengthLeft = end - destination; 175 const LChar* const simdEnd = end - (lengthLeft % memoryAccessSize); 176 do { 177 asm("vld2.8 { d0-d1 }, [%[SOURCE]] !\n\t" 178 "vst1.8 { d0 }, [%[DESTINATION],:64] !\n\t" 179 : [SOURCE]"+r" (source), [DESTINATION]"+r" (destination) 180 : 181 : "memory", "d0", "d1"); 182 } while (destination != simdEnd); 183 } 184 185 while (destination != end) 186 *destination++ = static_cast<LChar>(*source++); 187#else 188 for (size_t i = 0; i < length; ++i) { 189 ASSERT(!(source[i] & 0xff00)); 190 destination[i] = static_cast<LChar>(source[i]); 191 } 192#endif 193} 194 195} // namespace WTF 196 197#endif // ASCIIFastPath_h 198