1/*
2 * Copyright (C) 2011, 2012 Apple Inc. All rights reserved.
3 * Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
4 *
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
9 *
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13 * Library General Public License for more details.
14 *
15 * You should have received a copy of the GNU Library General Public License
16 * along with this library; see the file COPYING.LIB.  If not, write to
17 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
19 *
20 */
21
22#ifndef ASCIIFastPath_h
23#define ASCIIFastPath_h
24
25#include <stdint.h>
26#include <unicode/utypes.h>
27#include <wtf/StdLibExtras.h>
28#include <wtf/text/LChar.h>
29
30#if OS(DARWIN) && (CPU(X86) || CPU(X86_64))
31#include <emmintrin.h>
32#endif
33
34namespace WTF {
35
36template <uintptr_t mask>
37inline bool isAlignedTo(const void* pointer)
38{
39    return !(reinterpret_cast<uintptr_t>(pointer) & mask);
40}
41
42// Assuming that a pointer is the size of a "machine word", then
43// uintptr_t is an integer type that is also a machine word.
44typedef uintptr_t MachineWord;
45const uintptr_t machineWordAlignmentMask = sizeof(MachineWord) - 1;
46
47inline bool isAlignedToMachineWord(const void* pointer)
48{
49    return isAlignedTo<machineWordAlignmentMask>(pointer);
50}
51
52template<typename T> inline T* alignToMachineWord(T* pointer)
53{
54    return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(pointer) & ~machineWordAlignmentMask);
55}
56
57template<size_t size, typename CharacterType> struct NonASCIIMask;
58template<> struct NonASCIIMask<4, UChar> {
59    static inline uint32_t value() { return 0xFF80FF80U; }
60};
61template<> struct NonASCIIMask<4, LChar> {
62    static inline uint32_t value() { return 0x80808080U; }
63};
64template<> struct NonASCIIMask<8, UChar> {
65    static inline uint64_t value() { return 0xFF80FF80FF80FF80ULL; }
66};
67template<> struct NonASCIIMask<8, LChar> {
68    static inline uint64_t value() { return 0x8080808080808080ULL; }
69};
70
71
72template<typename CharacterType>
73inline bool isAllASCII(MachineWord word)
74{
75    return !(word & NonASCIIMask<sizeof(MachineWord), CharacterType>::value());
76}
77
78// Note: This function assume the input is likely all ASCII, and
79// does not leave early if it is not the case.
80template<typename CharacterType>
81inline bool charactersAreAllASCII(const CharacterType* characters, size_t length)
82{
83    MachineWord allCharBits = 0;
84    const CharacterType* end = characters + length;
85
86    // Prologue: align the input.
87    while (!isAlignedToMachineWord(characters) && characters != end) {
88        allCharBits |= *characters;
89        ++characters;
90    }
91
92    // Compare the values of CPU word size.
93    const CharacterType* wordEnd = alignToMachineWord(end);
94    const size_t loopIncrement = sizeof(MachineWord) / sizeof(CharacterType);
95    while (characters < wordEnd) {
96        allCharBits |= *(reinterpret_cast_ptr<const MachineWord*>(characters));
97        characters += loopIncrement;
98    }
99
100    // Process the remaining bytes.
101    while (characters != end) {
102        allCharBits |= *characters;
103        ++characters;
104    }
105
106    MachineWord nonASCIIBitMask = NonASCIIMask<sizeof(MachineWord), CharacterType>::value();
107    return !(allCharBits & nonASCIIBitMask);
108}
109
110inline void copyLCharsFromUCharSource(LChar* destination, const UChar* source, size_t length)
111{
112#if OS(DARWIN) && (CPU(X86) || CPU(X86_64))
113    const uintptr_t memoryAccessSize = 16; // Memory accesses on 16 byte (128 bit) alignment
114    const uintptr_t memoryAccessMask = memoryAccessSize - 1;
115
116    size_t i = 0;
117    for (;i < length && !isAlignedTo<memoryAccessMask>(&source[i]); ++i) {
118        ASSERT(!(source[i] & 0xff00));
119        destination[i] = static_cast<LChar>(source[i]);
120    }
121
122    const uintptr_t sourceLoadSize = 32; // Process 32 bytes (16 UChars) each iteration
123    const size_t ucharsPerLoop = sourceLoadSize / sizeof(UChar);
124    if (length > ucharsPerLoop) {
125        const size_t endLength = length - ucharsPerLoop + 1;
126        for (; i < endLength; i += ucharsPerLoop) {
127#ifndef NDEBUG
128            for (unsigned checkIndex = 0; checkIndex < ucharsPerLoop; ++checkIndex)
129                ASSERT(!(source[i+checkIndex] & 0xff00));
130#endif
131            __m128i first8UChars = _mm_load_si128(reinterpret_cast<const __m128i*>(&source[i]));
132            __m128i second8UChars = _mm_load_si128(reinterpret_cast<const __m128i*>(&source[i+8]));
133            __m128i packedChars = _mm_packus_epi16(first8UChars, second8UChars);
134            _mm_storeu_si128(reinterpret_cast<__m128i*>(&destination[i]), packedChars);
135        }
136    }
137
138    for (; i < length; ++i) {
139        ASSERT(!(source[i] & 0xff00));
140        destination[i] = static_cast<LChar>(source[i]);
141    }
142#elif COMPILER(GCC) && CPU(ARM64) && defined(NDEBUG)
143    const LChar* const end = destination + length;
144    const uintptr_t memoryAccessSize = 16;
145
146    if (length >= memoryAccessSize) {
147        const uintptr_t memoryAccessMask = memoryAccessSize - 1;
148
149        // Vector interleaved unpack, we only store the lower 8 bits.
150        const uintptr_t lengthLeft = end - destination;
151        const LChar* const simdEnd = destination + (lengthLeft & ~memoryAccessMask);
152        do {
153            asm("ld2   { v0.16B, v1.16B }, [%[SOURCE]], #32\n\t"
154                "st1   { v0.16B }, [%[DESTINATION]], #16\n\t"
155                : [SOURCE]"+r" (source), [DESTINATION]"+r" (destination)
156                :
157                : "memory", "v0", "v1");
158        } while (destination != simdEnd);
159    }
160
161    while (destination != end)
162        *destination++ = static_cast<LChar>(*source++);
163#elif COMPILER(GCC) && CPU(ARM_NEON) && !(CPU(BIG_ENDIAN) || CPU(MIDDLE_ENDIAN)) && defined(NDEBUG)
164    const LChar* const end = destination + length;
165    const uintptr_t memoryAccessSize = 8;
166
167    if (length >= (2 * memoryAccessSize) - 1) {
168        // Prefix: align dst on 64 bits.
169        const uintptr_t memoryAccessMask = memoryAccessSize - 1;
170        while (!isAlignedTo<memoryAccessMask>(destination))
171            *destination++ = static_cast<LChar>(*source++);
172
173        // Vector interleaved unpack, we only store the lower 8 bits.
174        const uintptr_t lengthLeft = end - destination;
175        const LChar* const simdEnd = end - (lengthLeft % memoryAccessSize);
176        do {
177            asm("vld2.8   { d0-d1 }, [%[SOURCE]] !\n\t"
178                "vst1.8   { d0 }, [%[DESTINATION],:64] !\n\t"
179                : [SOURCE]"+r" (source), [DESTINATION]"+r" (destination)
180                :
181                : "memory", "d0", "d1");
182        } while (destination != simdEnd);
183    }
184
185    while (destination != end)
186        *destination++ = static_cast<LChar>(*source++);
187#else
188    for (size_t i = 0; i < length; ++i) {
189        ASSERT(!(source[i] & 0xff00));
190        destination[i] = static_cast<LChar>(source[i]);
191    }
192#endif
193}
194
195} // namespace WTF
196
197#endif // ASCIIFastPath_h
198