1 | #pragma once |
2 | |
3 | #include <Core/Types.h> |
4 | #include <Common/BitHelpers.h> |
5 | |
6 | #ifdef __SSE2__ |
7 | #include <emmintrin.h> |
8 | #endif |
9 | |
10 | |
11 | namespace DB |
12 | { |
13 | |
14 | |
15 | namespace UTF8 |
16 | { |
17 | |
18 | static const UInt8 CONTINUATION_OCTET_MASK = 0b11000000u; |
19 | static const UInt8 CONTINUATION_OCTET = 0b10000000u; |
20 | |
21 | /// return true if `octet` binary repr starts with 10 (octet is a UTF-8 sequence continuation) |
22 | inline bool isContinuationOctet(const UInt8 octet) |
23 | { |
24 | return (octet & CONTINUATION_OCTET_MASK) == CONTINUATION_OCTET; |
25 | } |
26 | |
27 | /// moves `s` backward until either first non-continuation octet or begin |
28 | inline void syncBackward(const UInt8 * & s, const UInt8 * const begin) |
29 | { |
30 | while (isContinuationOctet(*s) && s > begin) |
31 | --s; |
32 | } |
33 | |
34 | /// moves `s` forward until either first non-continuation octet or string end is met |
35 | inline void syncForward(const UInt8 * & s, const UInt8 * const end) |
36 | { |
37 | while (s < end && isContinuationOctet(*s)) |
38 | ++s; |
39 | } |
40 | |
41 | /// returns UTF-8 code point sequence length judging by it's first octet |
42 | inline size_t seqLength(const UInt8 first_octet) |
43 | { |
44 | if (first_octet < 0x80u) |
45 | return 1; |
46 | |
47 | const size_t bits = 8; |
48 | const auto first_zero = bitScanReverse(static_cast<UInt8>(~first_octet)); |
49 | |
50 | return bits - 1 - first_zero; |
51 | } |
52 | |
53 | inline size_t countCodePoints(const UInt8 * data, size_t size) |
54 | { |
55 | size_t res = 0; |
56 | const auto end = data + size; |
57 | |
58 | #ifdef __SSE2__ |
59 | constexpr auto bytes_sse = sizeof(__m128i); |
60 | const auto src_end_sse = data + size / bytes_sse * bytes_sse; |
61 | |
62 | const auto threshold = _mm_set1_epi8(0xBF); |
63 | |
64 | for (; data < src_end_sse; data += bytes_sse) |
65 | res += __builtin_popcount(_mm_movemask_epi8( |
66 | _mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(data)), threshold))); |
67 | #endif |
68 | |
69 | for (; data < end; ++data) /// Skip UTF-8 continuation bytes. |
70 | res += static_cast<Int8>(*data) > static_cast<Int8>(0xBF); |
71 | |
72 | return res; |
73 | } |
74 | |
75 | /// returns UTF-8 wcswidth. Invalid sequence is treated as zero width character. |
76 | /// `prefix` is used to compute the `\t` width which extends the string before |
77 | /// and include `\t` to the nearest longer length with multiple of eight. |
78 | size_t computeWidth(const UInt8 * data, size_t size, size_t prefix = 0) noexcept; |
79 | |
80 | } |
81 | |
82 | |
83 | } |
84 | |