1#pragma once
2
3#include <Core/Types.h>
4#include <Common/BitHelpers.h>
5
6#ifdef __SSE2__
7#include <emmintrin.h>
8#endif
9
10
11namespace DB
12{
13
14
15namespace UTF8
16{
17
18static const UInt8 CONTINUATION_OCTET_MASK = 0b11000000u;
19static const UInt8 CONTINUATION_OCTET = 0b10000000u;
20
21/// return true if `octet` binary repr starts with 10 (octet is a UTF-8 sequence continuation)
22inline bool isContinuationOctet(const UInt8 octet)
23{
24 return (octet & CONTINUATION_OCTET_MASK) == CONTINUATION_OCTET;
25}
26
27/// moves `s` backward until either first non-continuation octet or begin
28inline void syncBackward(const UInt8 * & s, const UInt8 * const begin)
29{
30 while (isContinuationOctet(*s) && s > begin)
31 --s;
32}
33
34/// moves `s` forward until either first non-continuation octet or string end is met
35inline void syncForward(const UInt8 * & s, const UInt8 * const end)
36{
37 while (s < end && isContinuationOctet(*s))
38 ++s;
39}
40
41/// returns UTF-8 code point sequence length judging by it's first octet
42inline size_t seqLength(const UInt8 first_octet)
43{
44 if (first_octet < 0x80u)
45 return 1;
46
47 const size_t bits = 8;
48 const auto first_zero = bitScanReverse(static_cast<UInt8>(~first_octet));
49
50 return bits - 1 - first_zero;
51}
52
53inline size_t countCodePoints(const UInt8 * data, size_t size)
54{
55 size_t res = 0;
56 const auto end = data + size;
57
58#ifdef __SSE2__
59 constexpr auto bytes_sse = sizeof(__m128i);
60 const auto src_end_sse = data + size / bytes_sse * bytes_sse;
61
62 const auto threshold = _mm_set1_epi8(0xBF);
63
64 for (; data < src_end_sse; data += bytes_sse)
65 res += __builtin_popcount(_mm_movemask_epi8(
66 _mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast<const __m128i *>(data)), threshold)));
67#endif
68
69 for (; data < end; ++data) /// Skip UTF-8 continuation bytes.
70 res += static_cast<Int8>(*data) > static_cast<Int8>(0xBF);
71
72 return res;
73}
74
75/// returns UTF-8 wcswidth. Invalid sequence is treated as zero width character.
76/// `prefix` is used to compute the `\t` width which extends the string before
77/// and include `\t` to the nearest longer length with multiple of eight.
78size_t computeWidth(const UInt8 * data, size_t size, size_t prefix = 0) noexcept;
79
80}
81
82
83}
84