1#pragma once
2
3#include <city.h>
4#include <farmhash.h>
5#include <metrohash.h>
6#include <murmurhash2.h>
7#include <murmurhash3.h>
8
9#include <Common/SipHash.h>
10#include <Common/typeid_cast.h>
11#include <Common/HashTable/Hash.h>
12
13#include "config_functions.h"
14#if USE_XXHASH
15# include <xxhash.h>
16#endif
17
18#include "config_core.h"
19#if USE_SSL
20# include <openssl/md5.h>
21# include <openssl/sha.h>
22#endif
23
24#include <Poco/ByteOrder.h>
25#include <DataTypes/DataTypesNumber.h>
26#include <DataTypes/DataTypeString.h>
27#include <DataTypes/DataTypeDate.h>
28#include <DataTypes/DataTypeDateTime.h>
29#include <DataTypes/DataTypeArray.h>
30#include <DataTypes/DataTypeFixedString.h>
31#include <DataTypes/DataTypeEnum.h>
32#include <DataTypes/DataTypeTuple.h>
33#include <Columns/ColumnsNumber.h>
34#include <Columns/ColumnString.h>
35#include <Columns/ColumnConst.h>
36#include <Columns/ColumnFixedString.h>
37#include <Columns/ColumnArray.h>
38#include <Columns/ColumnTuple.h>
39#include <Functions/IFunctionImpl.h>
40#include <Functions/FunctionHelpers.h>
41#include <ext/range.h>
42#include <ext/bit_cast.h>
43
44
45namespace DB
46{
47
48namespace ErrorCodes
49{
50 extern const int LOGICAL_ERROR;
51 extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
52 extern const int NOT_IMPLEMENTED;
53 extern const int ILLEGAL_COLUMN;
54}
55
56
57/** Hashing functions.
58 *
59 * halfMD5: String -> UInt64
60 *
61 * A faster cryptographic hash function:
62 * sipHash64: String -> UInt64
63 *
64 * Fast non-cryptographic hash function for strings:
65 * cityHash64: String -> UInt64
66 *
67 * A non-cryptographic hashes from a tuple of values of any types (uses respective function for strings and intHash64 for numbers):
68 * cityHash64: any* -> UInt64
69 * sipHash64: any* -> UInt64
70 * halfMD5: any* -> UInt64
71 *
72 * Fast non-cryptographic hash function from any integer:
73 * intHash32: number -> UInt32
74 * intHash64: number -> UInt64
75 *
76 */
77
78struct IntHash32Impl
79{
80 using ReturnType = UInt32;
81
82 static UInt32 apply(UInt64 x)
83 {
84 /// seed is taken from /dev/urandom. It allows you to avoid undesirable dependencies with hashes in different data structures.
85 return intHash32<0x75D9543DE018BF45ULL>(x);
86 }
87};
88
89struct IntHash64Impl
90{
91 using ReturnType = UInt64;
92
93 static UInt64 apply(UInt64 x)
94 {
95 return intHash64(x ^ 0x4CF2D2BAAE6DA887ULL);
96 }
97};
98
99#if USE_SSL
100struct HalfMD5Impl
101{
102 static constexpr auto name = "halfMD5";
103 using ReturnType = UInt64;
104
105 static UInt64 apply(const char * begin, size_t size)
106 {
107 union
108 {
109 unsigned char char_data[16];
110 uint64_t uint64_data;
111 } buf;
112
113 MD5_CTX ctx;
114 MD5_Init(&ctx);
115 MD5_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
116 MD5_Final(buf.char_data, &ctx);
117
118 return Poco::ByteOrder::flipBytes(static_cast<Poco::UInt64>(buf.uint64_data)); /// Compatibility with existing code. Cast need for old poco AND macos where UInt64 != uint64_t
119 }
120
121 static UInt64 combineHashes(UInt64 h1, UInt64 h2)
122 {
123 UInt64 hashes[] = {h1, h2};
124 return apply(reinterpret_cast<const char *>(hashes), 16);
125 }
126
127 /// If true, it will use intHash32 or intHash64 to hash POD types. This behaviour is intended for better performance of some functions.
128 /// Otherwise it will hash bytes in memory as a string using corresponding hash function.
129
130 static constexpr bool use_int_hash_for_pods = false;
131};
132
133struct MD5Impl
134{
135 static constexpr auto name = "MD5";
136 enum { length = 16 };
137
138 static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
139 {
140 MD5_CTX ctx;
141 MD5_Init(&ctx);
142 MD5_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
143 MD5_Final(out_char_data, &ctx);
144 }
145};
146
147struct SHA1Impl
148{
149 static constexpr auto name = "SHA1";
150 enum { length = 20 };
151
152 static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
153 {
154 SHA_CTX ctx;
155 SHA1_Init(&ctx);
156 SHA1_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
157 SHA1_Final(out_char_data, &ctx);
158 }
159};
160
161struct SHA224Impl
162{
163 static constexpr auto name = "SHA224";
164 enum { length = 28 };
165
166 static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
167 {
168 SHA256_CTX ctx;
169 SHA224_Init(&ctx);
170 SHA224_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
171 SHA224_Final(out_char_data, &ctx);
172 }
173};
174
175struct SHA256Impl
176{
177 static constexpr auto name = "SHA256";
178 enum { length = 32 };
179
180 static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
181 {
182 SHA256_CTX ctx;
183 SHA256_Init(&ctx);
184 SHA256_Update(&ctx, reinterpret_cast<const unsigned char *>(begin), size);
185 SHA256_Final(out_char_data, &ctx);
186 }
187};
188#endif
189
190struct SipHash64Impl
191{
192 static constexpr auto name = "sipHash64";
193 using ReturnType = UInt64;
194
195 static UInt64 apply(const char * begin, size_t size)
196 {
197 return sipHash64(begin, size);
198 }
199
200 static UInt64 combineHashes(UInt64 h1, UInt64 h2)
201 {
202 UInt64 hashes[] = {h1, h2};
203 return apply(reinterpret_cast<const char *>(hashes), 16);
204 }
205
206 static constexpr bool use_int_hash_for_pods = false;
207};
208
209struct SipHash128Impl
210{
211 static constexpr auto name = "sipHash128";
212 enum { length = 16 };
213
214 static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
215 {
216 sipHash128(begin, size, reinterpret_cast<char*>(out_char_data));
217 }
218};
219
220
221/** Why we need MurmurHash2?
222 * MurmurHash2 is an outdated hash function, superseded by MurmurHash3 and subsequently by CityHash, xxHash, HighwayHash.
223 * Usually there is no reason to use MurmurHash.
224 * It is needed for the cases when you already have MurmurHash in some applications and you want to reproduce it
225 * in ClickHouse as is. For example, it is needed to reproduce the behaviour
226 * for NGINX a/b testing module: https://nginx.ru/en/docs/http/ngx_http_split_clients_module.html
227 */
228struct MurmurHash2Impl32
229{
230 static constexpr auto name = "murmurHash2_32";
231
232 using ReturnType = UInt32;
233
234 static UInt32 apply(const char * data, const size_t size)
235 {
236 return MurmurHash2(data, size, 0);
237 }
238
239 static UInt32 combineHashes(UInt32 h1, UInt32 h2)
240 {
241 return IntHash32Impl::apply(h1) ^ h2;
242 }
243
244 static constexpr bool use_int_hash_for_pods = false;
245};
246
247struct MurmurHash2Impl64
248{
249 static constexpr auto name = "murmurHash2_64";
250 using ReturnType = UInt64;
251
252 static UInt64 apply(const char * data, const size_t size)
253 {
254 return MurmurHash64A(data, size, 0);
255 }
256
257 static UInt64 combineHashes(UInt64 h1, UInt64 h2)
258 {
259 return IntHash64Impl::apply(h1) ^ h2;
260 }
261
262 static constexpr bool use_int_hash_for_pods = false;
263};
264
265/// To be compatible with gcc: https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191
266struct GccMurmurHashImpl
267{
268 static constexpr auto name = "gccMurmurHash";
269 using ReturnType = UInt64;
270
271 static UInt64 apply(const char * data, const size_t size)
272 {
273 return MurmurHash64A(data, size, 0xc70f6907UL);
274 }
275
276 static UInt64 combineHashes(UInt64 h1, UInt64 h2)
277 {
278 return IntHash64Impl::apply(h1) ^ h2;
279 }
280
281 static constexpr bool use_int_hash_for_pods = false;
282};
283
284struct MurmurHash3Impl32
285{
286 static constexpr auto name = "murmurHash3_32";
287 using ReturnType = UInt32;
288
289 static UInt32 apply(const char * data, const size_t size)
290 {
291 union
292 {
293 UInt32 h;
294 char bytes[sizeof(h)];
295 };
296 MurmurHash3_x86_32(data, size, 0, bytes);
297 return h;
298 }
299
300 static UInt32 combineHashes(UInt32 h1, UInt32 h2)
301 {
302 return IntHash32Impl::apply(h1) ^ h2;
303 }
304
305 static constexpr bool use_int_hash_for_pods = false;
306};
307
308struct MurmurHash3Impl64
309{
310 static constexpr auto name = "murmurHash3_64";
311 using ReturnType = UInt64;
312
313 static UInt64 apply(const char * data, const size_t size)
314 {
315 union
316 {
317 UInt64 h[2];
318 char bytes[16];
319 };
320 MurmurHash3_x64_128(data, size, 0, bytes);
321 return h[0] ^ h[1];
322 }
323
324 static UInt64 combineHashes(UInt64 h1, UInt64 h2)
325 {
326 return IntHash64Impl::apply(h1) ^ h2;
327 }
328
329 static constexpr bool use_int_hash_for_pods = false;
330};
331
332/// http://hg.openjdk.java.net/jdk8u/jdk8u/jdk/file/478a4add975b/src/share/classes/java/lang/String.java#l1452
333/// Care should be taken to do all calculation in unsigned integers (to avoid undefined behaviour on overflow)
334/// but obtain the same result as it is done in singed integers with two's complement arithmetic.
335struct JavaHashImpl
336{
337 static constexpr auto name = "javaHash";
338 using ReturnType = Int32;
339
340 static Int32 apply(const char * data, const size_t size)
341 {
342 UInt32 h = 0;
343 for (size_t i = 0; i < size; ++i)
344 h = 31 * h + static_cast<UInt32>(static_cast<Int8>(data[i]));
345 return static_cast<Int32>(h);
346 }
347
348 static Int32 combineHashes(Int32, Int32)
349 {
350 throw Exception("Java hash is not combineable for multiple arguments", ErrorCodes::NOT_IMPLEMENTED);
351 }
352
353 static constexpr bool use_int_hash_for_pods = false;
354};
355
356struct JavaHashUTF16LEImpl
357{
358 static constexpr auto name = "javaHashUTF16LE";
359 using ReturnType = Int32;
360
361 static Int32 apply(const char * raw_data, const size_t raw_size)
362 {
363 char * data = const_cast<char *>(raw_data);
364 size_t size = raw_size;
365
366 // Remove Byte-order-mark(0xFFFE) for UTF-16LE
367 if (size >= 2 && data[0] == '\xFF' && data[1] == '\xFE')
368 {
369 data += 2;
370 size -= 2;
371 }
372
373 if (size % 2 != 0)
374 throw Exception("Arguments for javaHashUTF16LE must be in the form of UTF-16", ErrorCodes::LOGICAL_ERROR);
375
376 UInt32 h = 0;
377 for (size_t i = 0; i < size; i += 2)
378 h = 31 * h + static_cast<UInt16>(static_cast<UInt8>(data[i]) | static_cast<UInt8>(data[i + 1]) << 8);
379
380 return static_cast<Int32>(h);
381 }
382
383 static Int32 combineHashes(Int32, Int32)
384 {
385 throw Exception("Java hash is not combineable for multiple arguments", ErrorCodes::NOT_IMPLEMENTED);
386 }
387
388 static constexpr bool use_int_hash_for_pods = false;
389};
390
391/// This is just JavaHash with zeroed out sign bit.
392/// This function is used in Hive for versions before 3.0,
393/// after 3.0, Hive uses murmur-hash3.
394struct HiveHashImpl
395{
396 static constexpr auto name = "hiveHash";
397 using ReturnType = Int32;
398
399 static Int32 apply(const char * data, const size_t size)
400 {
401 return static_cast<Int32>(0x7FFFFFFF & static_cast<UInt32>(JavaHashImpl::apply(data, size)));
402 }
403
404 static Int32 combineHashes(Int32, Int32)
405 {
406 throw Exception("Hive hash is not combineable for multiple arguments", ErrorCodes::NOT_IMPLEMENTED);
407 }
408
409 static constexpr bool use_int_hash_for_pods = false;
410};
411
412struct MurmurHash3Impl128
413{
414 static constexpr auto name = "murmurHash3_128";
415 enum { length = 16 };
416
417 static void apply(const char * begin, const size_t size, unsigned char * out_char_data)
418 {
419 MurmurHash3_x64_128(begin, size, 0, out_char_data);
420 }
421};
422
423struct ImplCityHash64
424{
425 static constexpr auto name = "cityHash64";
426 using ReturnType = UInt64;
427 using uint128_t = CityHash_v1_0_2::uint128;
428
429 static auto combineHashes(UInt64 h1, UInt64 h2) { return CityHash_v1_0_2::Hash128to64(uint128_t(h1, h2)); }
430 static auto apply(const char * s, const size_t len) { return CityHash_v1_0_2::CityHash64(s, len); }
431 static constexpr bool use_int_hash_for_pods = true;
432};
433
434// see farmhash.h for definition of NAMESPACE_FOR_HASH_FUNCTIONS
435struct ImplFarmHash64
436{
437 static constexpr auto name = "farmHash64";
438 using ReturnType = UInt64;
439 using uint128_t = NAMESPACE_FOR_HASH_FUNCTIONS::uint128_t;
440
441 static auto combineHashes(UInt64 h1, UInt64 h2) { return NAMESPACE_FOR_HASH_FUNCTIONS::Hash128to64(uint128_t(h1, h2)); }
442 static auto apply(const char * s, const size_t len) { return NAMESPACE_FOR_HASH_FUNCTIONS::Hash64(s, len); }
443 static constexpr bool use_int_hash_for_pods = true;
444};
445
446struct ImplMetroHash64
447{
448 static constexpr auto name = "metroHash64";
449 using ReturnType = UInt64;
450 using uint128_t = CityHash_v1_0_2::uint128;
451
452 static auto combineHashes(UInt64 h1, UInt64 h2) { return CityHash_v1_0_2::Hash128to64(uint128_t(h1, h2)); }
453 static auto apply(const char * s, const size_t len)
454 {
455 union
456 {
457 UInt64 u64;
458 UInt8 u8[sizeof(u64)];
459 };
460
461 metrohash64_1(reinterpret_cast<const UInt8 *>(s), len, 0, u8);
462
463 return u64;
464 }
465
466 static constexpr bool use_int_hash_for_pods = true;
467};
468
469
470#if USE_XXHASH
471
472struct ImplXxHash32
473{
474 static constexpr auto name = "xxHash32";
475 using ReturnType = UInt32;
476
477 static auto apply(const char * s, const size_t len) { return XXH32(s, len, 0); }
478 /**
479 * With current implementation with more than 1 arguments it will give the results
480 * non-reproducable from outside of CH.
481 *
482 * Proper way of combining several input is to use streaming mode of hash function
483 * https://github.com/Cyan4973/xxHash/issues/114#issuecomment-334908566
484 *
485 * In common case doable by init_state / update_state / finalize_state
486 */
487 static auto combineHashes(UInt32 h1, UInt32 h2) { return IntHash32Impl::apply(h1) ^ h2; }
488
489 static constexpr bool use_int_hash_for_pods = false;
490};
491
492
493struct ImplXxHash64
494{
495 static constexpr auto name = "xxHash64";
496 using ReturnType = UInt64;
497 using uint128_t = CityHash_v1_0_2::uint128;
498
499 static auto apply(const char * s, const size_t len) { return XXH64(s, len, 0); }
500
501 /*
502 With current implementation with more than 1 arguments it will give the results
503 non-reproducable from outside of CH. (see comment on ImplXxHash32).
504 */
505 static auto combineHashes(UInt64 h1, UInt64 h2) { return CityHash_v1_0_2::Hash128to64(uint128_t(h1, h2)); }
506
507 static constexpr bool use_int_hash_for_pods = false;
508};
509
510#endif
511
512
513template <typename Impl>
514class FunctionStringHashFixedString : public IFunction
515{
516public:
517 static constexpr auto name = Impl::name;
518 static FunctionPtr create(const Context &) { return std::make_shared<FunctionStringHashFixedString>(); }
519
520 String getName() const override
521 {
522 return name;
523 }
524
525 size_t getNumberOfArguments() const override { return 1; }
526
527 DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
528 {
529 if (!isString(arguments[0]))
530 throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
531 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
532
533 return std::make_shared<DataTypeFixedString>(Impl::length);
534 }
535
536 bool useDefaultImplementationForConstants() const override { return true; }
537
538 void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
539 {
540 if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(block.getByPosition(arguments[0]).column.get()))
541 {
542 auto col_to = ColumnFixedString::create(Impl::length);
543
544 const typename ColumnString::Chars & data = col_from->getChars();
545 const typename ColumnString::Offsets & offsets = col_from->getOffsets();
546 auto & chars_to = col_to->getChars();
547 const auto size = offsets.size();
548 chars_to.resize(size * Impl::length);
549
550 ColumnString::Offset current_offset = 0;
551 for (size_t i = 0; i < size; ++i)
552 {
553 Impl::apply(
554 reinterpret_cast<const char *>(&data[current_offset]),
555 offsets[i] - current_offset - 1,
556 &chars_to[i * Impl::length]);
557
558 current_offset = offsets[i];
559 }
560
561 block.getByPosition(result).column = std::move(col_to);
562 }
563 else
564 throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
565 + " of first argument of function " + getName(),
566 ErrorCodes::ILLEGAL_COLUMN);
567 }
568};
569
570
571template <typename Impl, typename Name>
572class FunctionIntHash : public IFunction
573{
574public:
575 static constexpr auto name = Name::name;
576 static FunctionPtr create(const Context &) { return std::make_shared<FunctionIntHash>(); }
577
578private:
579 using ToType = typename Impl::ReturnType;
580
581 template <typename FromType>
582 void executeType(Block & block, const ColumnNumbers & arguments, size_t result)
583 {
584 if (auto col_from = checkAndGetColumn<ColumnVector<FromType>>(block.getByPosition(arguments[0]).column.get()))
585 {
586 auto col_to = ColumnVector<ToType>::create();
587
588 const typename ColumnVector<FromType>::Container & vec_from = col_from->getData();
589 typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
590
591 size_t size = vec_from.size();
592 vec_to.resize(size);
593 for (size_t i = 0; i < size; ++i)
594 vec_to[i] = Impl::apply(vec_from[i]);
595
596 block.getByPosition(result).column = std::move(col_to);
597 }
598 else
599 throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
600 + " of first argument of function " + Name::name,
601 ErrorCodes::ILLEGAL_COLUMN);
602 }
603
604public:
605 String getName() const override
606 {
607 return name;
608 }
609
610 size_t getNumberOfArguments() const override { return 1; }
611
612 DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
613 {
614 if (!arguments[0]->isValueRepresentedByNumber())
615 throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
616 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
617
618 return std::make_shared<DataTypeNumber<typename Impl::ReturnType>>();
619 }
620
621 bool useDefaultImplementationForConstants() const override { return true; }
622
623 void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
624 {
625 const IDataType * from_type = block.getByPosition(arguments[0]).type.get();
626 WhichDataType which(from_type);
627
628 if (which.isUInt8()) executeType<UInt8>(block, arguments, result);
629 else if (which.isUInt16()) executeType<UInt16>(block, arguments, result);
630 else if (which.isUInt32()) executeType<UInt32>(block, arguments, result);
631 else if (which.isUInt64()) executeType<UInt64>(block, arguments, result);
632 else if (which.isInt8()) executeType<Int8>(block, arguments, result);
633 else if (which.isInt16()) executeType<Int16>(block, arguments, result);
634 else if (which.isInt32()) executeType<Int32>(block, arguments, result);
635 else if (which.isInt64()) executeType<Int64>(block, arguments, result);
636 else if (which.isDate()) executeType<UInt16>(block, arguments, result);
637 else if (which.isDateTime()) executeType<UInt32>(block, arguments, result);
638 else
639 throw Exception("Illegal type " + block.getByPosition(arguments[0]).type->getName() + " of argument of function " + getName(),
640 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
641 }
642};
643
644
645template <typename Impl>
646class FunctionAnyHash : public IFunction
647{
648public:
649 static constexpr auto name = Impl::name;
650 static FunctionPtr create(const Context &) { return std::make_shared<FunctionAnyHash>(); }
651
652private:
653 using ToType = typename Impl::ReturnType;
654
655 template <typename FromType, bool first>
656 void executeIntType(const IColumn * column, typename ColumnVector<ToType>::Container & vec_to)
657 {
658 if (const ColumnVector<FromType> * col_from = checkAndGetColumn<ColumnVector<FromType>>(column))
659 {
660 const typename ColumnVector<FromType>::Container & vec_from = col_from->getData();
661 size_t size = vec_from.size();
662 for (size_t i = 0; i < size; ++i)
663 {
664 ToType h;
665
666 if constexpr (Impl::use_int_hash_for_pods)
667 {
668 if constexpr (std::is_same_v<ToType, UInt64>)
669 h = IntHash64Impl::apply(ext::bit_cast<UInt64>(vec_from[i]));
670 else
671 h = IntHash32Impl::apply(ext::bit_cast<UInt32>(vec_from[i]));
672 }
673 else
674 {
675 h = Impl::apply(reinterpret_cast<const char *>(&vec_from[i]), sizeof(vec_from[i]));
676 }
677
678 if (first)
679 vec_to[i] = h;
680 else
681 vec_to[i] = Impl::combineHashes(vec_to[i], h);
682 }
683 }
684 else if (auto col_from_const = checkAndGetColumnConst<ColumnVector<FromType>>(column))
685 {
686 auto value = col_from_const->template getValue<FromType>();
687 ToType hash;
688 if constexpr (std::is_same_v<ToType, UInt64>)
689 hash = IntHash64Impl::apply(ext::bit_cast<UInt64>(value));
690 else
691 hash = IntHash32Impl::apply(ext::bit_cast<UInt32>(value));
692
693 size_t size = vec_to.size();
694 if (first)
695 {
696 vec_to.assign(size, hash);
697 }
698 else
699 {
700 for (size_t i = 0; i < size; ++i)
701 vec_to[i] = Impl::combineHashes(vec_to[i], hash);
702 }
703 }
704 else
705 throw Exception("Illegal column " + column->getName()
706 + " of argument of function " + getName(),
707 ErrorCodes::ILLEGAL_COLUMN);
708 }
709
710 template <bool first>
711 void executeGeneric(const IColumn * column, typename ColumnVector<ToType>::Container & vec_to)
712 {
713 for (size_t i = 0, size = column->size(); i < size; ++i)
714 {
715 StringRef bytes = column->getDataAt(i);
716 const ToType h = Impl::apply(bytes.data, bytes.size);
717 if (first)
718 vec_to[i] = h;
719 else
720 vec_to[i] = Impl::combineHashes(vec_to[i], h);
721 }
722 }
723
724 template <bool first>
725 void executeString(const IColumn * column, typename ColumnVector<ToType>::Container & vec_to)
726 {
727 if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(column))
728 {
729 const typename ColumnString::Chars & data = col_from->getChars();
730 const typename ColumnString::Offsets & offsets = col_from->getOffsets();
731 size_t size = offsets.size();
732
733 ColumnString::Offset current_offset = 0;
734 for (size_t i = 0; i < size; ++i)
735 {
736 const ToType h = Impl::apply(
737 reinterpret_cast<const char *>(&data[current_offset]),
738 offsets[i] - current_offset - 1);
739
740 if (first)
741 vec_to[i] = h;
742 else
743 vec_to[i] = Impl::combineHashes(vec_to[i], h);
744
745 current_offset = offsets[i];
746 }
747 }
748 else if (const ColumnFixedString * col_from_fixed = checkAndGetColumn<ColumnFixedString>(column))
749 {
750 const typename ColumnString::Chars & data = col_from_fixed->getChars();
751 size_t n = col_from_fixed->getN();
752 size_t size = data.size() / n;
753
754 for (size_t i = 0; i < size; ++i)
755 {
756 const ToType h = Impl::apply(reinterpret_cast<const char *>(&data[i * n]), n);
757 if (first)
758 vec_to[i] = h;
759 else
760 vec_to[i] = Impl::combineHashes(vec_to[i], h);
761 }
762 }
763 else if (const ColumnConst * col_from_const = checkAndGetColumnConstStringOrFixedString(column))
764 {
765 String value = col_from_const->getValue<String>().data();
766 const ToType hash = Impl::apply(value.data(), value.size());
767 const size_t size = vec_to.size();
768
769 if (first)
770 {
771 vec_to.assign(size, hash);
772 }
773 else
774 {
775 for (size_t i = 0; i < size; ++i)
776 {
777 vec_to[i] = Impl::combineHashes(vec_to[i], hash);
778 }
779 }
780 }
781 else
782 throw Exception("Illegal column " + column->getName()
783 + " of first argument of function " + getName(),
784 ErrorCodes::ILLEGAL_COLUMN);
785 }
786
787 template <bool first>
788 void executeArray(const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to)
789 {
790 const IDataType * nested_type = typeid_cast<const DataTypeArray *>(type)->getNestedType().get();
791
792 if (const ColumnArray * col_from = checkAndGetColumn<ColumnArray>(column))
793 {
794 const IColumn * nested_column = &col_from->getData();
795 const ColumnArray::Offsets & offsets = col_from->getOffsets();
796 const size_t nested_size = nested_column->size();
797
798 typename ColumnVector<ToType>::Container vec_temp(nested_size);
799 executeAny<true>(nested_type, nested_column, vec_temp);
800
801 const size_t size = offsets.size();
802
803 ColumnArray::Offset current_offset = 0;
804 for (size_t i = 0; i < size; ++i)
805 {
806 ColumnArray::Offset next_offset = offsets[i];
807
808 ToType h;
809 if constexpr (std::is_same_v<ToType, UInt64>)
810 h = IntHash64Impl::apply(next_offset - current_offset);
811 else
812 h = IntHash32Impl::apply(next_offset - current_offset);
813
814 if (first)
815 vec_to[i] = h;
816 else
817 vec_to[i] = Impl::combineHashes(vec_to[i], h);
818
819 for (size_t j = current_offset; j < next_offset; ++j)
820 vec_to[i] = Impl::combineHashes(vec_to[i], vec_temp[j]);
821
822 current_offset = offsets[i];
823 }
824 }
825 else if (const ColumnConst * col_from_const = checkAndGetColumnConst<ColumnArray>(column))
826 {
827 /// NOTE: here, of course, you can do without the materialization of the column.
828 ColumnPtr full_column = col_from_const->convertToFullColumn();
829 executeArray<first>(type, &*full_column, vec_to);
830 }
831 else
832 throw Exception("Illegal column " + column->getName()
833 + " of first argument of function " + getName(),
834 ErrorCodes::ILLEGAL_COLUMN);
835 }
836
837 template <bool first>
838 void executeAny(const IDataType * from_type, const IColumn * icolumn, typename ColumnVector<ToType>::Container & vec_to)
839 {
840 WhichDataType which(from_type);
841
842 if (which.isUInt8()) executeIntType<UInt8, first>(icolumn, vec_to);
843 else if (which.isUInt16()) executeIntType<UInt16, first>(icolumn, vec_to);
844 else if (which.isUInt32()) executeIntType<UInt32, first>(icolumn, vec_to);
845 else if (which.isUInt64()) executeIntType<UInt64, first>(icolumn, vec_to);
846 else if (which.isInt8()) executeIntType<Int8, first>(icolumn, vec_to);
847 else if (which.isInt16()) executeIntType<Int16, first>(icolumn, vec_to);
848 else if (which.isInt32()) executeIntType<Int32, first>(icolumn, vec_to);
849 else if (which.isInt64()) executeIntType<Int64, first>(icolumn, vec_to);
850 else if (which.isEnum8()) executeIntType<Int8, first>(icolumn, vec_to);
851 else if (which.isEnum16()) executeIntType<Int16, first>(icolumn, vec_to);
852 else if (which.isDate()) executeIntType<UInt16, first>(icolumn, vec_to);
853 else if (which.isDateTime()) executeIntType<UInt32, first>(icolumn, vec_to);
854 else if (which.isFloat32()) executeIntType<Float32, first>(icolumn, vec_to);
855 else if (which.isFloat64()) executeIntType<Float64, first>(icolumn, vec_to);
856 else if (which.isString()) executeString<first>(icolumn, vec_to);
857 else if (which.isFixedString()) executeString<first>(icolumn, vec_to);
858 else if (which.isArray()) executeArray<first>(from_type, icolumn, vec_to);
859 else
860 executeGeneric<first>(icolumn, vec_to);
861 }
862
863 void executeForArgument(const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, bool & is_first)
864 {
865 /// Flattening of tuples.
866 if (const ColumnTuple * tuple = typeid_cast<const ColumnTuple *>(column))
867 {
868 const auto & tuple_columns = tuple->getColumns();
869 const DataTypes & tuple_types = typeid_cast<const DataTypeTuple &>(*type).getElements();
870 size_t tuple_size = tuple_columns.size();
871 for (size_t i = 0; i < tuple_size; ++i)
872 executeForArgument(tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first);
873 }
874 else if (const ColumnTuple * tuple_const = checkAndGetColumnConstData<ColumnTuple>(column))
875 {
876 const auto & tuple_columns = tuple_const->getColumns();
877 const DataTypes & tuple_types = typeid_cast<const DataTypeTuple &>(*type).getElements();
878 size_t tuple_size = tuple_columns.size();
879 for (size_t i = 0; i < tuple_size; ++i)
880 {
881 auto tmp = ColumnConst::create(tuple_columns[i], column->size());
882 executeForArgument(tuple_types[i].get(), tmp.get(), vec_to, is_first);
883 }
884 }
885 else
886 {
887 if (is_first)
888 executeAny<true>(type, column, vec_to);
889 else
890 executeAny<false>(type, column, vec_to);
891 }
892
893 is_first = false;
894 }
895
896public:
897 String getName() const override
898 {
899 return name;
900 }
901
902 bool isVariadic() const override { return true; }
903 size_t getNumberOfArguments() const override { return 0; }
904 bool useDefaultImplementationForConstants() const override { return true; }
905
906 DataTypePtr getReturnTypeImpl(const DataTypes & /*arguments*/) const override
907 {
908 return std::make_shared<DataTypeNumber<ToType>>();
909 }
910
911 void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
912 {
913 size_t rows = input_rows_count;
914 auto col_to = ColumnVector<ToType>::create(rows);
915
916 typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
917
918 if (arguments.empty())
919 {
920 /// Constant random number from /dev/urandom is used as a hash value of empty list of arguments.
921 vec_to.assign(rows, static_cast<ToType>(0xe28dbde7fe22e41c));
922 }
923
924 /// The function supports arbitrary number of arguments of arbitrary types.
925
926 bool is_first_argument = true;
927 for (size_t i = 0; i < arguments.size(); ++i)
928 {
929 const ColumnWithTypeAndName & col = block.getByPosition(arguments[i]);
930 executeForArgument(col.type.get(), col.column.get(), vec_to, is_first_argument);
931 }
932
933 block.getByPosition(result).column = std::move(col_to);
934 }
935};
936
937
938struct URLHashImpl
939{
940 static UInt64 apply(const char * data, const size_t size)
941 {
942 /// do not take last slash, '?' or '#' character into account
943 if (size > 0 && (data[size - 1] == '/' || data[size - 1] == '?' || data[size - 1] == '#'))
944 return CityHash_v1_0_2::CityHash64(data, size - 1);
945
946 return CityHash_v1_0_2::CityHash64(data, size);
947 }
948};
949
950
951struct URLHierarchyHashImpl
952{
953 static size_t findLevelLength(const UInt64 level, const char * begin, const char * end)
954 {
955 auto pos = begin;
956
957 /// Let's parse everything that goes before the path
958
959 /// Suppose that the protocol has already been changed to lowercase.
960 while (pos < end && ((*pos > 'a' && *pos < 'z') || (*pos > '0' && *pos < '9')))
961 ++pos;
962
963 /** We will calculate the hierarchy only for URLs in which there is a protocol, and after it there are two slashes.
964 * (http, file - fit, mailto, magnet - do not fit), and after two slashes there is still something
965 * For the rest, simply return the full URL as the only element of the hierarchy.
966 */
967 if (pos == begin || pos == end || !(*pos++ == ':' && pos < end && *pos++ == '/' && pos < end && *pos++ == '/' && pos < end))
968 {
969 pos = end;
970 return 0 == level ? pos - begin : 0;
971 }
972
973 /// The domain for simplicity is everything that after the protocol and the two slashes, until the next slash or before `?` or `#`
974 while (pos < end && !(*pos == '/' || *pos == '?' || *pos == '#'))
975 ++pos;
976
977 if (pos != end)
978 ++pos;
979
980 if (0 == level)
981 return pos - begin;
982
983 UInt64 current_level = 0;
984
985 while (current_level != level && pos < end)
986 {
987 /// We go to the next `/` or `?` or `#`, skipping all at the beginning.
988 while (pos < end && (*pos == '/' || *pos == '?' || *pos == '#'))
989 ++pos;
990 if (pos == end)
991 break;
992 while (pos < end && !(*pos == '/' || *pos == '?' || *pos == '#'))
993 ++pos;
994
995 if (pos != end)
996 ++pos;
997
998 ++current_level;
999 }
1000
1001 return current_level == level ? pos - begin : 0;
1002 }
1003
1004 static UInt64 apply(const UInt64 level, const char * data, const size_t size)
1005 {
1006 return URLHashImpl::apply(data, findLevelLength(level, data, data + size));
1007 }
1008};
1009
1010
1011class FunctionURLHash : public IFunction
1012{
1013public:
1014 static constexpr auto name = "URLHash";
1015 static FunctionPtr create(const Context &) { return std::make_shared<FunctionURLHash>(); }
1016
1017 String getName() const override { return name; }
1018
1019 bool isVariadic() const override { return true; }
1020 size_t getNumberOfArguments() const override { return 0; }
1021
1022 DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
1023 {
1024 const auto arg_count = arguments.size();
1025 if (arg_count != 1 && arg_count != 2)
1026 throw Exception{"Number of arguments for function " + getName() + " doesn't match: passed " +
1027 toString(arg_count) + ", should be 1 or 2.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH};
1028
1029 const auto first_arg = arguments.front().get();
1030 if (!WhichDataType(first_arg).isString())
1031 throw Exception{"Illegal type " + first_arg->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
1032
1033 if (arg_count == 2)
1034 {
1035 const auto & second_arg = arguments.back();
1036 if (!isInteger(second_arg))
1037 throw Exception{"Illegal type " + second_arg->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
1038 }
1039
1040 return std::make_shared<DataTypeUInt64>();
1041 }
1042
1043 bool useDefaultImplementationForConstants() const override { return true; }
1044 ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
1045
1046 void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/) override
1047 {
1048 const auto arg_count = arguments.size();
1049
1050 if (arg_count == 1)
1051 executeSingleArg(block, arguments, result);
1052 else if (arg_count == 2)
1053 executeTwoArgs(block, arguments, result);
1054 else
1055 throw Exception{"got into IFunction::execute with unexpected number of arguments", ErrorCodes::LOGICAL_ERROR};
1056 }
1057
1058private:
1059 void executeSingleArg(Block & block, const ColumnNumbers & arguments, const size_t result) const
1060 {
1061 const auto col_untyped = block.getByPosition(arguments.front()).column.get();
1062
1063 if (const auto col_from = checkAndGetColumn<ColumnString>(col_untyped))
1064 {
1065 const auto size = col_from->size();
1066 auto col_to = ColumnUInt64::create(size);
1067
1068 const auto & chars = col_from->getChars();
1069 const auto & offsets = col_from->getOffsets();
1070 auto & out = col_to->getData();
1071
1072 ColumnString::Offset current_offset = 0;
1073 for (size_t i = 0; i < size; ++i)
1074 {
1075 out[i] = URLHashImpl::apply(
1076 reinterpret_cast<const char *>(&chars[current_offset]),
1077 offsets[i] - current_offset - 1);
1078
1079 current_offset = offsets[i];
1080 }
1081
1082 block.getByPosition(result).column = std::move(col_to);
1083 }
1084 else
1085 throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() +
1086 " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
1087 }
1088
1089 void executeTwoArgs(Block & block, const ColumnNumbers & arguments, const size_t result) const
1090 {
1091 const auto level_col = block.getByPosition(arguments.back()).column.get();
1092 if (!isColumnConst(*level_col))
1093 throw Exception{"Second argument of function " + getName() + " must be an integral constant", ErrorCodes::ILLEGAL_COLUMN};
1094
1095 const auto level = level_col->get64(0);
1096
1097 const auto col_untyped = block.getByPosition(arguments.front()).column.get();
1098 if (const auto col_from = checkAndGetColumn<ColumnString>(col_untyped))
1099 {
1100 const auto size = col_from->size();
1101 auto col_to = ColumnUInt64::create(size);
1102
1103 const auto & chars = col_from->getChars();
1104 const auto & offsets = col_from->getOffsets();
1105 auto & out = col_to->getData();
1106
1107 ColumnString::Offset current_offset = 0;
1108 for (size_t i = 0; i < size; ++i)
1109 {
1110 out[i] = URLHierarchyHashImpl::apply(
1111 level,
1112 reinterpret_cast<const char *>(&chars[current_offset]),
1113 offsets[i] - current_offset - 1);
1114
1115 current_offset = offsets[i];
1116 }
1117
1118 block.getByPosition(result).column = std::move(col_to);
1119 }
1120 else
1121 throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() +
1122 " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};
1123 }
1124};
1125
1126
1127struct NameIntHash32 { static constexpr auto name = "intHash32"; };
1128struct NameIntHash64 { static constexpr auto name = "intHash64"; };
1129
1130#if USE_SSL
1131using FunctionHalfMD5 = FunctionAnyHash<HalfMD5Impl>;
1132#endif
1133using FunctionSipHash64 = FunctionAnyHash<SipHash64Impl>;
1134using FunctionIntHash32 = FunctionIntHash<IntHash32Impl, NameIntHash32>;
1135using FunctionIntHash64 = FunctionIntHash<IntHash64Impl, NameIntHash64>;
1136#if USE_SSL
1137using FunctionMD5 = FunctionStringHashFixedString<MD5Impl>;
1138using FunctionSHA1 = FunctionStringHashFixedString<SHA1Impl>;
1139using FunctionSHA224 = FunctionStringHashFixedString<SHA224Impl>;
1140using FunctionSHA256 = FunctionStringHashFixedString<SHA256Impl>;
1141#endif
1142using FunctionSipHash128 = FunctionStringHashFixedString<SipHash128Impl>;
1143using FunctionCityHash64 = FunctionAnyHash<ImplCityHash64>;
1144using FunctionFarmHash64 = FunctionAnyHash<ImplFarmHash64>;
1145using FunctionMetroHash64 = FunctionAnyHash<ImplMetroHash64>;
1146using FunctionMurmurHash2_32 = FunctionAnyHash<MurmurHash2Impl32>;
1147using FunctionMurmurHash2_64 = FunctionAnyHash<MurmurHash2Impl64>;
1148using FunctionGccMurmurHash = FunctionAnyHash<GccMurmurHashImpl>;
1149using FunctionMurmurHash3_32 = FunctionAnyHash<MurmurHash3Impl32>;
1150using FunctionMurmurHash3_64 = FunctionAnyHash<MurmurHash3Impl64>;
1151using FunctionMurmurHash3_128 = FunctionStringHashFixedString<MurmurHash3Impl128>;
1152using FunctionJavaHash = FunctionAnyHash<JavaHashImpl>;
1153using FunctionJavaHashUTF16LE = FunctionAnyHash<JavaHashUTF16LEImpl>;
1154using FunctionHiveHash = FunctionAnyHash<HiveHashImpl>;
1155
1156#if USE_XXHASH
1157 using FunctionXxHash32 = FunctionAnyHash<ImplXxHash32>;
1158 using FunctionXxHash64 = FunctionAnyHash<ImplXxHash64>;
1159#endif
1160
1161}
1162