1#include <Functions/FunctionFactory.h>
2#include <Functions/FunctionsLogical.h>
3
4#include <Columns/IColumn.h>
5#include <Columns/ColumnVector.h>
6#include <Columns/ColumnsNumber.h>
7#include <Columns/ColumnConst.h>
8#include <Columns/ColumnNullable.h>
9#include <Common/FieldVisitors.h>
10#include <Common/typeid_cast.h>
11#include <DataTypes/DataTypeNullable.h>
12#include <DataTypes/DataTypesNumber.h>
13#include <Functions/FunctionHelpers.h>
14
15#include <algorithm>
16
17
18namespace DB
19{
20
21void registerFunctionsLogical(FunctionFactory & factory)
22{
23 factory.registerFunction<FunctionAnd>();
24 factory.registerFunction<FunctionOr>();
25 factory.registerFunction<FunctionXor>();
26 factory.registerFunction<FunctionNot>();
27}
28
29namespace ErrorCodes
30{
31 extern const int LOGICAL_ERROR;
32 extern const int ILLEGAL_TYPE_OF_ARGUMENT;
33 extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
34 extern const int ILLEGAL_COLUMN;
35}
36
37namespace
38{
39using namespace FunctionsLogicalDetail;
40
41using UInt8Container = ColumnUInt8::Container;
42using UInt8ColumnPtrs = std::vector<const ColumnUInt8 *>;
43
44
45MutableColumnPtr convertFromTernaryData(const UInt8Container & ternary_data, const bool make_nullable)
46{
47 const size_t rows_count = ternary_data.size();
48
49 auto new_column = ColumnUInt8::create(rows_count);
50 std::transform(
51 ternary_data.cbegin(), ternary_data.cend(), new_column->getData().begin(),
52 [](const auto x) { return x == Ternary::True; });
53
54 if (!make_nullable)
55 return new_column;
56
57 auto null_column = ColumnUInt8::create(rows_count);
58 std::transform(
59 ternary_data.cbegin(), ternary_data.cend(), null_column->getData().begin(),
60 [](const auto x) { return x == Ternary::Null; });
61
62 return ColumnNullable::create(std::move(new_column), std::move(null_column));
63}
64
65template <typename T>
66bool tryConvertColumnToUInt8(const IColumn * column, UInt8Container & res)
67{
68 const auto col = checkAndGetColumn<ColumnVector<T>>(column);
69 if (!col)
70 return false;
71
72 std::transform(
73 col->getData().cbegin(), col->getData().cend(), res.begin(),
74 [](const auto x) { return x != 0; });
75
76 return true;
77}
78
79void convertColumnToUInt8(const IColumn * column, UInt8Container & res)
80{
81 if (!tryConvertColumnToUInt8<Int8>(column, res) &&
82 !tryConvertColumnToUInt8<Int16>(column, res) &&
83 !tryConvertColumnToUInt8<Int32>(column, res) &&
84 !tryConvertColumnToUInt8<Int64>(column, res) &&
85 !tryConvertColumnToUInt8<UInt16>(column, res) &&
86 !tryConvertColumnToUInt8<UInt32>(column, res) &&
87 !tryConvertColumnToUInt8<UInt64>(column, res) &&
88 !tryConvertColumnToUInt8<Float32>(column, res) &&
89 !tryConvertColumnToUInt8<Float64>(column, res))
90 throw Exception("Unexpected type of column: " + column->getName(), ErrorCodes::ILLEGAL_COLUMN);
91}
92
93
94template <class Op, typename Func>
95static bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res, Func && func)
96{
97 bool has_res = false;
98
99 for (int i = static_cast<int>(in.size()) - 1; i >= 0; --i)
100 {
101 if (!isColumnConst(*in[i]))
102 continue;
103
104 UInt8 x = func((*in[i])[0]);
105 if (has_res)
106 {
107 res = Op::apply(res, x);
108 }
109 else
110 {
111 res = x;
112 has_res = true;
113 }
114
115 in.erase(in.begin() + i);
116 }
117
118 return has_res;
119}
120
121template <class Op>
122inline bool extractConstColumns(ColumnRawPtrs & in, UInt8 & res)
123{
124 return extractConstColumns<Op>(
125 in, res,
126 [](const Field & value)
127 {
128 return !value.isNull() && applyVisitor(FieldVisitorConvertToNumber<bool>(), value);
129 }
130 );
131}
132
133template <class Op>
134inline bool extractConstColumnsTernary(ColumnRawPtrs & in, UInt8 & res_3v)
135{
136 return extractConstColumns<Op>(
137 in, res_3v,
138 [](const Field & value)
139 {
140 return value.isNull()
141 ? Ternary::makeValue(false, true)
142 : Ternary::makeValue(applyVisitor(FieldVisitorConvertToNumber<bool>(), value));
143 }
144 );
145}
146
147
148template <typename Op, size_t N>
149class AssociativeApplierImpl
150{
151 using ResultValueType = typename Op::ResultType;
152
153public:
154 /// Remembers the last N columns from `in`.
155 AssociativeApplierImpl(const UInt8ColumnPtrs & in)
156 : vec(in[in.size() - N]->getData()), next(in) {}
157
158 /// Returns a combination of values in the i-th row of all columns stored in the constructor.
159 inline ResultValueType apply(const size_t i) const
160 {
161 const auto & a = vec[i];
162 if constexpr (Op::isSaturable())
163 return Op::isSaturatedValue(a) ? a : Op::apply(a, next.apply(i));
164 else
165 return Op::apply(a, next.apply(i));
166 }
167
168private:
169 const UInt8Container & vec;
170 const AssociativeApplierImpl<Op, N - 1> next;
171};
172
173template <typename Op>
174class AssociativeApplierImpl<Op, 1>
175{
176 using ResultValueType = typename Op::ResultType;
177
178public:
179 AssociativeApplierImpl(const UInt8ColumnPtrs & in)
180 : vec(in[in.size() - 1]->getData()) {}
181
182 inline ResultValueType apply(const size_t i) const { return vec[i]; }
183
184private:
185 const UInt8Container & vec;
186};
187
188
189/// A helper class used by AssociativeGenericApplierImpl
190/// Allows for on-the-fly conversion of any data type into intermediate ternary representation
191using ValueGetter = std::function<Ternary::ResultType (size_t)>;
192
193template <typename ... Types>
194struct ValueGetterBuilderImpl;
195
196template <typename Type, typename ...Types>
197struct ValueGetterBuilderImpl<Type, Types...>
198{
199 static ValueGetter build(const IColumn * x)
200 {
201 if (const auto nullable_column = typeid_cast<const ColumnNullable *>(x))
202 {
203 if (const auto nested_column = typeid_cast<const ColumnVector<Type> *>(nullable_column->getNestedColumnPtr().get()))
204 {
205 return [&null_data = nullable_column->getNullMapData(), &column_data = nested_column->getData()](size_t i)
206 { return Ternary::makeValue(column_data[i], null_data[i]); };
207 }
208 else
209 return ValueGetterBuilderImpl<Types...>::build(x);
210 }
211 else if (const auto column = typeid_cast<const ColumnVector<Type> *>(x))
212 return [&column_data = column->getData()](size_t i) { return Ternary::makeValue(column_data[i]); };
213 else
214 return ValueGetterBuilderImpl<Types...>::build(x);
215 }
216};
217
218template <>
219struct ValueGetterBuilderImpl<>
220{
221 static ValueGetter build(const IColumn * x)
222 {
223 throw Exception(
224 std::string("Unknown numeric column of type: ") + demangle(typeid(x).name()),
225 ErrorCodes::LOGICAL_ERROR);
226 }
227};
228
229using ValueGetterBuilder =
230 ValueGetterBuilderImpl<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
231
232/// This class together with helper class ValueGetterBuilder can be used with columns of arbitrary data type
233/// Allows for on-the-fly conversion of any type of data into intermediate ternary representation
234/// and eliminates the need to materialize data columns in intermediate representation
235template <typename Op, size_t N>
236class AssociativeGenericApplierImpl
237{
238 using ResultValueType = typename Op::ResultType;
239
240public:
241 /// Remembers the last N columns from `in`.
242 AssociativeGenericApplierImpl(const ColumnRawPtrs & in)
243 : val_getter{ValueGetterBuilder::build(in[in.size() - N])}, next{in} {}
244
245 /// Returns a combination of values in the i-th row of all columns stored in the constructor.
246 inline ResultValueType apply(const size_t i) const
247 {
248 const auto a = val_getter(i);
249 if constexpr (Op::isSaturable())
250 return Op::isSaturatedValue(a) ? a : Op::apply(a, next.apply(i));
251 else
252 return Op::apply(a, next.apply(i));
253 }
254
255private:
256 const ValueGetter val_getter;
257 const AssociativeGenericApplierImpl<Op, N - 1> next;
258};
259
260
261template <typename Op>
262class AssociativeGenericApplierImpl<Op, 1>
263{
264 using ResultValueType = typename Op::ResultType;
265
266public:
267 /// Remembers the last N columns from `in`.
268 AssociativeGenericApplierImpl(const ColumnRawPtrs & in)
269 : val_getter{ValueGetterBuilder::build(in[in.size() - 1])} {}
270
271 inline ResultValueType apply(const size_t i) const { return val_getter(i); }
272
273private:
274 const ValueGetter val_getter;
275};
276
277
278/// Apply target function by feeding it "batches" of N columns
279/// Combining 10 columns per pass is the fastest for large block sizes.
280/// For small block sizes - more columns is faster.
281template <
282 typename Op, template <typename, size_t> typename OperationApplierImpl, size_t N = 10>
283struct OperationApplier
284{
285 template <typename Columns, typename ResultColumn>
286 static void apply(Columns & in, ResultColumn & result)
287 {
288 while (in.size() > 1)
289 {
290 doBatchedApply(in, result->getData());
291 in.push_back(result.get());
292 }
293 }
294
295 template <typename Columns, typename ResultData>
296 static void NO_INLINE doBatchedApply(Columns & in, ResultData & result_data)
297 {
298 if (N > in.size())
299 {
300 OperationApplier<Op, OperationApplierImpl, N - 1>::doBatchedApply(in, result_data);
301 return;
302 }
303
304 const OperationApplierImpl<Op, N> operationApplierImpl(in);
305 size_t i = 0;
306 for (auto & res : result_data)
307 res = operationApplierImpl.apply(i++);
308
309 in.erase(in.end() - N, in.end());
310 }
311};
312
313template <
314 typename Op, template <typename, size_t> typename OperationApplierImpl>
315struct OperationApplier<Op, OperationApplierImpl, 1>
316{
317 template <typename Columns, typename Result>
318 static void NO_INLINE doBatchedApply(Columns &, Result &)
319 {
320 throw Exception(
321 "OperationApplier<...>::apply(...): not enough arguments to run this method",
322 ErrorCodes::LOGICAL_ERROR);
323 }
324};
325
326
327template <class Op>
328static void executeForTernaryLogicImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & result_info, size_t input_rows_count)
329{
330 /// Combine all constant columns into a single constant value.
331 UInt8 const_3v_value = 0;
332 const bool has_consts = extractConstColumnsTernary<Op>(arguments, const_3v_value);
333
334 /// If the constant value uniquely determines the result, return it.
335 if (has_consts && (arguments.empty() || (Op::isSaturable() && Op::isSaturatedValue(const_3v_value))))
336 {
337 result_info.column = ColumnConst::create(
338 convertFromTernaryData(UInt8Container({const_3v_value}), result_info.type->isNullable()),
339 input_rows_count
340 );
341 return;
342 }
343
344 const auto result_column = ColumnUInt8::create(input_rows_count);
345 MutableColumnPtr const_column_holder;
346 if (has_consts)
347 {
348 const_column_holder =
349 convertFromTernaryData(UInt8Container(input_rows_count, const_3v_value), const_3v_value == Ternary::Null);
350 arguments.push_back(const_column_holder.get());
351 }
352
353 OperationApplier<Op, AssociativeGenericApplierImpl>::apply(arguments, result_column);
354
355 result_info.column = convertFromTernaryData(result_column->getData(), result_info.type->isNullable());
356}
357
358
359template <typename Op, typename ... Types>
360struct TypedExecutorInvoker;
361
362template <typename Op>
363using FastApplierImpl =
364 TypedExecutorInvoker<Op, UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
365
366template <typename Op, typename Type, typename ... Types>
367struct TypedExecutorInvoker<Op, Type, Types ...>
368{
369 template <typename T, typename Result>
370 static void apply(const ColumnVector<T> & x, const IColumn & y, Result & result)
371 {
372 if (const auto column = typeid_cast<const ColumnVector<Type> *>(&y))
373 std::transform(
374 x.getData().cbegin(), x.getData().cend(),
375 column->getData().cbegin(), result.begin(),
376 [](const auto a, const auto b) { return Op::apply(!!a, !!b); });
377 else
378 TypedExecutorInvoker<Op, Types ...>::template apply<T>(x, y, result);
379 }
380
381 template <typename Result>
382 static void apply(const IColumn & x, const IColumn & y, Result & result)
383 {
384 if (const auto column = typeid_cast<const ColumnVector<Type> *>(&x))
385 FastApplierImpl<Op>::template apply<Type>(*column, y, result);
386 else
387 TypedExecutorInvoker<Op, Types ...>::apply(x, y, result);
388 }
389};
390
391template <typename Op>
392struct TypedExecutorInvoker<Op>
393{
394 template <typename T, typename Result>
395 static void apply(const ColumnVector<T> &, const IColumn & y, Result &)
396 {
397 throw Exception(std::string("Unknown numeric column y of type: ") + demangle(typeid(y).name()), ErrorCodes::LOGICAL_ERROR);
398 }
399
400 template <typename Result>
401 static void apply(const IColumn & x, const IColumn &, Result &)
402 {
403 throw Exception(std::string("Unknown numeric column x of type: ") + demangle(typeid(x).name()), ErrorCodes::LOGICAL_ERROR);
404 }
405};
406
407
408template <class Op>
409static void basicExecuteImpl(ColumnRawPtrs arguments, ColumnWithTypeAndName & result_info, size_t input_rows_count)
410{
411 /// Combine all constant columns into a single constant value.
412 UInt8 const_val = 0;
413 bool has_consts = extractConstColumns<Op>(arguments, const_val);
414
415 /// If the constant value uniquely determines the result, return it.
416 if (has_consts && (arguments.empty() || Op::apply(const_val, 0) == Op::apply(const_val, 1)))
417 {
418 if (!arguments.empty())
419 const_val = Op::apply(const_val, 0);
420 result_info.column = DataTypeUInt8().createColumnConst(input_rows_count, toField(const_val));
421 return;
422 }
423
424 /// If the constant value is a neutral element, let's forget about it.
425 if (has_consts && Op::apply(const_val, 0) == 0 && Op::apply(const_val, 1) == 1)
426 has_consts = false;
427
428 UInt8ColumnPtrs uint8_args;
429
430 auto col_res = ColumnUInt8::create();
431 UInt8Container & vec_res = col_res->getData();
432 if (has_consts)
433 {
434 vec_res.assign(input_rows_count, const_val);
435 uint8_args.push_back(col_res.get());
436 }
437 else
438 {
439 vec_res.resize(input_rows_count);
440 }
441
442 /// FastPath detection goes in here
443 if (arguments.size() == (has_consts ? 1 : 2))
444 {
445 if (has_consts)
446 FastApplierImpl<Op>::apply(*arguments[0], *col_res, col_res->getData());
447 else
448 FastApplierImpl<Op>::apply(*arguments[0], *arguments[1], col_res->getData());
449
450 result_info.column = std::move(col_res);
451 return;
452 }
453
454 /// Convert all columns to UInt8
455 Columns converted_columns;
456 for (const IColumn * column : arguments)
457 {
458 if (auto uint8_column = checkAndGetColumn<ColumnUInt8>(column))
459 uint8_args.push_back(uint8_column);
460 else
461 {
462 auto converted_column = ColumnUInt8::create(input_rows_count);
463 convertColumnToUInt8(column, converted_column->getData());
464 uint8_args.push_back(converted_column.get());
465 converted_columns.emplace_back(std::move(converted_column));
466 }
467 }
468
469 OperationApplier<Op, AssociativeApplierImpl>::apply(uint8_args, col_res);
470
471 /// This is possible if there is exactly one non-constant among the arguments, and it is of type UInt8.
472 if (uint8_args[0] != col_res.get())
473 vec_res.assign(uint8_args[0]->getData());
474
475 result_info.column = std::move(col_res);
476}
477
478}
479
480template <typename Impl, typename Name>
481DataTypePtr FunctionAnyArityLogical<Impl, Name>::getReturnTypeImpl(const DataTypes & arguments) const
482{
483 if (arguments.size() < 2)
484 throw Exception("Number of arguments for function \"" + getName() + "\" should be at least 2: passed "
485 + toString(arguments.size()),
486 ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION);
487
488 bool has_nullable_arguments = false;
489 for (size_t i = 0; i < arguments.size(); ++i)
490 {
491 const auto & arg_type = arguments[i];
492
493 if (!has_nullable_arguments)
494 {
495 has_nullable_arguments = arg_type->isNullable();
496 if (has_nullable_arguments && !Impl::specialImplementationForNulls())
497 throw Exception("Logical error: Unexpected type of argument for function \"" + getName() + "\": "
498 " argument " + toString(i + 1) + " is of type " + arg_type->getName(), ErrorCodes::LOGICAL_ERROR);
499 }
500
501 if (!(isNativeNumber(arg_type)
502 || (Impl::specialImplementationForNulls() && (arg_type->onlyNull() || isNativeNumber(removeNullable(arg_type))))))
503 throw Exception("Illegal type ("
504 + arg_type->getName()
505 + ") of " + toString(i + 1) + " argument of function " + getName(),
506 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
507 }
508
509 auto result_type = std::make_shared<DataTypeUInt8>();
510 return has_nullable_arguments
511 ? makeNullable(result_type)
512 : result_type;
513}
514
515template <typename Impl, typename Name>
516void FunctionAnyArityLogical<Impl, Name>::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_index, size_t input_rows_count)
517{
518 ColumnRawPtrs args_in;
519 for (const auto arg_index : arguments)
520 args_in.push_back(block.getByPosition(arg_index).column.get());
521
522 auto & result_info = block.getByPosition(result_index);
523 if (result_info.type->isNullable())
524 executeForTernaryLogicImpl<Impl>(std::move(args_in), result_info, input_rows_count);
525 else
526 basicExecuteImpl<Impl>(std::move(args_in), result_info, input_rows_count);
527}
528
529
530template <typename A, typename Op>
531struct UnaryOperationImpl
532{
533 using ResultType = typename Op::ResultType;
534 using ArrayA = typename ColumnVector<A>::Container;
535 using ArrayC = typename ColumnVector<ResultType>::Container;
536
537 static void NO_INLINE vector(const ArrayA & a, ArrayC & c)
538 {
539 std::transform(
540 a.cbegin(), a.cend(), c.begin(),
541 [](const auto x) { return Op::apply(x); });
542 }
543};
544
545template <template <typename> class Impl, typename Name>
546DataTypePtr FunctionUnaryLogical<Impl, Name>::getReturnTypeImpl(const DataTypes & arguments) const
547{
548 if (!isNativeNumber(arguments[0]))
549 throw Exception("Illegal type ("
550 + arguments[0]->getName()
551 + ") of argument of function " + getName(),
552 ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
553
554 return std::make_shared<DataTypeUInt8>();
555}
556
557template <template <typename> class Impl, typename T>
558bool functionUnaryExecuteType(Block & block, const ColumnNumbers & arguments, size_t result)
559{
560 if (auto col = checkAndGetColumn<ColumnVector<T>>(block.getByPosition(arguments[0]).column.get()))
561 {
562 auto col_res = ColumnUInt8::create();
563
564 typename ColumnUInt8::Container & vec_res = col_res->getData();
565 vec_res.resize(col->getData().size());
566 UnaryOperationImpl<T, Impl<T>>::vector(col->getData(), vec_res);
567
568 block.getByPosition(result).column = std::move(col_res);
569 return true;
570 }
571
572 return false;
573}
574
575template <template <typename> class Impl, typename Name>
576void FunctionUnaryLogical<Impl, Name>::executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /*input_rows_count*/)
577{
578 if (!(functionUnaryExecuteType<Impl, UInt8>(block, arguments, result)
579 || functionUnaryExecuteType<Impl, UInt16>(block, arguments, result)
580 || functionUnaryExecuteType<Impl, UInt32>(block, arguments, result)
581 || functionUnaryExecuteType<Impl, UInt64>(block, arguments, result)
582 || functionUnaryExecuteType<Impl, Int8>(block, arguments, result)
583 || functionUnaryExecuteType<Impl, Int16>(block, arguments, result)
584 || functionUnaryExecuteType<Impl, Int32>(block, arguments, result)
585 || functionUnaryExecuteType<Impl, Int64>(block, arguments, result)
586 || functionUnaryExecuteType<Impl, Float32>(block, arguments, result)
587 || functionUnaryExecuteType<Impl, Float64>(block, arguments, result)))
588 throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
589 + " of argument of function " + getName(),
590 ErrorCodes::ILLEGAL_COLUMN);
591}
592
593}
594