1#pragma once
2
3#include <vector>
4#include <Parsers/Lexer.h>
5
6
7namespace DB
8{
9
10/** Parser operates on lazy stream of tokens.
11 * It could do lookaheads of any depth.
12 */
13
14/** Used as an input for parsers.
15 * All whitespace and comment tokens are transparently skipped.
16 */
17class Tokens
18{
19private:
20 std::vector<Token> data;
21 Lexer lexer;
22
23public:
24 Tokens(const char * begin, const char * end, size_t max_query_size = 0) : lexer(begin, end, max_query_size) {}
25
26 const Token & operator[] (size_t index)
27 {
28 while (true)
29 {
30 if (index < data.size())
31 return data[index];
32
33 if (!data.empty() && data.back().isEnd())
34 return data.back();
35
36 Token token = lexer.nextToken();
37
38 if (token.isSignificant())
39 data.emplace_back(token);
40 }
41 }
42
43 const Token & max()
44 {
45 if (data.empty())
46 return (*this)[0];
47 return data.back();
48 }
49};
50
51
52/// To represent position in a token stream.
53class TokenIterator
54{
55private:
56 Tokens * tokens;
57 size_t index = 0;
58
59public:
60 explicit TokenIterator(Tokens & tokens_) : tokens(&tokens_) {}
61
62 const Token & get() { return (*tokens)[index]; }
63 const Token & operator*() { return get(); }
64 const Token * operator->() { return &get(); }
65
66 TokenIterator & operator++() { ++index; return *this; }
67 TokenIterator & operator--() { --index; return *this; }
68
69 bool operator< (const TokenIterator & rhs) const { return index < rhs.index; }
70 bool operator<= (const TokenIterator & rhs) const { return index <= rhs.index; }
71 bool operator== (const TokenIterator & rhs) const { return index == rhs.index; }
72 bool operator!= (const TokenIterator & rhs) const { return index != rhs.index; }
73
74 bool isValid() { return get().type < TokenType::EndOfStream; }
75
76 /// Rightmost token we had looked.
77 const Token & max() { return tokens->max(); }
78};
79
80
81/// Returns positions of unmatched parentheses.
82using UnmatchedParentheses = std::vector<Token>;
83UnmatchedParentheses checkUnmatchedParentheses(TokenIterator begin, Token * last);
84
85}
86