1 | #pragma once |
---|---|
2 | |
3 | #include <vector> |
4 | #include <Parsers/Lexer.h> |
5 | |
6 | |
7 | namespace DB |
8 | { |
9 | |
10 | /** Parser operates on lazy stream of tokens. |
11 | * It could do lookaheads of any depth. |
12 | */ |
13 | |
14 | /** Used as an input for parsers. |
15 | * All whitespace and comment tokens are transparently skipped. |
16 | */ |
17 | class Tokens |
18 | { |
19 | private: |
20 | std::vector<Token> data; |
21 | Lexer lexer; |
22 | |
23 | public: |
24 | Tokens(const char * begin, const char * end, size_t max_query_size = 0) : lexer(begin, end, max_query_size) {} |
25 | |
26 | const Token & operator[] (size_t index) |
27 | { |
28 | while (true) |
29 | { |
30 | if (index < data.size()) |
31 | return data[index]; |
32 | |
33 | if (!data.empty() && data.back().isEnd()) |
34 | return data.back(); |
35 | |
36 | Token token = lexer.nextToken(); |
37 | |
38 | if (token.isSignificant()) |
39 | data.emplace_back(token); |
40 | } |
41 | } |
42 | |
43 | const Token & max() |
44 | { |
45 | if (data.empty()) |
46 | return (*this)[0]; |
47 | return data.back(); |
48 | } |
49 | }; |
50 | |
51 | |
52 | /// To represent position in a token stream. |
53 | class TokenIterator |
54 | { |
55 | private: |
56 | Tokens * tokens; |
57 | size_t index = 0; |
58 | |
59 | public: |
60 | explicit TokenIterator(Tokens & tokens_) : tokens(&tokens_) {} |
61 | |
62 | const Token & get() { return (*tokens)[index]; } |
63 | const Token & operator*() { return get(); } |
64 | const Token * operator->() { return &get(); } |
65 | |
66 | TokenIterator & operator++() { ++index; return *this; } |
67 | TokenIterator & operator--() { --index; return *this; } |
68 | |
69 | bool operator< (const TokenIterator & rhs) const { return index < rhs.index; } |
70 | bool operator<= (const TokenIterator & rhs) const { return index <= rhs.index; } |
71 | bool operator== (const TokenIterator & rhs) const { return index == rhs.index; } |
72 | bool operator!= (const TokenIterator & rhs) const { return index != rhs.index; } |
73 | |
74 | bool isValid() { return get().type < TokenType::EndOfStream; } |
75 | |
76 | /// Rightmost token we had looked. |
77 | const Token & max() { return tokens->max(); } |
78 | }; |
79 | |
80 | |
81 | /// Returns positions of unmatched parentheses. |
82 | using UnmatchedParentheses = std::vector<Token>; |
83 | UnmatchedParentheses checkUnmatchedParentheses(TokenIterator begin, Token * last); |
84 | |
85 | } |
86 |