1 | // |
2 | // Token.h |
3 | // |
4 | // Library: Foundation |
5 | // Package: Streams |
6 | // Module: StreamTokenizer |
7 | // |
8 | // Definition of the Token class. |
9 | // |
10 | // Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. |
11 | // and Contributors. |
12 | // |
13 | // SPDX-License-Identifier: BSL-1.0 |
14 | // |
15 | |
16 | |
17 | #ifndef Foundation_Token_INCLUDED |
18 | #define Foundation_Token_INCLUDED |
19 | |
20 | |
21 | #include "Poco/Foundation.h" |
22 | #include <istream> |
23 | |
24 | |
25 | namespace Poco { |
26 | |
27 | |
28 | class Foundation_API Token |
29 | /// The base class for all token classes that can be |
30 | /// registered with the StreamTokenizer. |
31 | { |
32 | public: |
33 | enum Class |
34 | { |
35 | IDENTIFIER_TOKEN, |
36 | KEYWORD_TOKEN, |
37 | SEPARATOR_TOKEN, |
38 | OPERATOR_TOKEN, |
39 | STRING_LITERAL_TOKEN, |
40 | CHAR_LITERAL_TOKEN, |
41 | INTEGER_LITERAL_TOKEN, |
42 | LONG_INTEGER_LITERAL_TOKEN, |
43 | FLOAT_LITERAL_TOKEN, |
44 | DOUBLE_LITERAL_TOKEN, |
45 | , |
46 | , |
47 | PREPROCESSOR_TOKEN, |
48 | WHITESPACE_TOKEN, |
49 | EOF_TOKEN, |
50 | INVALID_TOKEN, |
51 | USER_TOKEN |
52 | }; |
53 | |
54 | Token(bool ignore = false); |
55 | /// Creates the Token. |
56 | |
57 | virtual ~Token(); |
58 | /// Destroys the Token. |
59 | |
60 | virtual bool start(char c, std::istream& istr); |
61 | /// Checks if the given character (and, optionally, |
62 | /// the next character in the input stream) start |
63 | /// a valid token. Returns true if so, false |
64 | /// otherwise. |
65 | /// |
66 | /// The current read position in istr must not be |
67 | /// changed. In other words, only the peek() method |
68 | /// of istream may be used. |
69 | /// |
70 | /// If the character starts the token, it should |
71 | /// be set as the token's value. |
72 | |
73 | virtual void finish(std::istream& istr); |
74 | /// Builds the token by reading and appending |
75 | /// the remaining characters from istr. |
76 | |
77 | virtual Class tokenClass() const; |
78 | /// Returns the kind of the token. |
79 | |
80 | const std::string& tokenString() const; |
81 | /// Returns the token's raw string. |
82 | |
83 | virtual std::string asString() const; |
84 | /// Returns a string representation of the token. |
85 | |
86 | #if defined(POCO_HAVE_INT64) |
87 | virtual Int64 asInteger64() const; |
88 | /// Returns a 64-bit integer representation of the token. |
89 | |
90 | virtual UInt64 asUnsignedInteger64() const; |
91 | /// Returns an unsigned 64-bit integer representation of the token. |
92 | #endif |
93 | |
94 | virtual int asInteger() const; |
95 | /// Returns an integer representation of the token. |
96 | |
97 | virtual unsigned asUnsignedInteger() const; |
98 | /// Returns an unsigned integer representation of the token. |
99 | |
100 | virtual double asFloat() const; |
101 | /// Returns a floating-point representation of the token. |
102 | |
103 | virtual char asChar() const; |
104 | /// Returns a char representation of the token. |
105 | |
106 | bool is(Class tokenClass) const; |
107 | /// Returns true iff the token has the given class. |
108 | |
109 | void ignore(bool ignored); |
110 | /// If ignored is true, the token will be marked |
111 | /// as ignorable, which means that next() will |
112 | /// not return it. |
113 | /// If ignored is false, the token will be marked |
114 | /// as acceptable, which means that next() will |
115 | /// return it. |
116 | |
117 | bool ignored() const; |
118 | /// return if the token is ignored or not |
119 | |
120 | protected: |
121 | std::string _value; |
122 | bool _ignored; |
123 | |
124 | private: |
125 | Token(const Token&); |
126 | Token& operator = (const Token&); |
127 | }; |
128 | |
129 | |
130 | class Foundation_API InvalidToken: public Token |
131 | /// This token class is used for signaling that |
132 | /// an invalid character sequence has been encountered |
133 | /// in the input stream. |
134 | { |
135 | public: |
136 | InvalidToken(); |
137 | ~InvalidToken(); |
138 | Class tokenClass() const; |
139 | }; |
140 | |
141 | |
142 | class Foundation_API EOFToken: public Token |
143 | /// This token class is used to signal the |
144 | /// end of the input stream. |
145 | { |
146 | public: |
147 | EOFToken(); |
148 | ~EOFToken(); |
149 | Class tokenClass() const; |
150 | }; |
151 | |
152 | |
153 | class Foundation_API WhitespaceToken: public Token |
154 | /// This pseudo token class is used to eat |
155 | /// up whitespace in between real tokens. |
156 | { |
157 | public: |
158 | WhitespaceToken(); |
159 | ~WhitespaceToken(); |
160 | Class tokenClass() const; |
161 | bool start(char c, std::istream& istr); |
162 | void finish(std::istream& istr); |
163 | }; |
164 | |
165 | |
166 | // |
167 | // inlines |
168 | // |
169 | inline const std::string& Token::tokenString() const |
170 | { |
171 | return _value; |
172 | } |
173 | |
174 | |
175 | inline bool Token::is(Token::Class cls) const |
176 | { |
177 | return tokenClass() == cls; |
178 | } |
179 | |
180 | |
181 | } // namespace Poco |
182 | |
183 | |
184 | #endif // Foundation_Token_INCLUDED |
185 | |