1//
2// Token.h
3//
4// Library: Foundation
5// Package: Streams
6// Module: StreamTokenizer
7//
8// Definition of the Token class.
9//
10// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH.
11// and Contributors.
12//
13// SPDX-License-Identifier: BSL-1.0
14//
15
16
17#ifndef Foundation_Token_INCLUDED
18#define Foundation_Token_INCLUDED
19
20
21#include "Poco/Foundation.h"
22#include <istream>
23
24
25namespace Poco {
26
27
28class Foundation_API Token
29 /// The base class for all token classes that can be
30 /// registered with the StreamTokenizer.
31{
32public:
33 enum Class
34 {
35 IDENTIFIER_TOKEN,
36 KEYWORD_TOKEN,
37 SEPARATOR_TOKEN,
38 OPERATOR_TOKEN,
39 STRING_LITERAL_TOKEN,
40 CHAR_LITERAL_TOKEN,
41 INTEGER_LITERAL_TOKEN,
42 LONG_INTEGER_LITERAL_TOKEN,
43 FLOAT_LITERAL_TOKEN,
44 DOUBLE_LITERAL_TOKEN,
45 COMMENT_TOKEN,
46 SPECIAL_COMMENT_TOKEN,
47 PREPROCESSOR_TOKEN,
48 WHITESPACE_TOKEN,
49 EOF_TOKEN,
50 INVALID_TOKEN,
51 USER_TOKEN
52 };
53
54 Token(bool ignore = false);
55 /// Creates the Token.
56
57 virtual ~Token();
58 /// Destroys the Token.
59
60 virtual bool start(char c, std::istream& istr);
61 /// Checks if the given character (and, optionally,
62 /// the next character in the input stream) start
63 /// a valid token. Returns true if so, false
64 /// otherwise.
65 ///
66 /// The current read position in istr must not be
67 /// changed. In other words, only the peek() method
68 /// of istream may be used.
69 ///
70 /// If the character starts the token, it should
71 /// be set as the token's value.
72
73 virtual void finish(std::istream& istr);
74 /// Builds the token by reading and appending
75 /// the remaining characters from istr.
76
77 virtual Class tokenClass() const;
78 /// Returns the kind of the token.
79
80 const std::string& tokenString() const;
81 /// Returns the token's raw string.
82
83 virtual std::string asString() const;
84 /// Returns a string representation of the token.
85
86#if defined(POCO_HAVE_INT64)
87 virtual Int64 asInteger64() const;
88 /// Returns a 64-bit integer representation of the token.
89
90 virtual UInt64 asUnsignedInteger64() const;
91 /// Returns an unsigned 64-bit integer representation of the token.
92#endif
93
94 virtual int asInteger() const;
95 /// Returns an integer representation of the token.
96
97 virtual unsigned asUnsignedInteger() const;
98 /// Returns an unsigned integer representation of the token.
99
100 virtual double asFloat() const;
101 /// Returns a floating-point representation of the token.
102
103 virtual char asChar() const;
104 /// Returns a char representation of the token.
105
106 bool is(Class tokenClass) const;
107 /// Returns true iff the token has the given class.
108
109 void ignore(bool ignored);
110 /// If ignored is true, the token will be marked
111 /// as ignorable, which means that next() will
112 /// not return it.
113 /// If ignored is false, the token will be marked
114 /// as acceptable, which means that next() will
115 /// return it.
116
117 bool ignored() const;
118 /// return if the token is ignored or not
119
120protected:
121 std::string _value;
122 bool _ignored;
123
124private:
125 Token(const Token&);
126 Token& operator = (const Token&);
127};
128
129
130class Foundation_API InvalidToken: public Token
131 /// This token class is used for signaling that
132 /// an invalid character sequence has been encountered
133 /// in the input stream.
134{
135public:
136 InvalidToken();
137 ~InvalidToken();
138 Class tokenClass() const;
139};
140
141
142class Foundation_API EOFToken: public Token
143 /// This token class is used to signal the
144 /// end of the input stream.
145{
146public:
147 EOFToken();
148 ~EOFToken();
149 Class tokenClass() const;
150};
151
152
153class Foundation_API WhitespaceToken: public Token
154 /// This pseudo token class is used to eat
155 /// up whitespace in between real tokens.
156{
157public:
158 WhitespaceToken();
159 ~WhitespaceToken();
160 Class tokenClass() const;
161 bool start(char c, std::istream& istr);
162 void finish(std::istream& istr);
163};
164
165
166//
167// inlines
168//
169inline const std::string& Token::tokenString() const
170{
171 return _value;
172}
173
174
175inline bool Token::is(Token::Class cls) const
176{
177 return tokenClass() == cls;
178}
179
180
181} // namespace Poco
182
183
184#endif // Foundation_Token_INCLUDED
185