1//
2// StreamTokenizerTest.cpp
3//
4// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH.
5// and Contributors.
6//
7// SPDX-License-Identifier: BSL-1.0
8//
9
10
11#include "StreamTokenizerTest.h"
12#include "Poco/CppUnit/TestCaller.h"
13#include "Poco/CppUnit/TestSuite.h"
14#include "Poco/StreamTokenizer.h"
15#include "Poco/Token.h"
16#include "Poco/Ascii.h"
17#include <sstream>
18
19
20using Poco::StreamTokenizer;
21using Poco::Token;
22using Poco::InvalidToken;
23using Poco::EOFToken;
24using Poco::WhitespaceToken;
25using Poco::Ascii;
26
27
28class IdentifierToken: public Token
29{
30public:
31 IdentifierToken()
32 {
33 }
34
35 ~IdentifierToken()
36 {
37 }
38
39 Class tokenClass() const
40 {
41 return Token::IDENTIFIER_TOKEN;
42 }
43
44 bool start(char c, std::istream& istr)
45 {
46 if (c != -1 && Ascii::isAlpha(c))
47 {
48 _value = c;
49 return true;
50 }
51 else return false;
52 }
53
54 void finish(std::istream& istr)
55 {
56 int c = istr.peek();
57 while (c != -1 && Ascii::isAlphaNumeric(c))
58 {
59 istr.get();
60 _value += c;
61 c = istr.peek();
62 }
63 }
64};
65
66
67class IntLiteralToken: public Token
68{
69public:
70 IntLiteralToken()
71 {
72 }
73
74 ~IntLiteralToken()
75 {
76 }
77
78 Class tokenClass() const
79 {
80 return Token::INTEGER_LITERAL_TOKEN;
81 }
82
83 bool start(char c, std::istream& istr)
84 {
85 if (c != -1 && Ascii::isDigit(c))
86 {
87 _value = c;
88 return true;
89 }
90 else return false;
91 }
92
93 void finish(std::istream& istr)
94 {
95 int c = istr.peek();
96 while (c != -1 && Ascii::isDigit(c))
97 {
98 istr.get();
99 _value += c;
100 c = istr.peek();
101 }
102 }
103};
104
105
106StreamTokenizerTest::StreamTokenizerTest(const std::string& rName): CppUnit::TestCase(rName)
107{
108}
109
110
111StreamTokenizerTest::~StreamTokenizerTest()
112{
113}
114
115
116void StreamTokenizerTest::testTokenizer1()
117{
118 std::string data = "";
119 std::istringstream istr(data);
120 StreamTokenizer tokenizer(istr);
121 tokenizer.addToken(new WhitespaceToken());
122 tokenizer.addToken(new IdentifierToken());
123 tokenizer.addToken(new IntLiteralToken());
124
125 const Token* next = tokenizer.next();
126 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
127}
128
129
130void StreamTokenizerTest::testTokenizer2()
131{
132 std::string data = "foo";
133 std::istringstream istr(data);
134 StreamTokenizer tokenizer(istr);
135 tokenizer.addToken(new WhitespaceToken());
136 tokenizer.addToken(new IdentifierToken());
137 tokenizer.addToken(new IntLiteralToken());
138
139 const Token* next = tokenizer.next();
140 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
141 assertTrue (next->tokenString() == "foo");
142
143 next = tokenizer.next();
144 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
145}
146
147
148void StreamTokenizerTest::testTokenizer3()
149{
150 std::string data = "foo bar";
151 std::istringstream istr(data);
152 StreamTokenizer tokenizer(istr);
153 tokenizer.addToken(new WhitespaceToken());
154 tokenizer.addToken(new IdentifierToken());
155 tokenizer.addToken(new IntLiteralToken());
156
157 const Token* next = tokenizer.next();
158 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
159 assertTrue (next->tokenString() == "foo");
160
161 next = tokenizer.next();
162 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
163 assertTrue (next->tokenString() == "bar");
164
165 next = tokenizer.next();
166 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
167}
168
169
170void StreamTokenizerTest::testTokenizer4()
171{
172 std::string data = "foo 123";
173 std::istringstream istr(data);
174 StreamTokenizer tokenizer(istr);
175 tokenizer.addToken(new WhitespaceToken());
176 tokenizer.addToken(new IdentifierToken());
177 tokenizer.addToken(new IntLiteralToken());
178
179 const Token* next = tokenizer.next();
180 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
181 assertTrue (next->tokenString() == "foo");
182
183 next = tokenizer.next();
184 assertTrue (next->tokenClass() == Token::INTEGER_LITERAL_TOKEN);
185 assertTrue (next->asInteger() == 123);
186
187 next = tokenizer.next();
188 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
189}
190
191
192void StreamTokenizerTest::testTokenizer5()
193{
194 std::string data = "foo # 123";
195 std::istringstream istr(data);
196 StreamTokenizer tokenizer(istr);
197 tokenizer.addToken(new WhitespaceToken());
198 tokenizer.addToken(new IdentifierToken());
199 tokenizer.addToken(new IntLiteralToken());
200
201 const Token* next = tokenizer.next();
202 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
203 assertTrue (next->tokenString() == "foo");
204
205 next = tokenizer.next();
206 assertTrue (next->tokenClass() == Token::INVALID_TOKEN);
207 assertTrue (next->tokenString() == "#");
208
209 next = tokenizer.next();
210 assertTrue (next->tokenClass() == Token::INTEGER_LITERAL_TOKEN);
211 assertTrue (next->asInteger() == 123);
212
213 next = tokenizer.next();
214 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
215}
216
217
218void StreamTokenizerTest::testTokenizer6()
219{
220 std::string data = "foo 123 #";
221 std::istringstream istr(data);
222 StreamTokenizer tokenizer(istr);
223 tokenizer.addToken(new WhitespaceToken());
224 tokenizer.addToken(new IdentifierToken());
225 tokenizer.addToken(new IntLiteralToken());
226
227 const Token* next = tokenizer.next();
228 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
229 assertTrue (next->tokenString() == "foo");
230
231 next = tokenizer.next();
232 assertTrue (next->tokenClass() == Token::INTEGER_LITERAL_TOKEN);
233 assertTrue (next->asInteger() == 123);
234
235 next = tokenizer.next();
236 assertTrue (next->tokenClass() == Token::INVALID_TOKEN);
237 assertTrue (next->tokenString() == "#");
238
239 next = tokenizer.next();
240 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
241}
242
243
244void StreamTokenizerTest::testTokenizer7()
245{
246 std::string data = " foo 123 ";
247 std::istringstream istr(data);
248 StreamTokenizer tokenizer(istr);
249 tokenizer.addToken(new WhitespaceToken());
250 tokenizer.addToken(new IdentifierToken());
251 tokenizer.addToken(new IntLiteralToken());
252
253 const Token* next = tokenizer.next();
254 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
255 assertTrue (next->tokenString() == "foo");
256
257 next = tokenizer.next();
258 assertTrue (next->tokenClass() == Token::INTEGER_LITERAL_TOKEN);
259 assertTrue (next->asInteger() == 123);
260
261 next = tokenizer.next();
262 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
263}
264
265
266void StreamTokenizerTest::testTokenizer8()
267{
268 std::string data = " foo 123 ";
269 std::istringstream istr(data);
270 StreamTokenizer tokenizer(istr);
271 Token* whitespace = new WhitespaceToken();
272 tokenizer.addToken(whitespace);
273 tokenizer.addToken(new IdentifierToken());
274 tokenizer.addToken(new IntLiteralToken());
275
276 const Token* next = tokenizer.next();
277 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
278 assertTrue (next->tokenString() == "foo");
279
280 whitespace->ignore(false);
281 next = tokenizer.next();
282 assertTrue (next->tokenClass() == Token::WHITESPACE_TOKEN);
283 assertTrue (next->asString() == " ");
284
285 next = tokenizer.next();
286 assertTrue (next->tokenClass() == Token::INTEGER_LITERAL_TOKEN);
287 assertTrue (next->asInteger() == 123);
288
289 next = tokenizer.next();
290 assertTrue (next->tokenClass() == Token::WHITESPACE_TOKEN);
291 assertTrue (next->asString() == " ");
292
293 next = tokenizer.next();
294 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
295}
296
297
298void StreamTokenizerTest::testTokenizer9()
299{
300 std::string data = " foo 123 ";
301 std::istringstream istr(data);
302 StreamTokenizer tokenizer(istr);
303 Token* whitespace = new WhitespaceToken();
304 tokenizer.addToken(whitespace);
305 tokenizer.addToken(new IdentifierToken());
306 tokenizer.addToken(new IntLiteralToken());
307
308 const Token* next = tokenizer.next();
309 assertTrue (next->tokenClass() == Token::IDENTIFIER_TOKEN);
310 assertTrue (next->tokenString() == "foo");
311
312 whitespace->ignore(false);
313 next = tokenizer.next();
314 assertTrue (next->tokenClass() == Token::WHITESPACE_TOKEN);
315 assertTrue (next->asString() == " ");
316
317 next = tokenizer.next();
318 assertTrue (next->tokenClass() == Token::INTEGER_LITERAL_TOKEN);
319 assertTrue (next->asInteger() == 123);
320
321 whitespace->ignore(true);
322 next = tokenizer.next();
323 assertTrue (next->tokenClass() == Token::EOF_TOKEN);
324}
325
326void StreamTokenizerTest::setUp()
327{
328}
329
330
331void StreamTokenizerTest::tearDown()
332{
333}
334
335
336CppUnit::Test* StreamTokenizerTest::suite()
337{
338 CppUnit::TestSuite* pSuite = new CppUnit::TestSuite("StreamTokenizerTest");
339
340 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer1);
341 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer2);
342 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer3);
343 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer4);
344 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer5);
345 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer6);
346 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer7);
347 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer8);
348 CppUnit_addTest(pSuite, StreamTokenizerTest, testTokenizer9);
349
350 return pSuite;
351}
352