1//
2// StreamTokenizer.cpp
3//
4// Library: Foundation
5// Package: Streams
6// Module: StreamTokenizer
7//
8// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH.
9// and Contributors.
10//
11// SPDX-License-Identifier: BSL-1.0
12//
13
14
15#include "Poco/StreamTokenizer.h"
16#include "Poco/Token.h"
17
18
19namespace Poco {
20
21
22StreamTokenizer::StreamTokenizer():
23 _pIstr(0)
24{
25}
26
27
28StreamTokenizer::StreamTokenizer(std::istream& istr):
29 _pIstr(&istr)
30{
31}
32
33
34StreamTokenizer::~StreamTokenizer()
35{
36 for (TokenVec::iterator it = _tokens.begin(); it != _tokens.end(); ++it)
37 {
38 delete *it;
39 }
40}
41
42
43void StreamTokenizer::attachToStream(std::istream& istr)
44{
45 _pIstr = &istr;
46}
47
48
49void StreamTokenizer::addToken(Token* pToken)
50{
51 poco_check_ptr (pToken);
52
53 pToken->ignore(pToken->tokenClass() == Token::COMMENT_TOKEN || pToken->tokenClass() == Token::WHITESPACE_TOKEN);
54 _tokens.push_back(pToken);
55}
56
57
58void StreamTokenizer::addToken(Token* pToken, bool ignore)
59{
60 poco_check_ptr (pToken);
61
62 pToken->ignore(ignore);
63 _tokens.push_back(pToken);
64}
65
66
67const Token* StreamTokenizer::next()
68{
69 poco_check_ptr (_pIstr);
70
71 static const int eof = std::char_traits<char>::eof();
72
73 int first = _pIstr->get();
74 TokenVec::const_iterator it = _tokens.begin();
75 while (first != eof && it != _tokens.end())
76 {
77 Token* ti = *it;
78 if (ti->start((char) first, *_pIstr))
79 {
80 ti->finish(*_pIstr);
81 if (ti->ignored())
82 {
83 first = _pIstr->get();
84 it = _tokens.begin();
85 }
86 else return *it;
87 }
88 else ++it;
89 }
90 if (first == eof)
91 {
92 return &_eofToken;
93 }
94 else
95 {
96 _invalidToken.start((char) first, *_pIstr);
97 return &_invalidToken;
98 }
99}
100
101
102} // namespace Poco
103