1//
2// StreamTokenizer.h
3//
4// Library: Foundation
5// Package: Streams
6// Module: StreamTokenizer
7//
8// Definition of the StreamTokenizer class.
9//
10// Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH.
11// and Contributors.
12//
13// SPDX-License-Identifier: BSL-1.0
14//
15
16
17#ifndef Foundation_StreamTokenizer_INCLUDED
18#define Foundation_StreamTokenizer_INCLUDED
19
20
21#include "Poco/Foundation.h"
22#include "Poco/Token.h"
23#include <istream>
24#include <vector>
25
26
27namespace Poco {
28
29
30class Foundation_API StreamTokenizer
31 /// A stream tokenizer splits an input stream
32 /// into a sequence of tokens of different kinds.
33 /// Various token kinds can be registered with
34 /// the tokenizer.
35{
36public:
37 StreamTokenizer();
38 /// Creates a StreamTokenizer with no attached stream.
39
40 StreamTokenizer(std::istream& istr);
41 /// Creates a StreamTokenizer with no attached stream.
42
43 virtual ~StreamTokenizer();
44 /// Destroys the StreamTokenizer and deletes all
45 /// registered tokens.
46
47 void attachToStream(std::istream& istr);
48 /// Attaches the tokenizer to an input stream.
49
50 void addToken(Token* pToken);
51 /// Adds a token class to the tokenizer. The
52 /// tokenizer takes ownership of the token and
53 /// deletes it when no longer needed. Comment
54 /// and whitespace tokens will be marked as
55 /// ignorable, which means that next() will not
56 /// return them.
57
58 void addToken(Token* pToken, bool ignore);
59 /// Adds a token class to the tokenizer. The
60 /// tokenizer takes ownership of the token and
61 /// deletes it when no longer needed.
62 /// If ignore is true, the token will be marked
63 /// as ignorable, which means that next() will
64 /// not return it.
65
66 const Token* next();
67 /// Extracts the next token from the input stream.
68 /// Returns a pointer to an EOFToken if there are
69 /// no more characters to read.
70 /// Returns a pointer to an InvalidToken if an
71 /// invalid character is encountered.
72 /// If a token is marked as ignorable, it will not
73 /// be returned, and the next token will be
74 /// examined.
75 /// Never returns a NULL pointer.
76 /// You must not delete the token returned by next().
77
78private:
79 typedef std::vector<Token*> TokenVec;
80
81 TokenVec _tokens;
82 std::istream* _pIstr;
83 InvalidToken _invalidToken;
84 EOFToken _eofToken;
85};
86
87
88} // namespace Poco
89
90
91#endif // Foundation_StreamTokenizer_INCLUDED
92