1 | // |
2 | // StreamTokenizer.h |
3 | // |
4 | // Library: Foundation |
5 | // Package: Streams |
6 | // Module: StreamTokenizer |
7 | // |
8 | // Definition of the StreamTokenizer class. |
9 | // |
10 | // Copyright (c) 2004-2006, Applied Informatics Software Engineering GmbH. |
11 | // and Contributors. |
12 | // |
13 | // SPDX-License-Identifier: BSL-1.0 |
14 | // |
15 | |
16 | |
17 | #ifndef Foundation_StreamTokenizer_INCLUDED |
18 | #define Foundation_StreamTokenizer_INCLUDED |
19 | |
20 | |
21 | #include "Poco/Foundation.h" |
22 | #include "Poco/Token.h" |
23 | #include <istream> |
24 | #include <vector> |
25 | |
26 | |
27 | namespace Poco { |
28 | |
29 | |
30 | class Foundation_API StreamTokenizer |
31 | /// A stream tokenizer splits an input stream |
32 | /// into a sequence of tokens of different kinds. |
33 | /// Various token kinds can be registered with |
34 | /// the tokenizer. |
35 | { |
36 | public: |
37 | StreamTokenizer(); |
38 | /// Creates a StreamTokenizer with no attached stream. |
39 | |
40 | StreamTokenizer(std::istream& istr); |
41 | /// Creates a StreamTokenizer with no attached stream. |
42 | |
43 | virtual ~StreamTokenizer(); |
44 | /// Destroys the StreamTokenizer and deletes all |
45 | /// registered tokens. |
46 | |
47 | void attachToStream(std::istream& istr); |
48 | /// Attaches the tokenizer to an input stream. |
49 | |
50 | void addToken(Token* pToken); |
51 | /// Adds a token class to the tokenizer. The |
52 | /// tokenizer takes ownership of the token and |
53 | /// deletes it when no longer needed. Comment |
54 | /// and whitespace tokens will be marked as |
55 | /// ignorable, which means that next() will not |
56 | /// return them. |
57 | |
58 | void addToken(Token* pToken, bool ignore); |
59 | /// Adds a token class to the tokenizer. The |
60 | /// tokenizer takes ownership of the token and |
61 | /// deletes it when no longer needed. |
62 | /// If ignore is true, the token will be marked |
63 | /// as ignorable, which means that next() will |
64 | /// not return it. |
65 | |
66 | const Token* next(); |
67 | /// Extracts the next token from the input stream. |
68 | /// Returns a pointer to an EOFToken if there are |
69 | /// no more characters to read. |
70 | /// Returns a pointer to an InvalidToken if an |
71 | /// invalid character is encountered. |
72 | /// If a token is marked as ignorable, it will not |
73 | /// be returned, and the next token will be |
74 | /// examined. |
75 | /// Never returns a NULL pointer. |
76 | /// You must not delete the token returned by next(). |
77 | |
78 | private: |
79 | typedef std::vector<Token*> TokenVec; |
80 | |
81 | TokenVec _tokens; |
82 | std::istream* _pIstr; |
83 | InvalidToken _invalidToken; |
84 | EOFToken _eofToken; |
85 | }; |
86 | |
87 | |
88 | } // namespace Poco |
89 | |
90 | |
91 | #endif // Foundation_StreamTokenizer_INCLUDED |
92 | |