1//
2// Tokenizer.cpp
3//
4// Library: CppParser
5// Package: CppParser
6// Module: Tokenizer
7//
8// Copyright (c) 2006, Applied Informatics Software Engineering GmbH.
9// and Contributors.
10//
11// SPDX-License-Identifier: BSL-1.0
12//
13
14
15#include "Poco/CppParser/Tokenizer.h"
16#include "Poco/CppParser/CppToken.h"
17
18
19using Poco::StreamTokenizer;
20using Poco::WhitespaceToken;
21
22
23namespace Poco {
24namespace CppParser {
25
26
27Tokenizer::Tokenizer(std::istream& istr):
28 StreamTokenizer(istr)
29{
30 addToken(new OperatorToken);
31 addToken(new IdentifierToken);
32 addToken(new StringLiteralToken);
33 addToken(new CharLiteralToken);
34 addToken(new NumberLiteralToken);
35 addToken(new CommentToken, false);
36 addToken(new PreprocessorToken);
37 addToken(new WhitespaceToken);
38}
39
40
41Tokenizer::~Tokenizer()
42{
43}
44
45
46} } // namespace Poco::CppParser
47