1/*
2 * Copyright (C) 2020-2022 Roy Qu (royqh1979@gmail.com)
3 *
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <https://www.gnu.org/licenses/>.
16 */
17#ifndef CPPTOKENIZER_H
18#define CPPTOKENIZER_H
19
20#include <QObject>
21#include "parserutils.h"
22
23class CppTokenizer
24{
25public:
26 struct Token {
27 QString text;
28 int line;
29 };
30 using PToken = std::shared_ptr<Token>;
31 using TokenList = QVector<PToken>;
32 explicit CppTokenizer();
33
34 void reset();
35 void tokenize(const QStringList& buffer);
36 void dumpTokens(const QString& fileName);
37 const TokenList& tokens();
38 PToken operator[](int i);
39 int tokenCount();
40 bool isIdentChar(const QChar& ch);
41private:
42 void addToken(const QString& sText, int iLine);
43 void advance();
44 void countLines();
45 PToken getToken(int index);
46
47 QString getArguments();
48 QString getForInit();
49 QString getNextToken(
50 bool bSkipParenthesis = false,
51 bool bSkipArray = false,
52 bool bSkipBlock = false);
53 QString getNumber();
54 QString getPreprocessor();
55 QString getWord(
56 bool bSkipParenthesis = false,
57 bool bSkipArray = false,
58 bool bSkipBlock = false);
59 bool isArguments();
60 bool isForInit();
61 bool isNumber();
62 bool isPreprocessor();
63 bool isWord();
64 void simplify(QString& output);
65 void simplifyArgs(QString& output);
66 void skipAssignment();
67 void skipDoubleQuotes();
68 void skipPair(const QChar& cStart, const QChar cEnd, const QSet<QChar>& failChars = QSet<QChar>());
69 void skipRawString();
70 void skipSingleQuote();
71 void skipSplitLine();
72 void skipTemplateArgs();
73 void skipToEOL();
74 void skipToNextToken();
75 bool openFile(const QString& fileName);
76 bool isLetterChar(const QChar& ch);
77 bool isHexChar(const QChar& ch);
78 bool isDigitChar(const QChar& ch);
79 bool isSpaceChar(const QChar& ch);
80 bool isLineChar(const QChar& ch);
81 bool isBlankChar(const QChar& ch);
82 bool isOperatorChar(const QChar& ch);
83
84 bool currentWordEquals(QChar* wordStart, QChar *wordEnd, const QString& text);
85
86private:
87 QStringList mBuffer;
88 QString mBufferStr;
89 QChar* mStart;
90 QChar* mCurrent;
91 QChar* mLineCount;
92 int mCurrentLine;
93 QString mLastToken;
94 TokenList mTokenList;
95};
96
97#endif // CPPTOKENIZER_H
98