1/****************************************************************************
2**
3** Copyright (C) 2018 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39
40#include "qtaptestlogger_p.h"
41
42#include "qtestlog_p.h"
43#include "qtestresult_p.h"
44#include "qtestassert.h"
45
46#if QT_CONFIG(regularexpression)
47# include <QtCore/qregularexpression.h>
48#endif
49
50QT_BEGIN_NAMESPACE
51
52QTapTestLogger::QTapTestLogger(const char *filename)
53 : QAbstractTestLogger(filename)
54 , m_wasExpectedFail(false)
55{
56}
57
58QTapTestLogger::~QTapTestLogger() = default;
59
60void QTapTestLogger::startLogging()
61{
62 QAbstractTestLogger::startLogging();
63
64 QTestCharBuffer preamble;
65 QTest::qt_asprintf(&preamble, "TAP version 13\n"
66 // By convention, test suite names are output as diagnostics lines
67 // This is a pretty poor convention, as consumers will then treat
68 // actual diagnostics, e.g. qDebug, as test suite names o_O
69 "# %s\n", QTestResult::currentTestObjectName());
70 outputString(preamble.data());
71}
72
73void QTapTestLogger::stopLogging()
74{
75 const int total = QTestLog::totalCount();
76
77 QTestCharBuffer testPlanAndStats;
78 QTest::qt_asprintf(&testPlanAndStats,
79 "1..%d\n"
80 "# tests %d\n"
81 "# pass %d\n"
82 "# fail %d\n",
83 total, total, QTestLog::passCount(), QTestLog::failCount());
84 outputString(testPlanAndStats.data());
85
86 QAbstractTestLogger::stopLogging();
87}
88
89void QTapTestLogger::enterTestFunction(const char *function)
90{
91 Q_UNUSED(function);
92 m_wasExpectedFail = false;
93}
94
95void QTapTestLogger::enterTestData(QTestData *data)
96{
97 Q_UNUSED(data);
98 m_wasExpectedFail = false;
99}
100
101using namespace QTestPrivate;
102
103void QTapTestLogger::outputTestLine(bool ok, int testNumber, QTestCharBuffer &directive)
104{
105 QTestCharBuffer testIdentifier;
106 QTestPrivate::generateTestIdentifier(&testIdentifier, TestFunction | TestDataTag);
107
108 QTestCharBuffer testLine;
109 QTest::qt_asprintf(&testLine, "%s %d - %s%s\n",
110 ok ? "ok" : "not ok", testNumber, testIdentifier.data(), directive.data());
111
112 outputString(testLine.data());
113}
114
115void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
116 const char *file, int line)
117{
118 if (m_wasExpectedFail && type == Pass) {
119 // XFail comes with a corresponding Pass incident, but we only want
120 // to emit a single test point for it, so skip the this pass.
121 return;
122 }
123
124 bool ok = type == Pass || type == XPass || type == BlacklistedPass || type == BlacklistedXPass;
125
126 QTestCharBuffer directive;
127 if (type == XFail || type == XPass || type == BlacklistedFail || type == BlacklistedPass
128 || type == BlacklistedXFail || type == BlacklistedXPass) {
129 // We treat expected or blacklisted failures/passes as TODO-failures/passes,
130 // which should be treated as soft issues by consumers. Not all do though :/
131 QTest::qt_asprintf(&directive, " # TODO %s", description);
132 }
133
134 int testNumber = QTestLog::totalCount();
135 if (type == XFail) {
136 // The global test counter hasn't been updated yet for XFail
137 testNumber += 1;
138 }
139
140 outputTestLine(ok, testNumber, directive);
141
142 if (!ok) {
143 // All failures need a diagnostics sections to not confuse consumers
144
145 // The indent needs to be two spaces for maximum compatibility
146 #define YAML_INDENT " "
147
148 outputString(YAML_INDENT "---\n");
149
150 if (type != XFail) {
151#if QT_CONFIG(regularexpression)
152 // This is fragile, but unfortunately testlib doesn't plumb
153 // the expected and actual values to the loggers (yet).
154 static QRegularExpression verifyRegex(
155 QLatin1String("^'(?<actualexpression>.*)' returned (?<actual>\\w+).+\\((?<message>.*)\\)$"));
156
157 static QRegularExpression comparRegex(
158 QLatin1String("^(?<message>.*)\n"
159 "\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n"
160 "\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: (?<expected>.*)$"));
161
162 QString descriptionString = QString::fromUtf8(description);
163 QRegularExpressionMatch match = verifyRegex.match(descriptionString);
164 if (!match.hasMatch())
165 match = comparRegex.match(descriptionString);
166
167 if (match.hasMatch()) {
168 bool isVerify = match.regularExpression() == verifyRegex;
169 QString message = match.captured(QLatin1String("message"));
170 QString expected;
171 QString actual;
172
173 if (isVerify) {
174 QString expression = QLatin1String(" (")
175 % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')') ;
176 actual = match.captured(QLatin1String("actual")).toLower() % expression;
177 expected = (actual.startsWith(QLatin1String("true")) ? QLatin1String("false") : QLatin1String("true")) % expression;
178 if (message.isEmpty())
179 message = QLatin1String("Verification failed");
180 } else {
181 expected = match.captured(QLatin1String("expected"))
182 % QLatin1String(" (") % match.captured(QLatin1String("expectedexpresssion")) % QLatin1Char(')');
183 actual = match.captured(QLatin1String("actual"))
184 % QLatin1String(" (") % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')');
185 }
186
187 QTestCharBuffer diagnosticsYamlish;
188 QTest::qt_asprintf(&diagnosticsYamlish,
189 YAML_INDENT "type: %s\n"
190 YAML_INDENT "message: %s\n"
191
192 // Some consumers understand 'wanted/found', while others need
193 // 'expected/actual', so we do both for maximum compatibility.
194 YAML_INDENT "wanted: %s\n"
195 YAML_INDENT "found: %s\n"
196 YAML_INDENT "expected: %s\n"
197 YAML_INDENT "actual: %s\n",
198
199 isVerify ? "QVERIFY" : "QCOMPARE",
200 qPrintable(message),
201 qPrintable(expected), qPrintable(actual),
202 qPrintable(expected), qPrintable(actual)
203 );
204
205 outputString(diagnosticsYamlish.data());
206 } else {
207 QTestCharBuffer unparsableDescription;
208 QTest::qt_asprintf(&unparsableDescription,
209 YAML_INDENT "# %s\n", description);
210 outputString(unparsableDescription.data());
211 }
212#else
213 QTestCharBuffer unparsableDescription;
214 QTest::qt_asprintf(&unparsableDescription,
215 YAML_INDENT "# %s\n", description);
216 outputString(unparsableDescription.data());
217#endif
218 }
219
220 if (file) {
221 QTestCharBuffer location;
222 QTest::qt_asprintf(&location,
223 // The generic 'at' key is understood by most consumers.
224 YAML_INDENT "at: %s::%s() (%s:%d)\n"
225
226 // The file and line keys are for consumers that are able
227 // to read more granular location info.
228 YAML_INDENT "file: %s\n"
229 YAML_INDENT "line: %d\n",
230
231 QTestResult::currentTestObjectName(),
232 QTestResult::currentTestFunction(),
233 file, line, file, line
234 );
235 outputString(location.data());
236 }
237
238 outputString(YAML_INDENT "...\n");
239 }
240
241 m_wasExpectedFail = type == XFail;
242}
243
244void QTapTestLogger::addMessage(MessageTypes type, const QString &message,
245 const char *file, int line)
246{
247 Q_UNUSED(file);
248 Q_UNUSED(line);
249
250 if (type == Skip) {
251 QTestCharBuffer directive;
252 QTest::qt_asprintf(&directive, " # SKIP %s", message.toUtf8().constData());
253 outputTestLine(/* ok = */ true, QTestLog::totalCount(), directive);
254 return;
255 }
256
257 QTestCharBuffer diagnostics;
258 QTest::qt_asprintf(&diagnostics, "# %s\n", qPrintable(message));
259 outputString(diagnostics.data());
260}
261
262QT_END_NAMESPACE
263
264