1/****************************************************************************
2**
3** Copyright (C) 2020 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39
40#include <QtTest/qtestassert.h>
41
42#include <QtTest/private/qtestlog_p.h>
43#include <QtTest/private/qtestresult_p.h>
44#include <QtTest/private/qabstracttestlogger_p.h>
45#include <QtTest/private/qplaintestlogger_p.h>
46#include <QtTest/private/qcsvbenchmarklogger_p.h>
47#include <QtTest/private/qjunittestlogger_p.h>
48#include <QtTest/private/qxmltestlogger_p.h>
49#include <QtTest/private/qteamcitylogger_p.h>
50#include <QtTest/private/qtaptestlogger_p.h>
51#if defined(HAVE_XCTEST)
52#include <QtTest/private/qxctestlogger_p.h>
53#endif
54
55#if defined(Q_OS_DARWIN)
56#include <QtTest/private/qappletestlogger_p.h>
57#endif
58
59#include <QtCore/qatomic.h>
60#include <QtCore/qbytearray.h>
61#include <QtCore/qelapsedtimer.h>
62#include <QtCore/qlist.h>
63#include <QtCore/qvariant.h>
64#if QT_CONFIG(regularexpression)
65#include <QtCore/QRegularExpression>
66#endif
67
68#include <stdlib.h>
69#include <string.h>
70#include <limits.h>
71
72QT_BEGIN_NAMESPACE
73
74static void saveCoverageTool(const char * appname, bool testfailed, bool installedTestCoverage)
75{
76#ifdef __COVERAGESCANNER__
77# if QT_CONFIG(testlib_selfcover)
78 __coveragescanner_teststate(QTestLog::failCount() > 0 ? "FAILED" :
79 QTestLog::passCount() > 0 ? "PASSED" : "SKIPPED");
80# else
81 if (!installedTestCoverage)
82 return;
83 // install again to make sure the filename is correct.
84 // without this, a plugin or similar may have changed the filename.
85 __coveragescanner_install(appname);
86 __coveragescanner_teststate(testfailed ? "FAILED" : "PASSED");
87 __coveragescanner_save();
88 __coveragescanner_testname("");
89 __coveragescanner_clear();
90 unsetenv("QT_TESTCOCOON_ACTIVE");
91# endif // testlib_selfcover
92#else
93 Q_UNUSED(appname);
94 Q_UNUSED(testfailed);
95 Q_UNUSED(installedTestCoverage);
96#endif
97}
98
99static QElapsedTimer elapsedFunctionTime;
100static QElapsedTimer elapsedTotalTime;
101
102#define FOREACH_TEST_LOGGER for (QAbstractTestLogger *logger : QTest::loggers)
103
104namespace QTest {
105
106 int fails = 0;
107 int passes = 0;
108 int skips = 0;
109 int blacklists = 0;
110
111 struct IgnoreResultList
112 {
113 inline IgnoreResultList(QtMsgType tp, const QVariant &patternIn)
114 : type(tp), pattern(patternIn) {}
115
116 static inline void clearList(IgnoreResultList *&list)
117 {
118 while (list) {
119 IgnoreResultList *current = list;
120 list = list->next;
121 delete current;
122 }
123 }
124
125 static void append(IgnoreResultList *&list, QtMsgType type, const QVariant &patternIn)
126 {
127 QTest::IgnoreResultList *item = new QTest::IgnoreResultList(type, patternIn);
128
129 if (!list) {
130 list = item;
131 return;
132 }
133 IgnoreResultList *last = list;
134 for ( ; last->next; last = last->next) ;
135 last->next = item;
136 }
137
138 static bool stringsMatch(const QString &expected, const QString &actual)
139 {
140 if (expected == actual)
141 return true;
142
143 // ignore an optional whitespace at the end of str
144 // (the space was added automatically by ~QDebug() until Qt 5.3,
145 // so autotests still might expect it)
146 if (expected.endsWith(QLatin1Char(' ')))
147 return actual == QStringView{expected}.left(expected.length() - 1);
148
149 return false;
150 }
151
152 inline bool matches(QtMsgType tp, const QString &message) const
153 {
154 return tp == type
155 && (pattern.userType() == QMetaType::QString ?
156 stringsMatch(pattern.toString(), message) :
157#if QT_CONFIG(regularexpression)
158 pattern.toRegularExpression().match(message).hasMatch());
159#else
160 false);
161#endif
162 }
163
164 QtMsgType type;
165 QVariant pattern;
166 IgnoreResultList *next = nullptr;
167 };
168
169 static IgnoreResultList *ignoreResultList = nullptr;
170
171 static QList<QAbstractTestLogger *> loggers;
172 static bool loggerUsingStdout = false;
173
174 static int verbosity = 0;
175 static int maxWarnings = 2002;
176 static bool installedTestCoverage = true;
177
178 static QtMessageHandler oldMessageHandler;
179
180 static bool handleIgnoredMessage(QtMsgType type, const QString &message)
181 {
182 if (!ignoreResultList)
183 return false;
184 IgnoreResultList *last = nullptr;
185 IgnoreResultList *list = ignoreResultList;
186 while (list) {
187 if (list->matches(type, message)) {
188 // remove the item from the list
189 if (last)
190 last->next = list->next;
191 else if (list->next)
192 ignoreResultList = list->next;
193 else
194 ignoreResultList = nullptr;
195
196 delete list;
197 return true;
198 }
199
200 last = list;
201 list = list->next;
202 }
203 return false;
204 }
205
206 static void messageHandler(QtMsgType type, const QMessageLogContext & context, const QString &message)
207 {
208 static QBasicAtomicInt counter = Q_BASIC_ATOMIC_INITIALIZER(QTest::maxWarnings);
209
210 if (QTestLog::loggerCount() == 0) {
211 // if this goes wrong, something is seriously broken.
212 qInstallMessageHandler(oldMessageHandler);
213 QTEST_ASSERT(QTestLog::loggerCount() != 0);
214 }
215
216 if (handleIgnoredMessage(type, message)) {
217 // the message is expected, so just swallow it.
218 return;
219 }
220
221 if (type != QtFatalMsg) {
222 if (counter.loadRelaxed() <= 0)
223 return;
224
225 if (!counter.deref()) {
226 FOREACH_TEST_LOGGER {
227 logger->addMessage(QAbstractTestLogger::QSystem,
228 QStringLiteral("Maximum amount of warnings exceeded. Use -maxwarnings to override."));
229 }
230 return;
231 }
232 }
233
234 FOREACH_TEST_LOGGER
235 logger->addMessage(type, context, message);
236
237 if (type == QtFatalMsg) {
238 /* Right now, we're inside the custom message handler and we're
239 * being qt_message_output in qglobal.cpp. After we return from
240 * this function, it will proceed with calling exit() and abort()
241 * and hence crash. Therefore, we call these logging functions such
242 * that we wrap up nicely, and in particular produce well-formed XML. */
243 QTestResult::addFailure("Received a fatal error.", "Unknown file", 0);
244 QTestLog::leaveTestFunction();
245 QTestLog::stopLogging();
246 }
247 }
248}
249
250void QTestLog::enterTestFunction(const char* function)
251{
252 elapsedFunctionTime.restart();
253 if (printAvailableTags)
254 return;
255
256 QTEST_ASSERT(function);
257
258 FOREACH_TEST_LOGGER
259 logger->enterTestFunction(function);
260}
261
262void QTestLog::enterTestData(QTestData *data)
263{
264 QTEST_ASSERT(data);
265
266 FOREACH_TEST_LOGGER
267 logger->enterTestData(data);
268}
269
270int QTestLog::unhandledIgnoreMessages()
271{
272 int i = 0;
273 QTest::IgnoreResultList *list = QTest::ignoreResultList;
274 while (list) {
275 ++i;
276 list = list->next;
277 }
278 return i;
279}
280
281void QTestLog::leaveTestFunction()
282{
283 if (printAvailableTags)
284 return;
285
286 FOREACH_TEST_LOGGER
287 logger->leaveTestFunction();
288}
289
290void QTestLog::printUnhandledIgnoreMessages()
291{
292 QString message;
293 QTest::IgnoreResultList *list = QTest::ignoreResultList;
294 while (list) {
295 if (list->pattern.userType() == QMetaType::QString) {
296 message = QStringLiteral("Did not receive message: \"") + list->pattern.toString() + QLatin1Char('"');
297 } else {
298#if QT_CONFIG(regularexpression)
299 message = QStringLiteral("Did not receive any message matching: \"") + list->pattern.toRegularExpression().pattern() + QLatin1Char('"');
300#endif
301 }
302 FOREACH_TEST_LOGGER
303 logger->addMessage(QAbstractTestLogger::Info, message);
304
305 list = list->next;
306 }
307}
308
309void QTestLog::clearIgnoreMessages()
310{
311 QTest::IgnoreResultList::clearList(QTest::ignoreResultList);
312}
313
314void QTestLog::addPass(const char *msg)
315{
316 if (printAvailableTags)
317 return;
318
319 QTEST_ASSERT(msg);
320
321 ++QTest::passes;
322
323 FOREACH_TEST_LOGGER
324 logger->addIncident(QAbstractTestLogger::Pass, msg);
325}
326
327void QTestLog::addFail(const char *msg, const char *file, int line)
328{
329 QTEST_ASSERT(msg);
330
331 ++QTest::fails;
332
333 FOREACH_TEST_LOGGER
334 logger->addIncident(QAbstractTestLogger::Fail, msg, file, line);
335}
336
337void QTestLog::addXFail(const char *msg, const char *file, int line)
338{
339 QTEST_ASSERT(msg);
340 QTEST_ASSERT(file);
341
342 FOREACH_TEST_LOGGER
343 logger->addIncident(QAbstractTestLogger::XFail, msg, file, line);
344}
345
346void QTestLog::addXPass(const char *msg, const char *file, int line)
347{
348 QTEST_ASSERT(msg);
349 QTEST_ASSERT(file);
350
351 ++QTest::fails;
352
353 FOREACH_TEST_LOGGER
354 logger->addIncident(QAbstractTestLogger::XPass, msg, file, line);
355}
356
357void QTestLog::addBPass(const char *msg)
358{
359 QTEST_ASSERT(msg);
360
361 ++QTest::blacklists;
362
363 FOREACH_TEST_LOGGER
364 logger->addIncident(QAbstractTestLogger::BlacklistedPass, msg);
365}
366
367void QTestLog::addBFail(const char *msg, const char *file, int line)
368{
369 QTEST_ASSERT(msg);
370 QTEST_ASSERT(file);
371
372 ++QTest::blacklists;
373
374 FOREACH_TEST_LOGGER
375 logger->addIncident(QAbstractTestLogger::BlacklistedFail, msg, file, line);
376}
377
378void QTestLog::addBXPass(const char *msg, const char *file, int line)
379{
380 QTEST_ASSERT(msg);
381 QTEST_ASSERT(file);
382
383 ++QTest::blacklists;
384
385 FOREACH_TEST_LOGGER
386 logger->addIncident(QAbstractTestLogger::BlacklistedXPass, msg, file, line);
387}
388
389void QTestLog::addBXFail(const char *msg, const char *file, int line)
390{
391 QTEST_ASSERT(msg);
392 QTEST_ASSERT(file);
393
394 ++QTest::blacklists;
395
396 FOREACH_TEST_LOGGER
397 logger->addIncident(QAbstractTestLogger::BlacklistedXFail, msg, file, line);
398}
399
400void QTestLog::addSkip(const char *msg, const char *file, int line)
401{
402 QTEST_ASSERT(msg);
403 QTEST_ASSERT(file);
404
405 ++QTest::skips;
406
407 FOREACH_TEST_LOGGER
408 logger->addMessage(QAbstractTestLogger::Skip, QString::fromUtf8(msg), file, line);
409}
410
411void QTestLog::addBenchmarkResult(const QBenchmarkResult &result)
412{
413 FOREACH_TEST_LOGGER
414 logger->addBenchmarkResult(result);
415}
416
417void QTestLog::startLogging()
418{
419 elapsedTotalTime.start();
420 elapsedFunctionTime.start();
421 FOREACH_TEST_LOGGER
422 logger->startLogging();
423 QTest::oldMessageHandler = qInstallMessageHandler(QTest::messageHandler);
424}
425
426void QTestLog::stopLogging()
427{
428 qInstallMessageHandler(QTest::oldMessageHandler);
429 FOREACH_TEST_LOGGER {
430 logger->stopLogging();
431 delete logger;
432 }
433 QTest::loggers.clear();
434 QTest::loggerUsingStdout = false;
435 saveCoverageTool(QTestResult::currentAppName(), failCount() != 0, QTestLog::installedTestCoverage());
436}
437
438void QTestLog::addLogger(LogMode mode, const char *filename)
439{
440 if (filename && strcmp(filename, "-") == 0)
441 filename = nullptr;
442 if (!filename)
443 QTest::loggerUsingStdout = true;
444
445 QAbstractTestLogger *logger = nullptr;
446 switch (mode) {
447 case QTestLog::Plain:
448 logger = new QPlainTestLogger(filename);
449 break;
450 case QTestLog::CSV:
451 logger = new QCsvBenchmarkLogger(filename);
452 break;
453 case QTestLog::XML:
454 logger = new QXmlTestLogger(QXmlTestLogger::Complete, filename);
455 break;
456 case QTestLog::LightXML:
457 logger = new QXmlTestLogger(QXmlTestLogger::Light, filename);
458 break;
459 case QTestLog::JUnitXML:
460 logger = new QJUnitTestLogger(filename);
461 break;
462 case QTestLog::TeamCity:
463 logger = new QTeamCityLogger(filename);
464 break;
465 case QTestLog::TAP:
466 logger = new QTapTestLogger(filename);
467 break;
468#if defined(QT_USE_APPLE_UNIFIED_LOGGING)
469 case QTestLog::Apple:
470 logger = new QAppleTestLogger;
471 break;
472#endif
473#if defined(HAVE_XCTEST)
474 case QTestLog::XCTest:
475 logger = new QXcodeTestLogger;
476 break;
477#endif
478 }
479
480 QTEST_ASSERT(logger);
481 QTest::loggers.append(logger);
482}
483
484int QTestLog::loggerCount()
485{
486 return QTest::loggers.size();
487}
488
489bool QTestLog::loggerUsingStdout()
490{
491 return QTest::loggerUsingStdout;
492}
493
494void QTestLog::warn(const char *msg, const char *file, int line)
495{
496 QTEST_ASSERT(msg);
497
498 FOREACH_TEST_LOGGER
499 logger->addMessage(QAbstractTestLogger::Warn, QString::fromUtf8(msg), file, line);
500}
501
502void QTestLog::info(const char *msg, const char *file, int line)
503{
504 QTEST_ASSERT(msg);
505
506 FOREACH_TEST_LOGGER
507 logger->addMessage(QAbstractTestLogger::Info, QString::fromUtf8(msg), file, line);
508}
509
510void QTestLog::setVerboseLevel(int level)
511{
512 QTest::verbosity = level;
513}
514
515int QTestLog::verboseLevel()
516{
517 return QTest::verbosity;
518}
519
520void QTestLog::ignoreMessage(QtMsgType type, const char *msg)
521{
522 QTEST_ASSERT(msg);
523
524 QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QString::fromUtf8(msg));
525}
526
527#if QT_CONFIG(regularexpression)
528void QTestLog::ignoreMessage(QtMsgType type, const QRegularExpression &expression)
529{
530 QTEST_ASSERT(expression.isValid());
531
532 QTest::IgnoreResultList::append(QTest::ignoreResultList, type, QVariant(expression));
533}
534#endif // QT_CONFIG(regularexpression)
535
536void QTestLog::setMaxWarnings(int m)
537{
538 QTest::maxWarnings = m <= 0 ? INT_MAX : m + 2;
539}
540
541bool QTestLog::printAvailableTags = false;
542
543void QTestLog::setPrintAvailableTagsMode()
544{
545 printAvailableTags = true;
546}
547
548int QTestLog::passCount()
549{
550 return QTest::passes;
551}
552
553int QTestLog::failCount()
554{
555 return QTest::fails;
556}
557
558int QTestLog::skipCount()
559{
560 return QTest::skips;
561}
562
563int QTestLog::blacklistCount()
564{
565 return QTest::blacklists;
566}
567
568int QTestLog::totalCount()
569{
570 return passCount() + failCount() + skipCount() + blacklistCount();
571}
572
573void QTestLog::resetCounters()
574{
575 QTest::passes = 0;
576 QTest::fails = 0;
577 QTest::skips = 0;
578}
579
580void QTestLog::setInstalledTestCoverage(bool installed)
581{
582 QTest::installedTestCoverage = installed;
583}
584
585bool QTestLog::installedTestCoverage()
586{
587 return QTest::installedTestCoverage;
588}
589
590qint64 QTestLog::nsecsTotalTime()
591{
592 return elapsedTotalTime.nsecsElapsed();
593}
594
595qint64 QTestLog::nsecsFunctionTime()
596{
597 return elapsedFunctionTime.nsecsElapsed();
598}
599
600QT_END_NAMESPACE
601
602#include "moc_qtestlog_p.cpp"
603