1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39#include "qtestblacklist_p.h"
40#include "qtestresult_p.h"
41
42#include <QtTest/qtestcase.h>
43#include <QtCore/qbytearray.h>
44#include <QtCore/qfile.h>
45#include <QtCore/qset.h>
46#include <QtCore/qcoreapplication.h>
47#include <QtCore/qvariant.h>
48#include <QtCore/QSysInfo>
49
50#include <set>
51
52QT_BEGIN_NAMESPACE
53
54/*
55 The BLACKLIST file format is a grouped listing of keywords.
56
57 Blank lines and everything after # is simply ignored. An initial #-line
58 referring to this documentation is kind to readers. Comments can also be used
59 to indicate the reasons for ignoring particular cases.
60
61 The key "ci" applies only when run by COIN.
62 The key "cmake" applies when Qt is built using CMake. Other keys name platforms,
63 operating systems, distributions, tool-chains or architectures; a ! prefix
64 reverses what it checks. A version, joined to a key (at present, only for
65 distributions and for msvc) with a hyphen, limits the key to the specific
66 version. A keyword line matches if every key on it applies to the present
67 run. Successive lines are alternate conditions for ignoring a test.
68
69 Ungrouped lines at the beginning of a file apply to the whole testcase.
70 A group starts with a [square-bracketed] identification of a test function,
71 optionally with (after a colon, the name of) a specific data set, to ignore.
72 Subsequent lines give conditions for ignoring this test.
73
74 # See qtbase/src/testlib/qtestblacklist.cpp for format
75 # Test doesn't work on QNX at all
76 qnx
77
78 # QTBUG-12345
79 [testFunction]
80 linux
81 windows 64bit
82
83 # Flaky in COIN on macOS, not reproducible by developers
84 [testSlowly]
85 ci osx
86
87 # Needs basic C++11 support
88 [testfunction2:testData]
89 msvc-2010
90
91 QML test functions are identified using the following format:
92
93 <TestCase name>::<function name>:<data tag>
94
95 For example, to blacklist a QML test on RHEL 7.6:
96
97 # QTBUG-12345
98 [Button::test_display:TextOnly]
99 ci rhel-7.6
100
101 Keys are lower-case. Distribution name and version are supported if
102 QSysInfo's productType() and productVersion() return them.
103
104 Keys can be added via the space-separated QTEST_ENVIRONMENT
105 environment variable:
106
107 QTEST_ENVIRONMENT=ci ./tst_stuff
108
109 This can be used to "mock" a test environment. In the example above,
110 we add "ci" to the list of keys for the test environment, making it
111 possible to test BLACKLIST files that blacklist tests in a CI environment.
112
113 The other known keys are listed below:
114*/
115
116static QSet<QByteArray> keywords()
117{
118 // this list can be extended with new keywords as required
119 QSet<QByteArray> set = QSet<QByteArray>()
120 << "*"
121#ifdef Q_OS_LINUX
122 << "linux"
123#endif
124#ifdef Q_OS_MACOS
125 << "osx"
126 << "macos"
127#endif
128#if defined(Q_OS_WIN)
129 << "windows"
130#endif
131#ifdef Q_OS_IOS
132 << "ios"
133#endif
134#ifdef Q_OS_TVOS
135 << "tvos"
136#endif
137#ifdef Q_OS_WATCHOS
138 << "watchos"
139#endif
140#ifdef Q_OS_ANDROID
141 << "android"
142#endif
143#ifdef Q_OS_QNX
144 << "qnx"
145#endif
146
147#if QT_POINTER_SIZE == 8
148 << "64bit"
149#else
150 << "32bit"
151#endif
152
153#ifdef Q_CC_GNU
154 << "gcc"
155#endif
156#ifdef Q_CC_CLANG
157 << "clang"
158#endif
159#ifdef Q_CC_MSVC
160 << "msvc"
161# if _MSC_VER <= 1600
162 << "msvc-2010"
163# elif _MSC_VER <= 1700
164 << "msvc-2012"
165# elif _MSC_VER <= 1800
166 << "msvc-2013"
167# elif _MSC_VER <= 1900
168 << "msvc-2015"
169# elif _MSC_VER <= 1916
170 << "msvc-2017"
171# else
172 << "msvc-2019"
173# endif
174#endif
175
176#ifdef Q_PROCESSOR_X86
177 << "x86"
178#endif
179#ifdef Q_PROCESSOR_ARM
180 << "arm"
181#endif
182
183#ifdef QT_BUILD_INTERNAL
184 << "developer-build"
185#endif
186
187#ifdef QT_CMAKE_BUILD
188 << "cmake"
189#endif
190 ;
191
192#if QT_CONFIG(properties)
193 QCoreApplication *app = QCoreApplication::instance();
194 if (app) {
195 const QVariant platformName = app->property("platformName");
196 if (platformName.isValid())
197 set << platformName.toByteArray();
198 }
199#endif
200
201 return set;
202}
203
204static QSet<QByteArray> activeConditions()
205{
206 QSet<QByteArray> result = keywords();
207
208 QByteArray distributionName = QSysInfo::productType().toLower().toUtf8();
209 QByteArray distributionRelease = QSysInfo::productVersion().toLower().toUtf8();
210 if (!distributionName.isEmpty()) {
211 if (result.find(distributionName) == result.end())
212 result.insert(distributionName);
213 // backwards compatibility with Qt 5
214 if (distributionName == "macos") {
215 if (result.find(distributionName) == result.end())
216 result.insert("osx");
217 }
218 if (!distributionRelease.isEmpty()) {
219 QByteArray versioned = distributionName + "-" + distributionRelease;
220 if (result.find(versioned) == result.end())
221 result.insert(versioned);
222 if (distributionName == "macos") {
223 QByteArray versioned = "osx-" + distributionRelease;
224 if (result.find(versioned) == result.end())
225 result.insert(versioned);
226 }
227 }
228 }
229
230 if (qEnvironmentVariableIsSet("QTEST_ENVIRONMENT")) {
231 for (const QByteArray &k : qgetenv("QTEST_ENVIRONMENT").split(' '))
232 result.insert(k);
233 }
234
235 return result;
236}
237
238static bool checkCondition(const QByteArray &condition)
239{
240 static const QSet<QByteArray> matchedConditions = activeConditions();
241 QList<QByteArray> conds = condition.split(' ');
242
243 for (QByteArray c : conds) {
244 bool result = c.startsWith('!');
245 if (result)
246 c.remove(0, 1);
247
248 result ^= matchedConditions.contains(c);
249 if (!result)
250 return false;
251 }
252 return true;
253}
254
255static bool ignoreAll = false;
256static std::set<QByteArray> *ignoredTests = nullptr;
257
258namespace QTestPrivate {
259
260void parseBlackList()
261{
262 QString filename = QTest::qFindTestData(QStringLiteral("BLACKLIST"));
263 if (filename.isEmpty())
264 return;
265 QFile ignored(filename);
266 if (!ignored.open(QIODevice::ReadOnly))
267 return;
268
269 QByteArray function;
270
271 while (!ignored.atEnd()) {
272 QByteArray line = ignored.readLine();
273 const int commentPosition = line.indexOf('#');
274 if (commentPosition >= 0)
275 line.truncate(commentPosition);
276 line = line.simplified();
277 if (line.isEmpty())
278 continue;
279 if (line.startsWith('[')) {
280 function = line.mid(1, line.length() - 2);
281 continue;
282 }
283 bool condition = checkCondition(line);
284 if (condition) {
285 if (!function.size()) {
286 ignoreAll = true;
287 } else {
288 if (!ignoredTests)
289 ignoredTests = new std::set<QByteArray>;
290 ignoredTests->insert(function);
291 }
292 }
293 }
294}
295
296void checkBlackLists(const char *slot, const char *data)
297{
298 bool ignore = ignoreAll;
299
300 if (!ignore && ignoredTests) {
301 QByteArray s = slot;
302 ignore = (ignoredTests->find(s) != ignoredTests->end());
303 if (!ignore && data) {
304 s += ':';
305 s += data;
306 ignore = (ignoredTests->find(s) != ignoredTests->end());
307 }
308 }
309
310 QTestResult::setBlacklistCurrentTest(ignore);
311}
312
313} // QTestPrivate
314
315QT_END_NAMESPACE
316