1/*
2 * ECMA Test 262 Runner for QuickJS
3 *
4 * Copyright (c) 2017-2021 Fabrice Bellard
5 * Copyright (c) 2017-2021 Charlie Gordon
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 * THE SOFTWARE.
24 */
25#include <stdlib.h>
26#include <stdio.h>
27#include <stdarg.h>
28#include <inttypes.h>
29#include <string.h>
30#include <assert.h>
31#include <ctype.h>
32#include <unistd.h>
33#include <errno.h>
34#include <time.h>
35#include <dirent.h>
36#include <ftw.h>
37
38#include "cutils.h"
39#include "list.h"
40#include "quickjs-libc.h"
41
42/* enable test262 thread support to test SharedArrayBuffer and Atomics */
43#define CONFIG_AGENT
44
45#define CMD_NAME "run-test262"
46
47typedef struct namelist_t {
48 char **array;
49 int count;
50 int size;
51 unsigned int sorted : 1;
52} namelist_t;
53
54namelist_t test_list;
55namelist_t exclude_list;
56namelist_t exclude_dir_list;
57
58FILE *outfile;
59enum test_mode_t {
60 TEST_DEFAULT_NOSTRICT, /* run tests as nostrict unless test is flagged as strictonly */
61 TEST_DEFAULT_STRICT, /* run tests as strict unless test is flagged as nostrict */
62 TEST_NOSTRICT, /* run tests as nostrict, skip strictonly tests */
63 TEST_STRICT, /* run tests as strict, skip nostrict tests */
64 TEST_ALL, /* run tests in both strict and nostrict, unless restricted by spec */
65} test_mode = TEST_DEFAULT_NOSTRICT;
66int compact;
67int show_timings;
68int skip_async;
69int skip_module;
70int new_style;
71int dump_memory;
72int stats_count;
73JSMemoryUsage stats_all, stats_avg, stats_min, stats_max;
74char *stats_min_filename;
75char *stats_max_filename;
76int verbose;
77char *harness_dir;
78char *harness_exclude;
79char *harness_features;
80char *harness_skip_features;
81char *error_filename;
82char *error_file;
83FILE *error_out;
84char *report_filename;
85int update_errors;
86int test_count, test_failed, test_index, test_skipped, test_excluded;
87int new_errors, changed_errors, fixed_errors;
88int async_done;
89
90void warning(const char *, ...) __attribute__((__format__(__printf__, 1, 2)));
91void fatal(int, const char *, ...) __attribute__((__format__(__printf__, 2, 3)));
92
93void warning(const char *fmt, ...)
94{
95 va_list ap;
96
97 fflush(stdout);
98 fprintf(stderr, "%s: ", CMD_NAME);
99 va_start(ap, fmt);
100 vfprintf(stderr, fmt, ap);
101 va_end(ap);
102 fputc('\n', stderr);
103}
104
105void fatal(int errcode, const char *fmt, ...)
106{
107 va_list ap;
108
109 fflush(stdout);
110 fprintf(stderr, "%s: ", CMD_NAME);
111 va_start(ap, fmt);
112 vfprintf(stderr, fmt, ap);
113 va_end(ap);
114 fputc('\n', stderr);
115
116 exit(errcode);
117}
118
119void perror_exit(int errcode, const char *s)
120{
121 fflush(stdout);
122 fprintf(stderr, "%s: ", CMD_NAME);
123 perror(s);
124 exit(errcode);
125}
126
127char *strdup_len(const char *str, int len)
128{
129 char *p = malloc(len + 1);
130 memcpy(p, str, len);
131 p[len] = '\0';
132 return p;
133}
134
135static inline int str_equal(const char *a, const char *b) {
136 return !strcmp(a, b);
137}
138
139char *str_append(char **pp, const char *sep, const char *str) {
140 char *res, *p;
141 size_t len = 0;
142 p = *pp;
143 if (p) {
144 len = strlen(p) + strlen(sep);
145 }
146 res = malloc(len + strlen(str) + 1);
147 if (p) {
148 strcpy(res, p);
149 strcat(res, sep);
150 }
151 strcpy(res + len, str);
152 free(p);
153 return *pp = res;
154}
155
156char *str_strip(char *p)
157{
158 size_t len = strlen(p);
159 while (len > 0 && isspace((unsigned char)p[len - 1]))
160 p[--len] = '\0';
161 while (isspace((unsigned char)*p))
162 p++;
163 return p;
164}
165
166int has_prefix(const char *str, const char *prefix)
167{
168 return !strncmp(str, prefix, strlen(prefix));
169}
170
171char *skip_prefix(const char *str, const char *prefix)
172{
173 int i;
174 for (i = 0;; i++) {
175 if (prefix[i] == '\0') { /* skip the prefix */
176 str += i;
177 break;
178 }
179 if (str[i] != prefix[i])
180 break;
181 }
182 return (char *)str;
183}
184
185char *get_basename(const char *filename)
186{
187 char *p;
188
189 p = strrchr(filename, '/');
190 if (!p)
191 return NULL;
192 return strdup_len(filename, p - filename);
193}
194
195char *compose_path(const char *path, const char *name)
196{
197 int path_len, name_len;
198 char *d, *q;
199
200 if (!path || path[0] == '\0' || *name == '/') {
201 d = strdup(name);
202 } else {
203 path_len = strlen(path);
204 name_len = strlen(name);
205 d = malloc(path_len + 1 + name_len + 1);
206 if (d) {
207 q = d;
208 memcpy(q, path, path_len);
209 q += path_len;
210 if (path[path_len - 1] != '/')
211 *q++ = '/';
212 memcpy(q, name, name_len + 1);
213 }
214 }
215 return d;
216}
217
218int namelist_cmp(const char *a, const char *b)
219{
220 /* compare strings in modified lexicographical order */
221 for (;;) {
222 int ca = (unsigned char)*a++;
223 int cb = (unsigned char)*b++;
224 if (isdigit(ca) && isdigit(cb)) {
225 int na = ca - '0';
226 int nb = cb - '0';
227 while (isdigit(ca = (unsigned char)*a++))
228 na = na * 10 + ca - '0';
229 while (isdigit(cb = (unsigned char)*b++))
230 nb = nb * 10 + cb - '0';
231 if (na < nb)
232 return -1;
233 if (na > nb)
234 return +1;
235 }
236 if (ca < cb)
237 return -1;
238 if (ca > cb)
239 return +1;
240 if (ca == '\0')
241 return 0;
242 }
243}
244
245int namelist_cmp_indirect(const void *a, const void *b)
246{
247 return namelist_cmp(*(const char **)a, *(const char **)b);
248}
249
250void namelist_sort(namelist_t *lp)
251{
252 int i, count;
253 if (lp->count > 1) {
254 qsort(lp->array, lp->count, sizeof(*lp->array), namelist_cmp_indirect);
255 /* remove duplicates */
256 for (count = i = 1; i < lp->count; i++) {
257 if (namelist_cmp(lp->array[count - 1], lp->array[i]) == 0) {
258 free(lp->array[i]);
259 } else {
260 lp->array[count++] = lp->array[i];
261 }
262 }
263 lp->count = count;
264 }
265 lp->sorted = 1;
266}
267
268int namelist_find(namelist_t *lp, const char *name)
269{
270 int a, b, m, cmp;
271
272 if (!lp->sorted) {
273 namelist_sort(lp);
274 }
275 for (a = 0, b = lp->count; a < b;) {
276 m = a + (b - a) / 2;
277 cmp = namelist_cmp(lp->array[m], name);
278 if (cmp < 0)
279 a = m + 1;
280 else if (cmp > 0)
281 b = m;
282 else
283 return m;
284 }
285 return -1;
286}
287
288void namelist_add(namelist_t *lp, const char *base, const char *name)
289{
290 char *s;
291
292 s = compose_path(base, name);
293 if (!s)
294 goto fail;
295 if (lp->count == lp->size) {
296 size_t newsize = lp->size + (lp->size >> 1) + 4;
297 char **a = realloc(lp->array, sizeof(lp->array[0]) * newsize);
298 if (!a)
299 goto fail;
300 lp->array = a;
301 lp->size = newsize;
302 }
303 lp->array[lp->count] = s;
304 lp->count++;
305 return;
306fail:
307 fatal(1, "allocation failure\n");
308}
309
310void namelist_load(namelist_t *lp, const char *filename)
311{
312 char buf[1024];
313 char *base_name;
314 FILE *f;
315
316 f = fopen(filename, "rb");
317 if (!f) {
318 perror_exit(1, filename);
319 }
320 base_name = get_basename(filename);
321
322 while (fgets(buf, sizeof(buf), f) != NULL) {
323 char *p = str_strip(buf);
324 if (*p == '#' || *p == ';' || *p == '\0')
325 continue; /* line comment */
326
327 namelist_add(lp, base_name, p);
328 }
329 free(base_name);
330 fclose(f);
331}
332
333void namelist_add_from_error_file(namelist_t *lp, const char *file)
334{
335 const char *p, *p0;
336 char *pp;
337
338 for (p = file; (p = strstr(p, ".js:")) != NULL; p++) {
339 for (p0 = p; p0 > file && p0[-1] != '\n'; p0--)
340 continue;
341 pp = strdup_len(p0, p + 3 - p0);
342 namelist_add(lp, NULL, pp);
343 free(pp);
344 }
345}
346
347void namelist_free(namelist_t *lp)
348{
349 while (lp->count > 0) {
350 free(lp->array[--lp->count]);
351 }
352 free(lp->array);
353 lp->array = NULL;
354 lp->size = 0;
355}
356
357static int add_test_file(const char *filename, const struct stat *ptr, int flag)
358{
359 namelist_t *lp = &test_list;
360 if (has_suffix(filename, ".js") && !has_suffix(filename, "_FIXTURE.js"))
361 namelist_add(lp, NULL, filename);
362 return 0;
363}
364
365/* find js files from the directory tree and sort the list */
366static void enumerate_tests(const char *path)
367{
368 namelist_t *lp = &test_list;
369 int start = lp->count;
370 ftw(path, add_test_file, 100);
371 qsort(lp->array + start, lp->count - start, sizeof(*lp->array),
372 namelist_cmp_indirect);
373}
374
375static JSValue js_print(JSContext *ctx, JSValueConst this_val,
376 int argc, JSValueConst *argv)
377{
378 int i;
379 const char *str;
380
381 if (outfile) {
382 for (i = 0; i < argc; i++) {
383 if (i != 0)
384 fputc(' ', outfile);
385 str = JS_ToCString(ctx, argv[i]);
386 if (!str)
387 return JS_EXCEPTION;
388 if (!strcmp(str, "Test262:AsyncTestComplete")) {
389 async_done++;
390 } else if (strstart(str, "Test262:AsyncTestFailure", NULL)) {
391 async_done = 2; /* force an error */
392 }
393 fputs(str, outfile);
394 JS_FreeCString(ctx, str);
395 }
396 fputc('\n', outfile);
397 }
398 return JS_UNDEFINED;
399}
400
401static JSValue js_detachArrayBuffer(JSContext *ctx, JSValue this_val,
402 int argc, JSValue *argv)
403{
404 JS_DetachArrayBuffer(ctx, argv[0]);
405 return JS_UNDEFINED;
406}
407
408static JSValue js_evalScript(JSContext *ctx, JSValue this_val,
409 int argc, JSValue *argv)
410{
411 const char *str;
412 size_t len;
413 JSValue ret;
414 str = JS_ToCStringLen(ctx, &len, argv[0]);
415 if (!str)
416 return JS_EXCEPTION;
417 ret = JS_Eval(ctx, str, len, "<evalScript>", JS_EVAL_TYPE_GLOBAL);
418 JS_FreeCString(ctx, str);
419 return ret;
420}
421
422#ifdef CONFIG_AGENT
423
424#include <pthread.h>
425
426typedef struct {
427 struct list_head link;
428 pthread_t tid;
429 char *script;
430 JSValue broadcast_func;
431 BOOL broadcast_pending;
432 JSValue broadcast_sab; /* in the main context */
433 uint8_t *broadcast_sab_buf;
434 size_t broadcast_sab_size;
435 int32_t broadcast_val;
436} Test262Agent;
437
438typedef struct {
439 struct list_head link;
440 char *str;
441} AgentReport;
442
443static JSValue add_helpers1(JSContext *ctx);
444static void add_helpers(JSContext *ctx);
445
446static pthread_mutex_t agent_mutex = PTHREAD_MUTEX_INITIALIZER;
447static pthread_cond_t agent_cond = PTHREAD_COND_INITIALIZER;
448/* list of Test262Agent.link */
449static struct list_head agent_list = LIST_HEAD_INIT(agent_list);
450
451static pthread_mutex_t report_mutex = PTHREAD_MUTEX_INITIALIZER;
452/* list of AgentReport.link */
453static struct list_head report_list = LIST_HEAD_INIT(report_list);
454
455static void *agent_start(void *arg)
456{
457 Test262Agent *agent = arg;
458 JSRuntime *rt;
459 JSContext *ctx;
460 JSValue ret_val;
461 int ret;
462
463 rt = JS_NewRuntime();
464 if (rt == NULL) {
465 fatal(1, "JS_NewRuntime failure");
466 }
467 ctx = JS_NewContext(rt);
468 if (ctx == NULL) {
469 JS_FreeRuntime(rt);
470 fatal(1, "JS_NewContext failure");
471 }
472 JS_SetContextOpaque(ctx, agent);
473 JS_SetRuntimeInfo(rt, "agent");
474 JS_SetCanBlock(rt, TRUE);
475
476 add_helpers(ctx);
477 ret_val = JS_Eval(ctx, agent->script, strlen(agent->script),
478 "<evalScript>", JS_EVAL_TYPE_GLOBAL);
479 free(agent->script);
480 agent->script = NULL;
481 if (JS_IsException(ret_val))
482 js_std_dump_error(ctx);
483 JS_FreeValue(ctx, ret_val);
484
485 for(;;) {
486 JSContext *ctx1;
487 ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
488 if (ret < 0) {
489 js_std_dump_error(ctx);
490 break;
491 } else if (ret == 0) {
492 if (JS_IsUndefined(agent->broadcast_func)) {
493 break;
494 } else {
495 JSValue args[2];
496
497 pthread_mutex_lock(&agent_mutex);
498 while (!agent->broadcast_pending) {
499 pthread_cond_wait(&agent_cond, &agent_mutex);
500 }
501
502 agent->broadcast_pending = FALSE;
503 pthread_cond_signal(&agent_cond);
504
505 pthread_mutex_unlock(&agent_mutex);
506
507 args[0] = JS_NewArrayBuffer(ctx, agent->broadcast_sab_buf,
508 agent->broadcast_sab_size,
509 NULL, NULL, TRUE);
510 args[1] = JS_NewInt32(ctx, agent->broadcast_val);
511 ret_val = JS_Call(ctx, agent->broadcast_func, JS_UNDEFINED,
512 2, (JSValueConst *)args);
513 JS_FreeValue(ctx, args[0]);
514 JS_FreeValue(ctx, args[1]);
515 if (JS_IsException(ret_val))
516 js_std_dump_error(ctx);
517 JS_FreeValue(ctx, ret_val);
518 JS_FreeValue(ctx, agent->broadcast_func);
519 agent->broadcast_func = JS_UNDEFINED;
520 }
521 }
522 }
523 JS_FreeValue(ctx, agent->broadcast_func);
524
525 JS_FreeContext(ctx);
526 JS_FreeRuntime(rt);
527 return NULL;
528}
529
530static JSValue js_agent_start(JSContext *ctx, JSValue this_val,
531 int argc, JSValue *argv)
532{
533 const char *script;
534 Test262Agent *agent;
535 pthread_attr_t attr;
536
537 if (JS_GetContextOpaque(ctx) != NULL)
538 return JS_ThrowTypeError(ctx, "cannot be called inside an agent");
539
540 script = JS_ToCString(ctx, argv[0]);
541 if (!script)
542 return JS_EXCEPTION;
543 agent = malloc(sizeof(*agent));
544 memset(agent, 0, sizeof(*agent));
545 agent->broadcast_func = JS_UNDEFINED;
546 agent->broadcast_sab = JS_UNDEFINED;
547 agent->script = strdup(script);
548 JS_FreeCString(ctx, script);
549 list_add_tail(&agent->link, &agent_list);
550 pthread_attr_init(&attr);
551 // musl libc gives threads 80 kb stacks, much smaller than
552 // JS_DEFAULT_STACK_SIZE (256 kb)
553 pthread_attr_setstacksize(&attr, 2 << 20); // 2 MB, glibc default
554 pthread_create(&agent->tid, &attr, agent_start, agent);
555 pthread_attr_destroy(&attr);
556 return JS_UNDEFINED;
557}
558
559static void js_agent_free(JSContext *ctx)
560{
561 struct list_head *el, *el1;
562 Test262Agent *agent;
563
564 list_for_each_safe(el, el1, &agent_list) {
565 agent = list_entry(el, Test262Agent, link);
566 pthread_join(agent->tid, NULL);
567 JS_FreeValue(ctx, agent->broadcast_sab);
568 list_del(&agent->link);
569 free(agent);
570 }
571}
572
573static JSValue js_agent_leaving(JSContext *ctx, JSValue this_val,
574 int argc, JSValue *argv)
575{
576 Test262Agent *agent = JS_GetContextOpaque(ctx);
577 if (!agent)
578 return JS_ThrowTypeError(ctx, "must be called inside an agent");
579 /* nothing to do */
580 return JS_UNDEFINED;
581}
582
583static BOOL is_broadcast_pending(void)
584{
585 struct list_head *el;
586 Test262Agent *agent;
587 list_for_each(el, &agent_list) {
588 agent = list_entry(el, Test262Agent, link);
589 if (agent->broadcast_pending)
590 return TRUE;
591 }
592 return FALSE;
593}
594
595static JSValue js_agent_broadcast(JSContext *ctx, JSValue this_val,
596 int argc, JSValue *argv)
597{
598 JSValueConst sab = argv[0];
599 struct list_head *el;
600 Test262Agent *agent;
601 uint8_t *buf;
602 size_t buf_size;
603 int32_t val;
604
605 if (JS_GetContextOpaque(ctx) != NULL)
606 return JS_ThrowTypeError(ctx, "cannot be called inside an agent");
607
608 buf = JS_GetArrayBuffer(ctx, &buf_size, sab);
609 if (!buf)
610 return JS_EXCEPTION;
611 if (JS_ToInt32(ctx, &val, argv[1]))
612 return JS_EXCEPTION;
613
614 /* broadcast the values and wait until all agents have started
615 calling their callbacks */
616 pthread_mutex_lock(&agent_mutex);
617 list_for_each(el, &agent_list) {
618 agent = list_entry(el, Test262Agent, link);
619 agent->broadcast_pending = TRUE;
620 /* the shared array buffer is used by the thread, so increment
621 its refcount */
622 agent->broadcast_sab = JS_DupValue(ctx, sab);
623 agent->broadcast_sab_buf = buf;
624 agent->broadcast_sab_size = buf_size;
625 agent->broadcast_val = val;
626 }
627 pthread_cond_broadcast(&agent_cond);
628
629 while (is_broadcast_pending()) {
630 pthread_cond_wait(&agent_cond, &agent_mutex);
631 }
632 pthread_mutex_unlock(&agent_mutex);
633 return JS_UNDEFINED;
634}
635
636static JSValue js_agent_receiveBroadcast(JSContext *ctx, JSValue this_val,
637 int argc, JSValue *argv)
638{
639 Test262Agent *agent = JS_GetContextOpaque(ctx);
640 if (!agent)
641 return JS_ThrowTypeError(ctx, "must be called inside an agent");
642 if (!JS_IsFunction(ctx, argv[0]))
643 return JS_ThrowTypeError(ctx, "expecting function");
644 JS_FreeValue(ctx, agent->broadcast_func);
645 agent->broadcast_func = JS_DupValue(ctx, argv[0]);
646 return JS_UNDEFINED;
647}
648
649static JSValue js_agent_sleep(JSContext *ctx, JSValue this_val,
650 int argc, JSValue *argv)
651{
652 uint32_t duration;
653 if (JS_ToUint32(ctx, &duration, argv[0]))
654 return JS_EXCEPTION;
655 usleep(duration * 1000);
656 return JS_UNDEFINED;
657}
658
659static int64_t get_clock_ms(void)
660{
661 struct timespec ts;
662 clock_gettime(CLOCK_MONOTONIC, &ts);
663 return (uint64_t)ts.tv_sec * 1000 + (ts.tv_nsec / 1000000);
664}
665
666static JSValue js_agent_monotonicNow(JSContext *ctx, JSValue this_val,
667 int argc, JSValue *argv)
668{
669 return JS_NewInt64(ctx, get_clock_ms());
670}
671
672static JSValue js_agent_getReport(JSContext *ctx, JSValue this_val,
673 int argc, JSValue *argv)
674{
675 AgentReport *rep;
676 JSValue ret;
677
678 pthread_mutex_lock(&report_mutex);
679 if (list_empty(&report_list)) {
680 rep = NULL;
681 } else {
682 rep = list_entry(report_list.next, AgentReport, link);
683 list_del(&rep->link);
684 }
685 pthread_mutex_unlock(&report_mutex);
686 if (rep) {
687 ret = JS_NewString(ctx, rep->str);
688 free(rep->str);
689 free(rep);
690 } else {
691 ret = JS_NULL;
692 }
693 return ret;
694}
695
696static JSValue js_agent_report(JSContext *ctx, JSValue this_val,
697 int argc, JSValue *argv)
698{
699 const char *str;
700 AgentReport *rep;
701
702 str = JS_ToCString(ctx, argv[0]);
703 if (!str)
704 return JS_EXCEPTION;
705 rep = malloc(sizeof(*rep));
706 rep->str = strdup(str);
707 JS_FreeCString(ctx, str);
708
709 pthread_mutex_lock(&report_mutex);
710 list_add_tail(&rep->link, &report_list);
711 pthread_mutex_unlock(&report_mutex);
712 return JS_UNDEFINED;
713}
714
715static const JSCFunctionListEntry js_agent_funcs[] = {
716 /* only in main */
717 JS_CFUNC_DEF("start", 1, js_agent_start ),
718 JS_CFUNC_DEF("getReport", 0, js_agent_getReport ),
719 JS_CFUNC_DEF("broadcast", 2, js_agent_broadcast ),
720 /* only in agent */
721 JS_CFUNC_DEF("report", 1, js_agent_report ),
722 JS_CFUNC_DEF("leaving", 0, js_agent_leaving ),
723 JS_CFUNC_DEF("receiveBroadcast", 1, js_agent_receiveBroadcast ),
724 /* in both */
725 JS_CFUNC_DEF("sleep", 1, js_agent_sleep ),
726 JS_CFUNC_DEF("monotonicNow", 0, js_agent_monotonicNow ),
727};
728
729static JSValue js_new_agent(JSContext *ctx)
730{
731 JSValue agent;
732 agent = JS_NewObject(ctx);
733 JS_SetPropertyFunctionList(ctx, agent, js_agent_funcs,
734 countof(js_agent_funcs));
735 return agent;
736}
737#endif
738
739static JSValue js_createRealm(JSContext *ctx, JSValue this_val,
740 int argc, JSValue *argv)
741{
742 JSContext *ctx1;
743 JSValue ret;
744
745 ctx1 = JS_NewContext(JS_GetRuntime(ctx));
746 if (!ctx1)
747 return JS_ThrowOutOfMemory(ctx);
748 ret = add_helpers1(ctx1);
749 /* ctx1 has a refcount so it stays alive */
750 JS_FreeContext(ctx1);
751 return ret;
752}
753
754static JSValue js_IsHTMLDDA(JSContext *ctx, JSValue this_val,
755 int argc, JSValue *argv)
756{
757 return JS_NULL;
758}
759
760static JSValue js_gc(JSContext *ctx, JSValueConst this_val,
761 int argc, JSValueConst *argv)
762{
763 JS_RunGC(JS_GetRuntime(ctx));
764 return JS_UNDEFINED;
765}
766
767static JSValue add_helpers1(JSContext *ctx)
768{
769 JSValue global_obj;
770 JSValue obj262, obj;
771
772 global_obj = JS_GetGlobalObject(ctx);
773
774 JS_SetPropertyStr(ctx, global_obj, "print",
775 JS_NewCFunction(ctx, js_print, "print", 1));
776
777 /* $262 special object used by the tests */
778 obj262 = JS_NewObject(ctx);
779 JS_SetPropertyStr(ctx, obj262, "detachArrayBuffer",
780 JS_NewCFunction(ctx, js_detachArrayBuffer,
781 "detachArrayBuffer", 1));
782 JS_SetPropertyStr(ctx, obj262, "evalScript",
783 JS_NewCFunction(ctx, js_evalScript,
784 "evalScript", 1));
785 JS_SetPropertyStr(ctx, obj262, "codePointRange",
786 JS_NewCFunction(ctx, js_string_codePointRange,
787 "codePointRange", 2));
788#ifdef CONFIG_AGENT
789 JS_SetPropertyStr(ctx, obj262, "agent", js_new_agent(ctx));
790#endif
791
792 JS_SetPropertyStr(ctx, obj262, "global",
793 JS_DupValue(ctx, global_obj));
794 JS_SetPropertyStr(ctx, obj262, "createRealm",
795 JS_NewCFunction(ctx, js_createRealm,
796 "createRealm", 0));
797 obj = JS_NewCFunction(ctx, js_IsHTMLDDA, "IsHTMLDDA", 0);
798 JS_SetIsHTMLDDA(ctx, obj);
799 JS_SetPropertyStr(ctx, obj262, "IsHTMLDDA", obj);
800 JS_SetPropertyStr(ctx, obj262, "gc",
801 JS_NewCFunction(ctx, js_gc, "gc", 0));
802
803 JS_SetPropertyStr(ctx, global_obj, "$262", JS_DupValue(ctx, obj262));
804
805 JS_FreeValue(ctx, global_obj);
806 return obj262;
807}
808
809static void add_helpers(JSContext *ctx)
810{
811 JS_FreeValue(ctx, add_helpers1(ctx));
812}
813
814static char *load_file(const char *filename, size_t *lenp)
815{
816 char *buf;
817 size_t buf_len;
818 buf = (char *)js_load_file(NULL, &buf_len, filename);
819 if (!buf)
820 perror_exit(1, filename);
821 if (lenp)
822 *lenp = buf_len;
823 return buf;
824}
825
826static JSModuleDef *js_module_loader_test(JSContext *ctx,
827 const char *module_name, void *opaque)
828{
829 size_t buf_len;
830 uint8_t *buf;
831 JSModuleDef *m;
832 JSValue func_val;
833 char *filename, *slash, path[1024];
834
835 // interpret import("bar.js") from path/to/foo.js as
836 // import("path/to/bar.js") but leave import("./bar.js") untouched
837 filename = opaque;
838 if (!strchr(module_name, '/')) {
839 slash = strrchr(filename, '/');
840 if (slash) {
841 snprintf(path, sizeof(path), "%.*s/%s",
842 (int)(slash - filename), filename, module_name);
843 module_name = path;
844 }
845 }
846
847 buf = js_load_file(ctx, &buf_len, module_name);
848 if (!buf) {
849 JS_ThrowReferenceError(ctx, "could not load module filename '%s'",
850 module_name);
851 return NULL;
852 }
853
854 /* compile the module */
855 func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name,
856 JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY);
857 js_free(ctx, buf);
858 if (JS_IsException(func_val))
859 return NULL;
860 /* the module is already referenced, so we must free it */
861 m = JS_VALUE_GET_PTR(func_val);
862 JS_FreeValue(ctx, func_val);
863 return m;
864}
865
866int is_line_sep(char c)
867{
868 return (c == '\0' || c == '\n' || c == '\r');
869}
870
871char *find_line(const char *str, const char *line)
872{
873 if (str) {
874 const char *p;
875 int len = strlen(line);
876 for (p = str; (p = strstr(p, line)) != NULL; p += len + 1) {
877 if ((p == str || is_line_sep(p[-1])) && is_line_sep(p[len]))
878 return (char *)p;
879 }
880 }
881 return NULL;
882}
883
884int is_word_sep(char c)
885{
886 return (c == '\0' || isspace((unsigned char)c) || c == ',');
887}
888
889char *find_word(const char *str, const char *word)
890{
891 const char *p;
892 int len = strlen(word);
893 if (str && len) {
894 for (p = str; (p = strstr(p, word)) != NULL; p += len) {
895 if ((p == str || is_word_sep(p[-1])) && is_word_sep(p[len]))
896 return (char *)p;
897 }
898 }
899 return NULL;
900}
901
902/* handle exclude directories */
903void update_exclude_dirs(void)
904{
905 namelist_t *lp = &test_list;
906 namelist_t *ep = &exclude_list;
907 namelist_t *dp = &exclude_dir_list;
908 char *name;
909 int i, j, count;
910
911 /* split directpries from exclude_list */
912 for (count = i = 0; i < ep->count; i++) {
913 name = ep->array[i];
914 if (has_suffix(name, "/")) {
915 namelist_add(dp, NULL, name);
916 free(name);
917 } else {
918 ep->array[count++] = name;
919 }
920 }
921 ep->count = count;
922
923 namelist_sort(dp);
924
925 /* filter out excluded directories */
926 for (count = i = 0; i < lp->count; i++) {
927 name = lp->array[i];
928 for (j = 0; j < dp->count; j++) {
929 if (has_prefix(name, dp->array[j])) {
930 test_excluded++;
931 free(name);
932 name = NULL;
933 break;
934 }
935 }
936 if (name) {
937 lp->array[count++] = name;
938 }
939 }
940 lp->count = count;
941}
942
943void load_config(const char *filename, const char *ignore)
944{
945 char buf[1024];
946 FILE *f;
947 char *base_name;
948 enum {
949 SECTION_NONE = 0,
950 SECTION_CONFIG,
951 SECTION_EXCLUDE,
952 SECTION_FEATURES,
953 SECTION_TESTS,
954 } section = SECTION_NONE;
955 int lineno = 0;
956
957 f = fopen(filename, "rb");
958 if (!f) {
959 perror_exit(1, filename);
960 }
961 base_name = get_basename(filename);
962
963 while (fgets(buf, sizeof(buf), f) != NULL) {
964 char *p, *q;
965 lineno++;
966 p = str_strip(buf);
967 if (*p == '#' || *p == ';' || *p == '\0')
968 continue; /* line comment */
969
970 if (*p == "[]"[0]) {
971 /* new section */
972 p++;
973 p[strcspn(p, "]")] = '\0';
974 if (str_equal(p, "config"))
975 section = SECTION_CONFIG;
976 else if (str_equal(p, "exclude"))
977 section = SECTION_EXCLUDE;
978 else if (str_equal(p, "features"))
979 section = SECTION_FEATURES;
980 else if (str_equal(p, "tests"))
981 section = SECTION_TESTS;
982 else
983 section = SECTION_NONE;
984 continue;
985 }
986 q = strchr(p, '=');
987 if (q) {
988 /* setting: name=value */
989 *q++ = '\0';
990 q = str_strip(q);
991 }
992 switch (section) {
993 case SECTION_CONFIG:
994 if (!q) {
995 printf("%s:%d: syntax error\n", filename, lineno);
996 continue;
997 }
998 if (strstr(ignore, p)) {
999 printf("%s:%d: ignoring %s=%s\n", filename, lineno, p, q);
1000 continue;
1001 }
1002 if (str_equal(p, "style")) {
1003 new_style = str_equal(q, "new");
1004 continue;
1005 }
1006 if (str_equal(p, "testdir")) {
1007 char *testdir = compose_path(base_name, q);
1008 enumerate_tests(testdir);
1009 free(testdir);
1010 continue;
1011 }
1012 if (str_equal(p, "harnessdir")) {
1013 harness_dir = compose_path(base_name, q);
1014 continue;
1015 }
1016 if (str_equal(p, "harnessexclude")) {
1017 str_append(&harness_exclude, " ", q);
1018 continue;
1019 }
1020 if (str_equal(p, "features")) {
1021 str_append(&harness_features, " ", q);
1022 continue;
1023 }
1024 if (str_equal(p, "skip-features")) {
1025 str_append(&harness_skip_features, " ", q);
1026 continue;
1027 }
1028 if (str_equal(p, "mode")) {
1029 if (str_equal(q, "default") || str_equal(q, "default-nostrict"))
1030 test_mode = TEST_DEFAULT_NOSTRICT;
1031 else if (str_equal(q, "default-strict"))
1032 test_mode = TEST_DEFAULT_STRICT;
1033 else if (str_equal(q, "nostrict"))
1034 test_mode = TEST_NOSTRICT;
1035 else if (str_equal(q, "strict"))
1036 test_mode = TEST_STRICT;
1037 else if (str_equal(q, "all") || str_equal(q, "both"))
1038 test_mode = TEST_ALL;
1039 else
1040 fatal(2, "unknown test mode: %s", q);
1041 continue;
1042 }
1043 if (str_equal(p, "strict")) {
1044 if (str_equal(q, "skip") || str_equal(q, "no"))
1045 test_mode = TEST_NOSTRICT;
1046 continue;
1047 }
1048 if (str_equal(p, "nostrict")) {
1049 if (str_equal(q, "skip") || str_equal(q, "no"))
1050 test_mode = TEST_STRICT;
1051 continue;
1052 }
1053 if (str_equal(p, "async")) {
1054 skip_async = !str_equal(q, "yes");
1055 continue;
1056 }
1057 if (str_equal(p, "module")) {
1058 skip_module = !str_equal(q, "yes");
1059 continue;
1060 }
1061 if (str_equal(p, "verbose")) {
1062 verbose = str_equal(q, "yes");
1063 continue;
1064 }
1065 if (str_equal(p, "errorfile")) {
1066 error_filename = compose_path(base_name, q);
1067 continue;
1068 }
1069 if (str_equal(p, "excludefile")) {
1070 char *path = compose_path(base_name, q);
1071 namelist_load(&exclude_list, path);
1072 free(path);
1073 continue;
1074 }
1075 if (str_equal(p, "reportfile")) {
1076 report_filename = compose_path(base_name, q);
1077 continue;
1078 }
1079 case SECTION_EXCLUDE:
1080 namelist_add(&exclude_list, base_name, p);
1081 break;
1082 case SECTION_FEATURES:
1083 if (!q || str_equal(q, "yes"))
1084 str_append(&harness_features, " ", p);
1085 else
1086 str_append(&harness_skip_features, " ", p);
1087 break;
1088 case SECTION_TESTS:
1089 namelist_add(&test_list, base_name, p);
1090 break;
1091 default:
1092 /* ignore settings in other sections */
1093 break;
1094 }
1095 }
1096 fclose(f);
1097 free(base_name);
1098}
1099
1100char *find_error(const char *filename, int *pline, int is_strict)
1101{
1102 if (error_file) {
1103 size_t len = strlen(filename);
1104 const char *p, *q, *r;
1105 int line;
1106
1107 for (p = error_file; (p = strstr(p, filename)) != NULL; p += len) {
1108 if ((p == error_file || p[-1] == '\n' || p[-1] == '(') && p[len] == ':') {
1109 q = p + len;
1110 line = 1;
1111 if (*q == ':') {
1112 line = strtol(q + 1, (char**)&q, 10);
1113 if (*q == ':')
1114 q++;
1115 }
1116 while (*q == ' ') {
1117 q++;
1118 }
1119 /* check strict mode indicator */
1120 if (!strstart(q, "strict mode: ", &q) != !is_strict)
1121 continue;
1122 r = q = skip_prefix(q, "unexpected error: ");
1123 r += strcspn(r, "\n");
1124 while (r[0] == '\n' && r[1] && strncmp(r + 1, filename, 8)) {
1125 r++;
1126 r += strcspn(r, "\n");
1127 }
1128 if (pline)
1129 *pline = line;
1130 return strdup_len(q, r - q);
1131 }
1132 }
1133 }
1134 return NULL;
1135}
1136
1137int skip_comments(const char *str, int line, int *pline)
1138{
1139 const char *p;
1140 int c;
1141
1142 p = str;
1143 while ((c = (unsigned char)*p++) != '\0') {
1144 if (isspace(c)) {
1145 if (c == '\n')
1146 line++;
1147 continue;
1148 }
1149 if (c == '/' && *p == '/') {
1150 while (*++p && *p != '\n')
1151 continue;
1152 continue;
1153 }
1154 if (c == '/' && *p == '*') {
1155 for (p += 1; *p; p++) {
1156 if (*p == '\n') {
1157 line++;
1158 continue;
1159 }
1160 if (*p == '*' && p[1] == '/') {
1161 p += 2;
1162 break;
1163 }
1164 }
1165 continue;
1166 }
1167 break;
1168 }
1169 if (pline)
1170 *pline = line;
1171
1172 return p - str;
1173}
1174
1175int longest_match(const char *str, const char *find, int pos, int *ppos, int line, int *pline)
1176{
1177 int len, maxlen;
1178
1179 maxlen = 0;
1180
1181 if (*find) {
1182 const char *p;
1183 for (p = str + pos; *p; p++) {
1184 if (*p == *find) {
1185 for (len = 1; p[len] && p[len] == find[len]; len++)
1186 continue;
1187 if (len > maxlen) {
1188 maxlen = len;
1189 if (ppos)
1190 *ppos = p - str;
1191 if (pline)
1192 *pline = line;
1193 if (!find[len])
1194 break;
1195 }
1196 }
1197 if (*p == '\n')
1198 line++;
1199 }
1200 }
1201 return maxlen;
1202}
1203
1204static int eval_buf(JSContext *ctx, const char *buf, size_t buf_len,
1205 const char *filename, int is_test, int is_negative,
1206 const char *error_type, FILE *outfile, int eval_flags,
1207 int is_async)
1208{
1209 JSValue res_val, exception_val;
1210 int ret, error_line, pos, pos_line;
1211 BOOL is_error, has_error_line, ret_promise;
1212 const char *error_name;
1213
1214 pos = skip_comments(buf, 1, &pos_line);
1215 error_line = pos_line;
1216 has_error_line = FALSE;
1217 exception_val = JS_UNDEFINED;
1218 error_name = NULL;
1219
1220 /* a module evaluation returns a promise */
1221 ret_promise = ((eval_flags & JS_EVAL_TYPE_MODULE) != 0);
1222 async_done = 0; /* counter of "Test262:AsyncTestComplete" messages */
1223
1224 res_val = JS_Eval(ctx, buf, buf_len, filename, eval_flags);
1225
1226 if ((is_async || ret_promise) && !JS_IsException(res_val)) {
1227 JSValue promise = JS_UNDEFINED;
1228 if (ret_promise) {
1229 promise = res_val;
1230 } else {
1231 JS_FreeValue(ctx, res_val);
1232 }
1233 for(;;) {
1234 JSContext *ctx1;
1235 ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
1236 if (ret < 0) {
1237 res_val = JS_EXCEPTION;
1238 break;
1239 } else if (ret == 0) {
1240 if (is_async) {
1241 /* test if the test called $DONE() once */
1242 if (async_done != 1) {
1243 res_val = JS_ThrowTypeError(ctx, "$DONE() not called");
1244 } else {
1245 res_val = JS_UNDEFINED;
1246 }
1247 } else {
1248 /* check that the returned promise is fulfilled */
1249 JSPromiseStateEnum state = JS_PromiseState(ctx, promise);
1250 if (state == JS_PROMISE_FULFILLED)
1251 res_val = JS_UNDEFINED;
1252 else if (state == JS_PROMISE_REJECTED)
1253 res_val = JS_Throw(ctx, JS_PromiseResult(ctx, promise));
1254 else
1255 res_val = JS_ThrowTypeError(ctx, "promise is pending");
1256 }
1257 break;
1258 }
1259 }
1260 JS_FreeValue(ctx, promise);
1261 }
1262
1263 if (JS_IsException(res_val)) {
1264 exception_val = JS_GetException(ctx);
1265 is_error = JS_IsError(ctx, exception_val);
1266 /* XXX: should get the filename and line number */
1267 if (outfile) {
1268 if (!is_error)
1269 fprintf(outfile, "%sThrow: ", (eval_flags & JS_EVAL_FLAG_STRICT) ?
1270 "strict mode: " : "");
1271 js_print(ctx, JS_NULL, 1, &exception_val);
1272 }
1273 if (is_error) {
1274 JSValue name, stack;
1275 const char *stack_str;
1276
1277 name = JS_GetPropertyStr(ctx, exception_val, "name");
1278 error_name = JS_ToCString(ctx, name);
1279 stack = JS_GetPropertyStr(ctx, exception_val, "stack");
1280 if (!JS_IsUndefined(stack)) {
1281 stack_str = JS_ToCString(ctx, stack);
1282 if (stack_str) {
1283 const char *p;
1284 int len;
1285
1286 if (outfile)
1287 fprintf(outfile, "%s", stack_str);
1288
1289 len = strlen(filename);
1290 p = strstr(stack_str, filename);
1291 if (p != NULL && p[len] == ':') {
1292 error_line = atoi(p + len + 1);
1293 has_error_line = TRUE;
1294 }
1295 JS_FreeCString(ctx, stack_str);
1296 }
1297 }
1298 JS_FreeValue(ctx, stack);
1299 JS_FreeValue(ctx, name);
1300 }
1301 if (is_negative) {
1302 ret = 0;
1303 if (error_type) {
1304 char *error_class;
1305 const char *msg;
1306
1307 msg = JS_ToCString(ctx, exception_val);
1308 error_class = strdup_len(msg, strcspn(msg, ":"));
1309 if (!str_equal(error_class, error_type))
1310 ret = -1;
1311 free(error_class);
1312 JS_FreeCString(ctx, msg);
1313 }
1314 } else {
1315 ret = -1;
1316 }
1317 } else {
1318 if (is_negative)
1319 ret = -1;
1320 else
1321 ret = 0;
1322 }
1323
1324 if (verbose && is_test) {
1325 JSValue msg_val = JS_UNDEFINED;
1326 const char *msg = NULL;
1327 int s_line;
1328 char *s = find_error(filename, &s_line, eval_flags & JS_EVAL_FLAG_STRICT);
1329 const char *strict_mode = (eval_flags & JS_EVAL_FLAG_STRICT) ? "strict mode: " : "";
1330
1331 if (!JS_IsUndefined(exception_val)) {
1332 msg_val = JS_ToString(ctx, exception_val);
1333 msg = JS_ToCString(ctx, msg_val);
1334 }
1335 if (is_negative) { // expect error
1336 if (ret == 0) {
1337 if (msg && s &&
1338 (str_equal(s, "expected error") ||
1339 strstart(s, "unexpected error type:", NULL) ||
1340 str_equal(s, msg))) { // did not have error yet
1341 if (!has_error_line) {
1342 longest_match(buf, msg, pos, &pos, pos_line, &error_line);
1343 }
1344 printf("%s:%d: %sOK, now has error %s\n",
1345 filename, error_line, strict_mode, msg);
1346 fixed_errors++;
1347 }
1348 } else {
1349 if (!s) { // not yet reported
1350 if (msg) {
1351 fprintf(error_out, "%s:%d: %sunexpected error type: %s\n",
1352 filename, error_line, strict_mode, msg);
1353 } else {
1354 fprintf(error_out, "%s:%d: %sexpected error\n",
1355 filename, error_line, strict_mode);
1356 }
1357 new_errors++;
1358 }
1359 }
1360 } else { // should not have error
1361 if (msg) {
1362 if (!s || !str_equal(s, msg)) {
1363 if (!has_error_line) {
1364 char *p = skip_prefix(msg, "Test262 Error: ");
1365 if (strstr(p, "Test case returned non-true value!")) {
1366 longest_match(buf, "runTestCase", pos, &pos, pos_line, &error_line);
1367 } else {
1368 longest_match(buf, p, pos, &pos, pos_line, &error_line);
1369 }
1370 }
1371 fprintf(error_out, "%s:%d: %s%s%s\n", filename, error_line, strict_mode,
1372 error_file ? "unexpected error: " : "", msg);
1373
1374 if (s && (!str_equal(s, msg) || error_line != s_line)) {
1375 printf("%s:%d: %sprevious error: %s\n", filename, s_line, strict_mode, s);
1376 changed_errors++;
1377 } else {
1378 new_errors++;
1379 }
1380 }
1381 } else {
1382 if (s) {
1383 printf("%s:%d: %sOK, fixed error: %s\n", filename, s_line, strict_mode, s);
1384 fixed_errors++;
1385 }
1386 }
1387 }
1388 JS_FreeValue(ctx, msg_val);
1389 JS_FreeCString(ctx, msg);
1390 free(s);
1391 }
1392 JS_FreeCString(ctx, error_name);
1393 JS_FreeValue(ctx, exception_val);
1394 JS_FreeValue(ctx, res_val);
1395 return ret;
1396}
1397
1398static int eval_file(JSContext *ctx, const char *base, const char *p,
1399 int eval_flags)
1400{
1401 char *buf;
1402 size_t buf_len;
1403 char *filename = compose_path(base, p);
1404
1405 buf = load_file(filename, &buf_len);
1406 if (!buf) {
1407 warning("cannot load %s", filename);
1408 goto fail;
1409 }
1410 if (eval_buf(ctx, buf, buf_len, filename, FALSE, FALSE, NULL, stderr,
1411 eval_flags, FALSE)) {
1412 warning("error evaluating %s", filename);
1413 goto fail;
1414 }
1415 free(buf);
1416 free(filename);
1417 return 0;
1418
1419fail:
1420 free(buf);
1421 free(filename);
1422 return 1;
1423}
1424
1425char *extract_desc(const char *buf, char style)
1426{
1427 const char *p, *desc_start;
1428 char *desc;
1429 int len;
1430
1431 p = buf;
1432 while (*p != '\0') {
1433 if (p[0] == '/' && p[1] == '*' && p[2] == style && p[3] != '/') {
1434 p += 3;
1435 desc_start = p;
1436 while (*p != '\0' && (p[0] != '*' || p[1] != '/'))
1437 p++;
1438 if (*p == '\0') {
1439 warning("Expecting end of desc comment");
1440 return NULL;
1441 }
1442 len = p - desc_start;
1443 desc = malloc(len + 1);
1444 memcpy(desc, desc_start, len);
1445 desc[len] = '\0';
1446 return desc;
1447 } else {
1448 p++;
1449 }
1450 }
1451 return NULL;
1452}
1453
1454static char *find_tag(char *desc, const char *tag, int *state)
1455{
1456 char *p;
1457 p = strstr(desc, tag);
1458 if (p) {
1459 p += strlen(tag);
1460 *state = 0;
1461 }
1462 return p;
1463}
1464
1465static char *get_option(char **pp, int *state)
1466{
1467 char *p, *p0, *option = NULL;
1468 if (*pp) {
1469 for (p = *pp;; p++) {
1470 switch (*p) {
1471 case '[':
1472 *state += 1;
1473 continue;
1474 case ']':
1475 *state -= 1;
1476 if (*state > 0)
1477 continue;
1478 p = NULL;
1479 break;
1480 case ' ':
1481 case '\t':
1482 case '\r':
1483 case ',':
1484 case '-':
1485 continue;
1486 case '\n':
1487 if (*state > 0 || p[1] == ' ')
1488 continue;
1489 p = NULL;
1490 break;
1491 case '\0':
1492 p = NULL;
1493 break;
1494 default:
1495 p0 = p;
1496 p += strcspn(p0, " \t\r\n,]");
1497 option = strdup_len(p0, p - p0);
1498 break;
1499 }
1500 break;
1501 }
1502 *pp = p;
1503 }
1504 return option;
1505}
1506
1507void update_stats(JSRuntime *rt, const char *filename) {
1508 JSMemoryUsage stats;
1509 JS_ComputeMemoryUsage(rt, &stats);
1510 if (stats_count++ == 0) {
1511 stats_avg = stats_all = stats_min = stats_max = stats;
1512 stats_min_filename = strdup(filename);
1513 stats_max_filename = strdup(filename);
1514 } else {
1515 if (stats_max.malloc_size < stats.malloc_size) {
1516 stats_max = stats;
1517 free(stats_max_filename);
1518 stats_max_filename = strdup(filename);
1519 }
1520 if (stats_min.malloc_size > stats.malloc_size) {
1521 stats_min = stats;
1522 free(stats_min_filename);
1523 stats_min_filename = strdup(filename);
1524 }
1525
1526#define update(f) stats_avg.f = (stats_all.f += stats.f) / stats_count
1527 update(malloc_count);
1528 update(malloc_size);
1529 update(memory_used_count);
1530 update(memory_used_size);
1531 update(atom_count);
1532 update(atom_size);
1533 update(str_count);
1534 update(str_size);
1535 update(obj_count);
1536 update(obj_size);
1537 update(prop_count);
1538 update(prop_size);
1539 update(shape_count);
1540 update(shape_size);
1541 update(js_func_count);
1542 update(js_func_size);
1543 update(js_func_code_size);
1544 update(js_func_pc2line_count);
1545 update(js_func_pc2line_size);
1546 update(c_func_count);
1547 update(array_count);
1548 update(fast_array_count);
1549 update(fast_array_elements);
1550 }
1551#undef update
1552}
1553
1554int run_test_buf(const char *filename, const char *harness, namelist_t *ip,
1555 char *buf, size_t buf_len, const char* error_type,
1556 int eval_flags, BOOL is_negative, BOOL is_async,
1557 BOOL can_block)
1558{
1559 JSRuntime *rt;
1560 JSContext *ctx;
1561 int i, ret;
1562
1563 rt = JS_NewRuntime();
1564 if (rt == NULL) {
1565 fatal(1, "JS_NewRuntime failure");
1566 }
1567 ctx = JS_NewContext(rt);
1568 if (ctx == NULL) {
1569 JS_FreeRuntime(rt);
1570 fatal(1, "JS_NewContext failure");
1571 }
1572 JS_SetRuntimeInfo(rt, filename);
1573
1574 JS_SetCanBlock(rt, can_block);
1575
1576 /* loader for ES6 modules */
1577 JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, (void *)filename);
1578
1579 add_helpers(ctx);
1580
1581 for (i = 0; i < ip->count; i++) {
1582 if (eval_file(ctx, harness, ip->array[i],
1583 JS_EVAL_TYPE_GLOBAL)) {
1584 fatal(1, "error including %s for %s", ip->array[i], filename);
1585 }
1586 }
1587
1588 ret = eval_buf(ctx, buf, buf_len, filename, TRUE, is_negative,
1589 error_type, outfile, eval_flags, is_async);
1590 ret = (ret != 0);
1591
1592 if (dump_memory) {
1593 update_stats(rt, filename);
1594 }
1595#ifdef CONFIG_AGENT
1596 js_agent_free(ctx);
1597#endif
1598 JS_FreeContext(ctx);
1599 JS_FreeRuntime(rt);
1600
1601 test_count++;
1602 if (ret) {
1603 test_failed++;
1604 if (outfile) {
1605 /* do not output a failure number to minimize diff */
1606 fprintf(outfile, " FAILED\n");
1607 }
1608 }
1609 return ret;
1610}
1611
1612int run_test(const char *filename, int index)
1613{
1614 char harnessbuf[1024];
1615 char *harness;
1616 char *buf;
1617 size_t buf_len;
1618 char *desc, *p;
1619 char *error_type;
1620 int ret, eval_flags, use_strict, use_nostrict;
1621 BOOL is_negative, is_nostrict, is_onlystrict, is_async, is_module, skip;
1622 BOOL can_block;
1623 namelist_t include_list = { 0 }, *ip = &include_list;
1624
1625 is_nostrict = is_onlystrict = is_negative = is_async = is_module = skip = FALSE;
1626 can_block = TRUE;
1627 error_type = NULL;
1628 buf = load_file(filename, &buf_len);
1629
1630 harness = harness_dir;
1631
1632 if (new_style) {
1633 if (!harness) {
1634 p = strstr(filename, "test/");
1635 if (p) {
1636 snprintf(harnessbuf, sizeof(harnessbuf), "%.*s%s",
1637 (int)(p - filename), filename, "harness");
1638 } else {
1639 pstrcpy(harnessbuf, sizeof(harnessbuf), "");
1640 }
1641 harness = harnessbuf;
1642 }
1643 namelist_add(ip, NULL, "sta.js");
1644 namelist_add(ip, NULL, "assert.js");
1645 /* extract the YAML frontmatter */
1646 desc = extract_desc(buf, '-');
1647 if (desc) {
1648 char *ifile, *option;
1649 int state;
1650 p = find_tag(desc, "includes:", &state);
1651 if (p) {
1652 while ((ifile = get_option(&p, &state)) != NULL) {
1653 // skip unsupported harness files
1654 if (find_word(harness_exclude, ifile)) {
1655 skip |= 1;
1656 } else {
1657 namelist_add(ip, NULL, ifile);
1658 }
1659 free(ifile);
1660 }
1661 }
1662 p = find_tag(desc, "flags:", &state);
1663 if (p) {
1664 while ((option = get_option(&p, &state)) != NULL) {
1665 if (str_equal(option, "noStrict") ||
1666 str_equal(option, "raw")) {
1667 is_nostrict = TRUE;
1668 skip |= (test_mode == TEST_STRICT);
1669 }
1670 else if (str_equal(option, "onlyStrict")) {
1671 is_onlystrict = TRUE;
1672 skip |= (test_mode == TEST_NOSTRICT);
1673 }
1674 else if (str_equal(option, "async")) {
1675 is_async = TRUE;
1676 skip |= skip_async;
1677 }
1678 else if (str_equal(option, "module")) {
1679 is_module = TRUE;
1680 skip |= skip_module;
1681 }
1682 else if (str_equal(option, "CanBlockIsFalse")) {
1683 can_block = FALSE;
1684 }
1685 free(option);
1686 }
1687 }
1688 p = find_tag(desc, "negative:", &state);
1689 if (p) {
1690 /* XXX: should extract the phase */
1691 char *q = find_tag(p, "type:", &state);
1692 if (q) {
1693 while (isspace((unsigned char)*q))
1694 q++;
1695 error_type = strdup_len(q, strcspn(q, " \n"));
1696 }
1697 is_negative = TRUE;
1698 }
1699 p = find_tag(desc, "features:", &state);
1700 if (p) {
1701 while ((option = get_option(&p, &state)) != NULL) {
1702 if (find_word(harness_features, option)) {
1703 /* feature is enabled */
1704 } else if (find_word(harness_skip_features, option)) {
1705 /* skip disabled feature */
1706 skip |= 1;
1707 } else {
1708 /* feature is not listed: skip and warn */
1709 printf("%s:%d: unknown feature: %s\n", filename, 1, option);
1710 skip |= 1;
1711 }
1712 free(option);
1713 }
1714 }
1715 free(desc);
1716 }
1717 if (is_async)
1718 namelist_add(ip, NULL, "doneprintHandle.js");
1719 } else {
1720 char *ifile;
1721
1722 if (!harness) {
1723 p = strstr(filename, "test/");
1724 if (p) {
1725 snprintf(harnessbuf, sizeof(harnessbuf), "%.*s%s",
1726 (int)(p - filename), filename, "test/harness");
1727 } else {
1728 pstrcpy(harnessbuf, sizeof(harnessbuf), "");
1729 }
1730 harness = harnessbuf;
1731 }
1732
1733 namelist_add(ip, NULL, "sta.js");
1734
1735 /* include extra harness files */
1736 for (p = buf; (p = strstr(p, "$INCLUDE(\"")) != NULL; p++) {
1737 p += 10;
1738 ifile = strdup_len(p, strcspn(p, "\""));
1739 // skip unsupported harness files
1740 if (find_word(harness_exclude, ifile)) {
1741 skip |= 1;
1742 } else {
1743 namelist_add(ip, NULL, ifile);
1744 }
1745 free(ifile);
1746 }
1747
1748 /* locate the old style configuration comment */
1749 desc = extract_desc(buf, '*');
1750 if (desc) {
1751 if (strstr(desc, "@noStrict")) {
1752 is_nostrict = TRUE;
1753 skip |= (test_mode == TEST_STRICT);
1754 }
1755 if (strstr(desc, "@onlyStrict")) {
1756 is_onlystrict = TRUE;
1757 skip |= (test_mode == TEST_NOSTRICT);
1758 }
1759 if (strstr(desc, "@negative")) {
1760 /* XXX: should extract the regex to check error type */
1761 is_negative = TRUE;
1762 }
1763 free(desc);
1764 }
1765 }
1766
1767 if (outfile && index >= 0) {
1768 fprintf(outfile, "%d: %s%s%s%s%s%s%s\n", index, filename,
1769 is_nostrict ? " @noStrict" : "",
1770 is_onlystrict ? " @onlyStrict" : "",
1771 is_async ? " async" : "",
1772 is_module ? " module" : "",
1773 is_negative ? " @negative" : "",
1774 skip ? " SKIPPED" : "");
1775 fflush(outfile);
1776 }
1777
1778 use_strict = use_nostrict = 0;
1779 /* XXX: should remove 'test_mode' or simplify it just to force
1780 strict or non strict mode for single file tests */
1781 switch (test_mode) {
1782 case TEST_DEFAULT_NOSTRICT:
1783 if (is_onlystrict)
1784 use_strict = 1;
1785 else
1786 use_nostrict = 1;
1787 break;
1788 case TEST_DEFAULT_STRICT:
1789 if (is_nostrict)
1790 use_nostrict = 1;
1791 else
1792 use_strict = 1;
1793 break;
1794 case TEST_NOSTRICT:
1795 if (!is_onlystrict)
1796 use_nostrict = 1;
1797 break;
1798 case TEST_STRICT:
1799 if (!is_nostrict)
1800 use_strict = 1;
1801 break;
1802 case TEST_ALL:
1803 if (is_module) {
1804 use_nostrict = 1;
1805 } else {
1806 if (!is_nostrict)
1807 use_strict = 1;
1808 if (!is_onlystrict)
1809 use_nostrict = 1;
1810 }
1811 break;
1812 }
1813
1814 if (skip || use_strict + use_nostrict == 0) {
1815 test_skipped++;
1816 ret = -2;
1817 } else {
1818 clock_t clocks;
1819
1820 if (is_module) {
1821 eval_flags = JS_EVAL_TYPE_MODULE;
1822 } else {
1823 eval_flags = JS_EVAL_TYPE_GLOBAL;
1824 }
1825 clocks = clock();
1826 ret = 0;
1827 if (use_nostrict) {
1828 ret = run_test_buf(filename, harness, ip, buf, buf_len,
1829 error_type, eval_flags, is_negative, is_async,
1830 can_block);
1831 }
1832 if (use_strict) {
1833 ret |= run_test_buf(filename, harness, ip, buf, buf_len,
1834 error_type, eval_flags | JS_EVAL_FLAG_STRICT,
1835 is_negative, is_async, can_block);
1836 }
1837 clocks = clock() - clocks;
1838 if (outfile && index >= 0 && clocks >= CLOCKS_PER_SEC / 10) {
1839 /* output timings for tests that take more than 100 ms */
1840 fprintf(outfile, " time: %d ms\n", (int)(clocks * 1000LL / CLOCKS_PER_SEC));
1841 }
1842 }
1843 namelist_free(&include_list);
1844 free(error_type);
1845 free(buf);
1846
1847 return ret;
1848}
1849
1850/* run a test when called by test262-harness+eshost */
1851int run_test262_harness_test(const char *filename, BOOL is_module)
1852{
1853 JSRuntime *rt;
1854 JSContext *ctx;
1855 char *buf;
1856 size_t buf_len;
1857 int eval_flags, ret_code, ret;
1858 JSValue res_val;
1859 BOOL can_block;
1860
1861 outfile = stdout; /* for js_print */
1862
1863 rt = JS_NewRuntime();
1864 if (rt == NULL) {
1865 fatal(1, "JS_NewRuntime failure");
1866 }
1867 ctx = JS_NewContext(rt);
1868 if (ctx == NULL) {
1869 JS_FreeRuntime(rt);
1870 fatal(1, "JS_NewContext failure");
1871 }
1872 JS_SetRuntimeInfo(rt, filename);
1873
1874 can_block = TRUE;
1875 JS_SetCanBlock(rt, can_block);
1876
1877 /* loader for ES6 modules */
1878 JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, (void *)filename);
1879
1880 add_helpers(ctx);
1881
1882 buf = load_file(filename, &buf_len);
1883
1884 if (is_module) {
1885 eval_flags = JS_EVAL_TYPE_MODULE;
1886 } else {
1887 eval_flags = JS_EVAL_TYPE_GLOBAL;
1888 }
1889 res_val = JS_Eval(ctx, buf, buf_len, filename, eval_flags);
1890 ret_code = 0;
1891 if (JS_IsException(res_val)) {
1892 js_std_dump_error(ctx);
1893 ret_code = 1;
1894 } else {
1895 JSValue promise = JS_UNDEFINED;
1896 if (is_module) {
1897 promise = res_val;
1898 } else {
1899 JS_FreeValue(ctx, res_val);
1900 }
1901 for(;;) {
1902 JSContext *ctx1;
1903 ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
1904 if (ret < 0) {
1905 js_std_dump_error(ctx1);
1906 ret_code = 1;
1907 } else if (ret == 0) {
1908 break;
1909 }
1910 }
1911 /* dump the error if the module returned an error. */
1912 if (is_module) {
1913 JSPromiseStateEnum state = JS_PromiseState(ctx, promise);
1914 if (state == JS_PROMISE_REJECTED) {
1915 JS_Throw(ctx, JS_PromiseResult(ctx, promise));
1916 js_std_dump_error(ctx);
1917 ret_code = 1;
1918 }
1919 }
1920 JS_FreeValue(ctx, promise);
1921 }
1922 free(buf);
1923#ifdef CONFIG_AGENT
1924 js_agent_free(ctx);
1925#endif
1926 JS_FreeContext(ctx);
1927 JS_FreeRuntime(rt);
1928 return ret_code;
1929}
1930
1931clock_t last_clock;
1932
1933void show_progress(int force) {
1934 clock_t t = clock();
1935 if (force || !last_clock || (t - last_clock) > CLOCKS_PER_SEC / 20) {
1936 last_clock = t;
1937 if (compact) {
1938 static int last_test_skipped;
1939 static int last_test_failed;
1940 static int dots;
1941 char c = '.';
1942 if (test_skipped > last_test_skipped)
1943 c = '-';
1944 if (test_failed > last_test_failed)
1945 c = '!';
1946 last_test_skipped = test_skipped;
1947 last_test_failed = test_failed;
1948 fputc(c, stderr);
1949 if (force || ++dots % 60 == 0) {
1950 fprintf(stderr, " %d/%d/%d\n",
1951 test_failed, test_count, test_skipped);
1952 }
1953 } else {
1954 /* output progress indicator: erase end of line and return to col 0 */
1955 fprintf(stderr, "%d/%d/%d\033[K\r",
1956 test_failed, test_count, test_skipped);
1957 }
1958 fflush(stderr);
1959 }
1960}
1961
1962static int slow_test_threshold;
1963
1964void run_test_dir_list(namelist_t *lp, int start_index, int stop_index)
1965{
1966 int i;
1967
1968 namelist_sort(lp);
1969 for (i = 0; i < lp->count; i++) {
1970 const char *p = lp->array[i];
1971 if (namelist_find(&exclude_list, p) >= 0) {
1972 test_excluded++;
1973 } else if (test_index < start_index) {
1974 test_skipped++;
1975 } else if (stop_index >= 0 && test_index > stop_index) {
1976 test_skipped++;
1977 } else {
1978 int ti;
1979 if (slow_test_threshold != 0) {
1980 ti = get_clock_ms();
1981 } else {
1982 ti = 0;
1983 }
1984 run_test(p, test_index);
1985 if (slow_test_threshold != 0) {
1986 ti = get_clock_ms() - ti;
1987 if (ti >= slow_test_threshold)
1988 fprintf(stderr, "\n%s (%d ms)\n", p, ti);
1989 }
1990 show_progress(FALSE);
1991 }
1992 test_index++;
1993 }
1994 show_progress(TRUE);
1995}
1996
1997void help(void)
1998{
1999 printf("run-test262 version " CONFIG_VERSION "\n"
2000 "usage: run-test262 [options] {-f file ... | [dir_list] [index range]}\n"
2001 "-h help\n"
2002 "-a run tests in strict and nostrict modes\n"
2003 "-m print memory usage summary\n"
2004 "-n use new style harness\n"
2005 "-N run test prepared by test262-harness+eshost\n"
2006 "-s run tests in strict mode, skip @nostrict tests\n"
2007 "-E only run tests from the error file\n"
2008 "-C use compact progress indicator\n"
2009 "-t show timings\n"
2010 "-u update error file\n"
2011 "-v verbose: output error messages\n"
2012 "-T duration display tests taking more than 'duration' ms\n"
2013 "-c file read configuration from 'file'\n"
2014 "-d dir run all test files in directory tree 'dir'\n"
2015 "-e file load the known errors from 'file'\n"
2016 "-f file execute single test from 'file'\n"
2017 "-r file set the report file name (default=none)\n"
2018 "-x file exclude tests listed in 'file'\n");
2019 exit(1);
2020}
2021
2022char *get_opt_arg(const char *option, char *arg)
2023{
2024 if (!arg) {
2025 fatal(2, "missing argument for option %s", option);
2026 }
2027 return arg;
2028}
2029
2030int main(int argc, char **argv)
2031{
2032 int optind, start_index, stop_index;
2033 BOOL is_dir_list;
2034 BOOL only_check_errors = FALSE;
2035 const char *filename;
2036 const char *ignore = "";
2037 BOOL is_test262_harness = FALSE;
2038 BOOL is_module = FALSE;
2039 clock_t clocks;
2040
2041#if !defined(_WIN32)
2042 compact = !isatty(STDERR_FILENO);
2043 /* Date tests assume California local time */
2044 setenv("TZ", "America/Los_Angeles", 1);
2045#endif
2046
2047 optind = 1;
2048 while (optind < argc) {
2049 char *arg = argv[optind];
2050 if (*arg != '-')
2051 break;
2052 optind++;
2053 if (strstr("-c -d -e -x -f -r -E -T", arg))
2054 optind++;
2055 if (strstr("-d -f", arg))
2056 ignore = "testdir"; // run only the tests from -d or -f
2057 }
2058
2059 /* cannot use getopt because we want to pass the command line to
2060 the script */
2061 optind = 1;
2062 is_dir_list = TRUE;
2063 while (optind < argc) {
2064 char *arg = argv[optind];
2065 if (*arg != '-')
2066 break;
2067 optind++;
2068 if (str_equal(arg, "-h")) {
2069 help();
2070 } else if (str_equal(arg, "-m")) {
2071 dump_memory++;
2072 } else if (str_equal(arg, "-n")) {
2073 new_style++;
2074 } else if (str_equal(arg, "-s")) {
2075 test_mode = TEST_STRICT;
2076 } else if (str_equal(arg, "-a")) {
2077 test_mode = TEST_ALL;
2078 } else if (str_equal(arg, "-t")) {
2079 show_timings++;
2080 } else if (str_equal(arg, "-u")) {
2081 update_errors++;
2082 } else if (str_equal(arg, "-v")) {
2083 verbose++;
2084 } else if (str_equal(arg, "-C")) {
2085 compact = 1;
2086 } else if (str_equal(arg, "-c")) {
2087 load_config(get_opt_arg(arg, argv[optind++]), ignore);
2088 } else if (str_equal(arg, "-d")) {
2089 enumerate_tests(get_opt_arg(arg, argv[optind++]));
2090 } else if (str_equal(arg, "-e")) {
2091 error_filename = get_opt_arg(arg, argv[optind++]);
2092 } else if (str_equal(arg, "-x")) {
2093 namelist_load(&exclude_list, get_opt_arg(arg, argv[optind++]));
2094 } else if (str_equal(arg, "-f")) {
2095 is_dir_list = FALSE;
2096 } else if (str_equal(arg, "-r")) {
2097 report_filename = get_opt_arg(arg, argv[optind++]);
2098 } else if (str_equal(arg, "-E")) {
2099 only_check_errors = TRUE;
2100 } else if (str_equal(arg, "-T")) {
2101 slow_test_threshold = atoi(get_opt_arg(arg, argv[optind++]));
2102 } else if (str_equal(arg, "-N")) {
2103 is_test262_harness = TRUE;
2104 } else if (str_equal(arg, "--module")) {
2105 is_module = TRUE;
2106 } else {
2107 fatal(1, "unknown option: %s", arg);
2108 break;
2109 }
2110 }
2111
2112 if (optind >= argc && !test_list.count)
2113 help();
2114
2115 if (is_test262_harness) {
2116 return run_test262_harness_test(argv[optind], is_module);
2117 }
2118
2119 error_out = stdout;
2120 if (error_filename) {
2121 error_file = load_file(error_filename, NULL);
2122 if (only_check_errors && error_file) {
2123 namelist_free(&test_list);
2124 namelist_add_from_error_file(&test_list, error_file);
2125 }
2126 if (update_errors) {
2127 free(error_file);
2128 error_file = NULL;
2129 error_out = fopen(error_filename, "w");
2130 if (!error_out) {
2131 perror_exit(1, error_filename);
2132 }
2133 }
2134 }
2135
2136 update_exclude_dirs();
2137
2138 clocks = clock();
2139
2140 if (is_dir_list) {
2141 if (optind < argc && !isdigit((unsigned char)argv[optind][0])) {
2142 filename = argv[optind++];
2143 namelist_load(&test_list, filename);
2144 }
2145 start_index = 0;
2146 stop_index = -1;
2147 if (optind < argc) {
2148 start_index = atoi(argv[optind++]);
2149 if (optind < argc) {
2150 stop_index = atoi(argv[optind++]);
2151 }
2152 }
2153 if (!report_filename || str_equal(report_filename, "none")) {
2154 outfile = NULL;
2155 } else if (str_equal(report_filename, "-")) {
2156 outfile = stdout;
2157 } else {
2158 outfile = fopen(report_filename, "wb");
2159 if (!outfile) {
2160 perror_exit(1, report_filename);
2161 }
2162 }
2163 run_test_dir_list(&test_list, start_index, stop_index);
2164
2165 if (outfile && outfile != stdout) {
2166 fclose(outfile);
2167 outfile = NULL;
2168 }
2169 } else {
2170 outfile = stdout;
2171 while (optind < argc) {
2172 run_test(argv[optind++], -1);
2173 }
2174 }
2175
2176 clocks = clock() - clocks;
2177
2178 if (dump_memory) {
2179 if (dump_memory > 1 && stats_count > 1) {
2180 printf("\nMininum memory statistics for %s:\n\n", stats_min_filename);
2181 JS_DumpMemoryUsage(stdout, &stats_min, NULL);
2182 printf("\nMaximum memory statistics for %s:\n\n", stats_max_filename);
2183 JS_DumpMemoryUsage(stdout, &stats_max, NULL);
2184 }
2185 printf("\nAverage memory statistics for %d tests:\n\n", stats_count);
2186 JS_DumpMemoryUsage(stdout, &stats_avg, NULL);
2187 printf("\n");
2188 }
2189
2190 if (is_dir_list) {
2191 fprintf(stderr, "Result: %d/%d error%s",
2192 test_failed, test_count, test_count != 1 ? "s" : "");
2193 if (test_excluded)
2194 fprintf(stderr, ", %d excluded", test_excluded);
2195 if (test_skipped)
2196 fprintf(stderr, ", %d skipped", test_skipped);
2197 if (error_file) {
2198 if (new_errors)
2199 fprintf(stderr, ", %d new", new_errors);
2200 if (changed_errors)
2201 fprintf(stderr, ", %d changed", changed_errors);
2202 if (fixed_errors)
2203 fprintf(stderr, ", %d fixed", fixed_errors);
2204 }
2205 fprintf(stderr, "\n");
2206 if (show_timings)
2207 fprintf(stderr, "Total time: %.3fs\n", (double)clocks / CLOCKS_PER_SEC);
2208 }
2209
2210 if (error_out && error_out != stdout) {
2211 fclose(error_out);
2212 error_out = NULL;
2213 }
2214
2215 namelist_free(&test_list);
2216 namelist_free(&exclude_list);
2217 namelist_free(&exclude_dir_list);
2218 free(harness_dir);
2219 free(harness_features);
2220 free(harness_exclude);
2221 free(error_file);
2222
2223 /* Signal that the error file is out of date. */
2224 return new_errors || changed_errors || fixed_errors;
2225}
2226