ref: 857ffd39b0f2facb4ce1a7a81fc0d2f4436ace51
parent: e00c9ba79ca5f6508e3270d529d144fd566ce158
author: Avi Halachmi (:avih) <avihpit@yahoo.com>
date: Sun Sep 2 19:41:06 EDT 2018
tools: add test262 and harness tools: add test262-harness.js to run a single test Usage: mujs <this-file> -- [-f] [-l file1.js -l ...] suit-root test-file -f: print full paths/stacktraces if possible -l: load a js file after the harness and before the test (to override things) tools: add test-262 launcher It can run the entire suite or just a sub-folder or one file, and by default it skips tests which are known to crash/hang. test262-harness: @negative: match regex if exists Some @negative tests add a regex which needs to match the error. This wasn't tested, and now it is. This results in few more failures. The actual string to compare is not documented, but it appears to be err.message for plain Error(..) where the message is always compared to "NotEarlyError" (equals/different), and err.name for anything else. test262 launcher: minor improvements and custom mujs path - Use getopts instead of custom arguments parsing - Support -m path/to/mujs - Change -s (skip bad tests) to -b, and add -B to run only bad tests (useful when trying to update the known bad tests list). - Combine stderr to stdout on a per-test basis (mujs function warnings) - Exit with error if failures > 0 test262: remove 5 tests which no longer crash/hang test262 and harness: support -s to print failed source test262 and harness: convert spaces to tabs
--- /dev/null
+++ b/tools/test262
@@ -1,0 +1,139 @@
+#!/bin/sh
+
+usage() {+ [ "${1-}" ] && { to=2; >&$to printf "Error: %s\n" "$1"; } || to=1+ >&$to echo "Usage: ${0##*/} [OPTIONS] test-file | test-dir"+ >&$to echo "Run test-262 ES5 test file or directory (at the test262 dir)."
+ >&$to echo " -s Print source code of failed tests."
+ >&$to echo " -p Print every test name before running it"
+ >&$to echo " -f Display full paths and full stack trace when possible"
+ >&$to echo " -l file.js Load file.js after the harness and before the test"
+ >&$to echo " -m MUJS MUJS is [path/to/]mujs binary to test"
+ >&$to echo " Default is $(dirname "$0")/../build/release/mujs or mujs at \$PATH"
+ >&$to echo " -b Don't skip known bad (crashing/hanging) tests"
+ >&$to echo " -B Run only known bad tests"
+ exit $((to-1))
+}
+
+KNOWN_BAD="
+ --hang-with-sta.js:
+ S15.1.3.2_A2.5_T1.js
+ S15.1.3.1_A2.5_T1.js
+
+ --Hang-(or-taking-more-than-few-seconds):
+ 15.4.4.18-3-14.js
+ 15.4.4.20-3-14.js
+ S15.4.4.10_A3_T2.js
+ S15.4.4.10_A3_T1.js
+ 15.4.4.19-3-29.js
+ 15.4.4.19-3-28.js
+ 15.4.4.19-3-8.js
+ 15.4.4.19-3-14.js
+ S15.4.4.8_A3_T3.js
+ 15.4.4.22-3-9.js
+ 15.4.4.22-3-7.js
+ 15.4.4.22-3-25.js
+ 15.4.4.22-3-14.js
+ 15.4.4.21-3-14.js
+ 15.4.4.15-3-28.js
+ 15.4.4.15-3-14.js
+ 15.4.4.15-3-7.js
+ 15.4.4.15-3-25.js
+ 15.4.4.15-3-9.js
+"
+
+SKIP_KNOWN=yes # "yes": skip bad "no": don't skip "neg": run only bad
+PRINT_ALL=
+EXTRA_ARGS=
+mujs=
+
+while getopts bBfhl:ps o; do
+ case $o in
+ h) usage ;;
+ b) SKIP_KNOWN=no ;;
+ B) SKIP_KNOWN=neg ;;
+ p) PRINT_ALL=yes ;;
+ s) EXTRA_ARGS="$EXTRA_ARGS -s" ;;
+ f) EXTRA_ARGS="$EXTRA_ARGS -f" ;;
+ l) EXTRA_ARGS="$EXTRA_ARGS -l $OPTARG"; shift ;;
+ m) mujs=$OPTARG;;
+ *) usage "unknown option -$o" ;;
+ esac
+done
+shift $((OPTIND-1))
+[ $# = 1 ] || usage "expecting one file/dir"
+
+BAD=
+if [ "$SKIP_KNOWN" != no ]; then
+ for b in $KNOWN_BAD; do
+ BAD="$BAD $b "
+ done
+fi
+
+find_root() {+ ROOT=$1
+ n=0
+ while ! [ -e "$ROOT"/test/harness/sta.js ]; do
+ ROOT=$ROOT/..
+ n=$((n+1))
+ [ $n -lt 10 ] || usage "can't find test-suite root"
+ done
+}
+
+if [ -d "$1" ]; then
+ find_root "$1"
+
+ if [ "$ROOT" = "$1" ]; then
+ FILES_CMD='find "$1/test/suite" -name "*.js" | sort -V'
+ else
+ FILES_CMD='find "$1" -name "*.js" | sort -V'
+ fi
+else
+ find_root "$(dirname "$1")"
+ FILES_CMD='printf "%s\n" "$1"'
+fi
+
+if ! [ "$mujs" ]; then
+ # try to use a recently built mujs rather than a global one
+ mujs=$(dirname "$0")/../build/release/mujs
+ [ -e "$mujs" ] || mujs=mujs
+fi
+jsharness=$(dirname "$0")/test262-harness.js
+
+total=0
+skipped=0
+failed=0
+
+eval "$FILES_CMD" | (
+ while read f && [ "$f" ]; do
+ total=$((total+1))
+ base=${f##*/}+
+ case $BAD in *" $base "*) bad=yes;; *) bad=no;; esac
+
+ case $bad-$SKIP_KNOWN in
+ yes-yes)
+ skipped=$((skipped+1))
+ printf "[Skipping: $base]\n\n"
+ ;;
+ no-neg) # not known bad and running only bad - don't print anything
+ skipped=$((skipped+1))
+ ;;
+ *)
+ [ "$PRINT_ALL" ] && echo "Testing: $f"
+ if ! $mujs $jsharness $EXTRA_ARGS "$ROOT" $f 2>&1; then
+ failed=$((failed+1))
+ echo
+ fi
+ esac
+ done
+
+ if [ $total -gt 1 ]; then
+ printf "Total: $total\n"
+ printf "Pass: %${#total}s\n" $((total - skipped - failed))+ printf "Skip: %${#total}s\n" $skipped+ printf "Fail: %${#total}s\n" $failed+ fi
+
+ [ "$failed" = 0 ]
+)
--- /dev/null
+++ b/tools/test262-harness.js
@@ -1,0 +1,152 @@
+/*
+ * Runs one test file from the ES5 test suite test-262
+ * Usage: mujs <this-file> [-s ] [-f] [-l file1.js -l ...] suit-root test-file
+ * -s: print test source on failure
+ * -f: print full paths/stacktraces if possible
+ * -l: load a js file after the harness and before the test (to override things)
+ *
+ * If there are errors, print them and exits with code 1, else exit code is 0.
+ *
+ * The test suite is at: https://github.com/tc39/test262.git
+ * The ES5 suite is at branch "es5-tests"
+ *
+ * - The test suite throws on any error, possibly with info at ex.message .
+ * - Some tests make irreversible changes to global attrubutes, therefore it's
+ * required to run each test file in a new mujs instance.
+ */
+
+(function(global) {+ "use strict";
+
+ // clean the global environment
+ var mujs = {};+
+ ["gc", "load", "compile", "print", "write", "read", "readline", "quit", "scriptArgs"]
+ .forEach(function(a) {+ mujs[a] = global[a];
+ delete global[a];
+ });
+
+ // restore the original Error.toString behavior - it's being tested too
+ Error.prototype.toString = function() {+ return this.name + ': ' + this.message;
+ }
+
+ function die_usage(str) {+ if (str)
+ mujs.print(str);
+ mujs.print("Usage: mujs <this-file> [-f] [-l file1.js -l ...] suit-root test-file");+ mujs.quit(1);
+ }
+
+ // our file loader
+ function load(str, as_filename) {+ try {+ var runtime_err = false;
+ var compiled = mujs.compile(str, as_filename);
+ runtime_err = true;
+ compiled();
+ return false;
+ } catch (e) {+ return {err: e, runtime: runtime_err};+ }
+ }
+
+ var args = mujs.scriptArgs;
+ var full_mode = false;
+ var print_src = false;
+ var overrides = [];
+ while ((""+args[0])[0] == "-") {+ switch (args[0]) {+ case "-f": full_mode = true;
+ break;
+ case "-s": print_src = true;
+ break;
+ case "-l": args.shift();
+ overrides.push(args[0]);
+ break;
+ default: die_usage("Unknown option " + args[0]);+ }
+ args.shift();
+ }
+ if (args.length != 2)
+ die_usage("Exactly 2 paths are expected");+ var root_path = args[0];
+ var test_path = args[1];
+
+ // load suite utils
+ ["sta.js", "testBuiltInObject.js", "testIntl.js"]
+ .forEach(function(u) {+ var path = root_path + "/test/harness/" + u;
+ var as_file = full_mode ? path : "test/harness/" + u;
+ var err = load(mujs.read(path), as_file);
+ if (err) throw (err.err);
+ });
+
+ // load user overrides (e.g. reduced getPrecision), with a global mujs
+ if (overrides.length) {+ global.mujs = mujs
+ overrides.forEach(function(f) {+ var err = load(mujs.read(f), f);
+ if (err) throw (err.err);
+ });
+ delete global.mujs;
+ }
+
+ // the actual test
+ var source = mujs.read(test_path);
+ var negative = !!source.match(/@negative/);
+ if (negative)
+ var neg_str = (source.match(/@negative (.*)/) || [])[1];
+ var as_file = test_path;
+ if (!full_mode) {+ as_file = test_path.replace(/\\/g, "/");
+ var sub = as_file.indexOf("/suite/");+ if (sub >= 0)
+ as_file = "test" + as_file.substring(sub);
+ }
+
+ var result = load(mujs.read(test_path), as_file);
+ if (!!result == negative) {+ // The docs don't really help about matching str, but this covers all cases
+ if (neg_str)
+ var err_for_match = /NotEarlyError/.test(neg_str) ? result.err.message : result.err.name;
+ if (!negative || !neg_str || RegExp(neg_str).exec(err_for_match))
+ mujs.quit(0);
+ }
+
+ // failed
+ // FIXME: @description can span lines. E.g. test/suite/bestPractice/Sbp_A3_T2.js
+ var desc = source.match(/@description (.*)/);
+ var info = "[File] " + as_file +
+ (desc ? "\n[Desc] " + desc[1] : "") +
+ "\n";
+
+ if (result) {+ var err = result.err;
+ var msg = !neg_str ? err : "[Mismatch @negative " + neg_str + "]" + "\n " + err;
+
+ info += (result.runtime ? "[run] " : "[load] ") + msg;
+ if (err && err.stackTrace && (result.runtime || full_mode)) {+ if (full_mode) {+ info += err.stackTrace;
+ } else {+ // trim the internal loader from the trace
+ var internal = err.stackTrace.indexOf("\n" + load("mujs_blahblah()").err.stackTrace.trim().split("\n")[1]);+ if (internal >= 0)
+ info += err.stackTrace.substring(0, internal);
+ else
+ info += err.stackTrace;
+ }
+ }
+ } else {+ info += "[run] [Error expected but none thrown]";
+ }
+
+ if (print_src)
+ info += "\n[Source]\n" + source;
+
+ mujs.print(info);
+ mujs.quit(1);
+
+})(this)