| 1 | #!/usr/bin/env python3
|
|---|
| 2 | """BNC regression test runner.
|
|---|
| 3 |
|
|---|
| 4 | Each test case is a directory tests/cases/<name>/ containing
|
|---|
| 5 |
|
|---|
| 6 | case.json - how to run BNC and which outputs to compare
|
|---|
| 7 | expected/ - gzipped, normalised reference outputs ("golden files")
|
|---|
| 8 |
|
|---|
| 9 | case.json keys:
|
|---|
| 10 | description free text
|
|---|
| 11 | config BNC config file, relative to the BNC root (or "/dev/null")
|
|---|
| 12 | raw optional raw capture, relative to tests/ -> passed as --file
|
|---|
| 13 | args optional list of extra command line arguments (e.g. -key pairs)
|
|---|
| 14 | exit_code expected exit code (default 0; replay via --file exits with 5)
|
|---|
| 15 | timeout seconds (default 600)
|
|---|
| 16 | outputs list of output specs, each either a glob string or an object
|
|---|
| 17 | glob path relative to the work directory; stdout.txt and
|
|---|
| 18 | stderr.txt hold BNC's console output
|
|---|
| 19 | name reference name; if given the glob must match exactly
|
|---|
| 20 | one file, otherwise every match is compared under its
|
|---|
| 21 | basename and missing/extra files are reported
|
|---|
| 22 | abs_tol compare numbers with this absolute tolerance
|
|---|
| 23 | instead of requiring identical text
|
|---|
| 24 | debug_trace true for BNC_DEBUG_SSR stderr traces: reported as
|
|---|
| 25 | SKIP instead of FAIL if the build has no trace output
|
|---|
| 26 | fakecaster optional local Ntrip caster for an upload during the run:
|
|---|
| 27 | {"port", "out" (raw file, relative to the work dir), "sta",
|
|---|
| 28 | "time" (epoch stamped on the raw file), "timeout"}
|
|---|
| 29 | post optional list of further BNC runs after the main run, each
|
|---|
| 30 | {"dir" (subdirectory of the work dir), "config", "raw"
|
|---|
| 31 | (relative to the work dir), "args", "exit_code", "timeout"};
|
|---|
| 32 | outputs may refer to their files as "<dir>/Output/...";
|
|---|
| 33 | instead of a BNC run, {"dir", "tool": [script in tools/, args]}
|
|---|
| 34 | runs a Python tool in the work directory, stdout to <dir>/stdout.txt
|
|---|
| 35 |
|
|---|
| 36 | BNC runs in tests/work/<name>/, which contains an "Input" symlink to
|
|---|
| 37 | Example_Configs/Input and an empty "Output" directory, so the example
|
|---|
| 38 | configurations work unchanged and never write into Example_Configs/.
|
|---|
| 39 |
|
|---|
| 40 | Usage:
|
|---|
| 41 | tests/run_tests.py run all cases
|
|---|
| 42 | tests/run_tests.py ssr_ rinex run cases whose name contains a pattern
|
|---|
| 43 | tests/run_tests.py --update NAME (re)generate the reference outputs; references of
|
|---|
| 44 | outputs no longer produced are removed
|
|---|
| 45 | tests/run_tests.py --list
|
|---|
| 46 | """
|
|---|
| 47 |
|
|---|
| 48 | import argparse
|
|---|
| 49 | import concurrent.futures
|
|---|
| 50 | import difflib
|
|---|
| 51 | import glob
|
|---|
| 52 | import gzip
|
|---|
| 53 | import json
|
|---|
| 54 | import os
|
|---|
| 55 | import re
|
|---|
| 56 | import shutil
|
|---|
| 57 | import subprocess
|
|---|
| 58 | import sys
|
|---|
| 59 | import time
|
|---|
| 60 |
|
|---|
| 61 | TESTS = os.path.dirname(os.path.abspath(__file__))
|
|---|
| 62 | ROOT = os.path.dirname(TESTS)
|
|---|
| 63 | CASES = os.path.join(TESTS, "cases")
|
|---|
| 64 | WORK = os.path.join(TESTS, "work")
|
|---|
| 65 |
|
|---|
| 66 | # Content that changes from run to run or with the BNC version
|
|---|
| 67 | DEFAULT_FILTERS = [
|
|---|
| 68 | # wall-clock prefix of BNC log lines "yy-mm-dd hh:mm:ss "
|
|---|
| 69 | (re.compile(r"^\d\d-\d\d-\d\d \d\d:\d\d:\d\d "), ""),
|
|---|
| 70 | (re.compile(r"BNC v?\d+\.\d+(\.\d+)?"), "BNC <VERSION>"),
|
|---|
| 71 | (re.compile(r"^.*(PGM / RUN BY / DATE)\s*$"), r"<MASKED> \1"),
|
|---|
| 72 | (re.compile(r"^(Run by\s*:).*$"), r"\1 <MASKED>"),
|
|---|
| 73 | (re.compile(r"^(Date\s*:).*$"), r"\1 <MASKED>"),
|
|---|
| 74 | ]
|
|---|
| 75 |
|
|---|
| 76 | # Lines dropped completely. The start line carries the wall-clock date and,
|
|---|
| 77 | # in --file replay, lands in a log file named after today's date, whereas all
|
|---|
| 78 | # other log lines go to a file named after the data date.
|
|---|
| 79 | DROP_LINES = re.compile(r"========== Start BNC")
|
|---|
| 80 |
|
|---|
| 81 | NUMBER = re.compile(r"[-+]?(?:\d+\.\d*|\.\d+|\d+)(?:[eEdD][-+]?\d+)?")
|
|---|
| 82 |
|
|---|
| 83 |
|
|---|
| 84 | def normalise(text):
|
|---|
| 85 | out = []
|
|---|
| 86 | for line in text.splitlines():
|
|---|
| 87 | if DROP_LINES.search(line):
|
|---|
| 88 | continue
|
|---|
| 89 | for rx, repl in DEFAULT_FILTERS:
|
|---|
| 90 | line = rx.sub(repl, line)
|
|---|
| 91 | out.append(line.rstrip())
|
|---|
| 92 | return "\n".join(out) + "\n" if out else ""
|
|---|
| 93 |
|
|---|
| 94 |
|
|---|
| 95 | def read_text(path):
|
|---|
| 96 | with open(path, "rb") as f:
|
|---|
| 97 | return f.read().decode("latin-1")
|
|---|
| 98 |
|
|---|
| 99 |
|
|---|
| 100 | def read_expected(path):
|
|---|
| 101 | with gzip.open(path, "rb") as f:
|
|---|
| 102 | return f.read().decode("latin-1")
|
|---|
| 103 |
|
|---|
| 104 |
|
|---|
| 105 | def write_expected(path, text):
|
|---|
| 106 | os.makedirs(os.path.dirname(path), exist_ok=True)
|
|---|
| 107 | # mtime=0 keeps the .gz byte-identical when the content is unchanged
|
|---|
| 108 | with open(path, "wb") as raw:
|
|---|
| 109 | with gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as f:
|
|---|
| 110 | f.write(text.encode("latin-1"))
|
|---|
| 111 |
|
|---|
| 112 |
|
|---|
| 113 | def numeric_equal(exp, act, tol):
|
|---|
| 114 | """Line-by-line compare; numbers may differ by tol, other text must match."""
|
|---|
| 115 | el, al = exp.splitlines(), act.splitlines()
|
|---|
| 116 | if len(el) != len(al):
|
|---|
| 117 | return False
|
|---|
| 118 | for e, a in zip(el, al):
|
|---|
| 119 | if NUMBER.sub("#", e) != NUMBER.sub("#", a):
|
|---|
| 120 | return False
|
|---|
| 121 | for ne, na in zip(NUMBER.findall(e), NUMBER.findall(a)):
|
|---|
| 122 | try:
|
|---|
| 123 | if abs(float(ne.replace("D", "e").replace("d", "e")) -
|
|---|
| 124 | float(na.replace("D", "e").replace("d", "e"))) > tol:
|
|---|
| 125 | return False
|
|---|
| 126 | except ValueError:
|
|---|
| 127 | if ne != na:
|
|---|
| 128 | return False
|
|---|
| 129 | return True
|
|---|
| 130 |
|
|---|
| 131 |
|
|---|
| 132 | class Result:
|
|---|
| 133 | def __init__(self, name):
|
|---|
| 134 | self.name = name
|
|---|
| 135 | self.status = "PASS"
|
|---|
| 136 | self.messages = []
|
|---|
| 137 | self.seconds = 0.0
|
|---|
| 138 |
|
|---|
| 139 | def fail(self, msg):
|
|---|
| 140 | self.status = "FAIL"
|
|---|
| 141 | self.messages.append(msg)
|
|---|
| 142 |
|
|---|
| 143 | def skip(self, msg):
|
|---|
| 144 | if self.status == "PASS":
|
|---|
| 145 | self.status = "SKIP"
|
|---|
| 146 | self.messages.append(msg)
|
|---|
| 147 |
|
|---|
| 148 |
|
|---|
| 149 | def load_case(name):
|
|---|
| 150 | with open(os.path.join(CASES, name, "case.json")) as f:
|
|---|
| 151 | return json.load(f)
|
|---|
| 152 |
|
|---|
| 153 |
|
|---|
| 154 | def resolve(base, path):
|
|---|
| 155 | return path if os.path.isabs(path) else os.path.normpath(os.path.join(base, path))
|
|---|
| 156 |
|
|---|
| 157 |
|
|---|
| 158 | def run_bnc(bnc, workdir, spec, rawBase):
|
|---|
| 159 | """Run BNC once in workdir (created with Input link and Output dir); spec
|
|---|
| 160 | holds config, optional raw (relative to tests/, or to rawBase if given),
|
|---|
| 161 | args and timeout. Returns the exit code or None on timeout."""
|
|---|
| 162 | os.makedirs(os.path.join(workdir, "Output"), exist_ok=True)
|
|---|
| 163 | if not os.path.exists(os.path.join(workdir, "Input")):
|
|---|
| 164 | os.symlink(os.path.join(ROOT, "Example_Configs", "Input"), os.path.join(workdir, "Input"))
|
|---|
| 165 | cmd = [bnc, "--nw", "--platform", "offscreen", "--conf", resolve(ROOT, spec["config"])]
|
|---|
| 166 | if spec.get("raw"):
|
|---|
| 167 | cmd += ["--file", resolve(rawBase or TESTS, spec["raw"])]
|
|---|
| 168 | cmd += spec.get("args", [])
|
|---|
| 169 | env = dict(os.environ, QT_QPA_PLATFORM="offscreen", LC_ALL="C", TZ="UTC")
|
|---|
| 170 | with open(os.path.join(workdir, "stdout.txt"), "wb") as out, \
|
|---|
| 171 | open(os.path.join(workdir, "stderr.txt"), "wb") as err:
|
|---|
| 172 | try:
|
|---|
| 173 | code = subprocess.run(cmd, cwd=workdir, stdout=out, stderr=err, env=env,
|
|---|
| 174 | timeout=spec.get("timeout", 600)).returncode
|
|---|
| 175 | except subprocess.TimeoutExpired:
|
|---|
| 176 | code = None
|
|---|
| 177 | with open(os.path.join(workdir, "command.txt"), "w") as f:
|
|---|
| 178 | f.write(" ".join(cmd) + "\n")
|
|---|
| 179 | return code
|
|---|
| 180 |
|
|---|
| 181 |
|
|---|
| 182 | def run_tool(work, spec):
|
|---|
| 183 | """Run tools/<script> with args in the work directory, stdout to
|
|---|
| 184 | <dir>/stdout.txt. Returns the exit code or None on timeout."""
|
|---|
| 185 | outdir = os.path.join(work, spec["dir"])
|
|---|
| 186 | os.makedirs(outdir, exist_ok=True)
|
|---|
| 187 | cmd = [sys.executable, "-I", os.path.join(TESTS, "tools", spec["tool"][0])] + spec["tool"][1:]
|
|---|
| 188 | with open(os.path.join(outdir, "stdout.txt"), "wb") as out:
|
|---|
| 189 | try:
|
|---|
| 190 | return subprocess.run(cmd, cwd=work, stdout=out, stderr=subprocess.STDOUT,
|
|---|
| 191 | timeout=spec.get("timeout", 600)).returncode
|
|---|
| 192 | except subprocess.TimeoutExpired:
|
|---|
| 193 | return None
|
|---|
| 194 |
|
|---|
| 195 |
|
|---|
| 196 | def run_case(name, bnc, update):
|
|---|
| 197 | res = Result(name)
|
|---|
| 198 | case = load_case(name)
|
|---|
| 199 | work = os.path.join(WORK, name)
|
|---|
| 200 | expdir = os.path.join(CASES, name, "expected")
|
|---|
| 201 |
|
|---|
| 202 | shutil.rmtree(work, ignore_errors=True)
|
|---|
| 203 | t0 = time.time()
|
|---|
| 204 |
|
|---|
| 205 | # Optional local Ntrip caster receiving an upload (see tools/fakecaster.py)
|
|---|
| 206 | fc = case.get("fakecaster")
|
|---|
| 207 | fcProc = None
|
|---|
| 208 | if fc:
|
|---|
| 209 | os.makedirs(os.path.join(work, "Output"))
|
|---|
| 210 | fcProc = subprocess.Popen(
|
|---|
| 211 | [sys.executable, os.path.join(TESTS, "tools", "fakecaster.py"),
|
|---|
| 212 | "--port", str(fc["port"]), "--out", os.path.join(work, fc["out"]),
|
|---|
| 213 | "--sta", fc.get("sta", "CLOCK"), "--time", fc["time"],
|
|---|
| 214 | "--timeout", str(fc.get("timeout", 10))],
|
|---|
| 215 | stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|---|
| 216 | fcProc.stdout.readline() # "listening"
|
|---|
| 217 |
|
|---|
| 218 | code = run_bnc(bnc, work, case, "")
|
|---|
| 219 | if fcProc:
|
|---|
| 220 | try:
|
|---|
| 221 | fcProc.communicate(timeout=fc.get("timeout", 10) + 60)
|
|---|
| 222 | except subprocess.TimeoutExpired:
|
|---|
| 223 | fcProc.kill()
|
|---|
| 224 | res.fail("fake caster did not finish")
|
|---|
| 225 |
|
|---|
| 226 | if code is None:
|
|---|
| 227 | res.fail("timeout after %ds" % case.get("timeout", 600))
|
|---|
| 228 | return res
|
|---|
| 229 | if code != case.get("exit_code", 0):
|
|---|
| 230 | res.fail("exit code %d, expected %d" % (code, case.get("exit_code", 0)))
|
|---|
| 231 |
|
|---|
| 232 | # Optional further BNC runs, e.g. decoding the uploaded stream; each runs
|
|---|
| 233 | # in its own subdirectory of the work directory, raw relative to work/
|
|---|
| 234 | for post in case.get("post", []):
|
|---|
| 235 | if "tool" in post:
|
|---|
| 236 | pcode = run_tool(work, post)
|
|---|
| 237 | else:
|
|---|
| 238 | pcode = run_bnc(bnc, os.path.join(work, post["dir"]), post, work)
|
|---|
| 239 | if pcode is None or pcode != post.get("exit_code", 0):
|
|---|
| 240 | res.fail("post step %s: exit code %s" % (post["dir"], pcode))
|
|---|
| 241 | res.seconds = time.time() - t0
|
|---|
| 242 |
|
|---|
| 243 | # Collect (reference name, actual file, spec) triples
|
|---|
| 244 | pairs = []
|
|---|
| 245 | for spec in case["outputs"]:
|
|---|
| 246 | if isinstance(spec, str):
|
|---|
| 247 | spec = {"glob": spec}
|
|---|
| 248 | matches = sorted(glob.glob(os.path.join(work, spec["glob"])))
|
|---|
| 249 | if "name" in spec:
|
|---|
| 250 | if len(matches) != 1:
|
|---|
| 251 | res.fail("%s: expected exactly one match, found %d" % (spec["glob"], len(matches)))
|
|---|
| 252 | continue
|
|---|
| 253 | pairs.append((spec["name"], matches[0], spec))
|
|---|
| 254 | else:
|
|---|
| 255 | names = {os.path.basename(m) for m in matches}
|
|---|
| 256 | if not update:
|
|---|
| 257 | pattern = os.path.basename(spec["glob"]) + ".gz"
|
|---|
| 258 | for e in glob.glob(os.path.join(expdir, pattern)):
|
|---|
| 259 | if os.path.basename(e)[:-3] not in names:
|
|---|
| 260 | res.fail("missing output %s" % os.path.basename(e)[:-3])
|
|---|
| 261 | if not matches:
|
|---|
| 262 | res.fail("%s: no output produced" % spec["glob"])
|
|---|
| 263 | for m in matches:
|
|---|
| 264 | pairs.append((os.path.basename(m), m, spec))
|
|---|
| 265 |
|
|---|
| 266 | written = set()
|
|---|
| 267 | for ref, path, spec in pairs:
|
|---|
| 268 | actual = normalise(read_text(path))
|
|---|
| 269 | expfile = os.path.join(expdir, ref + ".gz")
|
|---|
| 270 | if update:
|
|---|
| 271 | if spec.get("debug_trace") and not actual.strip() and os.path.exists(expfile):
|
|---|
| 272 | # build without BNC_DEBUG_SSR: keep the existing trace reference
|
|---|
| 273 | written.add(os.path.basename(expfile))
|
|---|
| 274 | res.skip("%s: no BNC_DEBUG_SSR trace in this build, reference kept" % ref)
|
|---|
| 275 | continue
|
|---|
| 276 | write_expected(expfile, actual)
|
|---|
| 277 | written.add(os.path.basename(expfile))
|
|---|
| 278 | continue
|
|---|
| 279 | if not os.path.exists(expfile):
|
|---|
| 280 | res.fail("%s: no reference output (run with --update)" % ref)
|
|---|
| 281 | continue
|
|---|
| 282 | expected = read_expected(expfile)
|
|---|
| 283 | if actual == expected:
|
|---|
| 284 | continue
|
|---|
| 285 | if spec.get("debug_trace") and not actual.strip() and expected.strip():
|
|---|
| 286 | res.skip("%s: no BNC_DEBUG_SSR trace in this build" % ref)
|
|---|
| 287 | continue
|
|---|
| 288 | if "abs_tol" in spec and numeric_equal(expected, actual, spec["abs_tol"]):
|
|---|
| 289 | continue
|
|---|
| 290 | diff = list(difflib.unified_diff(expected.splitlines(), actual.splitlines(),
|
|---|
| 291 | "expected/" + ref, "actual/" + ref, lineterm="", n=1))
|
|---|
| 292 | difffile = os.path.join(work, ref + ".diff")
|
|---|
| 293 | with open(difffile, "w") as f:
|
|---|
| 294 | f.write("\n".join(diff) + "\n")
|
|---|
| 295 | nchg = sum(1 for d in diff if d[:1] in "+-" and d[:3] not in ("+++", "---"))
|
|---|
| 296 | res.fail("%s differs (%d changed lines, see %s)\n%s" %
|
|---|
| 297 | (ref, nchg, os.path.relpath(difffile, ROOT),
|
|---|
| 298 | "\n".join(" " + d for d in diff[:12])))
|
|---|
| 299 |
|
|---|
| 300 | if update and res.status == "PASS":
|
|---|
| 301 | # references of outputs no longer produced (e.g. a new date in the
|
|---|
| 302 | # file name after a new raw capture) are removed
|
|---|
| 303 | for e in glob.glob(os.path.join(expdir, "*.gz")):
|
|---|
| 304 | if os.path.basename(e) not in written:
|
|---|
| 305 | os.remove(e)
|
|---|
| 306 | res.status = "UPDATED"
|
|---|
| 307 | return res
|
|---|
| 308 |
|
|---|
| 309 |
|
|---|
| 310 | def main():
|
|---|
| 311 | ap = argparse.ArgumentParser(description=__doc__,
|
|---|
| 312 | formatter_class=argparse.RawDescriptionHelpFormatter)
|
|---|
| 313 | ap.add_argument("patterns", nargs="*", help="run only cases whose name contains one of these")
|
|---|
| 314 | ap.add_argument("--bnc", default=os.path.join(ROOT, "bnc"), help="BNC binary (default: ./bnc)")
|
|---|
| 315 | ap.add_argument("--update", action="store_true", help="write actual outputs as new reference")
|
|---|
| 316 | ap.add_argument("-j", "--jobs", type=int, default=os.cpu_count() or 1)
|
|---|
| 317 | ap.add_argument("--list", action="store_true", help="list cases and exit")
|
|---|
| 318 | args = ap.parse_args()
|
|---|
| 319 |
|
|---|
| 320 | names = sorted(d for d in os.listdir(CASES)
|
|---|
| 321 | if os.path.isfile(os.path.join(CASES, d, "case.json")))
|
|---|
| 322 | if args.patterns:
|
|---|
| 323 | names = [n for n in names if any(p in n for p in args.patterns)]
|
|---|
| 324 | if args.list:
|
|---|
| 325 | for n in names:
|
|---|
| 326 | print("%-28s %s" % (n, load_case(n).get("description", "")))
|
|---|
| 327 | return 0
|
|---|
| 328 | if not names:
|
|---|
| 329 | print("no matching test cases")
|
|---|
| 330 | return 2
|
|---|
| 331 | if not os.access(args.bnc, os.X_OK):
|
|---|
| 332 | print("BNC binary not found: %s" % args.bnc)
|
|---|
| 333 | return 2
|
|---|
| 334 |
|
|---|
| 335 | t0 = time.time()
|
|---|
| 336 | results = []
|
|---|
| 337 | with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool:
|
|---|
| 338 | futures = {pool.submit(run_case, n, args.bnc, args.update): n for n in names}
|
|---|
| 339 | for fut in concurrent.futures.as_completed(futures):
|
|---|
| 340 | try:
|
|---|
| 341 | r = fut.result()
|
|---|
| 342 | except Exception as exc: # runner bug or unreadable case.json
|
|---|
| 343 | r = Result(futures[fut])
|
|---|
| 344 | r.fail("runner error: %r" % exc)
|
|---|
| 345 | results.append(r)
|
|---|
| 346 | print("%-7s %-28s %6.1fs" % (r.status, r.name, r.seconds), flush=True)
|
|---|
| 347 | for m in r.messages:
|
|---|
| 348 | print(" " + m)
|
|---|
| 349 |
|
|---|
| 350 | nfail = sum(r.status == "FAIL" for r in results)
|
|---|
| 351 | nskip = sum(r.status == "SKIP" for r in results)
|
|---|
| 352 | print("\n%d cases, %d failed, %d skipped, %.0fs" %
|
|---|
| 353 | (len(results), nfail, nskip, time.time() - t0))
|
|---|
| 354 | return 1 if nfail else 0
|
|---|
| 355 |
|
|---|
| 356 |
|
|---|
| 357 | if __name__ == "__main__":
|
|---|
| 358 | sys.exit(main())
|
|---|
