#!/usr/bin/env python3
"""BNC regression test runner.

Each test case is a directory tests/cases/<name>/ containing

  case.json   - how to run BNC and which outputs to compare
  expected/   - gzipped, normalised reference outputs ("golden files")

case.json keys:
  description  free text
  config       BNC config file, relative to the BNC root (or "/dev/null")
  raw          optional raw capture, relative to tests/ -> passed as --file
  args         optional list of extra command line arguments (e.g. -key pairs)
  exit_code    expected exit code (default 0; replay via --file exits with 5)
  timeout      seconds (default 600)
  outputs      list of output specs, each either a glob string or an object
                 glob       path relative to the work directory; stdout.txt and
                            stderr.txt hold BNC's console output
                 name       reference name; if given the glob must match exactly
                            one file, otherwise every match is compared under its
                            basename and missing/extra files are reported
                 abs_tol    compare numbers with this absolute tolerance
                            instead of requiring identical text
                 debug_trace  true for BNC_DEBUG_SSR stderr traces: reported as
                            SKIP instead of FAIL if the build has no trace output
  fakecaster   optional local Ntrip caster for an upload during the run:
               {"port", "out" (raw file, relative to the work dir), "sta",
                "time" (epoch stamped on the raw file), "timeout"}
  post         optional list of further BNC runs after the main run, each
               {"dir" (subdirectory of the work dir), "config", "raw"
                (relative to the work dir), "args", "exit_code", "timeout"};
               outputs may refer to their files as "<dir>/Output/...";
               instead of a BNC run, {"dir", "tool": [script in tools/, args]}
               runs a Python tool in the work directory, stdout to <dir>/stdout.txt

BNC runs in tests/work/<name>/, which contains an "Input" symlink to
Example_Configs/Input and an empty "Output" directory, so the example
configurations work unchanged and never write into Example_Configs/.

Usage:
  tests/run_tests.py                 run all cases
  tests/run_tests.py ssr_ rinex      run cases whose name contains a pattern
  tests/run_tests.py --update NAME   (re)generate the reference outputs; references of
                                     outputs no longer produced are removed
  tests/run_tests.py --list
"""

import argparse
import concurrent.futures
import difflib
import glob
import gzip
import json
import os
import re
import shutil
import subprocess
import sys
import time

TESTS = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(TESTS)
CASES = os.path.join(TESTS, "cases")
WORK = os.path.join(TESTS, "work")

# Content that changes from run to run or with the BNC version
DEFAULT_FILTERS = [
    # wall-clock prefix of BNC log lines "yy-mm-dd hh:mm:ss "
    (re.compile(r"^\d\d-\d\d-\d\d \d\d:\d\d:\d\d "), ""),
    (re.compile(r"BNC v?\d+\.\d+(\.\d+)?"), "BNC <VERSION>"),
    (re.compile(r"^.*(PGM / RUN BY / DATE)\s*$"), r"<MASKED> \1"),
    (re.compile(r"^(Run by\s*:).*$"), r"\1 <MASKED>"),
    (re.compile(r"^(Date\s*:).*$"), r"\1 <MASKED>"),
]

# Lines dropped completely. The start line carries the wall-clock date and,
# in --file replay, lands in a log file named after today's date, whereas all
# other log lines go to a file named after the data date.
DROP_LINES = re.compile(r"========== Start BNC")

NUMBER = re.compile(r"[-+]?(?:\d+\.\d*|\.\d+|\d+)(?:[eEdD][-+]?\d+)?")


def normalise(text):
    out = []
    for line in text.splitlines():
        if DROP_LINES.search(line):
            continue
        for rx, repl in DEFAULT_FILTERS:
            line = rx.sub(repl, line)
        out.append(line.rstrip())
    return "\n".join(out) + "\n" if out else ""


def read_text(path):
    with open(path, "rb") as f:
        return f.read().decode("latin-1")


def read_expected(path):
    with gzip.open(path, "rb") as f:
        return f.read().decode("latin-1")


def write_expected(path, text):
    os.makedirs(os.path.dirname(path), exist_ok=True)
    # mtime=0 keeps the .gz byte-identical when the content is unchanged
    with open(path, "wb") as raw:
        with gzip.GzipFile(fileobj=raw, mode="wb", mtime=0) as f:
            f.write(text.encode("latin-1"))


def numeric_equal(exp, act, tol):
    """Line-by-line compare; numbers may differ by tol, other text must match."""
    el, al = exp.splitlines(), act.splitlines()
    if len(el) != len(al):
        return False
    for e, a in zip(el, al):
        if NUMBER.sub("#", e) != NUMBER.sub("#", a):
            return False
        for ne, na in zip(NUMBER.findall(e), NUMBER.findall(a)):
            try:
                if abs(float(ne.replace("D", "e").replace("d", "e")) -
                       float(na.replace("D", "e").replace("d", "e"))) > tol:
                    return False
            except ValueError:
                if ne != na:
                    return False
    return True


class Result:
    def __init__(self, name):
        self.name = name
        self.status = "PASS"
        self.messages = []
        self.seconds = 0.0

    def fail(self, msg):
        self.status = "FAIL"
        self.messages.append(msg)

    def skip(self, msg):
        if self.status == "PASS":
            self.status = "SKIP"
        self.messages.append(msg)


def load_case(name):
    with open(os.path.join(CASES, name, "case.json")) as f:
        return json.load(f)


def resolve(base, path):
    return path if os.path.isabs(path) else os.path.normpath(os.path.join(base, path))


def run_bnc(bnc, workdir, spec, rawBase):
    """Run BNC once in workdir (created with Input link and Output dir); spec
    holds config, optional raw (relative to tests/, or to rawBase if given),
    args and timeout. Returns the exit code or None on timeout."""
    os.makedirs(os.path.join(workdir, "Output"), exist_ok=True)
    if not os.path.exists(os.path.join(workdir, "Input")):
        os.symlink(os.path.join(ROOT, "Example_Configs", "Input"), os.path.join(workdir, "Input"))
    cmd = [bnc, "--nw", "--platform", "offscreen", "--conf", resolve(ROOT, spec["config"])]
    if spec.get("raw"):
        cmd += ["--file", resolve(rawBase or TESTS, spec["raw"])]
    cmd += spec.get("args", [])
    env = dict(os.environ, QT_QPA_PLATFORM="offscreen", LC_ALL="C", TZ="UTC")
    with open(os.path.join(workdir, "stdout.txt"), "wb") as out, \
         open(os.path.join(workdir, "stderr.txt"), "wb") as err:
        try:
            code = subprocess.run(cmd, cwd=workdir, stdout=out, stderr=err, env=env,
                                  timeout=spec.get("timeout", 600)).returncode
        except subprocess.TimeoutExpired:
            code = None
    with open(os.path.join(workdir, "command.txt"), "w") as f:
        f.write(" ".join(cmd) + "\n")
    return code


def run_tool(work, spec):
    """Run tools/<script> with args in the work directory, stdout to
    <dir>/stdout.txt. Returns the exit code or None on timeout."""
    outdir = os.path.join(work, spec["dir"])
    os.makedirs(outdir, exist_ok=True)
    cmd = [sys.executable, "-I", os.path.join(TESTS, "tools", spec["tool"][0])] + spec["tool"][1:]
    with open(os.path.join(outdir, "stdout.txt"), "wb") as out:
        try:
            return subprocess.run(cmd, cwd=work, stdout=out, stderr=subprocess.STDOUT,
                                  timeout=spec.get("timeout", 600)).returncode
        except subprocess.TimeoutExpired:
            return None


def run_case(name, bnc, update):
    res = Result(name)
    case = load_case(name)
    work = os.path.join(WORK, name)
    expdir = os.path.join(CASES, name, "expected")

    shutil.rmtree(work, ignore_errors=True)
    t0 = time.time()

    # Optional local Ntrip caster receiving an upload (see tools/fakecaster.py)
    fc = case.get("fakecaster")
    fcProc = None
    if fc:
        os.makedirs(os.path.join(work, "Output"))
        fcProc = subprocess.Popen(
            [sys.executable, os.path.join(TESTS, "tools", "fakecaster.py"),
             "--port", str(fc["port"]), "--out", os.path.join(work, fc["out"]),
             "--sta", fc.get("sta", "CLOCK"), "--time", fc["time"],
             "--timeout", str(fc.get("timeout", 10))],
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        fcProc.stdout.readline()  # "listening"

    code = run_bnc(bnc, work, case, "")
    if fcProc:
        try:
            fcProc.communicate(timeout=fc.get("timeout", 10) + 60)
        except subprocess.TimeoutExpired:
            fcProc.kill()
            res.fail("fake caster did not finish")

    if code is None:
        res.fail("timeout after %ds" % case.get("timeout", 600))
        return res
    if code != case.get("exit_code", 0):
        res.fail("exit code %d, expected %d" % (code, case.get("exit_code", 0)))

    # Optional further BNC runs, e.g. decoding the uploaded stream; each runs
    # in its own subdirectory of the work directory, raw relative to work/
    for post in case.get("post", []):
        if "tool" in post:
            pcode = run_tool(work, post)
        else:
            pcode = run_bnc(bnc, os.path.join(work, post["dir"]), post, work)
        if pcode is None or pcode != post.get("exit_code", 0):
            res.fail("post step %s: exit code %s" % (post["dir"], pcode))
    res.seconds = time.time() - t0

    # Collect (reference name, actual file, spec) triples
    pairs = []
    for spec in case["outputs"]:
        if isinstance(spec, str):
            spec = {"glob": spec}
        matches = sorted(glob.glob(os.path.join(work, spec["glob"])))
        if "name" in spec:
            if len(matches) != 1:
                res.fail("%s: expected exactly one match, found %d" % (spec["glob"], len(matches)))
                continue
            pairs.append((spec["name"], matches[0], spec))
        else:
            names = {os.path.basename(m) for m in matches}
            if not update:
                pattern = os.path.basename(spec["glob"]) + ".gz"
                for e in glob.glob(os.path.join(expdir, pattern)):
                    if os.path.basename(e)[:-3] not in names:
                        res.fail("missing output %s" % os.path.basename(e)[:-3])
            if not matches:
                res.fail("%s: no output produced" % spec["glob"])
            for m in matches:
                pairs.append((os.path.basename(m), m, spec))

    written = set()
    for ref, path, spec in pairs:
        actual = normalise(read_text(path))
        expfile = os.path.join(expdir, ref + ".gz")
        if update:
            if spec.get("debug_trace") and not actual.strip() and os.path.exists(expfile):
                # build without BNC_DEBUG_SSR: keep the existing trace reference
                written.add(os.path.basename(expfile))
                res.skip("%s: no BNC_DEBUG_SSR trace in this build, reference kept" % ref)
                continue
            write_expected(expfile, actual)
            written.add(os.path.basename(expfile))
            continue
        if not os.path.exists(expfile):
            res.fail("%s: no reference output (run with --update)" % ref)
            continue
        expected = read_expected(expfile)
        if actual == expected:
            continue
        if spec.get("debug_trace") and not actual.strip() and expected.strip():
            res.skip("%s: no BNC_DEBUG_SSR trace in this build" % ref)
            continue
        if "abs_tol" in spec and numeric_equal(expected, actual, spec["abs_tol"]):
            continue
        diff = list(difflib.unified_diff(expected.splitlines(), actual.splitlines(),
                                         "expected/" + ref, "actual/" + ref, lineterm="", n=1))
        difffile = os.path.join(work, ref + ".diff")
        with open(difffile, "w") as f:
            f.write("\n".join(diff) + "\n")
        nchg = sum(1 for d in diff if d[:1] in "+-" and d[:3] not in ("+++", "---"))
        res.fail("%s differs (%d changed lines, see %s)\n%s" %
                 (ref, nchg, os.path.relpath(difffile, ROOT),
                  "\n".join("        " + d for d in diff[:12])))

    if update and res.status == "PASS":
        # references of outputs no longer produced (e.g. a new date in the
        # file name after a new raw capture) are removed
        for e in glob.glob(os.path.join(expdir, "*.gz")):
            if os.path.basename(e) not in written:
                os.remove(e)
        res.status = "UPDATED"
    return res


def main():
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("patterns", nargs="*", help="run only cases whose name contains one of these")
    ap.add_argument("--bnc", default=os.path.join(ROOT, "bnc"), help="BNC binary (default: ./bnc)")
    ap.add_argument("--update", action="store_true", help="write actual outputs as new reference")
    ap.add_argument("-j", "--jobs", type=int, default=os.cpu_count() or 1)
    ap.add_argument("--list", action="store_true", help="list cases and exit")
    args = ap.parse_args()

    names = sorted(d for d in os.listdir(CASES)
                   if os.path.isfile(os.path.join(CASES, d, "case.json")))
    if args.patterns:
        names = [n for n in names if any(p in n for p in args.patterns)]
    if args.list:
        for n in names:
            print("%-28s %s" % (n, load_case(n).get("description", "")))
        return 0
    if not names:
        print("no matching test cases")
        return 2
    if not os.access(args.bnc, os.X_OK):
        print("BNC binary not found: %s" % args.bnc)
        return 2

    t0 = time.time()
    results = []
    with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as pool:
        futures = {pool.submit(run_case, n, args.bnc, args.update): n for n in names}
        for fut in concurrent.futures.as_completed(futures):
            try:
                r = fut.result()
            except Exception as exc:  # runner bug or unreadable case.json
                r = Result(futures[fut])
                r.fail("runner error: %r" % exc)
            results.append(r)
            print("%-7s %-28s %6.1fs" % (r.status, r.name, r.seconds), flush=True)
            for m in r.messages:
                print("        " + m)

    nfail = sum(r.status == "FAIL" for r in results)
    nskip = sum(r.status == "SKIP" for r in results)
    print("\n%d cases, %d failed, %d skipped, %.0fs" %
          (len(results), nfail, nskip, time.time() - t0))
    return 1 if nfail else 0


if __name__ == "__main__":
    sys.exit(main())