Changeset 11127 in ntrip

Timestamp:

Oct 24, 2026, 11:15:22 PM (5 hours ago)

Author:

stuerze

Message:

test suite extended regardiing security aspects

Location:

trunk/BNC/tests

Files:

• README.md (modified) (3 diffs)
• cases/upload_rtnet/case.json (modified) (2 diffs)
• cases/upload_rtnet/expected/upload_crs.txt.gz (added)
• run_tests.py (modified) (4 diffs)
• tools/bncraw.py (modified) (4 diffs)
• tools/fakecaster.py (modified) (7 diffs)
• tools/ssl_check.py (added)
• tools/ssr_harness.cpp (modified) (4 diffs)

trunk/BNC/tests/README.md

@@ -26 +26 @@
 | `cases/<name>/expected/*.gz` | normalised reference outputs |
 | `data/*.raw` | short raw captures (BNC `rawOutFile` format) replayed via `--file` |
-| `tools/fakecaster.py` | local Ntrip caster recording an upload as a BNC raw file (case key `fakecaster`) |
-| `tools/bncraw.py` | `info` (streams, time span, RTCM3 message types) and `slice` for raw captures |
+| `tools/fakecaster.py` | local Ntrip caster recording an upload as a BNC raw file (case key `fakecaster`); optionally TLS and serving a stream |
+| `tools/bncraw.py` | `info` (streams, time span, RTCM3 message types), `slice` and `crs` (decode 1300/1301/1302) for raw captures |
 | `tools/mkcrafted.py` | writes `data/crafted_limits.raw`: crafted frames at the upper field limits |
 | `tools/ssr_harness.sh` | builds `tools/ssr_harness.cpp` with the SSR decoders under ASan/UBSan and runs it |
+| `tools/ssl_check.py` | TLS checks of the Ntrip 2s client and upload (not part of `run_tests.py`) |
 | `work/` | scratch output, not versioned |
 
@@ -75 +76 @@
   (`tools/fakecaster.py`, Ntrip 1/2, chunked), and a `post` step decodes the
   received stream again: `upload_rtnet` (IGS-SSR encoding, SP3/CLK/BIA files,
-  decoded `.ssr`), `upload_eph` (RTCM3 ephemerides to RINEX nav) and
+  decoded `.ssr`, CRS messages 1300/1302 decoded by `bncraw.py crs` in a
+  `tool` post step), `upload_eph` (RTCM3 ephemerides to RINEX nav) and
   `upload_raw` (raw forwarding to RINEX nav). `upload_rtnet_rtcm` encodes the same input to RTCM-SSR
   (message numbers 1057-1068, 1240-1270) and decodes it again.
@@ -114 +116 @@
 `crafted_limits.raw`: 7 entries with additional parameters of 40 and 0 bits,
 and 31 entries, which must be rejected), which `RTCM3Decoder.cpp` does not
-dispatch at present. Exit code 0 only without sanitizer errors.
+dispatch at present. Every decoded VTEC and metadata message is encoded
+again with the `Make*` function of the same class and decoded once more; the
+result must be identical. Exit code 0 only without sanitizer errors and
+differences.
+
+## TLS check
+
+```
+tests/tools/ssl_check.py [--bnc PATH]
+```
+
+Runs BNC against `tools/fakecaster.py` with TLS on port 443 (BNC uploads with
+Ntrip 2s always to 443) inside an unprivileged user and network namespace
+(`unshare -rn`), with a test CA, server and client certificates created by
+openssl in `work/ssl_check/`. Checks for upload and download: certificate of
+an unknown CA rejected, accepted with the CA in `sslCaCertPath`, accepted with
+`sslIgnoreErrors` and the ignored errors logged; for upload also a
+certificate for another host name (rejected) and a client certificate
+required by the caster, with and without `sslIgnoreErrors`. Takes about 3 min.
 
 ## Adding a replay case

trunk/BNC/tests/cases/upload_rtnet/case.json

@@ -1 +1 @@
 {
-  "description": "16_Upload replay: RTNET orbits/clocks/biases + BCEP00BKG0 encoded to IGS-SSR and uploaded (Ntrip 2, chunked) to a local fake caster; SP3/CLK/BIA written; the uploaded stream is decoded again (03_BrdcCorr), 5 min",
+  "description": "16_Upload replay: RTNET orbits/clocks/biases + BCEP00BKG0 encoded to IGS-SSR and uploaded (Ntrip 2, chunked) to a local fake caster; SP3/CLK/BIA written; the uploaded stream is decoded again (03_BrdcCorr), CRS messages 1300/1302 decoded by bncraw.py crs, 5 min",
   "config": "Example_Configs/16_Upload.bnc",
   "raw": "data/upload_rtnet_5min.raw",
@@ -10 +10 @@
                  "time": "2026-10-01T09:33:00", "timeout": 10},
   "post": [{"dir": "dec", "config": "Example_Configs/03_BrdcCorr.bnc",
-            "raw": "Output/upload.raw", "exit_code": 5, "timeout": 300}],
+            "raw": "Output/upload.raw", "exit_code": 5, "timeout": 300},
+           {"dir": "crs", "tool": ["bncraw.py", "crs", "Output/upload.raw"]}],
   "outputs": ["Output/*.SP3", "Output/*.CLK", "Output/*.BIA",
               {"glob": "Output/Upload.log_261001", "name": "Upload.log"},
               "dec/Output/*.ssr",
+              {"glob": "crs/stdout.txt", "name": "upload_crs.txt"},
               {"glob": "dec/stderr.txt", "name": "decode_trace.txt", "debug_trace": true}]
 }

trunk/BNC/tests/run_tests.py

@@ -30 +30 @@
                {"dir" (subdirectory of the work dir), "config", "raw"
                 (relative to the work dir), "args", "exit_code", "timeout"};
-               outputs may refer to their files as "<dir>/Output/..."
+               outputs may refer to their files as "<dir>/Output/...";
+               instead of a BNC run, {"dir", "tool": [script in tools/, args]}
+               runs a Python tool in the work directory, stdout to <dir>/stdout.txt
 
 BNC runs in tests/work/<name>/, which contains an "Input" symlink to
@@ -178 +180 @@
 
 
+def run_tool(work, spec):
+    """Run tools/<script> with args in the work directory, stdout to
+    <dir>/stdout.txt. Returns the exit code or None on timeout."""
+    outdir = os.path.join(work, spec["dir"])
+    os.makedirs(outdir, exist_ok=True)
+    cmd = [sys.executable, "-I", os.path.join(TESTS, "tools", spec["tool"][0])] + spec["tool"][1:]
+    with open(os.path.join(outdir, "stdout.txt"), "wb") as out:
+        try:
+            return subprocess.run(cmd, cwd=work, stdout=out, stderr=subprocess.STDOUT,
+                                  timeout=spec.get("timeout", 600)).returncode
+        except subprocess.TimeoutExpired:
+            return None
+
+
 def run_case(name, bnc, update):
     res = Result(name)
@@ -217 +233 @@
     # in its own subdirectory of the work directory, raw relative to work/
     for post in case.get("post", []):
-        pcode = run_bnc(bnc, os.path.join(work, post["dir"]), post, work)
+        if "tool" in post:
+            pcode = run_tool(work, post)
+        else:
+            pcode = run_bnc(bnc, os.path.join(work, post["dir"]), post, work)
         if pcode is None or pcode != post.get("exit_code", 0):
             res.fail("post step %s: exit code %s" % (post["dir"], pcode))
@@ -250 +269 @@
         expfile = os.path.join(expdir, ref + ".gz")
         if update:
+            if spec.get("debug_trace") and not actual.strip() and os.path.exists(expfile):
+                # build without BNC_DEBUG_SSR: keep the existing trace reference
+                written.add(os.path.basename(expfile))
+                res.skip("%s: no BNC_DEBUG_SSR trace in this build, reference kept" % ref)
+                continue
             write_expected(expfile, actual)
             written.add(os.path.basename(expfile))

trunk/BNC/tests/tools/bncraw.py

@@ -9 +9 @@
     info  <raw> [<raw> ...]          streams, byte counts, time span, RTCM3 message types
     slice <raw> <out> [options]      cut a time window / subset of stations
+    crs   <raw>                      decode RTCM 1300/1301/1302 (CRS messages)
 """
 
@@ -110 +111 @@
         return "4076_%03d" % sub
     return str(t)
+
+
+class BitReader:
+    def __init__(self, data):
+        self.data = data
+        self.pos = 0
+
+    def u(self, n):
+        v = 0
+        for _ in range(n):
+            if self.pos >= 8 * len(self.data):
+                raise ValueError("message too short")
+            v = (v << 1) | ((self.data[self.pos // 8] >> (7 - self.pos % 8)) & 1)
+            self.pos += 1
+        return v
+
+    def s(self, n):
+        v = self.u(n)
+        return v - (1 << n) if v >> (n - 1) else v
+
+    def text(self):
+        n = self.u(5)
+        return bytes(self.u(8) for _ in range(n)).decode("latin-1")
+
+    def left(self):
+        return 8 * len(self.data) - self.pos
+
+
+def decode_crs(p):
+    """One line per 1300/1301/1302 message (RTCM 10403.4, DF562-DF587)."""
+    r = BitReader(p)
+    t = r.u(12)
+    if t == 1300:
+        out = "1300 name %r CE %.2f" % (r.text(), r.u(16) / 100.0)
+    elif t == 1301:
+        src, tgt = r.text(), r.text()
+        out = "1301 source %r target %r sys %d util %d mjd %d par" % (
+            src, tgt, r.u(8), r.u(10), r.u(16) + 44244)
+        for n in (23, 23, 23, 32, 32, 32, 25, 17, 17, 17, 17, 17, 17, 14):
+            out += " %d" % r.s(n)
+    else:
+        out = "1302 name %r anchor %d plate %d links" % (r.text(), r.u(1), r.u(5))
+        out += " %r" % [r.text() for _ in range(r.u(3))]
+    if r.left() >= 8:  # more than the padding to the byte boundary
+        out += "  TRAILING %d bits" % r.left()
+    return out
 
 
@@ -166 +213 @@
 
 
+def cmd_crs(args):
+    """Distinct CRS message contents per stream, with their number."""
+    stats = collections.OrderedDict()
+    framers = {}
+    for t, sta, fmt, data in read_chunks(args.input):
+        for p in framers.setdefault(sta, Rtcm3Framer()).feed(data):
+            if msg_type(p) in ("1300", "1301", "1302"):
+                try:
+                    line = decode_crs(p)
+                except ValueError as e:
+                    line = "%s %s" % (msg_type(p), e)
+                key = (sta, line)
+                stats[key] = stats.get(key, 0) + 1
+    for (sta, line), n in stats.items():
+        print("%s %4dx %s" % (sta, n, line))
+
+
 def main():
     ap = argparse.ArgumentParser(description=__doc__,
@@ -183 +247 @@
     p.set_defaults(func=cmd_slice)
 
+    p = sub.add_parser("crs", help="decode CRS messages 1300/1301/1302")
+    p.add_argument("input")
+    p.set_defaults(func=cmd_crs)
+
     args = ap.parse_args()
     args.func(args)

trunk/BNC/tests/tools/fakecaster.py

@@ -11 +11 @@
 
     fakecaster.py --port 21011 --out upload.raw --sta CLOCK --time 2026-10-01T09:30:00
+
+With --tls-cert/--tls-key the caster speaks TLS (Ntrip 2s); --tls-client-ca
+additionally requires a client certificate signed by that CA. With --serve
+a GET request (BNC as Ntrip client) is answered with the RTCM bytes of the
+given raw file. Failed TLS handshakes are reported on stdout.
 """
 
 import argparse
+import os
 import socket
+import ssl
 import sys
 import threading
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
 
@@ -45 +54 @@
         self.head = "%s %s %s" % (time, sta, fmt)
         self.total = 0
+        self.handshakes = 0
+        self.served = 0
 
     def write(self, data):
@@ -56 +67 @@
 
 
-def serve(conn, store, timeout):
+def serve(conn, store, timeout, tls=None, served=None):
     conn.settimeout(timeout)
     buf = b""
     try:
+        if tls:
+            try:
+                conn = tls.wrap_socket(conn, server_side=True)
+            except (ssl.SSLError, OSError) as e:
+                print("tls handshake failed: %s" % e, flush=True)
+                return
+            store.handshakes += 1
         while b"\r\n\r\n" not in buf:
             d = conn.recv(4096)
@@ -67 +85 @@
         header, _, pending = buf.partition(b"\r\n\r\n")
         chunked = b"chunked" in header.lower()
+        if header.startswith(b"GET") and served is not None:
+            conn.sendall(b"HTTP/1.1 200 OK\r\nNtrip-Version: Ntrip/2.0\r\n"
+                         b"Content-Type: gnss/data\r\nConnection: close\r\n\r\n" + served)
+            store.served += len(served)
+            return
         if header.startswith(b"SOURCE"):
             conn.sendall(b"ICY 200 OK\r\n\r\n")
@@ -100 +123 @@
     ap.add_argument("--timeout", type=float, default=600.0,
                     help="stop after this many seconds without a new connection or data")
+    ap.add_argument("--tls-cert", help="server certificate (PEM): TLS")
+    ap.add_argument("--tls-key", help="server private key (PEM)")
+    ap.add_argument("--tls-client-ca", help="require a client certificate of this CA")
+    ap.add_argument("--serve", help="raw file whose RTCM bytes are sent on GET")
     args = ap.parse_args()
+
+    tls = None
+    if args.tls_cert:
+        tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        tls.load_cert_chain(args.tls_cert, args.tls_key)
+        if args.tls_client_ca:
+            tls.verify_mode = ssl.CERT_REQUIRED
+            tls.load_verify_locations(args.tls_client_ca)
+    served = None
+    if args.serve:
+        from bncraw import read_chunks  # noqa: E402
+        served = b"".join(d for t, sta, fmt, d in read_chunks(args.serve))
 
     store = Store(args.out, args.sta, args.format, args.time)
@@ -116 +155 @@
         except socket.timeout:
             break
-        t = threading.Thread(target=serve, args=(conn, store, args.timeout), daemon=True)
+        t = threading.Thread(target=serve, args=(conn, store, args.timeout, tls, served),
+                             daemon=True)
         t.start()
         threads.append(t)
@@ -122 +162 @@
         t.join(args.timeout)
     print("received %d bytes in %d connection(s)" % (store.total, len(threads)), flush=True)
+    if tls:
+        print("%d TLS handshake(s) completed" % store.handshakes, flush=True)
+    if served is not None:
+        print("served %d bytes" % store.served, flush=True)
     return 0
 

trunk/BNC/tests/tools/ssr_harness.cpp

@@ -6 +6 @@
 // struct is a separate heap block here, so a write beyond a struct is
 // reported by AddressSanitizer instead of silently corrupting the next member.
+//
+// Every decoded VTEC (all formats) and metadata message (new RTCM-SSR) is
+// encoded again with the Make* function of the same class and decoded once
+// more; the result must be identical (exit code 1 otherwise).
 //
 // Usage: ssr_harness <file>   (BNC raw file or plain RTCM3 byte stream)
@@ -33 +37 @@
 }
 
+struct t_target {
+  t_target()  : co(new SsrCorr::ClockOrbit()), cb(new SsrCorr::CodeBias()),
+                v(new SsrCorr::VTEC()), pb(new SsrCorr::PhaseBias()),
+                md(new SsrCorr::MetaData()), a(new SsrCorr::Antenna()) {}
+  ~t_target() {delete co; delete cb; delete v; delete pb; delete md; delete a;}
+  int decode(SsrCorr* ssr, const unsigned char* frame, size_t size) {
+    int bytesused = 0;
+    return ssr->GetSSR(co, cb, v, pb, md, a, (const char*)frame, size, &bytesused);
+  }
+  SsrCorr::ClockOrbit* co;
+  SsrCorr::CodeBias*   cb;
+  SsrCorr::VTEC*       v;
+  SsrCorr::PhaseBias*  pb;
+  SsrCorr::MetaData*   md;
+  SsrCorr::Antenna*    a;
+};
+
+static bool sameVTEC(const SsrCorr::VTEC* v1, const SsrCorr::VTEC* v2) {
+  if (v1->EpochTime != v2->EpochTime || v1->NumLayers != v2->NumLayers ||
+      v1->Quality != v2->Quality) {
+    return false;
+  }
+  for (unsigned l = 0; l < v1->NumLayers; l++) {
+    const SsrCorr::VTEC::IonoLayers& l1 = v1->Layers[l];
+    const SsrCorr::VTEC::IonoLayers& l2 = v2->Layers[l];
+    if (l1.Height != l2.Height || l1.Degree != l2.Degree || l1.Order != l2.Order) {
+      return false;
+    }
+    for (unsigned d = 0; d <= l1.Degree; d++) {
+      for (unsigned o = 0; o <= l1.Order; o++) {
+        if (l1.Cosinus[d][o] != l2.Cosinus[d][o] || l1.Sinus[d][o] != l2.Sinus[d][o]) {
+          return false;
+        }
+      }
+    }
+  }
+  return true;
+}
+
+static bool sameMetaData(const SsrCorr::MetaData* m1, const SsrCorr::MetaData* m2) {
+  if (m1->SSRIOD != m2->SSRIOD || m1->SSRProviderID != m2->SSRProviderID ||
+      m1->SSRSolutionID != m2->SSRSolutionID || m1->NumEntries != m2->NumEntries) {
+    return false;
+  }
+  for (unsigned i = 0; i < m1->NumEntries; i++) {
+    if (memcmp(&m1->Entries[i], &m2->Entries[i], sizeof(m1->Entries[i])) != 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+static int nFailed = 0;
+
+// Decode, print a summary, encode the result again and compare
 static void decode(const char* name, SsrCorr* ssr, const unsigned char* frame, size_t size) {
-  SsrCorr::ClockOrbit* co = new SsrCorr::ClockOrbit();
-  SsrCorr::CodeBias*   cb = new SsrCorr::CodeBias();
-  SsrCorr::VTEC*       v  = new SsrCorr::VTEC();
-  SsrCorr::PhaseBias*  pb = new SsrCorr::PhaseBias();
-  SsrCorr::MetaData*   md = new SsrCorr::MetaData();
-  SsrCorr::Antenna*    a  = new SsrCorr::Antenna();
-  int bytesused = 0;
-
-  int irc = ssr->GetSSR(co, cb, v, pb, md, a, (const char*)frame, size, &bytesused);
+  t_target t1;
+  int irc = t1.decode(ssr, frame, size);
   printf("  %-8s irc %3d", name, irc);
-  if (irc >= 0 && v->NumLayers) {
+  if (irc >= 0 && t1.v->NumLayers) {
     double sum = 0.0;
     printf("  VTEC");
-    for (unsigned l = 0; l < v->NumLayers; l++) {
-      const SsrCorr::VTEC::IonoLayers& layer = v->Layers[l];
+    for (unsigned l = 0; l < t1.v->NumLayers; l++) {
+      const SsrCorr::VTEC::IonoLayers& layer = t1.v->Layers[l];
       printf(" %u/%u", layer.Degree, layer.Order);
       for (unsigned d = 0; d <= layer.Degree; d++) {
@@ -57 +109 @@
     }
     printf("  sum(C-S) %.3f", sum);
+
+    char buffer[CLOCKORBIT_BUFFERSIZE];
+    size_t len = ssr->MakeVTEC(t1.v, 0, buffer, sizeof(buffer));
+    t_target t2;
+    bool ok = len > 0 && t2.decode(ssr, (const unsigned char*)buffer, len) >= 0 && sameVTEC(t1.v, t2.v);
+    printf("  re-encoded: %s", ok ? "identical" : "DIFFERENT");
+    nFailed += !ok;
   }
-  if (irc >= 0 && md->NumEntries) {
-    printf("  META %u entries, additional data", md->NumEntries);
-    for (unsigned i = 0; i < md->NumEntries; i++) {
-      if (md->Entries[i].additionalParameterIndicator) {
-        printf(" %u:0x%x", md->Entries[i].additionalParameterNumBits,
-               md->Entries[i].additionalParameterData);
+  if (irc >= 0 && t1.md->NumEntries) {
+    printf("  META %u entries, additional data", t1.md->NumEntries);
+    for (unsigned i = 0; i < t1.md->NumEntries; i++) {
+      if (t1.md->Entries[i].additionalParameterIndicator) {
+        printf(" %u:0x%x", t1.md->Entries[i].additionalParameterNumBits,
+               t1.md->Entries[i].additionalParameterData);
       }
+    }
+    SsrCorrRtcmNew* rtcmNew = dynamic_cast<SsrCorrRtcmNew*>(ssr);
+    if (rtcmNew) {
+      char buffer[CLOCKORBIT_BUFFERSIZE];
+      size_t len = rtcmNew->MakeMetaData(t1.md, buffer, sizeof(buffer));
+      t_target t2;
+      bool ok = len > 0 && t2.decode(ssr, (const unsigned char*)buffer, len) >= 0 && sameMetaData(t1.md, t2.md);
+      printf("  re-encoded: %s", ok ? "identical" : "DIFFERENT");
+      nFailed += !ok;
     }
   }
   printf("\n");
-
-  delete co; delete cb; delete v; delete pb; delete md; delete a;
 }
 
@@ -111 +177 @@
     ii += len + 6;
   }
+  if (nFailed) {
+    printf("%d decoded messages differ after re-encoding\n", nFailed);
+    return 1;
+  }
   return 0;
 }