MiPi_TEST/flicker_watch.py

#!/usr/bin/env python3
"""
flicker_watch.py — Continuous LP capture during video on/off cycles.

Operator watches the display.  Script keeps cycling the video stream on/off
and triggering LP captures in the background.  Files accumulate on the scope
without being transferred (fast).

Keys (no Enter needed):
    f  — flicker observed: transfer + archive + analyse recent captures
    g  — good baseline:    transfer + archive recent captures (no analysis)
    q  — quit

Captures are organised under data/flicker/{event_ts}/ or data/good/{event_ts}/.
"""

import json
import select
import shutil
import sys
import termios
import time
import tty
from datetime import datetime
from pathlib import Path

import requests
import vxi11

import ai_mgmt
from csv_preprocessor import analyze_lp_file

# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
SCOPE_IP    = "192.168.45.4"
DEVICE_BASE = "http://192.168.45.8:5000"
VIDEO_URL   = f"{DEVICE_BASE}/video"

DATA_DIR    = Path(__file__).parent / "data"
FLICKER_DIR = DATA_DIR / "flicker"
GOOD_DIR    = DATA_DIR / "good"

# LP capture parameters (matched to mipi_test_interactive.py)
LP_SCALE       = 1e-6        # 1 µs/div → 20 µs window
LP_POINTS      = 200_000
LP_TRIG_OFFSET = 9e-6        # 1 µs pre / 19 µs post-trigger
LP_V_SCALE     = 0.2
LP_V_OFFSET    = 0.6
LP_TRIG_LEVEL  = 0.6

# Segmented memory: capture N back-to-back LP triggers per :DIGitize, then
# dump the whole acquisition as a single H5 file.  Massively higher coverage
# than single-shot CSV captures.
SEGMENT_COUNT  = 100
SAVE_FORMAT    = "H5"        # Keysight native multi-segment format

CYCLE_S        = 10.0        # seconds video is on per cycle
# Filling N segments takes ~N × LP-trigger period.  LP triggers fire roughly
# at line rate (≈48 kHz) so 100 segments fill in ms, but allow margin.
TRIG_TIMEOUT_S = max(SEGMENT_COUNT * 0.020 + 5.0, 10.0)

# ---------------------------------------------------------------------------
# Scope setup
# ---------------------------------------------------------------------------
scope = vxi11.Instrument(SCOPE_IP)
scope.timeout = 30


def setup_scope() -> None:
    """One-shot scope init — channels, math, default trigger."""
    print("CONFIGURING SCOPE...")
    cmds = [
        "*RST", ":RUN", ":STOP",
        ":CHANnel1:DISPlay ON", ":CHANnel1:INPut DC50", ":CHANnel1:PROBe 19.2",
        ":CHANnel1:LABel 'CLK+'",
        ":CHANnel2:DISPlay ON", ":CHANnel2:INPut DC50", ":CHANnel2:PROBe 19.2",
        ":CHANnel2:LABel 'CLK-'",
        ":CHANnel3:DISPlay ON", ":CHANnel3:INPut DC50", ":CHANnel3:PROBe 19.2",
        ":CHANnel3:LABel 'DAT0+'",
        ":CHANnel4:DISPlay ON", ":CHANnel4:INPut DC50", ":CHANnel4:PROBe 19.2",
        ":CHANnel4:LABel 'DAT0-'",
        ":TIMebase:REFerence CENTer",
        ":TRIGger:MODE EDGE",
        ":ACQuire:MODE RTIMe", ":ACQuire:INTerpolate ON",
        ":DISPlay:LAYout STACKED",
    ]
    for c in cmds:
        scope.write(c)
        time.sleep(0.05)
    print("SCOPE READY.")


def configure_for_lp() -> None:
    """LP-mode + segmented memory: N back-to-back LP triggers per acquisition."""
    for ch in (1, 2, 3, 4):
        scope.write(f":CHANnel{ch}:SCALe {LP_V_SCALE:.3f}")
        scope.write(f":CHANnel{ch}:OFFSet {LP_V_OFFSET:.3f}")
    scope.write(":TRIGger:EDGE:SOURce CHANnel3")
    scope.write(":TRIGger:EDGE:SLOPe NEGative")
    scope.write(f":TRIGger:EDGE:LEVel {LP_TRIG_LEVEL:.3f}")
    scope.write(":TRIGger:SWEep NORMal")
    scope.write(f":TIMebase:SCALe {LP_SCALE:.3E}")
    scope.write(f":ACQuire:POINts {LP_POINTS}")
    scope.write(f":TIMebase:POSition {LP_TRIG_OFFSET:.2E}")
    # Segmented memory: fill N segments per :DIGitize.
    scope.write(":ACQuire:MODE SEGMented")
    scope.write(f":ACQuire:SEGMented:COUNt {SEGMENT_COUNT}")
    time.sleep(0.5)


def arm_and_wait(timeout_s: float) -> bool:
    """:DIGitize + *OPC?. Returns True if trigger fired within timeout."""
    global scope
    prev = scope.timeout
    try:
        scope.timeout = timeout_s + 2
        scope.write(":DIGitize")
        return scope.ask("*OPC?").strip() == "1"
    except Exception:
        # Trigger timed out or scope locked up — reconnect.
        try:
            scope.close()
        except Exception:
            pass
        time.sleep(1.0)
        scope = vxi11.Instrument(SCOPE_IP)
        scope.timeout = 30
        try:
            scope.write(":STOP")
        except Exception:
            pass
        return False
    finally:
        try:
            scope.timeout = prev
        except Exception:
            pass


def save_lp(base_name: str) -> None:
    """Save all N segments of Ch1 (CLK+) and Ch3 (DAT0+) as a single H5 each."""
    base = f"C:\\TEMP\\{base_name}"
    ext  = SAVE_FORMAT.lower()
    scope.write(f':DISK:SAVE:WAVeform CHANnel1,"{base}_clk.{ext}",{SAVE_FORMAT}')
    time.sleep(3.0)
    scope.write(f':DISK:SAVE:WAVeform CHANnel3,"{base}_dat.{ext}",{SAVE_FORMAT}')
    time.sleep(3.0)


# ---------------------------------------------------------------------------
# Non-blocking keyboard
# ---------------------------------------------------------------------------
class KeyReader:
    def __enter__(self):
        self.fd  = sys.stdin.fileno()
        self.old = termios.tcgetattr(self.fd)
        tty.setcbreak(self.fd)
        return self

    def get_key(self) -> str | None:
        if select.select([sys.stdin], [], [], 0)[0]:
            return sys.stdin.read(1).lower()
        return None

    def __exit__(self, *_):
        termios.tcsetattr(self.fd, termios.TCSADRAIN, self.old)


# ---------------------------------------------------------------------------
# Video control
# ---------------------------------------------------------------------------
def video_start() -> None:
    try:
        requests.put(VIDEO_URL,
                     json={"action": "start", "mode": "static-pink"},
                     timeout=3)
    except requests.exceptions.RequestException as e:
        print(f"  VIDEO START failed: {e}")


def video_stop() -> None:
    try:
        requests.put(VIDEO_URL, json={"action": "stop"}, timeout=3)
    except requests.exceptions.RequestException as e:
        print(f"  VIDEO STOP failed: {e}")


# ---------------------------------------------------------------------------
# H5 transfer (ai_mgmt only handles CSV — segmented mode produces .h5)
# ---------------------------------------------------------------------------
def _transfer_h5_files() -> int:
    """SMB-pull every .h5 from the scope share into DATA_DIR; delete on scope."""
    from smb.SMBConnection import SMBConnection
    import socket
    conn = SMBConnection(
        ai_mgmt.USERNAME, ai_mgmt.PASSWORD,
        socket.gethostname(), ai_mgmt.SERVER_NAME,
        use_ntlm_v2=True, is_direct_tcp=True,
    )
    if not conn.connect(ai_mgmt.SERVER, 445):
        print("  H5 transfer: could not connect to scope share")
        return 0
    count = 0
    try:
        h5_paths: list[str] = []
        def walk(path: str) -> None:
            for entry in conn.listPath(ai_mgmt.SHARE, path):
                if entry.filename in (".", ".."):
                    continue
                full = f"{path}/{entry.filename}"
                if entry.isDirectory:
                    walk(full)
                elif entry.filename.lower().endswith(".h5"):
                    h5_paths.append(full)
        walk("/")
        for remote in h5_paths:
            local = DATA_DIR / Path(remote).name
            try:
                with open(local, "wb") as fh:
                    conn.retrieveFile(ai_mgmt.SHARE, remote, fh)
                conn.deleteFiles(ai_mgmt.SHARE, remote)
                count += 1
            except Exception as e:
                print(f"  H5 transfer failed for {Path(remote).name}: {e}")
    finally:
        conn.close()
    return count


# ---------------------------------------------------------------------------
# Register snapshot from device (DSIM PHY + SN65DSI83)
# ---------------------------------------------------------------------------
def fetch_registers_snapshot(target_dir: Path, event_ts: str) -> None:
    """GET /registers + /sn65_registers, print key indicators, save JSON."""
    combined: dict = {}
    for endpoint, key in [("/registers", "dsim"),
                          ("/sn65_registers", "sn65")]:
        try:
            r = requests.get(f"{DEVICE_BASE}{endpoint}", timeout=5)
            r.raise_for_status()
            combined[key] = r.json()
        except Exception as e:
            print(f"  REGISTERS: {endpoint} failed — {e}")
            combined[key] = None

    # Quick-look indicators
    sn65 = combined.get("sn65") or {}
    regs = sn65.get("registers", {}) if isinstance(sn65, dict) else {}
    csr_0a = regs.get("csr_0a", {}) or {}
    csr_e5 = regs.get("csr_e5", {}) or {}

    if csr_0a:
        pll_str = "LOCKED" if csr_0a.get("pll_lock") else "*** UNLOCKED ***"
        clk_str = "detected" if csr_0a.get("clk_det") else "NOT detected"
        print(f"  SN65: PLL {pll_str}  CLK {clk_str}  (CSR 0x0A = {csr_0a.get('value')})")

    if csr_e5:
        flags = [
            ("pll_unlock",      "PLL_UNLOCK"),
            ("cha_sot_bit_err", "SOT_BIT_ERR"),
            ("cha_llp_err",     "LLP_ERR"),
            ("cha_ecc_err",     "ECC_ERR"),
            ("cha_lp_err",      "LP_ERR"),
            ("cha_crc_err",     "CRC_ERR"),
        ]
        active = [label for k, label in flags if csr_e5.get(k)]
        if active:
            print(f"  SN65: *** ERROR FLAGS: {', '.join(active)}  "
                  f"(CSR 0xE5 = {csr_e5.get('value')}) ***")
        else:
            print(f"  SN65: no error flags  (CSR 0xE5 = {csr_e5.get('value')})")

    out = target_dir / f"{event_ts}_registers.json"
    try:
        out.write_text(json.dumps(combined, indent=2))
        print(f"  registers → {out.relative_to(DATA_DIR.parent)}")
    except Exception as e:
        print(f"  REGISTERS save failed: {e}")


# ---------------------------------------------------------------------------
# Event handling: archive recent captures and (for flicker) analyse
# ---------------------------------------------------------------------------
def archive_and_analyse(event: str, since_iso: str) -> None:
    """
    Pull every CSV from the scope, move into data/{event}/{event_ts}/.
    For flicker events, run csv_preprocessor on each LP capture and print a
    summary table.  Always pulls a register snapshot from the device too.
    """
    event_ts = datetime.now().strftime("%Y%m%d_%H%M%S")
    target   = (FLICKER_DIR if event == "flicker" else GOOD_DIR) / event_ts
    target.mkdir(parents=True, exist_ok=True)

    print(f"\n  *** {event.upper()} EVENT @ {event_ts} ***")

    # Register snapshot first (fast, before scope transfer which takes longer)
    fetch_registers_snapshot(target, event_ts)

    print(f"  Transferring scope → {target} ...")
    try:
        copied, failed = ai_mgmt.transfer_csv_files()
    except Exception as e:
        print(f"  TRANSFER ERROR: {e}")
        return
    print(f"  {copied} file(s) transferred  ({failed} failed)")

    # ai_mgmt only fetches CSVs.  H5 (segmented) files need a separate pass.
    h5_count = _transfer_h5_files()
    if h5_count:
        print(f"  {h5_count} H5 file(s) transferred")

    # Move just-arrived files (csv + h5) out of data/ (flat) into the event folder.
    moved = 0
    for f in list(DATA_DIR.glob("*.csv")) + list(DATA_DIR.glob("*.h5")):
        if f.is_file():
            shutil.move(str(f), target / f.name)
            moved += 1
    print(f"  {moved} file(s) archived to {target.relative_to(DATA_DIR.parent)}")

    # Explode each H5 into per-segment CSVs so csv_preprocessor can analyse them.
    from explode_h5 import explode
    h5_files = sorted(target.glob("*_lp_*.h5"))
    seg_csv_count = 0
    for h5 in h5_files:
        try:
            csvs = explode(h5)
            seg_csv_count += len(csvs)
        except Exception as e:
            print(f"  EXPLODE error on {h5.name}: {e}")
    if h5_files:
        print(f"  exploded {len(h5_files)} H5 file(s) → {seg_csv_count} segment CSV(s)")

    if event != "flicker":
        return

    # Analyse every segment CSV.  Flag outliers.
    print("\n  Per-segment LP analysis:")
    rows = []
    for f in sorted(target.glob("*_lp_*_dat.csv")):
        try:
            m = analyze_lp_file(f)
            rows.append({
                "file":   f.name,
                "lp_low": float(m.lp_low_duration_ns) if m.lp_low_duration_ns is not None else None,
                "hs_amp": float(m.hs_amplitude_mv)   if m.hs_amplitude_mv  is not None else None,
                "hs_dur": float(m.hs_burst_dur_ns)   if m.hs_burst_dur_ns  is not None else None,
                "n_burst": int(m.n_hs_bursts)        if m.n_hs_bursts      is not None else None,
                "sus":    bool(m.flicker_suspect),
            })
        except Exception as e:
            rows.append({"file": f.name, "error": str(e)})

    n_total = len(rows)
    n_sus   = sum(1 for r in rows if r.get("sus"))
    print(f"  {n_total} segments analysed  ({n_sus} flagged as flicker_suspect)")

    # Outlier search across the segments themselves.
    def _outliers(field: str, lo_thresh: float | None = None,
                  hi_thresh: float | None = None) -> list[dict]:
        vals = sorted(r[field] for r in rows if r.get(field) is not None)
        if not vals:
            return []
        med = vals[len(vals) // 2]
        out = []
        for r in rows:
            v = r.get(field)
            if v is None: continue
            far = (lo_thresh is not None and v < lo_thresh) or \
                  (hi_thresh is not None and v > hi_thresh)
            if far:
                out.append({"file": r["file"], field: v, "median": med})
        return out

    print("\n  Anomalies vs segment-set median:")
    for label, field, lo, hi in [
        ("very-short LP-low (<50 ns)",      "lp_low", 50,    None),
        ("very-low HS amplitude (<50 mV)",  "hs_amp", 50,    None),
        ("very-high HS amplitude (>140 mV)","hs_amp", None,  140),
        ("short HS burst (<8000 ns)",       "hs_dur", 8000,  None),
    ]:
        ax = _outliers(field, lo, hi)
        if ax:
            print(f"    {label}: {len(ax)} segment(s)")
            for x in ax[:8]:
                print(f"      {x['file']}  {field}={x[field]:.1f}  "
                      f"(set median={x['median']:.1f})")
            if len(ax) > 8:
                print(f"      ... +{len(ax) - 8} more")
        else:
            print(f"    {label}: none")


# ---------------------------------------------------------------------------
# Main loop
# ---------------------------------------------------------------------------
def main() -> None:
    DATA_DIR.mkdir(exist_ok=True)
    FLICKER_DIR.mkdir(exist_ok=True)
    GOOD_DIR.mkdir(exist_ok=True)

    setup_scope()
    configure_for_lp()

    print("\n" + "=" * 64)
    print("  FLICKER WATCH — keys:  f=flicker  g=good  q=quit")
    print("=" * 64 + "\n")

    cycle = 0
    try:
        with KeyReader() as keys:
            while True:
                cycle += 1
                cycle_ts   = datetime.now().strftime("%Y%m%d_%H%M%S")
                cycle_caps = []
                cycle_end  = time.time() + CYCLE_S

                video_start()
                print(f"\n[cycle {cycle:03d}  {cycle_ts}]  video ON  "
                      f"({CYCLE_S:.0f}s window, {SEGMENT_COUNT} segs/acquire)",
                      flush=True)

                event = None
                last_tick = 0.0
                while time.time() < cycle_end:
                    seq  = len(cycle_caps) + 1
                    base = f"{cycle_ts}_lp_c{cycle:03d}_{seq:02d}"
                    remaining = lambda: max(0, cycle_end - time.time())

                    if arm_and_wait(TRIG_TIMEOUT_S):
                        try:
                            save_lp(base)
                            cycle_caps.append(base)
                            print(f"  + acq {seq:02d}  ({SEGMENT_COUNT} segs)  "
                                  f"[{remaining():4.1f}s left]", flush=True)
                        except Exception as e:
                            print(f"  save error: {e}", flush=True)
                    else:
                        # Trigger timed out — print a heartbeat at most every 2s
                        if time.time() - last_tick > 2.0:
                            print(f"  ... waiting for trigger  "
                                  f"[{remaining():4.1f}s left]", flush=True)
                            last_tick = time.time()

                    key = keys.get_key()
                    if key in ("f", "g", "q"):
                        event = key
                        break

                video_stop()
                if event is None:
                    print(f"[cycle {cycle:03d}]  ended  "
                          f"({len(cycle_caps)} acq(s) ≈ "
                          f"{len(cycle_caps) * SEGMENT_COUNT} segments, no event)",
                          flush=True)

                if event == "f":
                    archive_and_analyse("flicker", cycle_ts)
                elif event == "g":
                    archive_and_analyse("good", cycle_ts)
                elif event == "q":
                    print("\nQUIT requested.")
                    break

                # Brief pause before next cycle so video stop settles.
                time.sleep(0.5)

    except KeyboardInterrupt:
        print("\nInterrupted (Ctrl+C).")
    finally:
        try:
            video_stop()
        except Exception:
            pass


if __name__ == "__main__":
    main()