diff --git a/app/admin_api.py b/app/admin_api.py index ed551ef..0635d03 100644 --- a/app/admin_api.py +++ b/app/admin_api.py @@ -1,11 +1,16 @@ """ -admin_api.py - read/act API over the internal record store. +admin_api.py - read API over the internal record store. This is the seam the future scout admin panel plugs into. The panel talks HTTP to these endpoints; it never opens the SQLite file directly. That keeps the panel deployable anywhere (separate container, separate host) and keeps this app the only writer to its own database. +Read-only by design, for now. The old PATCH route set status/assigned_to/notes +on a lead - columns that were removed because they were a guess at an outreach +process nobody has designed. When that process exists it gets its own table and +its own write routes; until then there is nothing on a lead to mutate. + Auth: every route requires the X-Admin-Token header to match ADMIN_TOKEN. If ADMIN_TOKEN is unset the whole router returns 503 - it FAILS CLOSED. These endpoints expose parent names, emails and phone numbers for minors' families, @@ -16,7 +21,6 @@ import hmac import os from fastapi import APIRouter, Header, HTTPException, Query -from pydantic import BaseModel import store @@ -32,61 +36,39 @@ def _auth(token): raise HTTPException(401, "bad or missing X-Admin-Token") -class RecordPatch(BaseModel): - status: str | None = None - assigned_to: str | None = None - notes: str | None = None - actor: str | None = None - - @router.get("/summary") def get_summary(x_admin_token: str = Header(None)): _auth(x_admin_token) return store.summary() -@router.get("/records") -def get_records(kind: str = None, status: str = None, since: str = None, - q: str = None, limit: int = Query(100, ge=1, le=500), offset: int = 0, - x_admin_token: str = Header(None)): +@router.get("/leads") +def get_leads(since: str = None, q: str = None, + limit: int = Query(100, ge=1, le=500), offset: int = 0, + x_admin_token: str = Header(None)): _auth(x_admin_token) - return {"records": store.list_records(kind=kind, status=status, since=since, - q=q, limit=limit, offset=offset)} + return {"leads": store.list_leads(since=since, q=q, limit=limit, offset=offset)} -@router.get("/records/{record_id}") +@router.get("/leads/{record_id}") def get_one(record_id: str, x_admin_token: str = Header(None)): _auth(x_admin_token) - rec = store.get_record(record_id, with_audit=True) + rec = store.get_lead(record_id) if not rec: - raise HTTPException(404, "no such record") - return rec - - -@router.patch("/records/{record_id}") -def patch_one(record_id: str, patch: RecordPatch, x_admin_token: str = Header(None)): - _auth(x_admin_token) - try: - rec = store.update_record(record_id, actor=patch.actor or "admin", - status=patch.status, assigned_to=patch.assigned_to, - notes=patch.notes) - except ValueError as e: - raise HTTPException(422, str(e)) - if not rec: - raise HTTPException(404, "no such record") + raise HTTPException(404, "no such lead") return rec @router.get("/mirrors/failed") def failed(target: str = "google_sheet", x_admin_token: str = Header(None)): _auth(x_admin_token) - return {"target": target, "records": store.failed_mirror_records(target)} + return {"target": target, "leads": store.failed_mirror_records(target)} @router.post("/mirrors/retry") def retry(target: str = "google_sheet", x_admin_token: str = Header(None)): - """Replay records whose copy to an external target failed. Idempotent-ish: - a record already marked ok is never retried.""" + """Replay leads whose copy to an external target failed. Idempotent-ish: + a lead already marked ok is never retried.""" _auth(x_admin_token) if target != "google_sheet": raise HTTPException(422, "only google_sheet retry is implemented") @@ -96,7 +78,6 @@ def retry(target: str = "google_sheet", x_admin_token: str = Header(None)): try: main_app.sheet_append(rec["payload"]) store.set_mirror(rec["id"], target, True) - store.log(rec["id"], "admin", "mirror_retry_ok", target) done += 1 except Exception as e: store.set_mirror(rec["id"], target, False, e) diff --git a/app/app.py b/app/app.py index 4d8a6b4..faa02ad 100644 --- a/app/app.py +++ b/app/app.py @@ -623,10 +623,14 @@ def pipeline_notify(rec, sheet_error=None): def join_post(parent_name: str = Form(...), email: str = Form(...), phone: str = Form(""), children: str = Form(""), interested_in: str = Form(""), source: str = Form(""), source_place: str = Form(""), source_other: str = Form(""), message: str = Form("")): + heard_from = source.strip() + heard_from_detail = "" if source == "Other school or daycare" and source_place.strip(): - source = "School/Daycare: " + source_place.strip() + heard_from_detail = source_place.strip() + source = "School/Daycare: " + heard_from_detail elif source == "Other" and source_other.strip(): - source = "Other: " + source_other.strip() + heard_from_detail = source_other.strip() + source = "Other: " + heard_from_detail rec = {"ts": datetime.datetime.now().isoformat(timespec="seconds"), "parent_name": parent_name.strip(), "email": email.strip(), "phone": phone.strip(), "children": children.strip() or "(not given)", @@ -642,11 +646,11 @@ def join_post(parent_name: str = Form(...), email: str = Form(...), phone: str = # against the record so a failure is replayable instead of just shouted. record_id = None try: - record_id = store.insert_record( - kind="join_lead", source=rec["source"], payload=rec, - unit=store.unit_of(rec["interested_in"]), - display_name=rec["parent_name"], email=rec["email"], - phone=rec["phone"], created_at=rec["ts"]) + record_id = store.insert_lead( + parent_name=rec["parent_name"], email=rec["email"], phone=rec["phone"], + interested_in=rec["interested_in"], children=rec["children"], + heard_from=heard_from, heard_from_detail=heard_from_detail, + message=rec["message"], payload=rec, submitted_at=rec["ts"]) except Exception as e: print("STORE WRITE FAILED for %s <%s>: %s" % (rec["parent_name"], rec["email"], e), flush=True) sheet_error = None diff --git a/app/store.py b/app/store.py index bd63992..b59947c 100644 --- a/app/store.py +++ b/app/store.py @@ -1,24 +1,28 @@ """ store.py - internal record store for greenlanescouts73.org -Why this exists +What this holds --------------- -Until now a /join submission went straight to the Google Sheet, with a raw -append to /data/leads.jsonl as a fire-and-forget side effect wrapped in a bare -except. The sheet is the only queryable copy, it has no record IDs, no status, -no way to mark a lead handled, and a failed write survives only as an ntfy push. +One table per thing the site collects. Today that is `join_leads`: the /join +interest form, one row per submission, one column per form field. -This module makes a local SQLite database the source of truth. The sheet and -the ntfy push become *mirrors* of a record that already exists locally, and -each mirror's success or failure is recorded per record so it can be replayed. +A /join submission lands here FIRST. The Google Sheet and the ntfy push are +mirrors of a row that already exists locally, and each mirror's outcome is +recorded per row so a failure is replayable instead of just shouted. -Designed for a future admin panel ---------------------------------- -The table is deliberately NOT a leads table. It is a generic record store keyed -by `kind`, so RSVPs, volunteer signups, popcorn orders or anything else the -admin panel eventually covers land in the same place with the same workflow -columns (status / assigned_to / notes) and the same audit trail. Adding a new -form means picking a new `kind` string, not migrating a schema. +Why join_leads and not a generic table +------------------------------------- +This started as a generic `records` table keyed by `kind`, carrying +status/assigned_to/notes for a future admin panel. That was wrong twice over. +The PII columns (display_name/email/phone) only make sense for a lead, and the +workflow columns describe an outreach process that has not been designed yet - +one that is plainly one-to-many, since a family gets contacted more than once. +Guessing at it in three columns would have locked in the wrong shape. + +So: a table per form, columns for the form's own fields, and outreach gets its +own table when the process is actually known. Anything the form adds later that +is not worth a column still survives in `payload`, which holds the submission +verbatim. Stdlib only - sqlite3 ships with Python, so this adds no image dependencies. """ @@ -32,33 +36,28 @@ from pathlib import Path DB_PATH = Path(os.environ.get("STORE_DB", "/data/scout73.db")) -# Workflow states an admin panel may set. 'new' is the only one this app writes. -STATUSES = ("new", "contacted", "joined", "declined", "duplicate", "spam") - -# Mirror targets - external destinations a record is copied out to. +# Mirror targets - external destinations a row is copied out to. TARGETS = ("google_sheet", "ntfy") SCHEMA = """ PRAGMA journal_mode=WAL; -CREATE TABLE IF NOT EXISTS records ( - id TEXT PRIMARY KEY, - kind TEXT NOT NULL, - source TEXT NOT NULL, - unit TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - display_name TEXT, - email TEXT, - phone TEXT, - payload TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'new', - assigned_to TEXT, - notes TEXT +CREATE TABLE IF NOT EXISTS join_leads ( + id TEXT PRIMARY KEY, + submitted_at TEXT NOT NULL, + recorded_at TEXT NOT NULL, + parent_name TEXT NOT NULL, + email TEXT NOT NULL, + phone TEXT, + interested_in TEXT, + children TEXT, + heard_from TEXT, + heard_from_detail TEXT, + message TEXT, + payload TEXT NOT NULL ); -CREATE INDEX IF NOT EXISTS idx_records_kind_created ON records(kind, created_at DESC); -CREATE INDEX IF NOT EXISTS idx_records_status ON records(status); -CREATE INDEX IF NOT EXISTS idx_records_email ON records(email); +CREATE INDEX IF NOT EXISTS idx_join_leads_submitted ON join_leads(submitted_at DESC); +CREATE INDEX IF NOT EXISTS idx_join_leads_email ON join_leads(email); CREATE TABLE IF NOT EXISTS mirrors ( record_id TEXT NOT NULL, @@ -71,16 +70,6 @@ CREATE TABLE IF NOT EXISTS mirrors ( ); CREATE INDEX IF NOT EXISTS idx_mirrors_state ON mirrors(target, state); -CREATE TABLE IF NOT EXISTS audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - record_id TEXT, - at TEXT NOT NULL, - actor TEXT NOT NULL, - action TEXT NOT NULL, - detail TEXT -); -CREATE INDEX IF NOT EXISTS idx_audit_record ON audit(record_id, id); - CREATE TABLE IF NOT EXISTS meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL @@ -95,10 +84,10 @@ def _now(): def _norm(ts): """Normalise any incoming timestamp to UTC ISO-8601 with an offset. - app.py builds its record timestamp with a naive datetime.now(), i.e. local - Eastern time. Storing that next to UTC audit rows would silently break both - sorting and `since=` filters in the admin panel, so everything that lands in - the DB is converted here. + app.py builds its submission timestamp with a naive datetime.now(), i.e. + local Eastern time. Storing that next to UTC rows would silently break both + sorting and `since=` filters, so everything that lands in the DB is + converted here. """ if not ts: return _now() @@ -131,27 +120,27 @@ def init(backfill_jsonl=None): con.close() -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- # Writes -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- -def insert_record(kind, source, payload, unit=None, display_name=None, - email=None, phone=None, created_at=None, record_id=None): - """Insert a record and return its id. Raises on failure - callers decide.""" +def insert_lead(parent_name, email, phone=None, interested_in=None, children=None, + heard_from=None, heard_from_detail=None, message=None, + payload=None, submitted_at=None, record_id=None): + """Insert one /join submission and return its id. Raises on failure - + callers decide what a failed local write means.""" rid = record_id or str(uuid.uuid4()) - ts = _norm(created_at) + submitted = _norm(submitted_at) con = connect() try: con.execute( - "INSERT INTO records (id, kind, source, unit, created_at, updated_at," - " display_name, email, phone, payload, status)" - " VALUES (?,?,?,?,?,?,?,?,?,?, 'new')", - (rid, kind, source, unit, ts, ts, display_name, email, phone, - json.dumps(payload, ensure_ascii=False)), - ) - con.execute( - "INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)", - (rid, _now(), "system", "created", kind), + "INSERT INTO join_leads (id, submitted_at, recorded_at, parent_name, email," + " phone, interested_in, children, heard_from, heard_from_detail, message, payload)" + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (rid, submitted, _now(), parent_name, email, phone or None, + interested_in or None, children or None, heard_from or None, + heard_from_detail or None, message or None, + json.dumps(payload or {}, ensure_ascii=False)), ) con.commit() finally: @@ -160,7 +149,7 @@ def insert_record(kind, source, payload, unit=None, display_name=None, def set_mirror(record_id, target, ok, error=None): - """Record the outcome of an attempt to copy a record to an external target.""" + """Record the outcome of an attempt to copy a row to an external target.""" ts = _now() state = "ok" if ok else "failed" con = connect() @@ -180,59 +169,13 @@ def set_mirror(record_id, target, ok, error=None): con.close() -def update_record(record_id, actor="admin", status=None, assigned_to=None, notes=None): - """Workflow update from the admin panel. Returns the updated row, or None.""" - sets, vals = [], [] - if status is not None: - if status not in STATUSES: - raise ValueError("unknown status: %s" % status) - sets.append("status=?"); vals.append(status) - if assigned_to is not None: - sets.append("assigned_to=?"); vals.append(assigned_to or None) - if notes is not None: - sets.append("notes=?"); vals.append(notes or None) - if not sets: - return get_record(record_id) - ts = _now() - sets.append("updated_at=?"); vals.append(ts) - vals.append(record_id) - con = connect() - try: - cur = con.execute("UPDATE records SET %s WHERE id=?" % ", ".join(sets), vals) - if cur.rowcount == 0: - return None - detail = json.dumps({k: v for k, v in - (("status", status), ("assigned_to", assigned_to), ("notes", notes)) - if v is not None}, ensure_ascii=False) - con.execute( - "INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)", - (record_id, ts, actor, "updated", detail), - ) - con.commit() - finally: - con.close() - return get_record(record_id) - - -def log(record_id, actor, action, detail=None): - con = connect() - try: - con.execute( - "INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)", - (record_id, _now(), actor, action, detail), - ) - con.commit() - finally: - con.close() - - -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- # Reads -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- def _row(r): d = dict(r) - if "payload" in d and d["payload"]: + if d.get("payload"): try: d["payload"] = json.loads(d["payload"]) except Exception: @@ -240,40 +183,33 @@ def _row(r): return d -def get_record(record_id, with_audit=False): +def get_lead(record_id): con = connect() try: - r = con.execute("SELECT * FROM records WHERE id=?", (record_id,)).fetchone() + r = con.execute("SELECT * FROM join_leads WHERE id=?", (record_id,)).fetchone() if not r: return None out = _row(r) out["mirrors"] = [dict(m) for m in con.execute( "SELECT target, state, attempts, last_error, last_attempt_at" " FROM mirrors WHERE record_id=?", (record_id,))] - if with_audit: - out["audit"] = [dict(a) for a in con.execute( - "SELECT at, actor, action, detail FROM audit WHERE record_id=? ORDER BY id", - (record_id,))] return out finally: con.close() -def list_records(kind=None, status=None, since=None, q=None, limit=100, offset=0): +def list_leads(since=None, q=None, limit=100, offset=0): where, vals = [], [] - if kind: - where.append("kind=?"); vals.append(kind) - if status: - where.append("status=?"); vals.append(status) if since: - where.append("created_at>=?"); vals.append(since) + where.append("submitted_at>=?"); vals.append(since) if q: - where.append("(display_name LIKE ? OR email LIKE ? OR payload LIKE ?)") - vals += ["%%%s%%" % q] * 3 - sql = "SELECT * FROM records" + where.append("(parent_name LIKE ? OR email LIKE ? OR children LIKE ?" + " OR heard_from LIKE ? OR message LIKE ?)") + vals += ["%%%s%%" % q] * 5 + sql = "SELECT * FROM join_leads" if where: sql += " WHERE " + " AND ".join(where) - sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?" + sql += " ORDER BY submitted_at DESC LIMIT ? OFFSET ?" vals += [max(1, min(int(limit), 500)), max(0, int(offset))] con = connect() try: @@ -293,15 +229,16 @@ def list_records(kind=None, status=None, since=None, q=None, limit=100, offset=0 def summary(): - """Counts for a dashboard: by kind, by status, and failed mirrors.""" + """Counts for a dashboard: totals, interest split, referral sources, failed mirrors.""" con = connect() try: return { - "total": con.execute("SELECT COUNT(*) c FROM records").fetchone()["c"], - "by_kind": {r["kind"]: r["c"] for r in con.execute( - "SELECT kind, COUNT(*) c FROM records GROUP BY kind")}, - "by_status": {r["status"]: r["c"] for r in con.execute( - "SELECT status, COUNT(*) c FROM records GROUP BY status")}, + "total": con.execute("SELECT COUNT(*) c FROM join_leads").fetchone()["c"], + "by_interest": {r["interested_in"]: r["c"] for r in con.execute( + "SELECT interested_in, COUNT(*) c FROM join_leads GROUP BY interested_in")}, + "by_heard_from": {r["heard_from"]: r["c"] for r in con.execute( + "SELECT heard_from, COUNT(*) c FROM join_leads GROUP BY heard_from" + " ORDER BY c DESC")}, "failed_mirrors": {r["target"]: r["c"] for r in con.execute( "SELECT target, COUNT(*) c FROM mirrors WHERE state='failed' GROUP BY target")}, } @@ -313,16 +250,27 @@ def failed_mirror_records(target, limit=50): con = connect() try: return [_row(r) for r in con.execute( - "SELECT r.* FROM records r JOIN mirrors m ON m.record_id=r.id" - " WHERE m.target=? AND m.state='failed' ORDER BY r.created_at LIMIT ?", + "SELECT l.* FROM join_leads l JOIN mirrors m ON m.record_id=l.id" + " WHERE m.target=? AND m.state='failed' ORDER BY l.submitted_at LIMIT ?", (target, limit))] finally: con.close() -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- # One-time backfill of the pre-existing raw log -# --------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- + +def split_heard_from(source): + """app.py used to fold source_place / source_other into one composed string + ('Other: Chocolate booth'), destroying the raw answer. Recover both halves.""" + s = (source or "").strip() + if s.startswith("School/Daycare: "): + return "Other school or daycare", s[len("School/Daycare: "):].strip() + if s.startswith("Other: "): + return "Other", s[len("Other: "):].strip() + return s or None, None + def _backfill(con, path): """Import /data/leads.jsonl once, so history is not stranded outside the DB.""" @@ -339,37 +287,21 @@ def _backfill(con, path): rec = json.loads(line) except Exception: continue - rid = str(uuid.uuid4()) + heard, detail = split_heard_from(rec.get("source")) ts = _norm(rec.get("ts")) con.execute( - "INSERT INTO records (id, kind, source, unit, created_at, updated_at," - " display_name, email, phone, payload, status)" - " VALUES (?,?,?,?,?,?,?,?,?,?, 'new')", - (rid, "join_lead", rec.get("source") or "greenlanescouts73.org", - unit_of(rec.get("interested_in", "")), ts, ts, - rec.get("parent_name"), rec.get("email"), rec.get("phone"), + "INSERT INTO join_leads (id, submitted_at, recorded_at, parent_name, email," + " phone, interested_in, children, heard_from, heard_from_detail, message, payload)" + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (str(uuid.uuid4()), ts, _now(), + rec.get("parent_name") or "(unknown)", rec.get("email") or "", + rec.get("phone") or None, rec.get("interested_in") or None, + rec.get("children") or None, heard, detail, + rec.get("message") or None, json.dumps(rec, ensure_ascii=False)), ) - con.execute( - "INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)", - (rid, _now(), "system", "backfilled", "leads.jsonl"), - ) n += 1 con.execute("INSERT INTO meta (key, value) VALUES ('backfill_leads_jsonl', ?)", ("%s rows at %s" % (n, _now()),)) con.commit() print("store: backfilled %s rows from %s" % (n, path), flush=True) - - -def unit_of(interested_in): - """Best-effort unit tag so the admin panel can filter pack vs troop.""" - s = (interested_in or "").lower() - pack = "pack" in s or "cub" in s - troop = "troop" in s or "scouts bsa" in s - if pack and troop or "both" in s: - return "both" - if pack: - return "pack" - if troop: - return "troop" - return None