thethreemagi/scout-website
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace the generic records table with a lean join_leads table

Browse Source 
records was a leads table wearing a generic name. It carried display_name /
email / phone, which only mean anything for a lead, plus status / assigned_to /
notes, which were a guess at an outreach process that has not been designed.
Contacting a family is one-to-many, so three columns on the lead row was always
the wrong shape for it.

join_leads is one row per /join submission with one column per form field, two
timestamps, and payload holding the submission verbatim. Outreach gets its own
table when the process is actually known.

Also splits heard_from from heard_from_detail. app.py folded source_place /
source_other into a composed "Other: ..." string and threw the raw answer away,
which is the half that tells you which daycare the lead came from. The composed
value is still built for the Sheet and the ntfy push.

admin API: /records -> /leads, and PATCH is gone since a lead now has nothing
mutable on it. mirrors and meta are unchanged.
This commit is contained in:
Mike Wichers
2026-08-26 20:55:58 -04:00
parent 391e1bcb21
commit 781f991fbe
3 changed files with 126 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/admin_api.py
+16 -35
View File
@@ -1,11 +1,16 @@
"""
admin_api.py - read/act API over the internal record store.
admin_api.py - read API over the internal record store.
This is the seam the future scout admin panel plugs into. The panel talks HTTP
to these endpoints; it never opens the SQLite file directly. That keeps the
panel deployable anywhere (separate container, separate host) and keeps this
app the only writer to its own database.
Read-only by design, for now. The old PATCH route set status/assigned_to/notes
on a lead - columns that were removed because they were a guess at an outreach
process nobody has designed. When that process exists it gets its own table and
its own write routes; until then there is nothing on a lead to mutate.
Auth: every route requires the X-Admin-Token header to match ADMIN_TOKEN.
If ADMIN_TOKEN is unset the whole router returns 503 - it FAILS CLOSED. These
endpoints expose parent names, emails and phone numbers for minors' families,
@@ -16,7 +21,6 @@ import hmac
import os
from fastapi import APIRouter, Header, HTTPException, Query
from pydantic import BaseModel
import store
@@ -32,61 +36,39 @@ def _auth(token):
raise HTTPException(401, "bad or missing X-Admin-Token")
class RecordPatch(BaseModel):
status: str | None = None
assigned_to: str | None = None
notes: str | None = None
actor: str | None = None
@router.get("/summary")
def get_summary(x_admin_token: str = Header(None)):
_auth(x_admin_token)
return store.summary()
@router.get("/records")
def get_records(kind: str = None, status: str = None, since: str = None,
q: str = None, limit: int = Query(100, ge=1, le=500), offset: int = 0,
@router.get("/leads")
def get_leads(since: str = None, q: str = None,
limit: int = Query(100, ge=1, le=500), offset: int = 0,
x_admin_token: str = Header(None)):
_auth(x_admin_token)
return {"records": store.list_records(kind=kind, status=status, since=since,
q=q, limit=limit, offset=offset)}
return {"leads": store.list_leads(since=since, q=q, limit=limit, offset=offset)}
@router.get("/records/{record_id}")
@router.get("/leads/{record_id}")
def get_one(record_id: str, x_admin_token: str = Header(None)):
_auth(x_admin_token)
rec = store.get_record(record_id, with_audit=True)
rec = store.get_lead(record_id)
if not rec:
raise HTTPException(404, "no such record")
return rec
@router.patch("/records/{record_id}")
def patch_one(record_id: str, patch: RecordPatch, x_admin_token: str = Header(None)):
_auth(x_admin_token)
try:
rec = store.update_record(record_id, actor=patch.actor or "admin",
status=patch.status, assigned_to=patch.assigned_to,
notes=patch.notes)
except ValueError as e:
raise HTTPException(422, str(e))
if not rec:
raise HTTPException(404, "no such record")
raise HTTPException(404, "no such lead")
return rec
@router.get("/mirrors/failed")
def failed(target: str = "google_sheet", x_admin_token: str = Header(None)):
_auth(x_admin_token)
return {"target": target, "records": store.failed_mirror_records(target)}
return {"target": target, "leads": store.failed_mirror_records(target)}
@router.post("/mirrors/retry")
def retry(target: str = "google_sheet", x_admin_token: str = Header(None)):
"""Replay records whose copy to an external target failed. Idempotent-ish:
a record already marked ok is never retried."""
"""Replay leads whose copy to an external target failed. Idempotent-ish:
a lead already marked ok is never retried."""
_auth(x_admin_token)
if target != "google_sheet":
raise HTTPException(422, "only google_sheet retry is implemented")
@@ -96,7 +78,6 @@ def retry(target: str = "google_sheet", x_admin_token: str = Header(None)):
try:
main_app.sheet_append(rec["payload"])
store.set_mirror(rec["id"], target, True)
store.log(rec["id"], "admin", "mirror_retry_ok", target)
done += 1
except Exception as e:
store.set_mirror(rec["id"], target, False, e)
app/app.py
+11 -7
View File
@@ -623,10 +623,14 @@ def pipeline_notify(rec, sheet_error=None):
def join_post(parent_name: str = Form(...), email: str = Form(...), phone: str = Form(""),
children: str = Form(""), interested_in: str = Form(""), source: str = Form(""),
source_place: str = Form(""), source_other: str = Form(""), message: str = Form("")):
heard_from = source.strip()
heard_from_detail = ""
if source == "Other school or daycare" and source_place.strip():
source = "School/Daycare: " + source_place.strip()
heard_from_detail = source_place.strip()
source = "School/Daycare: " + heard_from_detail
elif source == "Other" and source_other.strip():
source = "Other: " + source_other.strip()
heard_from_detail = source_other.strip()
source = "Other: " + heard_from_detail
rec = {"ts": datetime.datetime.now().isoformat(timespec="seconds"),
"parent_name": parent_name.strip(), "email": email.strip(), "phone": phone.strip(),
"children": children.strip() or "(not given)",
@@ -642,11 +646,11 @@ def join_post(parent_name: str = Form(...), email: str = Form(...), phone: str =
# against the record so a failure is replayable instead of just shouted.
record_id = None
try:
record_id = store.insert_record(
kind="join_lead", source=rec["source"], payload=rec,
unit=store.unit_of(rec["interested_in"]),
display_name=rec["parent_name"], email=rec["email"],
phone=rec["phone"], created_at=rec["ts"])
record_id = store.insert_lead(
parent_name=rec["parent_name"], email=rec["email"], phone=rec["phone"],
interested_in=rec["interested_in"], children=rec["children"],
heard_from=heard_from, heard_from_detail=heard_from_detail,
message=rec["message"], payload=rec, submitted_at=rec["ts"])
except Exception as e:
print("STORE WRITE FAILED for %s <%s>: %s" % (rec["parent_name"], rec["email"], e), flush=True)
sheet_error = None
app/store.py
+96 -164
View File
@@ -1,24 +1,28 @@
"""
store.py - internal record store for greenlanescouts73.org
Why this exists
What this holds
---------------
Until now a /join submission went straight to the Google Sheet, with a raw
append to /data/leads.jsonl as a fire-and-forget side effect wrapped in a bare
except. The sheet is the only queryable copy, it has no record IDs, no status,
no way to mark a lead handled, and a failed write survives only as an ntfy push.
One table per thing the site collects. Today that is `join_leads`: the /join
interest form, one row per submission, one column per form field.
This module makes a local SQLite database the source of truth. The sheet and
the ntfy push become *mirrors* of a record that already exists locally, and
each mirror's success or failure is recorded per record so it can be replayed.
A /join submission lands here FIRST. The Google Sheet and the ntfy push are
mirrors of a row that already exists locally, and each mirror's outcome is
recorded per row so a failure is replayable instead of just shouted.
Designed for a future admin panel
---------------------------------
The table is deliberately NOT a leads table. It is a generic record store keyed
by `kind`, so RSVPs, volunteer signups, popcorn orders or anything else the
admin panel eventually covers land in the same place with the same workflow
columns (status / assigned_to / notes) and the same audit trail. Adding a new
form means picking a new `kind` string, not migrating a schema.
Why join_leads and not a generic table
-------------------------------------
This started as a generic `records` table keyed by `kind`, carrying
status/assigned_to/notes for a future admin panel. That was wrong twice over.
The PII columns (display_name/email/phone) only make sense for a lead, and the
workflow columns describe an outreach process that has not been designed yet -
one that is plainly one-to-many, since a family gets contacted more than once.
Guessing at it in three columns would have locked in the wrong shape.
So: a table per form, columns for the form's own fields, and outreach gets its
own table when the process is actually known. Anything the form adds later that
is not worth a column still survives in `payload`, which holds the submission
verbatim.
Stdlib only - sqlite3 ships with Python, so this adds no image dependencies.
"""
@@ -32,33 +36,28 @@ from pathlib import Path
DB_PATH = Path(os.environ.get("STORE_DB", "/data/scout73.db"))
# Workflow states an admin panel may set. 'new' is the only one this app writes.
STATUSES = ("new", "contacted", "joined", "declined", "duplicate", "spam")
# Mirror targets - external destinations a record is copied out to.
# Mirror targets - external destinations a row is copied out to.
TARGETS = ("google_sheet", "ntfy")
SCHEMA = """
PRAGMA journal_mode=WAL;
CREATE TABLE IF NOT EXISTS records (
CREATE TABLE IF NOT EXISTS join_leads (
id TEXT PRIMARY KEY,
kind TEXT NOT NULL,
source TEXT NOT NULL,
unit TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
display_name TEXT,
email TEXT,
submitted_at TEXT NOT NULL,
recorded_at TEXT NOT NULL,
parent_name TEXT NOT NULL,
email TEXT NOT NULL,
phone TEXT,
payload TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'new',
assigned_to TEXT,
notes TEXT
interested_in TEXT,
children TEXT,
heard_from TEXT,
heard_from_detail TEXT,
message TEXT,
payload TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_records_kind_created ON records(kind, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_records_status ON records(status);
CREATE INDEX IF NOT EXISTS idx_records_email ON records(email);
CREATE INDEX IF NOT EXISTS idx_join_leads_submitted ON join_leads(submitted_at DESC);
CREATE INDEX IF NOT EXISTS idx_join_leads_email ON join_leads(email);
CREATE TABLE IF NOT EXISTS mirrors (
record_id TEXT NOT NULL,
@@ -71,16 +70,6 @@ CREATE TABLE IF NOT EXISTS mirrors (
);
CREATE INDEX IF NOT EXISTS idx_mirrors_state ON mirrors(target, state);
CREATE TABLE IF NOT EXISTS audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
record_id TEXT,
at TEXT NOT NULL,
actor TEXT NOT NULL,
action TEXT NOT NULL,
detail TEXT
);
CREATE INDEX IF NOT EXISTS idx_audit_record ON audit(record_id, id);
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
@@ -95,10 +84,10 @@ def _now():
def _norm(ts):
"""Normalise any incoming timestamp to UTC ISO-8601 with an offset.
app.py builds its record timestamp with a naive datetime.now(), i.e. local
Eastern time. Storing that next to UTC audit rows would silently break both
sorting and `since=` filters in the admin panel, so everything that lands in
the DB is converted here.
app.py builds its submission timestamp with a naive datetime.now(), i.e.
local Eastern time. Storing that next to UTC rows would silently break both
sorting and `since=` filters, so everything that lands in the DB is
converted here.
"""
if not ts:
return _now()
@@ -131,27 +120,27 @@ def init(backfill_jsonl=None):
con.close()
# ---------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Writes
# ---------------------------------------------------------------------------
# ----------------------------------------------------------------------------
def insert_record(kind, source, payload, unit=None, display_name=None,
email=None, phone=None, created_at=None, record_id=None):
"""Insert a record and return its id. Raises on failure - callers decide."""
def insert_lead(parent_name, email, phone=None, interested_in=None, children=None,
heard_from=None, heard_from_detail=None, message=None,
payload=None, submitted_at=None, record_id=None):
"""Insert one /join submission and return its id. Raises on failure -
callers decide what a failed local write means."""
rid = record_id or str(uuid.uuid4())
ts = _norm(created_at)
submitted = _norm(submitted_at)
con = connect()
try:
con.execute(
"INSERT INTO records (id, kind, source, unit, created_at, updated_at,"
" display_name, email, phone, payload, status)"
" VALUES (?,?,?,?,?,?,?,?,?,?, 'new')",
(rid, kind, source, unit, ts, ts, display_name, email, phone,
json.dumps(payload, ensure_ascii=False)),
)
con.execute(
"INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)",
(rid, _now(), "system", "created", kind),
"INSERT INTO join_leads (id, submitted_at, recorded_at, parent_name, email,"
" phone, interested_in, children, heard_from, heard_from_detail, message, payload)"
" VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
(rid, submitted, _now(), parent_name, email, phone or None,
interested_in or None, children or None, heard_from or None,
heard_from_detail or None, message or None,
json.dumps(payload or {}, ensure_ascii=False)),
)
con.commit()
finally:
@@ -160,7 +149,7 @@ def insert_record(kind, source, payload, unit=None, display_name=None,
def set_mirror(record_id, target, ok, error=None):
"""Record the outcome of an attempt to copy a record to an external target."""
"""Record the outcome of an attempt to copy a row to an external target."""
ts = _now()
state = "ok" if ok else "failed"
con = connect()
@@ -180,59 +169,13 @@ def set_mirror(record_id, target, ok, error=None):
con.close()
def update_record(record_id, actor="admin", status=None, assigned_to=None, notes=None):
"""Workflow update from the admin panel. Returns the updated row, or None."""
sets, vals = [], []
if status is not None:
if status not in STATUSES:
raise ValueError("unknown status: %s" % status)
sets.append("status=?"); vals.append(status)
if assigned_to is not None:
sets.append("assigned_to=?"); vals.append(assigned_to or None)
if notes is not None:
sets.append("notes=?"); vals.append(notes or None)
if not sets:
return get_record(record_id)
ts = _now()
sets.append("updated_at=?"); vals.append(ts)
vals.append(record_id)
con = connect()
try:
cur = con.execute("UPDATE records SET %s WHERE id=?" % ", ".join(sets), vals)
if cur.rowcount == 0:
return None
detail = json.dumps({k: v for k, v in
(("status", status), ("assigned_to", assigned_to), ("notes", notes))
if v is not None}, ensure_ascii=False)
con.execute(
"INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)",
(record_id, ts, actor, "updated", detail),
)
con.commit()
finally:
con.close()
return get_record(record_id)
def log(record_id, actor, action, detail=None):
con = connect()
try:
con.execute(
"INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)",
(record_id, _now(), actor, action, detail),
)
con.commit()
finally:
con.close()
# ---------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Reads
# ---------------------------------------------------------------------------
# ----------------------------------------------------------------------------
def _row(r):
d = dict(r)
if "payload" in d and d["payload"]:
if d.get("payload"):
try:
d["payload"] = json.loads(d["payload"])
except Exception:
@@ -240,40 +183,33 @@ def _row(r):
return d
def get_record(record_id, with_audit=False):
def get_lead(record_id):
con = connect()
try:
r = con.execute("SELECT * FROM records WHERE id=?", (record_id,)).fetchone()
r = con.execute("SELECT * FROM join_leads WHERE id=?", (record_id,)).fetchone()
if not r:
return None
out = _row(r)
out["mirrors"] = [dict(m) for m in con.execute(
"SELECT target, state, attempts, last_error, last_attempt_at"
" FROM mirrors WHERE record_id=?", (record_id,))]
if with_audit:
out["audit"] = [dict(a) for a in con.execute(
"SELECT at, actor, action, detail FROM audit WHERE record_id=? ORDER BY id",
(record_id,))]
return out
finally:
con.close()
def list_records(kind=None, status=None, since=None, q=None, limit=100, offset=0):
def list_leads(since=None, q=None, limit=100, offset=0):
where, vals = [], []
if kind:
where.append("kind=?"); vals.append(kind)
if status:
where.append("status=?"); vals.append(status)
if since:
where.append("created_at>=?"); vals.append(since)
where.append("submitted_at>=?"); vals.append(since)
if q:
where.append("(display_name LIKE ? OR email LIKE ? OR payload LIKE ?)")
vals += ["%%%s%%" % q] * 3
sql = "SELECT * FROM records"
where.append("(parent_name LIKE ? OR email LIKE ? OR children LIKE ?"
" OR heard_from LIKE ? OR message LIKE ?)")
vals += ["%%%s%%" % q] * 5
sql = "SELECT * FROM join_leads"
if where:
sql += " WHERE " + " AND ".join(where)
sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
sql += " ORDER BY submitted_at DESC LIMIT ? OFFSET ?"
vals += [max(1, min(int(limit), 500)), max(0, int(offset))]
con = connect()
try:
@@ -293,15 +229,16 @@ def list_records(kind=None, status=None, since=None, q=None, limit=100, offset=0
def summary():
"""Counts for a dashboard: by kind, by status, and failed mirrors."""
"""Counts for a dashboard: totals, interest split, referral sources, failed mirrors."""
con = connect()
try:
return {
"total": con.execute("SELECT COUNT(*) c FROM records").fetchone()["c"],
"by_kind": {r["kind"]: r["c"] for r in con.execute(
"SELECT kind, COUNT(*) c FROM records GROUP BY kind")},
"by_status": {r["status"]: r["c"] for r in con.execute(
"SELECT status, COUNT(*) c FROM records GROUP BY status")},
"total": con.execute("SELECT COUNT(*) c FROM join_leads").fetchone()["c"],
"by_interest": {r["interested_in"]: r["c"] for r in con.execute(
"SELECT interested_in, COUNT(*) c FROM join_leads GROUP BY interested_in")},
"by_heard_from": {r["heard_from"]: r["c"] for r in con.execute(
"SELECT heard_from, COUNT(*) c FROM join_leads GROUP BY heard_from"
" ORDER BY c DESC")},
"failed_mirrors": {r["target"]: r["c"] for r in con.execute(
"SELECT target, COUNT(*) c FROM mirrors WHERE state='failed' GROUP BY target")},
}
@@ -313,16 +250,27 @@ def failed_mirror_records(target, limit=50):
con = connect()
try:
return [_row(r) for r in con.execute(
"SELECT r.* FROM records r JOIN mirrors m ON m.record_id=r.id"
" WHERE m.target=? AND m.state='failed' ORDER BY r.created_at LIMIT ?",
"SELECT l.* FROM join_leads l JOIN mirrors m ON m.record_id=l.id"
" WHERE m.target=? AND m.state='failed' ORDER BY l.submitted_at LIMIT ?",
(target, limit))]
finally:
con.close()
# ---------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# One-time backfill of the pre-existing raw log
# ---------------------------------------------------------------------------
# ----------------------------------------------------------------------------
def split_heard_from(source):
"""app.py used to fold source_place / source_other into one composed string
('Other: Chocolate booth'), destroying the raw answer. Recover both halves."""
s = (source or "").strip()
if s.startswith("School/Daycare: "):
return "Other school or daycare", s[len("School/Daycare: "):].strip()
if s.startswith("Other: "):
return "Other", s[len("Other: "):].strip()
return s or None, None
def _backfill(con, path):
"""Import /data/leads.jsonl once, so history is not stranded outside the DB."""
@@ -339,37 +287,21 @@ def _backfill(con, path):
rec = json.loads(line)
except Exception:
continue
rid = str(uuid.uuid4())
heard, detail = split_heard_from(rec.get("source"))
ts = _norm(rec.get("ts"))
con.execute(
"INSERT INTO records (id, kind, source, unit, created_at, updated_at,"
" display_name, email, phone, payload, status)"
" VALUES (?,?,?,?,?,?,?,?,?,?, 'new')",
(rid, "join_lead", rec.get("source") or "greenlanescouts73.org",
unit_of(rec.get("interested_in", "")), ts, ts,
rec.get("parent_name"), rec.get("email"), rec.get("phone"),
"INSERT INTO join_leads (id, submitted_at, recorded_at, parent_name, email,"
" phone, interested_in, children, heard_from, heard_from_detail, message, payload)"
" VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
(str(uuid.uuid4()), ts, _now(),
rec.get("parent_name") or "(unknown)", rec.get("email") or "",
rec.get("phone") or None, rec.get("interested_in") or None,
rec.get("children") or None, heard, detail,
rec.get("message") or None,
json.dumps(rec, ensure_ascii=False)),
)
con.execute(
"INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)",
(rid, _now(), "system", "backfilled", "leads.jsonl"),
)
n += 1
con.execute("INSERT INTO meta (key, value) VALUES ('backfill_leads_jsonl', ?)",
("%s rows at %s" % (n, _now()),))
con.commit()
print("store: backfilled %s rows from %s" % (n, path), flush=True)
def unit_of(interested_in):
"""Best-effort unit tag so the admin panel can filter pack vs troop."""
s = (interested_in or "").lower()
pack = "pack" in s or "cub" in s
troop = "troop" in s or "scouts bsa" in s
if pack and troop or "both" in s:
return "both"
if pack:
return "pack"
if troop:
return "troop"
return None