thethreemagi/scout-website
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace the generic records table with a lean join_leads table

Browse Source 
records was a leads table wearing a generic name. It carried display_name /
email / phone, which only mean anything for a lead, plus status / assigned_to /
notes, which were a guess at an outreach process that has not been designed.
Contacting a family is one-to-many, so three columns on the lead row was always
the wrong shape for it.

join_leads is one row per /join submission with one column per form field, two
timestamps, and payload holding the submission verbatim. Outreach gets its own
table when the process is actually known.

Also splits heard_from from heard_from_detail. app.py folded source_place /
source_other into a composed "Other: ..." string and threw the raw answer away,
which is the half that tells you which daycare the lead came from. The composed
value is still built for the Sheet and the ntfy push.

admin API: /records -> /leads, and PATCH is gone since a lead now has nothing
mutable on it. mirrors and meta are unchanged.
This commit is contained in:
Mike Wichers
2026-08-26 20:55:58 -04:00
parent 391e1bcb21
commit 781f991fbe
3 changed files with 126 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/admin_api.py
+17 -36
View File
@@ -1,11 +1,16 @@
""" """
admin_api.py - read/act API over the internal record store. admin_api.py - read API over the internal record store.
This is the seam the future scout admin panel plugs into. The panel talks HTTP This is the seam the future scout admin panel plugs into. The panel talks HTTP
to these endpoints; it never opens the SQLite file directly. That keeps the to these endpoints; it never opens the SQLite file directly. That keeps the
panel deployable anywhere (separate container, separate host) and keeps this panel deployable anywhere (separate container, separate host) and keeps this
app the only writer to its own database. app the only writer to its own database.
Read-only by design, for now. The old PATCH route set status/assigned_to/notes
on a lead - columns that were removed because they were a guess at an outreach
process nobody has designed. When that process exists it gets its own table and
its own write routes; until then there is nothing on a lead to mutate.
Auth: every route requires the X-Admin-Token header to match ADMIN_TOKEN. Auth: every route requires the X-Admin-Token header to match ADMIN_TOKEN.
If ADMIN_TOKEN is unset the whole router returns 503 - it FAILS CLOSED. These If ADMIN_TOKEN is unset the whole router returns 503 - it FAILS CLOSED. These
endpoints expose parent names, emails and phone numbers for minors' families, endpoints expose parent names, emails and phone numbers for minors' families,
@@ -16,7 +21,6 @@ import hmac
import os import os
from fastapi import APIRouter, Header, HTTPException, Query from fastapi import APIRouter, Header, HTTPException, Query
from pydantic import BaseModel
import store import store
@@ -32,61 +36,39 @@ def _auth(token):
raise HTTPException(401, "bad or missing X-Admin-Token") raise HTTPException(401, "bad or missing X-Admin-Token")
class RecordPatch(BaseModel):
status: str | None = None
assigned_to: str | None = None
notes: str | None = None
actor: str | None = None
@router.get("/summary") @router.get("/summary")
def get_summary(x_admin_token: str = Header(None)): def get_summary(x_admin_token: str = Header(None)):
_auth(x_admin_token) _auth(x_admin_token)
return store.summary() return store.summary()
@router.get("/records") @router.get("/leads")
def get_records(kind: str = None, status: str = None, since: str = None, def get_leads(since: str = None, q: str = None,
q: str = None, limit: int = Query(100, ge=1, le=500), offset: int = 0, limit: int = Query(100, ge=1, le=500), offset: int = 0,
x_admin_token: str = Header(None)): x_admin_token: str = Header(None)):
_auth(x_admin_token) _auth(x_admin_token)
return {"records": store.list_records(kind=kind, status=status, since=since, return {"leads": store.list_leads(since=since, q=q, limit=limit, offset=offset)}
q=q, limit=limit, offset=offset)}
@router.get("/records/{record_id}") @router.get("/leads/{record_id}")
def get_one(record_id: str, x_admin_token: str = Header(None)): def get_one(record_id: str, x_admin_token: str = Header(None)):
_auth(x_admin_token) _auth(x_admin_token)
rec = store.get_record(record_id, with_audit=True) rec = store.get_lead(record_id)
if not rec: if not rec:
raise HTTPException(404, "no such record") raise HTTPException(404, "no such lead")
return rec
@router.patch("/records/{record_id}")
def patch_one(record_id: str, patch: RecordPatch, x_admin_token: str = Header(None)):
_auth(x_admin_token)
try:
rec = store.update_record(record_id, actor=patch.actor or "admin",
status=patch.status, assigned_to=patch.assigned_to,
notes=patch.notes)
except ValueError as e:
raise HTTPException(422, str(e))
if not rec:
raise HTTPException(404, "no such record")
return rec return rec
@router.get("/mirrors/failed") @router.get("/mirrors/failed")
def failed(target: str = "google_sheet", x_admin_token: str = Header(None)): def failed(target: str = "google_sheet", x_admin_token: str = Header(None)):
_auth(x_admin_token) _auth(x_admin_token)
return {"target": target, "records": store.failed_mirror_records(target)} return {"target": target, "leads": store.failed_mirror_records(target)}
@router.post("/mirrors/retry") @router.post("/mirrors/retry")
def retry(target: str = "google_sheet", x_admin_token: str = Header(None)): def retry(target: str = "google_sheet", x_admin_token: str = Header(None)):
"""Replay records whose copy to an external target failed. Idempotent-ish: """Replay leads whose copy to an external target failed. Idempotent-ish:
a record already marked ok is never retried.""" a lead already marked ok is never retried."""
_auth(x_admin_token) _auth(x_admin_token)
if target != "google_sheet": if target != "google_sheet":
raise HTTPException(422, "only google_sheet retry is implemented") raise HTTPException(422, "only google_sheet retry is implemented")
@@ -96,7 +78,6 @@ def retry(target: str = "google_sheet", x_admin_token: str = Header(None)):
try: try:
main_app.sheet_append(rec["payload"]) main_app.sheet_append(rec["payload"])
store.set_mirror(rec["id"], target, True) store.set_mirror(rec["id"], target, True)
store.log(rec["id"], "admin", "mirror_retry_ok", target)
done += 1 done += 1
except Exception as e: except Exception as e:
store.set_mirror(rec["id"], target, False, e) store.set_mirror(rec["id"], target, False, e)
app/app.py
+11 -7
View File
@@ -623,10 +623,14 @@ def pipeline_notify(rec, sheet_error=None):
def join_post(parent_name: str = Form(...), email: str = Form(...), phone: str = Form(""), def join_post(parent_name: str = Form(...), email: str = Form(...), phone: str = Form(""),
children: str = Form(""), interested_in: str = Form(""), source: str = Form(""), children: str = Form(""), interested_in: str = Form(""), source: str = Form(""),
source_place: str = Form(""), source_other: str = Form(""), message: str = Form("")): source_place: str = Form(""), source_other: str = Form(""), message: str = Form("")):
heard_from = source.strip()
heard_from_detail = ""
if source == "Other school or daycare" and source_place.strip(): if source == "Other school or daycare" and source_place.strip():
source = "School/Daycare: " + source_place.strip() heard_from_detail = source_place.strip()
source = "School/Daycare: " + heard_from_detail
elif source == "Other" and source_other.strip(): elif source == "Other" and source_other.strip():
source = "Other: " + source_other.strip() heard_from_detail = source_other.strip()
source = "Other: " + heard_from_detail
rec = {"ts": datetime.datetime.now().isoformat(timespec="seconds"), rec = {"ts": datetime.datetime.now().isoformat(timespec="seconds"),
"parent_name": parent_name.strip(), "email": email.strip(), "phone": phone.strip(), "parent_name": parent_name.strip(), "email": email.strip(), "phone": phone.strip(),
"children": children.strip() or "(not given)", "children": children.strip() or "(not given)",
@@ -642,11 +646,11 @@ def join_post(parent_name: str = Form(...), email: str = Form(...), phone: str =
# against the record so a failure is replayable instead of just shouted. # against the record so a failure is replayable instead of just shouted.
record_id = None record_id = None
try: try:
record_id = store.insert_record( record_id = store.insert_lead(
kind="join_lead", source=rec["source"], payload=rec, parent_name=rec["parent_name"], email=rec["email"], phone=rec["phone"],
unit=store.unit_of(rec["interested_in"]), interested_in=rec["interested_in"], children=rec["children"],
display_name=rec["parent_name"], email=rec["email"], heard_from=heard_from, heard_from_detail=heard_from_detail,
phone=rec["phone"], created_at=rec["ts"]) message=rec["message"], payload=rec, submitted_at=rec["ts"])
except Exception as e: except Exception as e:
print("STORE WRITE FAILED for %s <%s>: %s" % (rec["parent_name"], rec["email"], e), flush=True) print("STORE WRITE FAILED for %s <%s>: %s" % (rec["parent_name"], rec["email"], e), flush=True)
sheet_error = None sheet_error = None
app/store.py
+98 -166
View File
@@ -1,24 +1,28 @@
""" """
store.py - internal record store for greenlanescouts73.org store.py - internal record store for greenlanescouts73.org
Why this exists What this holds
--------------- ---------------
Until now a /join submission went straight to the Google Sheet, with a raw One table per thing the site collects. Today that is `join_leads`: the /join
append to /data/leads.jsonl as a fire-and-forget side effect wrapped in a bare interest form, one row per submission, one column per form field.
except. The sheet is the only queryable copy, it has no record IDs, no status,
no way to mark a lead handled, and a failed write survives only as an ntfy push.
This module makes a local SQLite database the source of truth. The sheet and A /join submission lands here FIRST. The Google Sheet and the ntfy push are
the ntfy push become *mirrors* of a record that already exists locally, and mirrors of a row that already exists locally, and each mirror's outcome is
each mirror's success or failure is recorded per record so it can be replayed. recorded per row so a failure is replayable instead of just shouted.
Designed for a future admin panel Why join_leads and not a generic table
--------------------------------- -------------------------------------
The table is deliberately NOT a leads table. It is a generic record store keyed This started as a generic `records` table keyed by `kind`, carrying
by `kind`, so RSVPs, volunteer signups, popcorn orders or anything else the status/assigned_to/notes for a future admin panel. That was wrong twice over.
admin panel eventually covers land in the same place with the same workflow The PII columns (display_name/email/phone) only make sense for a lead, and the
columns (status / assigned_to / notes) and the same audit trail. Adding a new workflow columns describe an outreach process that has not been designed yet -
form means picking a new `kind` string, not migrating a schema. one that is plainly one-to-many, since a family gets contacted more than once.
Guessing at it in three columns would have locked in the wrong shape.
So: a table per form, columns for the form's own fields, and outreach gets its
own table when the process is actually known. Anything the form adds later that
is not worth a column still survives in `payload`, which holds the submission
verbatim.
Stdlib only - sqlite3 ships with Python, so this adds no image dependencies. Stdlib only - sqlite3 ships with Python, so this adds no image dependencies.
""" """
@@ -32,33 +36,28 @@ from pathlib import Path
DB_PATH = Path(os.environ.get("STORE_DB", "/data/scout73.db")) DB_PATH = Path(os.environ.get("STORE_DB", "/data/scout73.db"))
# Workflow states an admin panel may set. 'new' is the only one this app writes. # Mirror targets - external destinations a row is copied out to.
STATUSES = ("new", "contacted", "joined", "declined", "duplicate", "spam")
# Mirror targets - external destinations a record is copied out to.
TARGETS = ("google_sheet", "ntfy") TARGETS = ("google_sheet", "ntfy")
SCHEMA = """ SCHEMA = """
PRAGMA journal_mode=WAL; PRAGMA journal_mode=WAL;
CREATE TABLE IF NOT EXISTS records ( CREATE TABLE IF NOT EXISTS join_leads (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
kind TEXT NOT NULL, submitted_at TEXT NOT NULL,
source TEXT NOT NULL, recorded_at TEXT NOT NULL,
unit TEXT, parent_name TEXT NOT NULL,
created_at TEXT NOT NULL, email TEXT NOT NULL,
updated_at TEXT NOT NULL, phone TEXT,
display_name TEXT, interested_in TEXT,
email TEXT, children TEXT,
phone TEXT, heard_from TEXT,
payload TEXT NOT NULL, heard_from_detail TEXT,
status TEXT NOT NULL DEFAULT 'new', message TEXT,
assigned_to TEXT, payload TEXT NOT NULL
notes TEXT
); );
CREATE INDEX IF NOT EXISTS idx_records_kind_created ON records(kind, created_at DESC); CREATE INDEX IF NOT EXISTS idx_join_leads_submitted ON join_leads(submitted_at DESC);
CREATE INDEX IF NOT EXISTS idx_records_status ON records(status); CREATE INDEX IF NOT EXISTS idx_join_leads_email ON join_leads(email);
CREATE INDEX IF NOT EXISTS idx_records_email ON records(email);
CREATE TABLE IF NOT EXISTS mirrors ( CREATE TABLE IF NOT EXISTS mirrors (
record_id TEXT NOT NULL, record_id TEXT NOT NULL,
@@ -71,16 +70,6 @@ CREATE TABLE IF NOT EXISTS mirrors (
); );
CREATE INDEX IF NOT EXISTS idx_mirrors_state ON mirrors(target, state); CREATE INDEX IF NOT EXISTS idx_mirrors_state ON mirrors(target, state);
CREATE TABLE IF NOT EXISTS audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
record_id TEXT,
at TEXT NOT NULL,
actor TEXT NOT NULL,
action TEXT NOT NULL,
detail TEXT
);
CREATE INDEX IF NOT EXISTS idx_audit_record ON audit(record_id, id);
CREATE TABLE IF NOT EXISTS meta ( CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY, key TEXT PRIMARY KEY,
value TEXT NOT NULL value TEXT NOT NULL
@@ -95,10 +84,10 @@ def _now():
def _norm(ts): def _norm(ts):
"""Normalise any incoming timestamp to UTC ISO-8601 with an offset. """Normalise any incoming timestamp to UTC ISO-8601 with an offset.
app.py builds its record timestamp with a naive datetime.now(), i.e. local app.py builds its submission timestamp with a naive datetime.now(), i.e.
Eastern time. Storing that next to UTC audit rows would silently break both local Eastern time. Storing that next to UTC rows would silently break both
sorting and `since=` filters in the admin panel, so everything that lands in sorting and `since=` filters, so everything that lands in the DB is
the DB is converted here. converted here.
""" """
if not ts: if not ts:
return _now() return _now()
@@ -131,27 +120,27 @@ def init(backfill_jsonl=None):
con.close() con.close()
# --------------------------------------------------------------------------- # ----------------------------------------------------------------------------
# Writes # Writes
# --------------------------------------------------------------------------- # ----------------------------------------------------------------------------
def insert_record(kind, source, payload, unit=None, display_name=None, def insert_lead(parent_name, email, phone=None, interested_in=None, children=None,
email=None, phone=None, created_at=None, record_id=None): heard_from=None, heard_from_detail=None, message=None,
"""Insert a record and return its id. Raises on failure - callers decide.""" payload=None, submitted_at=None, record_id=None):
"""Insert one /join submission and return its id. Raises on failure -
callers decide what a failed local write means."""
rid = record_id or str(uuid.uuid4()) rid = record_id or str(uuid.uuid4())
ts = _norm(created_at) submitted = _norm(submitted_at)
con = connect() con = connect()
try: try:
con.execute( con.execute(
"INSERT INTO records (id, kind, source, unit, created_at, updated_at," "INSERT INTO join_leads (id, submitted_at, recorded_at, parent_name, email,"
" display_name, email, phone, payload, status)" " phone, interested_in, children, heard_from, heard_from_detail, message, payload)"
" VALUES (?,?,?,?,?,?,?,?,?,?, 'new')", " VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
(rid, kind, source, unit, ts, ts, display_name, email, phone, (rid, submitted, _now(), parent_name, email, phone or None,
json.dumps(payload, ensure_ascii=False)), interested_in or None, children or None, heard_from or None,
) heard_from_detail or None, message or None,
con.execute( json.dumps(payload or {}, ensure_ascii=False)),
"INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)",
(rid, _now(), "system", "created", kind),
) )
con.commit() con.commit()
finally: finally:
@@ -160,7 +149,7 @@ def insert_record(kind, source, payload, unit=None, display_name=None,
def set_mirror(record_id, target, ok, error=None): def set_mirror(record_id, target, ok, error=None):
"""Record the outcome of an attempt to copy a record to an external target.""" """Record the outcome of an attempt to copy a row to an external target."""
ts = _now() ts = _now()
state = "ok" if ok else "failed" state = "ok" if ok else "failed"
con = connect() con = connect()
@@ -180,59 +169,13 @@ def set_mirror(record_id, target, ok, error=None):
con.close() con.close()
def update_record(record_id, actor="admin", status=None, assigned_to=None, notes=None): # ----------------------------------------------------------------------------
"""Workflow update from the admin panel. Returns the updated row, or None."""
sets, vals = [], []
if status is not None:
if status not in STATUSES:
raise ValueError("unknown status: %s" % status)
sets.append("status=?"); vals.append(status)
if assigned_to is not None:
sets.append("assigned_to=?"); vals.append(assigned_to or None)
if notes is not None:
sets.append("notes=?"); vals.append(notes or None)
if not sets:
return get_record(record_id)
ts = _now()
sets.append("updated_at=?"); vals.append(ts)
vals.append(record_id)
con = connect()
try:
cur = con.execute("UPDATE records SET %s WHERE id=?" % ", ".join(sets), vals)
if cur.rowcount == 0:
return None
detail = json.dumps({k: v for k, v in
(("status", status), ("assigned_to", assigned_to), ("notes", notes))
if v is not None}, ensure_ascii=False)
con.execute(
"INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)",
(record_id, ts, actor, "updated", detail),
)
con.commit()
finally:
con.close()
return get_record(record_id)
def log(record_id, actor, action, detail=None):
con = connect()
try:
con.execute(
"INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)",
(record_id, _now(), actor, action, detail),
)
con.commit()
finally:
con.close()
# ---------------------------------------------------------------------------
# Reads # Reads
# --------------------------------------------------------------------------- # ----------------------------------------------------------------------------
def _row(r): def _row(r):
d = dict(r) d = dict(r)
if "payload" in d and d["payload"]: if d.get("payload"):
try: try:
d["payload"] = json.loads(d["payload"]) d["payload"] = json.loads(d["payload"])
except Exception: except Exception:
@@ -240,40 +183,33 @@ def _row(r):
return d return d
def get_record(record_id, with_audit=False): def get_lead(record_id):
con = connect() con = connect()
try: try:
r = con.execute("SELECT * FROM records WHERE id=?", (record_id,)).fetchone() r = con.execute("SELECT * FROM join_leads WHERE id=?", (record_id,)).fetchone()
if not r: if not r:
return None return None
out = _row(r) out = _row(r)
out["mirrors"] = [dict(m) for m in con.execute( out["mirrors"] = [dict(m) for m in con.execute(
"SELECT target, state, attempts, last_error, last_attempt_at" "SELECT target, state, attempts, last_error, last_attempt_at"
" FROM mirrors WHERE record_id=?", (record_id,))] " FROM mirrors WHERE record_id=?", (record_id,))]
if with_audit:
out["audit"] = [dict(a) for a in con.execute(
"SELECT at, actor, action, detail FROM audit WHERE record_id=? ORDER BY id",
(record_id,))]
return out return out
finally: finally:
con.close() con.close()
def list_records(kind=None, status=None, since=None, q=None, limit=100, offset=0): def list_leads(since=None, q=None, limit=100, offset=0):
where, vals = [], [] where, vals = [], []
if kind:
where.append("kind=?"); vals.append(kind)
if status:
where.append("status=?"); vals.append(status)
if since: if since:
where.append("created_at>=?"); vals.append(since) where.append("submitted_at>=?"); vals.append(since)
if q: if q:
where.append("(display_name LIKE ? OR email LIKE ? OR payload LIKE ?)") where.append("(parent_name LIKE ? OR email LIKE ? OR children LIKE ?"
vals += ["%%%s%%" % q] * 3 " OR heard_from LIKE ? OR message LIKE ?)")
sql = "SELECT * FROM records" vals += ["%%%s%%" % q] * 5
sql = "SELECT * FROM join_leads"
if where: if where:
sql += " WHERE " + " AND ".join(where) sql += " WHERE " + " AND ".join(where)
sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?" sql += " ORDER BY submitted_at DESC LIMIT ? OFFSET ?"
vals += [max(1, min(int(limit), 500)), max(0, int(offset))] vals += [max(1, min(int(limit), 500)), max(0, int(offset))]
con = connect() con = connect()
try: try:
@@ -293,15 +229,16 @@ def list_records(kind=None, status=None, since=None, q=None, limit=100, offset=0
def summary(): def summary():
"""Counts for a dashboard: by kind, by status, and failed mirrors.""" """Counts for a dashboard: totals, interest split, referral sources, failed mirrors."""
con = connect() con = connect()
try: try:
return { return {
"total": con.execute("SELECT COUNT(*) c FROM records").fetchone()["c"], "total": con.execute("SELECT COUNT(*) c FROM join_leads").fetchone()["c"],
"by_kind": {r["kind"]: r["c"] for r in con.execute( "by_interest": {r["interested_in"]: r["c"] for r in con.execute(
"SELECT kind, COUNT(*) c FROM records GROUP BY kind")}, "SELECT interested_in, COUNT(*) c FROM join_leads GROUP BY interested_in")},
"by_status": {r["status"]: r["c"] for r in con.execute( "by_heard_from": {r["heard_from"]: r["c"] for r in con.execute(
"SELECT status, COUNT(*) c FROM records GROUP BY status")}, "SELECT heard_from, COUNT(*) c FROM join_leads GROUP BY heard_from"
" ORDER BY c DESC")},
"failed_mirrors": {r["target"]: r["c"] for r in con.execute( "failed_mirrors": {r["target"]: r["c"] for r in con.execute(
"SELECT target, COUNT(*) c FROM mirrors WHERE state='failed' GROUP BY target")}, "SELECT target, COUNT(*) c FROM mirrors WHERE state='failed' GROUP BY target")},
} }
@@ -313,16 +250,27 @@ def failed_mirror_records(target, limit=50):
con = connect() con = connect()
try: try:
return [_row(r) for r in con.execute( return [_row(r) for r in con.execute(
"SELECT r.* FROM records r JOIN mirrors m ON m.record_id=r.id" "SELECT l.* FROM join_leads l JOIN mirrors m ON m.record_id=l.id"
" WHERE m.target=? AND m.state='failed' ORDER BY r.created_at LIMIT ?", " WHERE m.target=? AND m.state='failed' ORDER BY l.submitted_at LIMIT ?",
(target, limit))] (target, limit))]
finally: finally:
con.close() con.close()
# --------------------------------------------------------------------------- # ----------------------------------------------------------------------------
# One-time backfill of the pre-existing raw log # One-time backfill of the pre-existing raw log
# --------------------------------------------------------------------------- # ----------------------------------------------------------------------------
def split_heard_from(source):
"""app.py used to fold source_place / source_other into one composed string
('Other: Chocolate booth'), destroying the raw answer. Recover both halves."""
s = (source or "").strip()
if s.startswith("School/Daycare: "):
return "Other school or daycare", s[len("School/Daycare: "):].strip()
if s.startswith("Other: "):
return "Other", s[len("Other: "):].strip()
return s or None, None
def _backfill(con, path): def _backfill(con, path):
"""Import /data/leads.jsonl once, so history is not stranded outside the DB.""" """Import /data/leads.jsonl once, so history is not stranded outside the DB."""
@@ -339,37 +287,21 @@ def _backfill(con, path):
rec = json.loads(line) rec = json.loads(line)
except Exception: except Exception:
continue continue
rid = str(uuid.uuid4()) heard, detail = split_heard_from(rec.get("source"))
ts = _norm(rec.get("ts")) ts = _norm(rec.get("ts"))
con.execute( con.execute(
"INSERT INTO records (id, kind, source, unit, created_at, updated_at," "INSERT INTO join_leads (id, submitted_at, recorded_at, parent_name, email,"
" display_name, email, phone, payload, status)" " phone, interested_in, children, heard_from, heard_from_detail, message, payload)"
" VALUES (?,?,?,?,?,?,?,?,?,?, 'new')", " VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
(rid, "join_lead", rec.get("source") or "greenlanescouts73.org", (str(uuid.uuid4()), ts, _now(),
unit_of(rec.get("interested_in", "")), ts, ts, rec.get("parent_name") or "(unknown)", rec.get("email") or "",
rec.get("parent_name"), rec.get("email"), rec.get("phone"), rec.get("phone") or None, rec.get("interested_in") or None,
rec.get("children") or None, heard, detail,
rec.get("message") or None,
json.dumps(rec, ensure_ascii=False)), json.dumps(rec, ensure_ascii=False)),
) )
con.execute(
"INSERT INTO audit (record_id, at, actor, action, detail) VALUES (?,?,?,?,?)",
(rid, _now(), "system", "backfilled", "leads.jsonl"),
)
n += 1 n += 1
con.execute("INSERT INTO meta (key, value) VALUES ('backfill_leads_jsonl', ?)", con.execute("INSERT INTO meta (key, value) VALUES ('backfill_leads_jsonl', ?)",
("%s rows at %s" % (n, _now()),)) ("%s rows at %s" % (n, _now()),))
con.commit() con.commit()
print("store: backfilled %s rows from %s" % (n, path), flush=True) print("store: backfilled %s rows from %s" % (n, path), flush=True)
def unit_of(interested_in):
"""Best-effort unit tag so the admin panel can filter pack vs troop."""
s = (interested_in or "").lower()
pack = "pack" in s or "cub" in s
troop = "troop" in s or "scouts bsa" in s
if pack and troop or "both" in s:
return "both"
if pack:
return "pack"
if troop:
return "troop"
return None