From 51b98c9399cdd69032c80ef88c5937dbaf13c2ed Mon Sep 17 00:00:00 2001 From: Mike Wichers Date: Sun, 22 Feb 2026 10:23:40 -0500 Subject: [PATCH] refactor: Migrate file-based data storage to database --- cli/wizard.py | 12 +++---- core/config.py | 3 +- core/utils.py | 5 +-- story/state.py | 37 +++++++++++++++++--- story/style_persona.py | 8 +++-- web/db.py | 13 +++++++ web/routes/admin.py | 7 ++-- web/routes/persona.py | 77 ++++++++++++++++++++++-------------------- web/routes/project.py | 26 +++----------- 9 files changed, 108 insertions(+), 80 deletions(-) diff --git a/cli/wizard.py b/cli/wizard.py index fa3b502..fb6b4a9 100644 --- a/cli/wizard.py +++ b/cli/wizard.py @@ -80,9 +80,9 @@ class BookWizard: while True: self.clear() personas = {} - if os.path.exists(config.PERSONAS_FILE): + if os.path.exists(os.path.join(config.PERSONAS_DIR, "personas.json")): try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) + with open(os.path.join(config.PERSONAS_DIR, "personas.json"), 'r') as f: personas = json.load(f) except: pass console.print(Panel("[bold cyan]Manage Author Personas[/bold cyan]")) @@ -120,7 +120,7 @@ class BookWizard: if sub == 2: if Confirm.ask(f"Delete '{selected_key}'?", default=False): del personas[selected_key] - with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2) + with open(os.path.join(config.PERSONAS_DIR, "personas.json"), 'w') as f: json.dump(personas, f, indent=2) continue elif sub == 3: continue @@ -145,7 +145,7 @@ class BookWizard: if Confirm.ask("Save Persona?", default=True): personas[selected_key] = details - with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2) + with open(os.path.join(config.PERSONAS_DIR, "personas.json"), 'w') as f: json.dump(personas, f, indent=2) def select_mode(self): while True: @@ -322,9 +322,9 @@ class BookWizard: console.print("\n[bold blue]Project Details[/bold blue]") personas = {} - if os.path.exists(config.PERSONAS_FILE): + if os.path.exists(os.path.join(config.PERSONAS_DIR, "personas.json")): try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) + with open(os.path.join(config.PERSONAS_DIR, "personas.json"), 'r') as f: personas = json.load(f) except: pass author_details = {} diff --git a/core/config.py b/core/config.py index 40fc00e..af798f6 100644 --- a/core/config.py +++ b/core/config.py @@ -35,7 +35,8 @@ if not API_KEY: raise ValueError("CRITICAL ERROR: GEMINI_API_KEY not found in en DATA_DIR = os.path.join(BASE_DIR, "data") PROJECTS_DIR = os.path.join(DATA_DIR, "projects") PERSONAS_DIR = os.path.join(DATA_DIR, "personas") -PERSONAS_FILE = os.path.join(PERSONAS_DIR, "personas.json") +# PERSONAS_FILE is deprecated — persona data is now stored in the Persona DB table. +# PERSONAS_FILE = os.path.join(PERSONAS_DIR, "personas.json") FONTS_DIR = os.path.join(DATA_DIR, "fonts") # --- ENSURE DIRECTORIES EXIST --- diff --git a/core/utils.py b/core/utils.py index 38f9674..78ab0bb 100644 --- a/core/utils.py +++ b/core/utils.py @@ -129,11 +129,8 @@ def load_json(path): return json.load(open(path, 'r')) if os.path.exists(path) else None def create_default_personas(): + # Persona data is now stored in the Persona DB table; ensure the directory exists for sample files. if not os.path.exists(config.PERSONAS_DIR): os.makedirs(config.PERSONAS_DIR) - if not os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'w') as f: json.dump({}, f, indent=2) - except: pass def get_length_presets(): presets = {} diff --git a/story/state.py b/story/state.py index 0aca396..43275f7 100644 --- a/story/state.py +++ b/story/state.py @@ -8,17 +8,27 @@ def _empty_state(): return {"active_threads": [], "immediate_handoff": "", "resolved_threads": [], "chapter": 0} -def load_story_state(folder): - """Load structured story state from story_state.json, or return empty state.""" +def load_story_state(folder, project_id=None): + """Load structured story state from DB (if project_id given) or story_state.json fallback.""" + if project_id is not None: + try: + from web.db import StoryState + record = StoryState.query.filter_by(project_id=project_id).first() + if record and record.state_json: + return json.loads(record.state_json) or _empty_state() + except Exception: + pass # Fall through to file-based load if DB unavailable (e.g. CLI context) + path = os.path.join(folder, "story_state.json") if os.path.exists(path): return utils.load_json(path) or _empty_state() return _empty_state() -def update_story_state(chapter_text, chapter_num, current_state, folder): +def update_story_state(chapter_text, chapter_num, current_state, folder, project_id=None): """Use model_logic to extract structured story threads from the new chapter - and save the updated state to story_state.json. Returns the new state.""" + and save the updated state to the StoryState DB table and/or story_state.json. + Returns the new state.""" utils.log("STATE", f"Updating story state after Ch {chapter_num}...") prompt = f""" ROLE: Story State Tracker @@ -54,9 +64,28 @@ def update_story_state(chapter_text, chapter_num, current_state, folder): utils.log_usage(folder, ai_models.model_logic.name, response.usage_metadata) new_state = json.loads(utils.clean_json(response.text)) new_state['chapter'] = chapter_num + + # Write to DB if project_id is available + if project_id is not None: + try: + from web.db import db, StoryState + from datetime import datetime + record = StoryState.query.filter_by(project_id=project_id).first() + if record: + record.state_json = json.dumps(new_state) + record.updated_at = datetime.utcnow() + else: + record = StoryState(project_id=project_id, state_json=json.dumps(new_state)) + db.session.add(record) + db.session.commit() + except Exception as db_err: + utils.log("STATE", f" -> DB write failed: {db_err}. Falling back to file.") + + # Always write to file for backward compat with CLI path = os.path.join(folder, "story_state.json") with open(path, 'w') as f: json.dump(new_state, f, indent=2) + utils.log("STATE", f" -> Story state saved. Active threads: {len(new_state.get('active_threads', []))}") return new_state except Exception as e: diff --git a/story/style_persona.py b/story/style_persona.py index ee797cc..8178c2a 100644 --- a/story/style_persona.py +++ b/story/style_persona.py @@ -157,10 +157,12 @@ def update_persona_sample(bp, folder): author_name = meta.get('author', 'Unknown Author') + # Use a local file mirror for the engine context (runs outside Flask app context) + _personas_file = os.path.join(config.PERSONAS_DIR, "personas.json") personas = {} - if os.path.exists(config.PERSONAS_FILE): + if os.path.exists(_personas_file): try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) + with open(_personas_file, 'r') as f: personas = json.load(f) except: pass if author_name not in personas: @@ -189,4 +191,4 @@ def update_persona_sample(bp, folder): if filename not in personas[author_name]['sample_files']: personas[author_name]['sample_files'].append(filename) - with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2) + with open(_personas_file, 'w') as f: json.dump(personas, f, indent=2) diff --git a/web/db.py b/web/db.py index 49da570..f72e4fc 100644 --- a/web/db.py +++ b/web/db.py @@ -51,3 +51,16 @@ class LogEntry(db.Model): timestamp = db.Column(db.DateTime, default=datetime.utcnow) phase = db.Column(db.String(50)) message = db.Column(db.Text) + + +class StoryState(db.Model): + id = db.Column(db.Integer, primary_key=True) + project_id = db.Column(db.Integer, db.ForeignKey('project.id'), nullable=False) + state_json = db.Column(db.Text, nullable=True) + updated_at = db.Column(db.DateTime, default=datetime.utcnow) + + +class Persona(db.Model): + id = db.Column(db.Integer, primary_key=True) + name = db.Column(db.String(150), unique=True, nullable=False) + details_json = db.Column(db.Text, nullable=True) diff --git a/web/routes/admin.py b/web/routes/admin.py index 7d3cd12..d599c31 100644 --- a/web/routes/admin.py +++ b/web/routes/admin.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta from flask import Blueprint, render_template, request, redirect, url_for, flash, session, jsonify from flask_login import login_required, login_user, current_user from sqlalchemy import func -from web.db import db, User, Project, Run +from web.db import db, User, Project, Run, Persona from web.helpers import admin_required from core import config, utils from ai import models as ai_models @@ -83,10 +83,7 @@ def admin_factory_reset(): except: pass db.session.delete(u) - if os.path.exists(config.PERSONAS_FILE): - try: os.remove(config.PERSONAS_FILE) - except: pass - utils.create_default_personas() + Persona.query.delete() db.session.commit() flash("Factory Reset Complete. All other users and projects have been wiped.") diff --git a/web/routes/persona.py b/web/routes/persona.py index 20c5353..f90460f 100644 --- a/web/routes/persona.py +++ b/web/routes/persona.py @@ -1,22 +1,31 @@ -import os import json from flask import Blueprint, render_template, request, redirect, url_for, flash from flask_login import login_required -from core import config, utils +from core import utils from ai import models as ai_models from ai import setup as ai_setup +from web.db import db, Persona persona_bp = Blueprint('persona', __name__) +def _all_personas_dict(): + """Return all personas as a dict keyed by name, matching the old personas.json structure.""" + records = Persona.query.all() + result = {} + for rec in records: + try: + details = json.loads(rec.details_json) if rec.details_json else {} + except Exception: + details = {} + result[rec.name] = details + return result + + @persona_bp.route('/personas') @login_required def list_personas(): - personas = {} - if os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) - except: pass + personas = _all_personas_dict() return render_template('personas.html', personas=personas) @@ -29,17 +38,16 @@ def new_persona(): @persona_bp.route('/persona/') @login_required def edit_persona(name): - personas = {} - if os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) - except: pass - - persona = personas.get(name) - if not persona: + record = Persona.query.filter_by(name=name).first() + if not record: flash(f"Persona '{name}' not found.") return redirect(url_for('persona.list_personas')) + try: + persona = json.loads(record.details_json) if record.details_json else {} + except Exception: + persona = {} + return render_template('persona_edit.html', persona=persona, name=name) @@ -53,16 +61,7 @@ def save_persona(): flash("Persona name is required.") return redirect(url_for('persona.list_personas')) - personas = {} - if os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) - except: pass - - if old_name and old_name != name and old_name in personas: - del personas[old_name] - - persona = { + persona_data = { "name": name, "bio": request.form.get('bio'), "age": request.form.get('age'), @@ -75,10 +74,21 @@ def save_persona(): "style_inspirations": request.form.get('style_inspirations') } - personas[name] = persona + # If name changed, remove old record + if old_name and old_name != name: + old_record = Persona.query.filter_by(name=old_name).first() + if old_record: + db.session.delete(old_record) + db.session.flush() - with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2) + record = Persona.query.filter_by(name=name).first() + if record: + record.details_json = json.dumps(persona_data) + else: + record = Persona(name=name, details_json=json.dumps(persona_data)) + db.session.add(record) + db.session.commit() flash(f"Persona '{name}' saved.") return redirect(url_for('persona.list_personas')) @@ -86,15 +96,10 @@ def save_persona(): @persona_bp.route('/persona/delete/', methods=['POST']) @login_required def delete_persona(name): - personas = {} - if os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) - except: pass - - if name in personas: - del personas[name] - with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2) + record = Persona.query.filter_by(name=name).first() + if record: + db.session.delete(record) + db.session.commit() flash(f"Persona '{name}' deleted.") return redirect(url_for('persona.list_personas')) diff --git a/web/routes/project.py b/web/routes/project.py index e4c6738..0e7ecfd 100644 --- a/web/routes/project.py +++ b/web/routes/project.py @@ -4,7 +4,7 @@ import shutil from datetime import datetime from flask import Blueprint, render_template, request, redirect, url_for, flash from flask_login import login_required, current_user -from web.db import db, Project, Run +from web.db import db, Project, Run, Persona from web.helpers import is_project_locked from core import config, utils from ai import models as ai_models @@ -104,11 +104,7 @@ def project_setup_wizard(): flash(f"AI Analysis failed — fill in the details manually. ({e})", "warning") suggestions = _default_suggestions - personas = {} - if os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) - except: pass + personas = {rec.name: (json.loads(rec.details_json) if rec.details_json else {}) for rec in Persona.query.all()} return render_template('project_setup.html', s=suggestions, concept=concept, personas=personas, lengths=config.LENGTH_DEFINITIONS) @@ -149,11 +145,7 @@ def project_setup_refine(): flash(f"Refinement failed: {e}") return redirect(url_for('project.index')) - personas = {} - if os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) - except: pass + personas = {rec.name: (json.loads(rec.details_json) if rec.details_json else {}) for rec in Persona.query.all()} return render_template('project_setup.html', s=suggestions, concept=concept, personas=personas, lengths=config.LENGTH_DEFINITIONS) @@ -329,11 +321,7 @@ def view_project(id): has_draft = os.path.exists(draft_path) is_refining = os.path.exists(os.path.join(proj.folder_path, ".refining")) - personas = {} - if os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) - except: pass + personas = {rec.name: (json.loads(rec.details_json) if rec.details_json else {}) for rec in Persona.query.all()} runs = Run.query.filter_by(project_id=id).order_by(Run.id.desc()).all() latest_run = runs[0] if runs else None @@ -730,11 +718,7 @@ def set_project_persona(id): bible = utils.load_json(bible_path) if bible: - personas = {} - if os.path.exists(config.PERSONAS_FILE): - try: - with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f) - except: pass + personas = {rec.name: (json.loads(rec.details_json) if rec.details_json else {}) for rec in Persona.query.all()} if persona_name in personas: bible['project_metadata']['author_details'] = personas[persona_name]