refactor: Migrate file-based data storage to database

cli/wizard.py

@@ -80,9 +80,9 @@ class BookWizard:
         while True:
             self.clear()
             personas = {}
-            if os.path.exists(config.PERSONAS_FILE):
+            if os.path.exists(os.path.join(config.PERSONAS_DIR, "personas.json")):
                 try:
-                    with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
+                    with open(os.path.join(config.PERSONAS_DIR, "personas.json"), 'r') as f: personas = json.load(f)
                 except: pass
 
             console.print(Panel("[bold cyan]Manage Author Personas[/bold cyan]"))
@@ -120,7 +120,7 @@ class BookWizard:
                 if sub == 2:
                     if Confirm.ask(f"Delete '{selected_key}'?", default=False):
                         del personas[selected_key]
-                        with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2)
+                        with open(os.path.join(config.PERSONAS_DIR, "personas.json"), 'w') as f: json.dump(personas, f, indent=2)
                     continue
                 elif sub == 3:
                     continue
@@ -145,7 +145,7 @@ class BookWizard:
 
             if Confirm.ask("Save Persona?", default=True):
                 personas[selected_key] = details
-                with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2)
+                with open(os.path.join(config.PERSONAS_DIR, "personas.json"), 'w') as f: json.dump(personas, f, indent=2)
 
     def select_mode(self):
         while True:
@@ -322,9 +322,9 @@ class BookWizard:
         console.print("\n[bold blue]Project Details[/bold blue]")
 
         personas = {}
-        if os.path.exists(config.PERSONAS_FILE):
+        if os.path.exists(os.path.join(config.PERSONAS_DIR, "personas.json")):
             try:
-                with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
+                with open(os.path.join(config.PERSONAS_DIR, "personas.json"), 'r') as f: personas = json.load(f)
             except: pass
 
         author_details = {}

core/config.py

@@ -35,7 +35,8 @@ if not API_KEY: raise ValueError("CRITICAL ERROR: GEMINI_API_KEY not found in en
 DATA_DIR = os.path.join(BASE_DIR, "data")
 PROJECTS_DIR = os.path.join(DATA_DIR, "projects")
 PERSONAS_DIR = os.path.join(DATA_DIR, "personas")
-PERSONAS_FILE = os.path.join(PERSONAS_DIR, "personas.json")
+# PERSONAS_FILE is deprecated — persona data is now stored in the Persona DB table.
+# PERSONAS_FILE = os.path.join(PERSONAS_DIR, "personas.json")
 FONTS_DIR = os.path.join(DATA_DIR, "fonts")
 
 # --- ENSURE DIRECTORIES EXIST ---

core/utils.py

@@ -129,11 +129,8 @@ def load_json(path):
     return json.load(open(path, 'r')) if os.path.exists(path) else None
 
 def create_default_personas():
+    # Persona data is now stored in the Persona DB table; ensure the directory exists for sample files.
     if not os.path.exists(config.PERSONAS_DIR): os.makedirs(config.PERSONAS_DIR)
-    if not os.path.exists(config.PERSONAS_FILE):
-        try:
-            with open(config.PERSONAS_FILE, 'w') as f: json.dump({}, f, indent=2)
-        except: pass
 
 def get_length_presets():
     presets = {}

story/state.py

@@ -8,17 +8,27 @@ def _empty_state():
     return {"active_threads": [], "immediate_handoff": "", "resolved_threads": [], "chapter": 0}
 
 
-def load_story_state(folder):
+def load_story_state(folder, project_id=None):
-    """Load structured story state from story_state.json, or return empty state."""
+    """Load structured story state from DB (if project_id given) or story_state.json fallback."""
+    if project_id is not None:
+        try:
+            from web.db import StoryState
+            record = StoryState.query.filter_by(project_id=project_id).first()
+            if record and record.state_json:
+                return json.loads(record.state_json) or _empty_state()
+        except Exception:
+            pass  # Fall through to file-based load if DB unavailable (e.g. CLI context)
+
     path = os.path.join(folder, "story_state.json")
     if os.path.exists(path):
         return utils.load_json(path) or _empty_state()
     return _empty_state()
 
 
-def update_story_state(chapter_text, chapter_num, current_state, folder):
+def update_story_state(chapter_text, chapter_num, current_state, folder, project_id=None):
     """Use model_logic to extract structured story threads from the new chapter
-    and save the updated state to story_state.json. Returns the new state."""
+    and save the updated state to the StoryState DB table and/or story_state.json.
+    Returns the new state."""
     utils.log("STATE", f"Updating story state after Ch {chapter_num}...")
     prompt = f"""
     ROLE: Story State Tracker
@@ -54,9 +64,28 @@ def update_story_state(chapter_text, chapter_num, current_state, folder):
         utils.log_usage(folder, ai_models.model_logic.name, response.usage_metadata)
         new_state = json.loads(utils.clean_json(response.text))
         new_state['chapter'] = chapter_num
+
+        # Write to DB if project_id is available
+        if project_id is not None:
+            try:
+                from web.db import db, StoryState
+                from datetime import datetime
+                record = StoryState.query.filter_by(project_id=project_id).first()
+                if record:
+                    record.state_json = json.dumps(new_state)
+                    record.updated_at = datetime.utcnow()
+                else:
+                    record = StoryState(project_id=project_id, state_json=json.dumps(new_state))
+                    db.session.add(record)
+                db.session.commit()
+            except Exception as db_err:
+                utils.log("STATE", f"  -> DB write failed: {db_err}. Falling back to file.")
+
+        # Always write to file for backward compat with CLI
         path = os.path.join(folder, "story_state.json")
         with open(path, 'w') as f:
             json.dump(new_state, f, indent=2)
+
         utils.log("STATE", f"  -> Story state saved. Active threads: {len(new_state.get('active_threads', []))}")
         return new_state
     except Exception as e:

story/style_persona.py

@@ -157,10 +157,12 @@ def update_persona_sample(bp, folder):
 
     author_name = meta.get('author', 'Unknown Author')
 
+    # Use a local file mirror for the engine context (runs outside Flask app context)
+    _personas_file = os.path.join(config.PERSONAS_DIR, "personas.json")
     personas = {}
-    if os.path.exists(config.PERSONAS_FILE):
+    if os.path.exists(_personas_file):
         try:
-            with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
+            with open(_personas_file, 'r') as f: personas = json.load(f)
         except: pass
 
     if author_name not in personas:
@@ -189,4 +191,4 @@ def update_persona_sample(bp, folder):
         if filename not in personas[author_name]['sample_files']:
             personas[author_name]['sample_files'].append(filename)
 
-    with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2)
+    with open(_personas_file, 'w') as f: json.dump(personas, f, indent=2)

web/db.py

@@ -51,3 +51,16 @@ class LogEntry(db.Model):
     timestamp = db.Column(db.DateTime, default=datetime.utcnow)
     phase = db.Column(db.String(50))
     message = db.Column(db.Text)
+
+
+class StoryState(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    project_id = db.Column(db.Integer, db.ForeignKey('project.id'), nullable=False)
+    state_json = db.Column(db.Text, nullable=True)
+    updated_at = db.Column(db.DateTime, default=datetime.utcnow)
+
+
+class Persona(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    name = db.Column(db.String(150), unique=True, nullable=False)
+    details_json = db.Column(db.Text, nullable=True)

web/routes/admin.py

@@ -5,7 +5,7 @@ from datetime import datetime, timedelta
 from flask import Blueprint, render_template, request, redirect, url_for, flash, session, jsonify
 from flask_login import login_required, login_user, current_user
 from sqlalchemy import func
-from web.db import db, User, Project, Run
+from web.db import db, User, Project, Run, Persona
 from web.helpers import admin_required
 from core import config, utils
 from ai import models as ai_models
@@ -83,10 +83,7 @@ def admin_factory_reset():
             except: pass
         db.session.delete(u)
 
-    if os.path.exists(config.PERSONAS_FILE):
+    Persona.query.delete()
-        try: os.remove(config.PERSONAS_FILE)
-        except: pass
-    utils.create_default_personas()
 
     db.session.commit()
     flash("Factory Reset Complete. All other users and projects have been wiped.")

web/routes/persona.py

@@ -1,22 +1,31 @@
-import os
 import json
 from flask import Blueprint, render_template, request, redirect, url_for, flash
 from flask_login import login_required
-from core import config, utils
+from core import utils
 from ai import models as ai_models
 from ai import setup as ai_setup
+from web.db import db, Persona
 
 persona_bp = Blueprint('persona', __name__)
 
 
+def _all_personas_dict():
+    """Return all personas as a dict keyed by name, matching the old personas.json structure."""
+    records = Persona.query.all()
+    result = {}
+    for rec in records:
+        try:
+            details = json.loads(rec.details_json) if rec.details_json else {}
+        except Exception:
+            details = {}
+        result[rec.name] = details
+    return result
+
+
 @persona_bp.route('/personas')
 @login_required
 def list_personas():
-    personas = {}
+    personas = _all_personas_dict()
-    if os.path.exists(config.PERSONAS_FILE):
-        try:
-            with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
-        except: pass
     return render_template('personas.html', personas=personas)
 
 
@@ -29,17 +38,16 @@ def new_persona():
 @persona_bp.route('/persona/<string:name>')
 @login_required
 def edit_persona(name):
-    personas = {}
+    record = Persona.query.filter_by(name=name).first()
-    if os.path.exists(config.PERSONAS_FILE):
+    if not record:
-        try:
-            with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
-        except: pass
-
-    persona = personas.get(name)
-    if not persona:
         flash(f"Persona '{name}' not found.")
         return redirect(url_for('persona.list_personas'))
 
+    try:
+        persona = json.loads(record.details_json) if record.details_json else {}
+    except Exception:
+        persona = {}
+
     return render_template('persona_edit.html', persona=persona, name=name)
 
 
@@ -53,16 +61,7 @@ def save_persona():
         flash("Persona name is required.")
         return redirect(url_for('persona.list_personas'))
 
-    personas = {}
+    persona_data = {
-    if os.path.exists(config.PERSONAS_FILE):
-        try:
-            with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
-        except: pass
-
-    if old_name and old_name != name and old_name in personas:
-        del personas[old_name]
-
-    persona = {
         "name": name,
         "bio": request.form.get('bio'),
         "age": request.form.get('age'),
@@ -75,10 +74,21 @@ def save_persona():
         "style_inspirations": request.form.get('style_inspirations')
     }
 
-    personas[name] = persona
+    # If name changed, remove old record
+    if old_name and old_name != name:
+        old_record = Persona.query.filter_by(name=old_name).first()
+        if old_record:
+            db.session.delete(old_record)
+            db.session.flush()
 
-    with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2)
+    record = Persona.query.filter_by(name=name).first()
+    if record:
+        record.details_json = json.dumps(persona_data)
+    else:
+        record = Persona(name=name, details_json=json.dumps(persona_data))
+        db.session.add(record)
 
+    db.session.commit()
     flash(f"Persona '{name}' saved.")
     return redirect(url_for('persona.list_personas'))
 
@@ -86,15 +96,10 @@ def save_persona():
 @persona_bp.route('/persona/delete/<string:name>', methods=['POST'])
 @login_required
 def delete_persona(name):
-    personas = {}
+    record = Persona.query.filter_by(name=name).first()
-    if os.path.exists(config.PERSONAS_FILE):
+    if record:
-        try:
+        db.session.delete(record)
-            with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
+        db.session.commit()
-        except: pass
-
-    if name in personas:
-        del personas[name]
-        with open(config.PERSONAS_FILE, 'w') as f: json.dump(personas, f, indent=2)
         flash(f"Persona '{name}' deleted.")
 
     return redirect(url_for('persona.list_personas'))

web/routes/project.py

@@ -4,7 +4,7 @@ import shutil
 from datetime import datetime
 from flask import Blueprint, render_template, request, redirect, url_for, flash
 from flask_login import login_required, current_user
-from web.db import db, Project, Run
+from web.db import db, Project, Run, Persona
 from web.helpers import is_project_locked
 from core import config, utils
 from ai import models as ai_models
@@ -104,11 +104,7 @@ def project_setup_wizard():
             flash(f"AI Analysis failed — fill in the details manually. ({e})", "warning")
             suggestions = _default_suggestions
 
-    personas = {}
+    personas = {rec.name: (json.loads(rec.details_json) if rec.details_json else {}) for rec in Persona.query.all()}
-    if os.path.exists(config.PERSONAS_FILE):
-        try:
-            with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
-        except: pass
 
     return render_template('project_setup.html', s=suggestions, concept=concept, personas=personas, lengths=config.LENGTH_DEFINITIONS)
 
@@ -149,11 +145,7 @@ def project_setup_refine():
         flash(f"Refinement failed: {e}")
         return redirect(url_for('project.index'))
 
-    personas = {}
+    personas = {rec.name: (json.loads(rec.details_json) if rec.details_json else {}) for rec in Persona.query.all()}
-    if os.path.exists(config.PERSONAS_FILE):
-        try:
-            with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
-        except: pass
 
     return render_template('project_setup.html', s=suggestions, concept=concept, personas=personas, lengths=config.LENGTH_DEFINITIONS)
 
@@ -329,11 +321,7 @@ def view_project(id):
     has_draft = os.path.exists(draft_path)
     is_refining = os.path.exists(os.path.join(proj.folder_path, ".refining"))
 
-    personas = {}
+    personas = {rec.name: (json.loads(rec.details_json) if rec.details_json else {}) for rec in Persona.query.all()}
-    if os.path.exists(config.PERSONAS_FILE):
-        try:
-            with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
-        except: pass
 
     runs = Run.query.filter_by(project_id=id).order_by(Run.id.desc()).all()
     latest_run = runs[0] if runs else None
@@ -730,11 +718,7 @@ def set_project_persona(id):
     bible = utils.load_json(bible_path)
 
     if bible:
-        personas = {}
+        personas = {rec.name: (json.loads(rec.details_json) if rec.details_json else {}) for rec in Persona.query.all()}
-        if os.path.exists(config.PERSONAS_FILE):
-            try:
-                with open(config.PERSONAS_FILE, 'r') as f: personas = json.load(f)
-            except: pass
 
         if persona_name in personas:
             bible['project_metadata']['author_details'] = personas[persona_name]