Mike Wichers
81340a18ea
- web/db.py: Add last_heartbeat column to Run model - core/utils.py: Add set_heartbeat_callback() and send_heartbeat() - web/tasks.py: Add _robust_update_run_status() with 5-retry exponential backoff; add db_heartbeat_callback(); remove all bare except:pass on DB status updates; set start_time + last_heartbeat when marking run as 'running' - web/app.py: Add last_heartbeat column migration; add _stale_job_watcher() background thread (checks every 5 min, 15-min heartbeat threshold, 2-hr start_time threshold) - cli/engine.py: Add phase-level logging banners and try/except wrappers in process_book(); add utils.send_heartbeat() after each chapter save; add start/finish logging in run_generation() Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
457 lines
22 KiB
Python
457 lines
22 KiB
Python
import json
|
|
import os
|
|
import time
|
|
import sys
|
|
import shutil
|
|
from rich.prompt import Confirm
|
|
from core import config, utils
|
|
from ai import models as ai_models
|
|
from ai import setup as ai_setup
|
|
from story import planner, writer as story_writer, editor as story_editor
|
|
from story import style_persona, bible_tracker, state as story_state
|
|
from marketing import assets as marketing_assets
|
|
from export import exporter
|
|
|
|
|
|
def process_book(bp, folder, context="", resume=False, interactive=False):
|
|
# Create lock file to indicate active processing
|
|
lock_path = os.path.join(folder, ".in_progress")
|
|
with open(lock_path, "w") as f: f.write("running")
|
|
|
|
total_start = time.time()
|
|
|
|
try:
|
|
# 1. Check completion
|
|
if resume and os.path.exists(os.path.join(folder, "final_blueprint.json")):
|
|
utils.log("SYSTEM", f"Book in {folder} already finished. Skipping.")
|
|
if os.path.exists(lock_path): os.remove(lock_path)
|
|
return
|
|
|
|
# 2. Load or Create Blueprint
|
|
bp_path = os.path.join(folder, "blueprint_initial.json")
|
|
t_step = time.time()
|
|
utils.update_progress(5)
|
|
utils.log("SYSTEM", "--- Phase: Blueprint ---")
|
|
try:
|
|
if resume and os.path.exists(bp_path):
|
|
utils.log("RESUME", "Loading existing blueprint...")
|
|
saved_bp = utils.load_json(bp_path)
|
|
if saved_bp:
|
|
if 'book_metadata' in bp and 'book_metadata' in saved_bp:
|
|
for k in ['title', 'author', 'genre', 'target_audience', 'style', 'author_bio', 'author_details']:
|
|
if k in bp['book_metadata']:
|
|
saved_bp['book_metadata'][k] = bp['book_metadata'][k]
|
|
if 'series_metadata' in bp:
|
|
saved_bp['series_metadata'] = bp['series_metadata']
|
|
bp = saved_bp
|
|
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
|
|
else:
|
|
bp = planner.enrich(bp, folder, context)
|
|
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
|
|
|
|
# Ensure Persona Exists (Auto-create if missing)
|
|
if 'author_details' not in bp['book_metadata'] or not bp['book_metadata']['author_details']:
|
|
bp['book_metadata']['author_details'] = style_persona.create_initial_persona(bp, folder)
|
|
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
|
|
except Exception as _e:
|
|
utils.log("ERROR", f"Blueprint phase failed: {type(_e).__name__}: {_e}")
|
|
raise
|
|
|
|
utils.log("TIMING", f"Blueprint Phase: {time.time() - t_step:.1f}s")
|
|
|
|
# 3. Events (Plan & Expand)
|
|
events_path = os.path.join(folder, "events.json")
|
|
t_step = time.time()
|
|
utils.update_progress(10)
|
|
utils.log("SYSTEM", "--- Phase: Story Structure & Events ---")
|
|
try:
|
|
if resume and os.path.exists(events_path):
|
|
utils.log("RESUME", "Loading existing events...")
|
|
events = utils.load_json(events_path)
|
|
else:
|
|
events = planner.plan_structure(bp, folder)
|
|
depth = bp['length_settings']['depth']
|
|
target_chaps = bp['length_settings']['chapters']
|
|
for d in range(1, depth+1):
|
|
utils.log("SYSTEM", f" Expanding story structure depth {d}/{depth}...")
|
|
events = planner.expand(events, d, target_chaps, bp, folder)
|
|
time.sleep(1)
|
|
with open(events_path, "w") as f: json.dump(events, f, indent=2)
|
|
except Exception as _e:
|
|
utils.log("ERROR", f"Events/Structure phase failed: {type(_e).__name__}: {_e}")
|
|
raise
|
|
utils.log("TIMING", f"Structure & Expansion: {time.time() - t_step:.1f}s")
|
|
|
|
# 4. Chapter Plan
|
|
chapters_path = os.path.join(folder, "chapters.json")
|
|
t_step = time.time()
|
|
utils.update_progress(15)
|
|
utils.log("SYSTEM", "--- Phase: Chapter Planning ---")
|
|
try:
|
|
if resume and os.path.exists(chapters_path):
|
|
utils.log("RESUME", "Loading existing chapter plan...")
|
|
chapters = utils.load_json(chapters_path)
|
|
else:
|
|
chapters = planner.create_chapter_plan(events, bp, folder)
|
|
with open(chapters_path, "w") as f: json.dump(chapters, f, indent=2)
|
|
except Exception as _e:
|
|
utils.log("ERROR", f"Chapter planning phase failed: {type(_e).__name__}: {_e}")
|
|
raise
|
|
utils.log("TIMING", f"Chapter Planning: {time.time() - t_step:.1f}s")
|
|
|
|
# 5. Writing Loop
|
|
ms_path = os.path.join(folder, "manuscript.json")
|
|
loaded_ms = utils.load_json(ms_path) if (resume and os.path.exists(ms_path)) else []
|
|
ms = loaded_ms if loaded_ms is not None else []
|
|
|
|
# Load Tracking
|
|
events_track_path = os.path.join(folder, "tracking_events.json")
|
|
chars_track_path = os.path.join(folder, "tracking_characters.json")
|
|
warn_track_path = os.path.join(folder, "tracking_warnings.json")
|
|
lore_track_path = os.path.join(folder, "tracking_lore.json")
|
|
|
|
tracking = {"events": [], "characters": {}, "content_warnings": [], "lore": {}}
|
|
if resume:
|
|
if os.path.exists(events_track_path):
|
|
tracking['events'] = utils.load_json(events_track_path)
|
|
if os.path.exists(chars_track_path):
|
|
tracking['characters'] = utils.load_json(chars_track_path)
|
|
if os.path.exists(warn_track_path):
|
|
tracking['content_warnings'] = utils.load_json(warn_track_path)
|
|
if os.path.exists(lore_track_path):
|
|
tracking['lore'] = utils.load_json(lore_track_path) or {}
|
|
|
|
# Load structured story state
|
|
current_story_state = story_state.load_story_state(folder)
|
|
|
|
summary = "The story begins."
|
|
if ms:
|
|
utils.log("RESUME", f"Rebuilding story context from {len(ms)} existing chapters...")
|
|
try:
|
|
selected = ms[:1] + ms[-4:] if len(ms) > 5 else ms
|
|
combined_text = "\n".join([f"Chapter {c['num']}: {c['content'][:3000]}" for c in selected])
|
|
resp_sum = ai_models.model_writer.generate_content(f"""
|
|
ROLE: Series Historian
|
|
TASK: Create a cumulative 'Story So Far' summary.
|
|
INPUT_TEXT:
|
|
{combined_text}
|
|
INSTRUCTIONS: Use dense, factual bullet points. Focus on character meetings, relationships, and known information.
|
|
OUTPUT: Summary text.
|
|
""")
|
|
utils.log_usage(folder, ai_models.model_writer.name, resp_sum.usage_metadata)
|
|
summary = resp_sum.text
|
|
except: summary = "The story continues."
|
|
|
|
utils.log("SYSTEM", f"--- Phase: Writing ({len(chapters)} chapters planned) ---")
|
|
t_step = time.time()
|
|
session_chapters = 0
|
|
session_time = 0
|
|
|
|
i = len(ms)
|
|
while i < len(chapters):
|
|
ch_start = time.time()
|
|
ch = chapters[i]
|
|
|
|
# Check for stop signal from Web UI
|
|
run_dir = os.path.dirname(folder)
|
|
if os.path.exists(os.path.join(run_dir, ".stop")):
|
|
utils.log("SYSTEM", "Stop signal detected. Aborting generation.")
|
|
break
|
|
|
|
# Robust Resume: Check if this specific chapter number is already in the manuscript
|
|
if any(str(c.get('num')) == str(ch['chapter_number']) for c in ms):
|
|
i += 1
|
|
continue
|
|
|
|
# Progress Banner
|
|
utils.update_progress(15 + int((i / len(chapters)) * 75))
|
|
utils.log_banner("WRITER", f"Chapter {ch['chapter_number']}/{len(chapters)}: {ch['title']}")
|
|
|
|
prev_content = ms[-1]['content'] if ms else None
|
|
|
|
while True:
|
|
try:
|
|
# Build context: use structured state if available, fall back to summary blob
|
|
structured_ctx = story_state.format_for_prompt(current_story_state, ch.get('beats', []))
|
|
if structured_ctx:
|
|
summary_ctx = structured_ctx
|
|
else:
|
|
summary_ctx = summary[-8000:] if len(summary) > 8000 else summary
|
|
next_hint = chapters[i+1]['title'] if i + 1 < len(chapters) else ""
|
|
txt = story_writer.write_chapter(ch, bp, folder, summary_ctx, tracking, prev_content, next_chapter_hint=next_hint)
|
|
except Exception as e:
|
|
utils.log("SYSTEM", f"Chapter generation failed: {e}")
|
|
if interactive:
|
|
if Confirm.ask("Generation failed (quality/error). Retry?", default=True):
|
|
continue
|
|
raise e
|
|
|
|
if interactive:
|
|
print(f"\n--- Chapter {ch['chapter_number']} Preview ---\n{txt[:800]}...\n-------------------------------")
|
|
if Confirm.ask(f"Accept Chapter {ch['chapter_number']}?", default=True):
|
|
break
|
|
else:
|
|
utils.log("SYSTEM", "Regenerating chapter...")
|
|
else:
|
|
break
|
|
|
|
# Refine Persona to match the actual output (every 5 chapters)
|
|
if (i == 0 or i % 5 == 0) and txt:
|
|
bp['book_metadata']['author_details'] = style_persona.refine_persona(bp, txt, folder)
|
|
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
|
|
|
|
# Look ahead for context
|
|
next_info = ""
|
|
if i + 1 < len(chapters):
|
|
next_ch = chapters[i+1]
|
|
next_info = f"\nUPCOMING CONTEXT (Prioritize details relevant to this): {next_ch.get('title')} - {json.dumps(next_ch.get('beats', []))}"
|
|
|
|
try:
|
|
update_prompt = f"""
|
|
ROLE: Series Historian
|
|
TASK: Update the 'Story So Far' summary to include the events of this new chapter.
|
|
|
|
INPUT_DATA:
|
|
- CURRENT_SUMMARY:
|
|
{summary}
|
|
- NEW_CHAPTER_TEXT:
|
|
{txt}
|
|
- UPCOMING_CONTEXT_HINT: {next_info}
|
|
|
|
INSTRUCTIONS:
|
|
1. STYLE: Dense, factual, chronological bullet points. Avoid narrative prose.
|
|
2. CUMULATIVE: Do NOT remove old events. Append and integrate new information.
|
|
3. TRACKING: Explicitly note who met whom, who knows what, and current locations.
|
|
4. RELEVANCE: Ensure details needed for the UPCOMING CONTEXT are preserved.
|
|
|
|
OUTPUT: Updated summary text.
|
|
"""
|
|
resp_sum = ai_models.model_writer.generate_content(update_prompt)
|
|
utils.log_usage(folder, ai_models.model_writer.name, resp_sum.usage_metadata)
|
|
summary = resp_sum.text
|
|
except:
|
|
try:
|
|
resp_fallback = ai_models.model_writer.generate_content(f"ROLE: Summarizer\nTASK: Summarize plot points.\nTEXT: {txt}\nOUTPUT: Bullet points.")
|
|
utils.log_usage(folder, ai_models.model_writer.name, resp_fallback.usage_metadata)
|
|
summary += f"\n\nChapter {ch['chapter_number']}: " + resp_fallback.text
|
|
except: summary += f"\n\nChapter {ch['chapter_number']}: [Content processed]"
|
|
|
|
ms.append({'num': ch['chapter_number'], 'title': ch['title'], 'pov_character': ch.get('pov_character'), 'content': txt})
|
|
|
|
with open(ms_path, "w") as f: json.dump(ms, f, indent=2)
|
|
utils.send_heartbeat() # Signal that the task is still alive
|
|
|
|
# Update Tracking
|
|
tracking = bible_tracker.update_tracking(folder, ch['chapter_number'], txt, tracking)
|
|
with open(events_track_path, "w") as f: json.dump(tracking['events'], f, indent=2)
|
|
with open(chars_track_path, "w") as f: json.dump(tracking['characters'], f, indent=2)
|
|
with open(warn_track_path, "w") as f: json.dump(tracking.get('content_warnings', []), f, indent=2)
|
|
|
|
# Update Lore Index (Item 8: RAG-Lite)
|
|
tracking['lore'] = bible_tracker.update_lore_index(folder, txt, tracking.get('lore', {}))
|
|
with open(lore_track_path, "w") as f: json.dump(tracking['lore'], f, indent=2)
|
|
|
|
# Update Structured Story State (Item 9: Thread Tracking)
|
|
current_story_state = story_state.update_story_state(txt, ch['chapter_number'], current_story_state, folder)
|
|
|
|
# Dynamic Pacing Check (every other chapter)
|
|
remaining = chapters[i+1:]
|
|
if remaining and len(remaining) >= 2 and i % 2 == 1:
|
|
pacing = story_editor.check_pacing(bp, summary, txt, ch, remaining, folder)
|
|
if pacing and pacing.get('status') == 'add_bridge':
|
|
new_data = pacing.get('new_chapter', {})
|
|
if chapters:
|
|
avg_words = int(sum(c.get('estimated_words', 1500) for c in chapters) / len(chapters))
|
|
else:
|
|
avg_words = 1500
|
|
new_ch = {
|
|
"chapter_number": ch['chapter_number'] + 1,
|
|
"title": new_data.get('title', 'Bridge Chapter'),
|
|
"pov_character": new_data.get('pov_character', ch.get('pov_character')),
|
|
"pacing": "Slow",
|
|
"estimated_words": avg_words,
|
|
"beats": new_data.get('beats', [])
|
|
}
|
|
chapters.insert(i+1, new_ch)
|
|
for k in range(i+1, len(chapters)): chapters[k]['chapter_number'] = k + 1
|
|
with open(chapters_path, "w") as f: json.dump(chapters, f, indent=2)
|
|
utils.log("ARCHITECT", f" -> Pacing Intervention: Added bridge chapter '{new_ch['title']}' to fix rushing.")
|
|
|
|
elif pacing and pacing.get('status') == 'cut_next':
|
|
removed = chapters.pop(i+1)
|
|
for k in range(i+1, len(chapters)): chapters[k]['chapter_number'] = k + 1
|
|
with open(chapters_path, "w") as f: json.dump(chapters, f, indent=2)
|
|
utils.log("ARCHITECT", f" -> Pacing Intervention: Removed redundant chapter '{removed['title']}'.")
|
|
elif pacing:
|
|
utils.log("ARCHITECT", f" -> Pacing OK. {pacing.get('reason', '')[:100]}")
|
|
|
|
# Increment loop
|
|
i += 1
|
|
|
|
duration = time.time() - ch_start
|
|
session_chapters += 1
|
|
session_time += duration
|
|
avg_time = session_time / session_chapters
|
|
eta = avg_time * (len(chapters) - (i + 1))
|
|
|
|
prog = 15 + int((i / len(chapters)) * 75)
|
|
utils.update_progress(prog)
|
|
|
|
word_count = len(txt.split()) if txt else 0
|
|
utils.log("TIMING", f" -> Ch {ch['chapter_number']} done in {duration:.1f}s | {word_count:,} words | Avg: {avg_time:.1f}s | ETA: {int(eta//60)}m {int(eta%60)}s")
|
|
|
|
utils.log("TIMING", f"Writing Phase: {time.time() - t_step:.1f}s")
|
|
|
|
# Post-Processing
|
|
t_step = time.time()
|
|
utils.log("SYSTEM", "--- Phase: Post-Processing (Harvest, Cover, Export) ---")
|
|
try:
|
|
utils.update_progress(92)
|
|
utils.log("SYSTEM", " Harvesting metadata from manuscript...")
|
|
bp = bible_tracker.harvest_metadata(bp, folder, ms)
|
|
with open(os.path.join(folder, "final_blueprint.json"), "w") as f: json.dump(bp, f, indent=2)
|
|
|
|
utils.update_progress(95)
|
|
utils.log("SYSTEM", " Generating cover and marketing assets...")
|
|
marketing_assets.create_marketing_assets(bp, folder, tracking, interactive=interactive)
|
|
|
|
utils.log("SYSTEM", " Updating author persona sample...")
|
|
style_persona.update_persona_sample(bp, folder)
|
|
|
|
utils.update_progress(98)
|
|
utils.log("SYSTEM", " Compiling final export files...")
|
|
exporter.compile_files(bp, ms, folder)
|
|
except Exception as _e:
|
|
utils.log("ERROR", f"Post-processing phase failed: {type(_e).__name__}: {_e}")
|
|
raise
|
|
|
|
utils.log("TIMING", f"Post-Processing: {time.time() - t_step:.1f}s")
|
|
utils.log("SYSTEM", f"Book Finished. Total Time: {time.time() - total_start:.1f}s")
|
|
|
|
finally:
|
|
if os.path.exists(lock_path): os.remove(lock_path)
|
|
|
|
|
|
def run_generation(target=None, specific_run_id=None, interactive=False):
|
|
utils.log("SYSTEM", "=== run_generation: Initialising AI models ===")
|
|
ai_setup.init_models()
|
|
|
|
if not target: target = config.DEFAULT_BLUEPRINT
|
|
data = utils.load_json(target)
|
|
|
|
if not data:
|
|
utils.log("ERROR", f"Could not load bible/target: {target}")
|
|
return
|
|
|
|
utils.log("SYSTEM", f"=== Starting Series Generation: {data.get('project_metadata', {}).get('title', 'Untitled')} ===")
|
|
|
|
project_dir = os.path.dirname(os.path.abspath(target))
|
|
runs_base = os.path.join(project_dir, "runs")
|
|
|
|
run_dir = None
|
|
resume_mode = False
|
|
|
|
if specific_run_id:
|
|
run_dir = os.path.join(runs_base, f"run_{specific_run_id}")
|
|
if not os.path.exists(run_dir): os.makedirs(run_dir)
|
|
resume_mode = True
|
|
else:
|
|
latest_run = utils.get_latest_run_folder(runs_base)
|
|
if latest_run:
|
|
has_lock = False
|
|
for root, dirs, files in os.walk(latest_run):
|
|
if ".in_progress" in files:
|
|
has_lock = True
|
|
break
|
|
|
|
if has_lock:
|
|
if Confirm.ask(f"Found incomplete run '{os.path.basename(latest_run)}'. Resume generation?", default=True):
|
|
run_dir = latest_run
|
|
resume_mode = True
|
|
elif Confirm.ask(f"Delete artifacts in '{os.path.basename(latest_run)}' and start over?", default=False):
|
|
shutil.rmtree(latest_run)
|
|
os.makedirs(latest_run)
|
|
run_dir = latest_run
|
|
|
|
if not run_dir: run_dir = utils.get_run_folder(runs_base)
|
|
utils.log("SYSTEM", f"Run Directory: {run_dir}")
|
|
|
|
previous_context = ""
|
|
|
|
for i, book in enumerate(data['books']):
|
|
utils.log("SERIES", f"Processing Book {book.get('book_number')}: {book.get('title')}")
|
|
|
|
if os.path.exists(os.path.join(run_dir, ".stop")):
|
|
utils.log("SYSTEM", "Stop signal detected. Aborting series generation.")
|
|
break
|
|
|
|
meta = data['project_metadata']
|
|
bp = {
|
|
"book_metadata": {
|
|
"title": book.get('title'),
|
|
"filename": book.get('filename'),
|
|
"author": meta.get('author'),
|
|
"genre": meta.get('genre'),
|
|
"target_audience": meta.get('target_audience'),
|
|
"style": meta.get('style', {}),
|
|
"author_details": meta.get('author_details', {}),
|
|
"author_bio": meta.get('author_bio', ''),
|
|
},
|
|
"length_settings": meta.get('length_settings', {}),
|
|
"characters": data.get('characters', []),
|
|
"manual_instruction": book.get('manual_instruction', ''),
|
|
"plot_beats": book.get('plot_beats', []),
|
|
"series_metadata": {
|
|
"is_series": meta.get('is_series', False),
|
|
"series_title": meta.get('title', ''),
|
|
"book_number": book.get('book_number', i+1),
|
|
"total_books": len(data['books'])
|
|
}
|
|
}
|
|
|
|
safe_title = utils.sanitize_filename(book.get('title', f"Book_{i+1}"))
|
|
book_folder = os.path.join(run_dir, f"Book_{book.get('book_number', i+1)}_{safe_title}")
|
|
os.makedirs(book_folder, exist_ok=True)
|
|
|
|
utils.log("SYSTEM", f"--- Starting process_book for '{book.get('title')}' in {book_folder} ---")
|
|
try:
|
|
process_book(bp, book_folder, context=previous_context, resume=resume_mode, interactive=interactive)
|
|
except Exception as _e:
|
|
utils.log("ERROR", f"process_book failed for Book {book.get('book_number')}: {type(_e).__name__}: {_e}")
|
|
raise
|
|
utils.log("SYSTEM", f"--- Finished process_book for '{book.get('title')}' ---")
|
|
|
|
final_bp_path = os.path.join(book_folder, "final_blueprint.json")
|
|
if os.path.exists(final_bp_path):
|
|
final_bp = utils.load_json(final_bp_path)
|
|
|
|
new_chars = final_bp.get('characters', [])
|
|
|
|
if os.path.exists(target):
|
|
current_bible = utils.load_json(target)
|
|
|
|
existing_names = {c['name'].lower() for c in current_bible.get('characters', [])}
|
|
for char in new_chars:
|
|
if char['name'].lower() not in existing_names:
|
|
current_bible['characters'].append(char)
|
|
|
|
for b in current_bible.get('books', []):
|
|
if b.get('book_number') == book.get('book_number'):
|
|
b['title'] = final_bp['book_metadata'].get('title', b.get('title'))
|
|
b['plot_beats'] = final_bp.get('plot_beats', b.get('plot_beats'))
|
|
b['manual_instruction'] = final_bp.get('manual_instruction', b.get('manual_instruction'))
|
|
break
|
|
|
|
with open(target, 'w') as f: json.dump(current_bible, f, indent=2)
|
|
utils.log("SERIES", "Updated World Bible with new characters and plot data.")
|
|
|
|
last_beat = final_bp.get('plot_beats', [])[-1] if final_bp.get('plot_beats') else "End of book."
|
|
previous_context = f"PREVIOUS BOOK SUMMARY: {last_beat}\nCHARACTERS: {json.dumps(final_bp.get('characters', []))}"
|
|
|
|
return
|
|
|
|
|
|
if __name__ == "__main__":
|
|
target_arg = sys.argv[1] if len(sys.argv) > 1 else None
|
|
run_generation(target_arg, interactive=True)
|