Files
bookapp/cli/engine.py
Mike Wichers dc39930da4 feat: Implement ai_blueprint.md Steps 1 & 2 — bible-tracking merge and character voice profiles
Step 1 (Bible-Tracking Merge):
- Added merge_tracking_to_bible() to story/bible_tracker.py — merges character
  tracking state and lore back into bible dict after each chapter, making
  blueprint_initial.json the single persistent source of truth.
- Integrated in cli/engine.py after each chapter's update_tracking + update_lore_index
  calls so the persisted bible is always up-to-date.

Step 2 (Character-Specific Voice Profiles):
- story/writer.py: write_chapter now checks bp['characters'] for a voice_profile on
  the POV character before falling back to the prebuilt_persona cache.
- story/style_persona.py: refine_persona() accepts pov_character=None; when a POV
  character with a voice_profile is supplied it refines that profile's bio instead of
  the global author_details bio.
- cli/engine.py: refine_persona call now passes ch.get('pov_character') so per-chapter
  persona refinement targets the correct voice.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-22 22:45:54 -05:00

503 lines
25 KiB
Python

import json
import os
import time
import sys
import shutil
from rich.prompt import Confirm
from core import config, utils
from ai import models as ai_models
from ai import setup as ai_setup
from story import planner, writer as story_writer, editor as story_editor
from story import style_persona, bible_tracker, state as story_state
from story.writer import build_persona_info
from marketing import assets as marketing_assets
from export import exporter
def process_book(bp, folder, context="", resume=False, interactive=False):
# Create lock file to indicate active processing
lock_path = os.path.join(folder, ".in_progress")
with open(lock_path, "w") as f: f.write("running")
total_start = time.time()
try:
# 1. Check completion
if resume and os.path.exists(os.path.join(folder, "final_blueprint.json")):
utils.log("SYSTEM", f"Book in {folder} already finished. Skipping.")
if os.path.exists(lock_path): os.remove(lock_path)
return
# 2. Load or Create Blueprint
bp_path = os.path.join(folder, "blueprint_initial.json")
t_step = time.time()
utils.update_progress(5)
utils.log("SYSTEM", "--- Phase: Blueprint ---")
try:
if resume and os.path.exists(bp_path):
utils.log("RESUME", "Loading existing blueprint...")
saved_bp = utils.load_json(bp_path)
if saved_bp:
if 'book_metadata' in bp and 'book_metadata' in saved_bp:
for k in ['title', 'author', 'genre', 'target_audience', 'style', 'author_bio', 'author_details']:
if k in bp['book_metadata']:
saved_bp['book_metadata'][k] = bp['book_metadata'][k]
if 'series_metadata' in bp:
saved_bp['series_metadata'] = bp['series_metadata']
bp = saved_bp
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
else:
bp = planner.enrich(bp, folder, context)
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
# Ensure Persona Exists (Auto-create + Exp 6: Validate before accepting)
if 'author_details' not in bp['book_metadata'] or not bp['book_metadata']['author_details']:
max_persona_attempts = 3
for persona_attempt in range(1, max_persona_attempts + 1):
candidate_persona = style_persona.create_initial_persona(bp, folder)
is_valid, p_score = style_persona.validate_persona(bp, candidate_persona, folder)
if is_valid or persona_attempt == max_persona_attempts:
if not is_valid:
utils.log("SYSTEM", f" ⚠️ Persona accepted after {max_persona_attempts} attempts despite low score ({p_score}/10). Voice drift risk elevated.")
bp['book_metadata']['author_details'] = candidate_persona
break
utils.log("SYSTEM", f" -> Persona attempt {persona_attempt}/{max_persona_attempts} scored {p_score}/10. Regenerating...")
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
except Exception as _e:
utils.log("ERROR", f"Blueprint phase failed: {type(_e).__name__}: {_e}")
raise
utils.log("TIMING", f"Blueprint Phase: {time.time() - t_step:.1f}s")
# 3. Events (Plan & Expand)
events_path = os.path.join(folder, "events.json")
t_step = time.time()
utils.update_progress(10)
utils.log("SYSTEM", "--- Phase: Story Structure & Events ---")
try:
if resume and os.path.exists(events_path):
utils.log("RESUME", "Loading existing events...")
events = utils.load_json(events_path)
else:
events = planner.plan_structure(bp, folder)
depth = bp['length_settings']['depth']
target_chaps = bp['length_settings']['chapters']
for d in range(1, depth+1):
utils.log("SYSTEM", f" Expanding story structure depth {d}/{depth}...")
events = planner.expand(events, d, target_chaps, bp, folder)
time.sleep(1)
with open(events_path, "w") as f: json.dump(events, f, indent=2)
except Exception as _e:
utils.log("ERROR", f"Events/Structure phase failed: {type(_e).__name__}: {_e}")
raise
utils.log("TIMING", f"Structure & Expansion: {time.time() - t_step:.1f}s")
# 4. Chapter Plan
chapters_path = os.path.join(folder, "chapters.json")
t_step = time.time()
utils.update_progress(15)
utils.log("SYSTEM", "--- Phase: Chapter Planning ---")
try:
if resume and os.path.exists(chapters_path):
utils.log("RESUME", "Loading existing chapter plan...")
chapters = utils.load_json(chapters_path)
else:
chapters = planner.create_chapter_plan(events, bp, folder)
with open(chapters_path, "w") as f: json.dump(chapters, f, indent=2)
except Exception as _e:
utils.log("ERROR", f"Chapter planning phase failed: {type(_e).__name__}: {_e}")
raise
utils.log("TIMING", f"Chapter Planning: {time.time() - t_step:.1f}s")
# 4b. Outline Validation Gate (Alt 2-B: pre-generation quality check)
if chapters and not resume:
try:
planner.validate_outline(events, chapters, bp, folder)
except Exception as _e:
utils.log("ARCHITECT", f"Outline validation skipped: {_e}")
# 5. Writing Loop
ms_path = os.path.join(folder, "manuscript.json")
loaded_ms = utils.load_json(ms_path) if (resume and os.path.exists(ms_path)) else []
ms = loaded_ms if loaded_ms is not None else []
# Load Tracking
events_track_path = os.path.join(folder, "tracking_events.json")
chars_track_path = os.path.join(folder, "tracking_characters.json")
warn_track_path = os.path.join(folder, "tracking_warnings.json")
lore_track_path = os.path.join(folder, "tracking_lore.json")
tracking = {"events": [], "characters": {}, "content_warnings": [], "lore": {}}
if resume:
if os.path.exists(events_track_path):
tracking['events'] = utils.load_json(events_track_path)
if os.path.exists(chars_track_path):
tracking['characters'] = utils.load_json(chars_track_path)
if os.path.exists(warn_track_path):
tracking['content_warnings'] = utils.load_json(warn_track_path)
if os.path.exists(lore_track_path):
tracking['lore'] = utils.load_json(lore_track_path) or {}
# Load structured story state
current_story_state = story_state.load_story_state(folder)
summary = "The story begins."
if ms:
utils.log("RESUME", f"Rebuilding story context from {len(ms)} existing chapters...")
try:
selected = ms[:1] + ms[-4:] if len(ms) > 5 else ms
combined_text = "\n".join([f"Chapter {c['num']}: {c['content'][:3000]}" for c in selected])
resp_sum = ai_models.model_writer.generate_content(f"""
ROLE: Series Historian
TASK: Create a cumulative 'Story So Far' summary.
INPUT_TEXT:
{combined_text}
INSTRUCTIONS: Use dense, factual bullet points. Focus on character meetings, relationships, and known information.
OUTPUT: Summary text.
""")
utils.log_usage(folder, ai_models.model_writer.name, resp_sum.usage_metadata)
summary = resp_sum.text
except: summary = "The story continues."
utils.log("SYSTEM", f"--- Phase: Writing ({len(chapters)} chapters planned) ---")
t_step = time.time()
session_chapters = 0
session_time = 0
# Pre-load persona once for the entire writing phase (Alt 3-D: persona cache)
# Rebuilt after each refine_persona() call to pick up bio updates.
cached_persona = build_persona_info(bp)
i = len(ms)
while i < len(chapters):
ch_start = time.time()
ch = chapters[i]
# Check for stop signal from Web UI
run_dir = os.path.dirname(folder)
if os.path.exists(os.path.join(run_dir, ".stop")):
utils.log("SYSTEM", "Stop signal detected. Aborting generation.")
break
# Robust Resume: Check if this specific chapter number is already in the manuscript
if any(str(c.get('num')) == str(ch['chapter_number']) for c in ms):
i += 1
continue
# Progress Banner
utils.update_progress(15 + int((i / len(chapters)) * 75))
utils.log_banner("WRITER", f"Chapter {ch['chapter_number']}/{len(chapters)}: {ch['title']}")
prev_content = ms[-1]['content'] if ms else None
while True:
try:
# Build context: use structured state if available, fall back to summary blob
structured_ctx = story_state.format_for_prompt(current_story_state, ch.get('beats', []))
if structured_ctx:
summary_ctx = structured_ctx
else:
summary_ctx = summary[-8000:] if len(summary) > 8000 else summary
next_hint = chapters[i+1]['title'] if i + 1 < len(chapters) else ""
chap_pos = i / max(len(chapters) - 1, 1) if len(chapters) > 1 else 0.5
txt = story_writer.write_chapter(ch, bp, folder, summary_ctx, tracking, prev_content, next_chapter_hint=next_hint, prebuilt_persona=cached_persona, chapter_position=chap_pos)
except Exception as e:
utils.log("SYSTEM", f"Chapter generation failed: {e}")
if interactive:
if Confirm.ask("Generation failed (quality/error). Retry?", default=True):
continue
raise e
if interactive:
print(f"\n--- Chapter {ch['chapter_number']} Preview ---\n{txt[:800]}...\n-------------------------------")
if Confirm.ask(f"Accept Chapter {ch['chapter_number']}?", default=True):
break
else:
utils.log("SYSTEM", "Regenerating chapter...")
else:
break
# Refine Persona to match the actual output (every 5 chapters)
if (i == 0 or i % 5 == 0) and txt:
pov_char = ch.get('pov_character')
bp['book_metadata']['author_details'] = style_persona.refine_persona(bp, txt, folder, pov_character=pov_char)
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
cached_persona = build_persona_info(bp) # Rebuild cache with updated bio
# Look ahead for context
next_info = ""
if i + 1 < len(chapters):
next_ch = chapters[i+1]
next_info = f"\nUPCOMING CONTEXT (Prioritize details relevant to this): {next_ch.get('title')} - {json.dumps(next_ch.get('beats', []))}"
try:
update_prompt = f"""
ROLE: Series Historian
TASK: Update the 'Story So Far' summary to include the events of this new chapter.
INPUT_DATA:
- CURRENT_SUMMARY:
{summary}
- NEW_CHAPTER_TEXT:
{txt}
- UPCOMING_CONTEXT_HINT: {next_info}
INSTRUCTIONS:
1. STYLE: Dense, factual, chronological bullet points. Avoid narrative prose.
2. CUMULATIVE: Do NOT remove old events. Append and integrate new information.
3. TRACKING: Explicitly note who met whom, who knows what, and current locations.
4. RELEVANCE: Ensure details needed for the UPCOMING CONTEXT are preserved.
OUTPUT: Updated summary text.
"""
resp_sum = ai_models.model_writer.generate_content(update_prompt)
utils.log_usage(folder, ai_models.model_writer.name, resp_sum.usage_metadata)
summary = resp_sum.text
except:
try:
resp_fallback = ai_models.model_writer.generate_content(f"ROLE: Summarizer\nTASK: Summarize plot points.\nTEXT: {txt}\nOUTPUT: Bullet points.")
utils.log_usage(folder, ai_models.model_writer.name, resp_fallback.usage_metadata)
summary += f"\n\nChapter {ch['chapter_number']}: " + resp_fallback.text
except: summary += f"\n\nChapter {ch['chapter_number']}: [Content processed]"
ms.append({'num': ch['chapter_number'], 'title': ch['title'], 'pov_character': ch.get('pov_character'), 'content': txt})
with open(ms_path, "w") as f: json.dump(ms, f, indent=2)
utils.send_heartbeat() # Signal that the task is still alive
# Update Tracking
tracking = bible_tracker.update_tracking(folder, ch['chapter_number'], txt, tracking)
with open(events_track_path, "w") as f: json.dump(tracking['events'], f, indent=2)
with open(chars_track_path, "w") as f: json.dump(tracking['characters'], f, indent=2)
with open(warn_track_path, "w") as f: json.dump(tracking.get('content_warnings', []), f, indent=2)
# Update Lore Index (Item 8: RAG-Lite) — every 3 chapters (lore is stable after ch 1-3)
if i == 0 or i % 3 == 0:
tracking['lore'] = bible_tracker.update_lore_index(folder, txt, tracking.get('lore', {}))
with open(lore_track_path, "w") as f: json.dump(tracking['lore'], f, indent=2)
# Persist dynamic tracking changes back to the bible (Step 1: Bible-Tracking Merge)
bp = bible_tracker.merge_tracking_to_bible(bp, tracking)
with open(bp_path, "w") as f: json.dump(bp, f, indent=2)
# Update Structured Story State (Item 9: Thread Tracking)
current_story_state = story_state.update_story_state(txt, ch['chapter_number'], current_story_state, folder)
# Exp 5: Mid-gen Consistency Snapshot (every 10 chapters)
# Sample: first 2 + last 8 chapters to keep token cost bounded regardless of book length
if len(ms) > 0 and len(ms) % 10 == 0:
utils.log("EDITOR", f"--- Mid-gen consistency check after chapter {ch['chapter_number']} ({len(ms)} written) ---")
try:
ms_sample = (ms[:2] + ms[-8:]) if len(ms) > 10 else ms
consistency = story_editor.analyze_consistency(bp, ms_sample, folder)
issues = consistency.get('issues', [])
if issues:
for issue in issues:
utils.log("EDITOR", f" ⚠️ {issue}")
c_score = consistency.get('score', 'N/A')
c_summary = consistency.get('summary', '')
utils.log("EDITOR", f" Consistency score: {c_score}/10 — {c_summary}")
except Exception as _ce:
utils.log("EDITOR", f" Mid-gen consistency check failed (non-blocking): {_ce}")
# Dynamic Pacing Check (every other chapter)
remaining = chapters[i+1:]
if remaining and len(remaining) >= 2 and i % 2 == 1:
pacing = story_editor.check_pacing(bp, summary, txt, ch, remaining, folder)
if pacing and pacing.get('status') == 'add_bridge':
new_data = pacing.get('new_chapter', {})
if chapters:
avg_words = int(sum(c.get('estimated_words', 1500) for c in chapters) / len(chapters))
else:
avg_words = 1500
new_ch = {
"chapter_number": ch['chapter_number'] + 1,
"title": new_data.get('title', 'Bridge Chapter'),
"pov_character": new_data.get('pov_character', ch.get('pov_character')),
"pacing": "Slow",
"estimated_words": avg_words,
"beats": new_data.get('beats', [])
}
chapters.insert(i+1, new_ch)
for k in range(i+1, len(chapters)): chapters[k]['chapter_number'] = k + 1
with open(chapters_path, "w") as f: json.dump(chapters, f, indent=2)
utils.log("ARCHITECT", f" -> Pacing Intervention: Added bridge chapter '{new_ch['title']}' to fix rushing.")
elif pacing and pacing.get('status') == 'cut_next':
removed = chapters.pop(i+1)
for k in range(i+1, len(chapters)): chapters[k]['chapter_number'] = k + 1
with open(chapters_path, "w") as f: json.dump(chapters, f, indent=2)
utils.log("ARCHITECT", f" -> Pacing Intervention: Removed redundant chapter '{removed['title']}'.")
elif pacing:
utils.log("ARCHITECT", f" -> Pacing OK. {pacing.get('reason', '')[:100]}")
# Increment loop
i += 1
duration = time.time() - ch_start
session_chapters += 1
session_time += duration
avg_time = session_time / session_chapters
eta = avg_time * (len(chapters) - (i + 1))
prog = 15 + int((i / len(chapters)) * 75)
utils.update_progress(prog)
word_count = len(txt.split()) if txt else 0
utils.log("TIMING", f" -> Ch {ch['chapter_number']} done in {duration:.1f}s | {word_count:,} words | Avg: {avg_time:.1f}s | ETA: {int(eta//60)}m {int(eta%60)}s")
utils.log("TIMING", f"Writing Phase: {time.time() - t_step:.1f}s")
# Post-Processing
t_step = time.time()
utils.log("SYSTEM", "--- Phase: Post-Processing (Harvest, Cover, Export) ---")
try:
utils.update_progress(92)
utils.log("SYSTEM", " Harvesting metadata from manuscript...")
bp = bible_tracker.harvest_metadata(bp, folder, ms)
with open(os.path.join(folder, "final_blueprint.json"), "w") as f: json.dump(bp, f, indent=2)
utils.update_progress(95)
utils.log("SYSTEM", " Generating cover and marketing assets...")
marketing_assets.create_marketing_assets(bp, folder, tracking, interactive=interactive)
utils.log("SYSTEM", " Updating author persona sample...")
style_persona.update_persona_sample(bp, folder)
utils.update_progress(98)
utils.log("SYSTEM", " Compiling final export files...")
exporter.compile_files(bp, ms, folder)
except Exception as _e:
utils.log("ERROR", f"Post-processing phase failed: {type(_e).__name__}: {_e}")
raise
utils.log("TIMING", f"Post-Processing: {time.time() - t_step:.1f}s")
utils.log("SYSTEM", f"Book Finished. Total Time: {time.time() - total_start:.1f}s")
finally:
if os.path.exists(lock_path): os.remove(lock_path)
def run_generation(target=None, specific_run_id=None, interactive=False):
utils.log("SYSTEM", "=== run_generation: Initialising AI models ===")
ai_setup.init_models()
if not target: target = config.DEFAULT_BLUEPRINT
data = utils.load_json(target)
if not data:
utils.log("ERROR", f"Could not load bible/target: {target}")
return
utils.log("SYSTEM", f"=== Starting Series Generation: {data.get('project_metadata', {}).get('title', 'Untitled')} ===")
project_dir = os.path.dirname(os.path.abspath(target))
runs_base = os.path.join(project_dir, "runs")
run_dir = None
resume_mode = False
if specific_run_id:
run_dir = os.path.join(runs_base, f"run_{specific_run_id}")
if not os.path.exists(run_dir): os.makedirs(run_dir)
resume_mode = True
else:
latest_run = utils.get_latest_run_folder(runs_base)
if latest_run:
has_lock = False
for root, dirs, files in os.walk(latest_run):
if ".in_progress" in files:
has_lock = True
break
if has_lock:
if Confirm.ask(f"Found incomplete run '{os.path.basename(latest_run)}'. Resume generation?", default=True):
run_dir = latest_run
resume_mode = True
elif Confirm.ask(f"Delete artifacts in '{os.path.basename(latest_run)}' and start over?", default=False):
shutil.rmtree(latest_run)
os.makedirs(latest_run)
run_dir = latest_run
if not run_dir: run_dir = utils.get_run_folder(runs_base)
utils.log("SYSTEM", f"Run Directory: {run_dir}")
previous_context = ""
for i, book in enumerate(data['books']):
utils.log("SERIES", f"Processing Book {book.get('book_number')}: {book.get('title')}")
if os.path.exists(os.path.join(run_dir, ".stop")):
utils.log("SYSTEM", "Stop signal detected. Aborting series generation.")
break
meta = data['project_metadata']
bp = {
"book_metadata": {
"title": book.get('title'),
"filename": book.get('filename'),
"author": meta.get('author'),
"genre": meta.get('genre'),
"target_audience": meta.get('target_audience'),
"style": meta.get('style', {}),
"author_details": meta.get('author_details', {}),
"author_bio": meta.get('author_bio', ''),
},
"length_settings": meta.get('length_settings', {}),
"characters": data.get('characters', []),
"manual_instruction": book.get('manual_instruction', ''),
"plot_beats": book.get('plot_beats', []),
"series_metadata": {
"is_series": meta.get('is_series', False),
"series_title": meta.get('title', ''),
"book_number": book.get('book_number', i+1),
"total_books": len(data['books'])
}
}
safe_title = utils.sanitize_filename(book.get('title', f"Book_{i+1}"))
book_folder = os.path.join(run_dir, f"Book_{book.get('book_number', i+1)}_{safe_title}")
os.makedirs(book_folder, exist_ok=True)
utils.log("SYSTEM", f"--- Starting process_book for '{book.get('title')}' in {book_folder} ---")
try:
process_book(bp, book_folder, context=previous_context, resume=resume_mode, interactive=interactive)
except Exception as _e:
utils.log("ERROR", f"process_book failed for Book {book.get('book_number')}: {type(_e).__name__}: {_e}")
raise
utils.log("SYSTEM", f"--- Finished process_book for '{book.get('title')}' ---")
final_bp_path = os.path.join(book_folder, "final_blueprint.json")
if os.path.exists(final_bp_path):
final_bp = utils.load_json(final_bp_path)
new_chars = final_bp.get('characters', [])
if os.path.exists(target):
current_bible = utils.load_json(target)
existing_names = {c['name'].lower() for c in current_bible.get('characters', [])}
for char in new_chars:
if char['name'].lower() not in existing_names:
current_bible['characters'].append(char)
for b in current_bible.get('books', []):
if b.get('book_number') == book.get('book_number'):
b['title'] = final_bp['book_metadata'].get('title', b.get('title'))
b['plot_beats'] = final_bp.get('plot_beats', b.get('plot_beats'))
b['manual_instruction'] = final_bp.get('manual_instruction', b.get('manual_instruction'))
break
with open(target, 'w') as f: json.dump(current_bible, f, indent=2)
utils.log("SERIES", "Updated World Bible with new characters and plot data.")
last_beat = final_bp.get('plot_beats', [])[-1] if final_bp.get('plot_beats') else "End of book."
previous_context = f"PREVIOUS BOOK SUMMARY: {last_beat}\nCHARACTERS: {json.dumps(final_bp.get('characters', []))}"
return
if __name__ == "__main__":
target_arg = sys.argv[1] if len(sys.argv) > 1 else None
run_generation(target_arg, interactive=True)