v1.4.0: Organic writing, speed, and log improvements

Organic book quality: - write_chapter: strip key_events spoilers from character context so the writer doesn't know planned future events when writing early chapters - write_chapter: added next_chapter_hint — seeds anticipation for the next scene in the final paragraphs of each chapter for natural story flow - write_chapter: added DIALOGUE VOICE instruction referencing CHARACTER TRACKING speech styles so every character sounds distinctly different - Lowered SCORE_AUTO_ACCEPT 9→8 to stop over-refining already-professional drafts Speed improvements: - check_pacing: reduced from every chapter to every other chapter (~50% fewer calls) - refine_persona: reduced from every 3 to every 5 chapters (~40% fewer calls) - Resume summary rebuild: uses first + last-4 chapters instead of all chapters to avoid massive prompts when resuming mid-book - Summary context sent to writer capped at 8000 chars (most-recent events) - update_tracking text cap lowered 500000→20000 (covers any realistic chapter) Logging and progress bars: - Progress bar updates at chapter START, not just after completion - Chapter banner logged before each write so the log shows which chapter is active - Word count logged after first draft (e.g. "Draft: 2,341 words (target: ~2200)") - Word count added to chapter completion TIMING line - Pacing check now logs "Pacing OK" with reason when no intervention needed - utils: added log_banner() helper for phase separator lines UI: - run_details.html: log lines are now phase-coloured (WRITER=cyan, ARCHITECT=green, TIMING=gray, SYSTEM=yellow, TRACKER=purple, RESUME=orange, etc.) - Status bar shows current active phase (e.g. "Status: Running — WRITER") Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-20 10:59:08 -05:00
parent 958a6d0ea0
commit edabc4d4fa
5 changed files with 105 additions and 29 deletions
--- a/config.py
+++ b/config.py
@@ -65,4 +65,4 @@ LENGTH_DEFINITIONS = {
 }

 # --- SYSTEM ---
-VERSION = "1.3.1"
+VERSION = "1.4.0"
--- a/main.py
+++ b/main.py
@@ -97,10 +97,11 @@ def process_book(bp, folder, context="", resume=False, interactive=False):
        
        summary = "The story begins."
        if ms:
-            # Generate summary from ALL written chapters to maintain continuity
-            utils.log("RESUME", "Rebuilding 'Story So Far' from existing manuscript...")
-            try: 
-                combined_text = "\n".join([f"Chapter {c['num']}: {c['content']}" for c in ms])
+            # Efficient rebuild: first chapter (setup) + last 4 (recent events) avoids huge prompts
+            utils.log("RESUME", f"Rebuilding story context from {len(ms)} existing chapters...")
+            try:
+                selected = ms[:1] + ms[-4:] if len(ms) > 5 else ms
+                combined_text = "\n".join([f"Chapter {c['num']}: {c['content'][:3000]}" for c in selected])
                resp_sum = ai.model_writer.generate_content(f"""
                ROLE: Series Historian
                TASK: Create a cumulative 'Story So Far' summary.
@@ -133,13 +134,20 @@ def process_book(bp, folder, context="", resume=False, interactive=False):
            if any(str(c.get('num')) == str(ch['chapter_number']) for c in ms):
                i += 1
                continue
-            
+
+            # Progress Banner — update bar and log chapter header before writing begins
+            utils.update_progress(15 + int((i / len(chapters)) * 75))
+            utils.log_banner("WRITER", f"Chapter {ch['chapter_number']}/{len(chapters)}: {ch['title']}")
+
            # Pass previous chapter content for continuity if available
            prev_content = ms[-1]['content'] if ms else None
            
            while True:
                try:
-                    txt = story.write_chapter(ch, bp, folder, summary, tracking, prev_content)
+                    # Cap summary to most-recent 8000 chars; pass next chapter title as hook hint
+                    summary_ctx = summary[-8000:] if len(summary) > 8000 else summary
+                    next_hint = chapters[i+1]['title'] if i + 1 < len(chapters) else ""
+                    txt = story.write_chapter(ch, bp, folder, summary_ctx, tracking, prev_content, next_chapter_hint=next_hint)
                except Exception as e:
                    utils.log("SYSTEM", f"Chapter generation failed: {e}")
                    if interactive:
@@ -156,8 +164,8 @@ def process_book(bp, folder, context="", resume=False, interactive=False):
                else:
                    break
            
-            # Refine Persona to match the actual output (Consistency Loop)
-            if (i == 0 or i % 3 == 0) and txt:
+            # Refine Persona to match the actual output (every 5 chapters to save API calls)
+            if (i == 0 or i % 5 == 0) and txt:
                bp['book_metadata']['author_details'] = story.refine_persona(bp, txt, folder)
                with open(bp_path, "w") as f: json.dump(bp, f, indent=2)

@@ -207,9 +215,9 @@ def process_book(bp, folder, context="", resume=False, interactive=False):
            with open(chars_track_path, "w") as f: json.dump(tracking['characters'], f, indent=2)
            with open(warn_track_path, "w") as f: json.dump(tracking.get('content_warnings', []), f, indent=2)
            
-            # --- DYNAMIC PACING CHECK ---
+            # --- DYNAMIC PACING CHECK (every other chapter to halve API overhead) ---
            remaining = chapters[i+1:]
-            if remaining:
+            if remaining and len(remaining) >= 2 and i % 2 == 1:
                pacing = story.check_pacing(bp, summary, txt, ch, remaining, folder)
                if pacing and pacing.get('status') == 'add_bridge':
                    new_data = pacing.get('new_chapter', {})
@@ -237,10 +245,12 @@ def process_book(bp, folder, context="", resume=False, interactive=False):
                    removed = chapters.pop(i+1)
                    # Renumber subsequent chapters
                    for k in range(i+1, len(chapters)): chapters[k]['chapter_number'] = k + 1
-                    
+
                    with open(chapters_path, "w") as f: json.dump(chapters, f, indent=2)
                    utils.log("ARCHITECT", f"  -> ⚠️ Pacing Intervention: Removed redundant chapter '{removed['title']}'.")
-            
+                elif pacing:
+                    utils.log("ARCHITECT", f"  -> Pacing OK. {pacing.get('reason', '')[:100]}")
+
            # Increment loop
            i += 1

@@ -254,7 +264,8 @@ def process_book(bp, folder, context="", resume=False, interactive=False):
            prog = 15 + int((i / len(chapters)) * 75)
            utils.update_progress(prog)
            
-            utils.log("TIMING", f"  -> Chapter {ch['chapter_number']} finished in {duration:.1f}s | Avg: {avg_time:.1f}s | ETA: {int(eta//60)}m {int(eta%60)}s")
+            word_count = len(txt.split()) if txt else 0
+            utils.log("TIMING", f"  -> Ch {ch['chapter_number']} done in {duration:.1f}s | {word_count:,} words | Avg: {avg_time:.1f}s | ETA: {int(eta//60)}m {int(eta%60)}s")
            
        utils.log("TIMING", f"Writing Phase: {time.time() - t_step:.1f}s")

--- a/modules/story.py
+++ b/modules/story.py
@@ -411,7 +411,7 @@ def update_tracking(folder, chapter_num, chapter_text, current_tracking):
    {json.dumps(current_tracking)}
    
    NEW_TEXT:
-    {chapter_text[:500000]} 
+    {chapter_text[:20000]}
    
    OPERATIONS:
    1. EVENTS: Append 1-3 key plot points to 'events'.
@@ -594,7 +594,7 @@ def refine_persona(bp, text, folder):
    except: pass
    return ad

-def write_chapter(chap, bp, folder, prev_sum, tracking=None, prev_content=None):
+def write_chapter(chap, bp, folder, prev_sum, tracking=None, prev_content=None, next_chapter_hint=""):
    pacing = chap.get('pacing', 'Standard')
    est_words = chap.get('estimated_words', 'Flexible')
    utils.log("WRITER", f"Drafting Ch {chap['chapter_number']} ({pacing} | ~{est_words} words): {chap['title']}")
@@ -662,6 +662,13 @@ def write_chapter(chap, bp, folder, prev_sum, tracking=None, prev_content=None):
        trunc_content = prev_content[-3000:] if len(prev_content) > 3000 else prev_content
        prev_context_block = f"\nPREVIOUS CHAPTER TEXT (For Tone & Continuity):\n{trunc_content}\n"

+    # Strip future planning notes (key_events) from character context — the writer
+    # should not know what is *planned* to happen; only name, role, and description.
+    chars_for_writer = [
+        {"name": c.get("name"), "role": c.get("role"), "description": c.get("description", "")}
+        for c in bp.get('characters', [])
+    ]
+
    total_chapters = ls.get('chapters', '?')
    prompt = f"""
    ROLE: Fiction Writer
@@ -705,7 +712,9 @@ def write_chapter(chap, bp, folder, prev_sum, tracking=None, prev_content=None):
    - CHARACTER INTERACTIONS: If characters are meeting for the first time in the summary, treat them as strangers.
    - SENTENCE VARIETY: Avoid repetitive sentence structures (e.g. starting multiple sentences with "He" or "She"). Vary sentence length to create rhythm.
    - GENRE CONSISTENCY: Ensure all introductions of characters, places, items, or actions are strictly appropriate for the {genre} genre. Avoid anachronisms or tonal clashes.
-    
+    - DIALOGUE VOICE: Every character speaks with their own distinct voice (see CHARACTER TRACKING for speech styles). No two characters may sound the same. Vary sentence length, vocabulary, and register per character.
+    - CHAPTER HOOK: End this chapter with unresolved tension — a decision pending, a threat imminent, or a question unanswered.{f" Seed subtle anticipation for the next scene: '{next_chapter_hint}'." if next_chapter_hint else " Do not neatly resolve all threads."}
+
    QUALITY_CRITERIA:
    1. ENGAGEMENT & TENSION: Grip the reader. Ensure conflict/tension in every scene.
    2. SCENE EXECUTION: Flesh out the middle. Avoid summarizing key moments.
@@ -724,7 +733,7 @@ def write_chapter(chap, bp, folder, prev_sum, tracking=None, prev_content=None):
    CONTEXT:
    - STORY_SO_FAR: {prev_sum}
    {prev_context_block}
-    - CHARACTERS: {json.dumps(bp['characters'])}
+    - CHARACTERS: {json.dumps(chars_for_writer)}
    {char_visuals}
    - SCENE_BEATS: {json.dumps(chap['beats'])}
    
@@ -735,13 +744,15 @@ def write_chapter(chap, bp, folder, prev_sum, tracking=None, prev_content=None):
        resp_draft = ai.model_writer.generate_content(prompt)
        utils.log_usage(folder, ai.model_writer.name, resp_draft.usage_metadata)
        current_text = resp_draft.text
+        draft_words = len(current_text.split()) if current_text else 0
+        utils.log("WRITER", f"  -> Draft: {draft_words:,} words (target: ~{est_words})")
    except Exception as e:
        utils.log("WRITER", f"⚠️ Failed Ch {chap['chapter_number']}: {e}")
        return f"## Chapter {chap['chapter_number']} Failed\n\nError: {e}"

    # Refinement Loop
    max_attempts = 5
-    SCORE_AUTO_ACCEPT = 9
+    SCORE_AUTO_ACCEPT = 8   # 8 = professional quality; no marginal gain from extra refinement
    SCORE_PASSING = 7
    SCORE_REWRITE_THRESHOLD = 6
    
--- a/modules/utils.py
+++ b/modules/utils.py
@@ -71,7 +71,11 @@ def get_sorted_book_folders(run_dir):
    return sorted(subdirs, key=sort_key)

 # --- SHARED UTILS ---
-def log(phase, msg): 
+def log_banner(phase, title):
+    """Log a visually distinct phase separator line."""
+    log(phase, f"{'─' * 18} {title} {'─' * 18}")
+
+def log(phase, msg):
    timestamp = datetime.datetime.now().strftime('%H:%M:%S')
    line = f"[{timestamp}] {phase:<15} | {msg}"
    print(line)
--- a/templates/run_details.html
+++ b/templates/run_details.html
@@ -337,20 +337,69 @@
    const statusText = document.getElementById('status-text');
    const statusBar = document.getElementById('status-bar');
    const costEl = document.getElementById('run-cost');
-    
+
+    let lastLog = '';
+
+    // Phase → colour mapping (matches utils.log phase labels)
+    const PHASE_COLORS = {
+        'WRITER':    '#4fc3f7',
+        'ARCHITECT': '#81c784',
+        'TIMING':    '#78909c',
+        'SYSTEM':    '#fff176',
+        'TRACKER':   '#ce93d8',
+        'RESUME':    '#ffb74d',
+        'SERIES':    '#64b5f6',
+        'ENRICHER':  '#4dd0e1',
+        'HARVESTER': '#ff8a65',
+        'EDITOR':    '#f48fb1',
+    };
+
+    function escapeHtml(str) {
+        return str.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+    }
+
+    function colorizeLog(logText) {
+        if (!logText) return '';
+        return logText.split('\n').map(line => {
+            const m = line.match(/^(\[[\d:]+\])\s+(\w+)\s+\|(.*)$/);
+            if (!m) return '<span style="color:#666">' + escapeHtml(line) + '</span>';
+            const [, ts, phase, msg] = m;
+            const color = PHASE_COLORS[phase] || '#aaaaaa';
+            return '<span style="color:#555">' + escapeHtml(ts) + '</span> '
+                 + '<span style="color:' + color + ';font-weight:bold">' + phase.padEnd(14) + '</span>'
+                 + '<span style="color:#ccc">|' + escapeHtml(msg) + '</span>';
+        }).join('\n');
+    }
+
+    function getCurrentPhase(logText) {
+        if (!logText) return '';
+        const lines = logText.split('\n').filter(l => l.trim());
+        for (let k = lines.length - 1; k >= 0; k--) {
+            const m = lines[k].match(/\]\s+(\w+)\s+\|/);
+            if (m) return m[1];
+        }
+        return '';
+    }
+
    function updateLog() {
        fetch(`/run/${runId}/status`)
            .then(response => response.json())
            .then(data => {
-                // Update Status Text
-                statusText.innerText = "Status: " + data.status.charAt(0).toUpperCase() + data.status.slice(1);
+                // Update Status Text + current phase
+                const statusLabel = data.status.charAt(0).toUpperCase() + data.status.slice(1);
+                if (data.status === 'running') {
+                    const phase = getCurrentPhase(data.log);
+                    statusText.innerText = 'Status: Running' + (phase ? ' — ' + phase : '');
+                } else {
+                    statusText.innerText = 'Status: ' + statusLabel;
+                }
                costEl.innerText = '$' + parseFloat(data.cost).toFixed(4);
-                
+
                // Update Status Bar
                if (data.status === 'running' || data.status === 'queued') {
                    statusBar.className = "progress-bar progress-bar-striped progress-bar-animated";
                    statusBar.style.width = (data.percent || 5) + "%";
-                    
+
                    let label = (data.percent || 0) + "%";
                    if (data.status === 'running' && data.percent > 2 && data.start_time) {
                        const elapsed = (Date.now() / 1000) - data.start_time;
@@ -371,15 +420,16 @@
                    statusBar.innerText = "";
                }

-                // Update Log (only if changed to avoid scroll jitter)
-                if (consoleEl.innerText !== data.log) {
+                // Update Log with phase colorization (only if changed to avoid scroll jitter)
+                if (lastLog !== data.log) {
+                    lastLog = data.log;
                    const isScrolledToBottom = consoleEl.scrollHeight - consoleEl.clientHeight <= consoleEl.scrollTop + 50;
-                    consoleEl.innerText = data.log;
+                    consoleEl.innerHTML = colorizeLog(data.log);
                    if (isScrolledToBottom) {
                        consoleEl.scrollTop = consoleEl.scrollHeight;
                    }
                }
-                
+
                // Poll if running
                if (data.status === 'running' || data.status === 'queued') {
                    setTimeout(updateLog, 2000);