Persist discovery answers to CSV for reprocessing and recovery

Save each discovery conversation's prompts and answers to durable CSV
files (per-conversation + append-only master log) on both save and
completion, so answers survive an extraction error, can be re-fed to the
AI, and can be reviewed/resumed by the user.

- app/services/answer_store.py: canonical prompt list + atomic CSV writes,
  master append, and read-back helpers (DB stays system of record; CSV
  failures are logged, never fatal).
- discovery router: write CSV on /respond and /complete; new endpoints
  GET /answers, GET /answers.csv, POST /reprocess (shared extraction
  helper; locked profiles return 409).
- discovery.html: prefill/resume from saved answers after an error and a
  "Re-run analysis" button wired to /reprocess.
- scripts/reprocess_csv.py: offline CLI to re-run extraction from a CSV
  (print or --write-db).
- QUESTIONS_DIR / QUESTIONS_MASTER_CSV config, .gitignore, README, tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Dg6XWUwprmP5QCL18HxssY

app/routers/discovery.py

@@ -11,6 +11,7 @@ import uuid
 from datetime import datetime, timezone
 
 from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import Response
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -24,6 +25,8 @@ from app.models import (
     ReflectionMessage,
     User,
 )
+from app.services import answer_store
+from app.services.answer_store import DISCOVERY_PROMPTS
 from app.services.extractor import DiscoveryExtractionError, DiscoveryExtractor
 from app.services.reflector import (
     EDITABLE_FIELDS,
@@ -66,6 +69,60 @@ def _record_revision(
     )
 
 
+async def _generate_profile(
+    db: AsyncSession,
+    conversation: DiscoveryConversation,
+    source: str,
+) -> tuple[DiscoveryProfile, str | None]:
+    """Run the extractor over a conversation's answers and stage a new profile
+    (plus a revision snapshot) on the session. The caller commits.
+
+    Shared by ``/complete`` (source="extraction") and ``/reprocess``
+    (source="reprocess"). Raises HTTPException(400) when there is nothing to
+    analyze and HTTPException(502) on an extractor failure.
+    """
+    responses = {
+        p["key"]: getattr(conversation, p["column"], None) or ""
+        for p in DISCOVERY_PROMPTS
+    }
+    if not any(text.strip() for text in responses.values()):
+        raise HTTPException(
+            status_code=400, detail="No responses available to analyze"
+        )
+
+    api_key = os.getenv("ANTHROPIC_API_KEY")
+    model = os.getenv("ANTHROPIC_MODEL", "claude-sonnet-4-6")
+    try:
+        extractor = DiscoveryExtractor(api_key=api_key, model=model)
+        data = await extractor.extract(responses)
+    except DiscoveryExtractionError as exc:
+        raise HTTPException(status_code=502, detail=str(exc)) from exc
+
+    profile = DiscoveryProfile(
+        id=str(uuid.uuid4()),
+        user_id=conversation.user_id,
+        conversation_id=conversation.id,
+        generated_at=_now(),
+        triad=data.get("triad"),
+        probable_type=_as_int(data.get("probable_type")),
+        wing=_as_int(data.get("wing")),
+        instinctual_variant=data.get("instinctual_variant"),
+        instinctual_stack=data.get("instinctual_stack"),
+        love_summary=data.get("love_summary"),
+        strength_summary=data.get("strength_summary"),
+        mission_summary=data.get("mission_summary"),
+        vocation_summary=data.get("vocation_summary"),
+        overlap_narrative=data.get("overlap_narrative"),
+        short_term_goals=data.get("short_term_goals"),
+        long_term_goals=data.get("long_term_goals"),
+        confidence_json=json.dumps(data.get("confidence", {})),
+        locked=False,
+    )
+    db.add(profile)
+    _record_revision(db, profile, source=source)
+    return profile, data.get("extraction_notes")
+
+
 def _to_profile_response(
     profile: DiscoveryProfile, extraction_notes: str | None = None
 ) -> schemas.ProfileResponse:
@@ -161,6 +218,10 @@ async def save_responses(
     conversation.prompt_goals_long = payload.prompt_goals_long
     await db.commit()
 
+    # Durable CSV copy, written before extraction runs so the answers survive
+    # an extraction error and can be reviewed or re-processed later.
+    answer_store.save_answers(conversation, user, status="responses_saved")
+
     return schemas.RespondResponse(
         conversation_id=conversation_id, status="responses_saved"
     )
@@ -176,59 +237,99 @@ async def complete_conversation(
 ):
     conversation = await _owned_conversation(db, conversation_id, user)
 
-    responses = {
-        "alive": conversation.prompt_alive or "",
-        "friction": conversation.prompt_friction or "",
-        "pull": conversation.prompt_pull or "",
-        "recognition": conversation.prompt_recognition or "",
-        "future": conversation.prompt_future or "",
-        "goals_short": conversation.prompt_goals_short or "",
-        "goals_long": conversation.prompt_goals_long or "",
-    }
-    if not any(text.strip() for text in responses.values()):
-        raise HTTPException(
-            status_code=400, detail="No responses available to analyze"
-        )
-
-    api_key = os.getenv("ANTHROPIC_API_KEY")
-    model = os.getenv("ANTHROPIC_MODEL", "claude-sonnet-4-6")
-
-    try:
-        extractor = DiscoveryExtractor(api_key=api_key, model=model)
-        data = await extractor.extract(responses)
-    except DiscoveryExtractionError as exc:
-        raise HTTPException(status_code=502, detail=str(exc)) from exc
-
-    profile = DiscoveryProfile(
-        id=str(uuid.uuid4()),
-        user_id=conversation.user_id,
-        conversation_id=conversation.id,
-        generated_at=_now(),
-        triad=data.get("triad"),
-        probable_type=_as_int(data.get("probable_type")),
-        wing=_as_int(data.get("wing")),
-        instinctual_variant=data.get("instinctual_variant"),
-        instinctual_stack=data.get("instinctual_stack"),
-        love_summary=data.get("love_summary"),
-        strength_summary=data.get("strength_summary"),
-        mission_summary=data.get("mission_summary"),
-        vocation_summary=data.get("vocation_summary"),
-        overlap_narrative=data.get("overlap_narrative"),
-        short_term_goals=data.get("short_term_goals"),
-        long_term_goals=data.get("long_term_goals"),
-        confidence_json=json.dumps(data.get("confidence", {})),
-        locked=False,
+    profile, extraction_notes = await _generate_profile(
+        db, conversation, source="extraction"
     )
     conversation.completed_at = _now()
-    db.add(profile)
-    _record_revision(db, profile, source="extraction")
     await db.commit()
 
-    return _to_profile_response(
-        profile, extraction_notes=data.get("extraction_notes")
+    # Refresh the durable CSV copy now that the conversation is complete.
+    answer_store.save_answers(conversation, user, status="completed")
+
+    return _to_profile_response(profile, extraction_notes=extraction_notes)
+
+
+@router.get(
+    "/{conversation_id}/answers", response_model=schemas.AnswersResponse
+)
+async def get_answers(
+    conversation_id: str,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(get_current_user),
+):
+    """The saved prompts and answers for a conversation, so the person can
+    review or resume from their original responses (e.g. after an error)."""
+    conversation = await _owned_conversation(db, conversation_id, user)
+    answers = [
+        schemas.AnswerItem(
+            prompt_key=p["key"],
+            prompt_title=p["title"],
+            answer=getattr(conversation, p["column"], None) or "",
+        )
+        for p in DISCOVERY_PROMPTS
+    ]
+    return schemas.AnswersResponse(
+        conversation_id=conversation.id,
+        started_at=conversation.started_at,
+        completed_at=conversation.completed_at,
+        answers=answers,
     )
 
 
+@router.get("/{conversation_id}/answers.csv")
+async def download_answers_csv(
+    conversation_id: str,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(get_current_user),
+):
+    """Download a conversation's answers as a CSV file (built from the DB so it
+    works even if the on-disk copy was never written)."""
+    conversation = await _owned_conversation(db, conversation_id, user)
+    status = "completed" if conversation.completed_at else "responses_saved"
+    csv_text = answer_store.conversation_csv_text(conversation, user, status)
+    filename = f"discovery-{conversation_id}.csv"
+    return Response(
+        content=csv_text,
+        media_type="text/csv",
+        headers={
+            "Content-Disposition": f'attachment; filename="{filename}"'
+        },
+    )
+
+
+@router.post(
+    "/{conversation_id}/reprocess", response_model=schemas.ProfileResponse
+)
+async def reprocess_conversation(
+    conversation_id: str,
+    db: AsyncSession = Depends(get_db),
+    user: User = Depends(get_current_user),
+):
+    """Re-run AI extraction over a conversation's saved answers, producing a
+    fresh profile. Used to recover from an extraction error or to regenerate a
+    profile after the answers were re-fed. The latest profile must be unlocked.
+    """
+    conversation = await _owned_conversation(db, conversation_id, user)
+
+    existing = await _latest_profile(db, user.id)
+    if existing is not None and existing.locked:
+        raise HTTPException(
+            status_code=409,
+            detail="Profile is affirmed and locked; it cannot be reprocessed.",
+        )
+
+    profile, extraction_notes = await _generate_profile(
+        db, conversation, source="reprocess"
+    )
+    if conversation.completed_at is None:
+        conversation.completed_at = _now()
+    await db.commit()
+
+    answer_store.save_answers(conversation, user, status="completed")
+
+    return _to_profile_response(profile, extraction_notes=extraction_notes)
+
+
 @router.get("/profile/me", response_model=schemas.ProfileResponse)
 async def get_my_profile(
     db: AsyncSession = Depends(get_db),