Persist discovery answers to CSV for reprocessing and recovery

Save each discovery conversation's prompts and answers to durable CSV files (per-conversation + append-only master log) on both save and completion, so answers survive an extraction error, can be re-fed to the AI, and can be reviewed/resumed by the user. - app/services/answer_store.py: canonical prompt list + atomic CSV writes, master append, and read-back helpers (DB stays system of record; CSV failures are logged, never fatal). - discovery router: write CSV on /respond and /complete; new endpoints GET /answers, GET /answers.csv, POST /reprocess (shared extraction helper; locked profiles return 409). - discovery.html: prefill/resume from saved answers after an error and a "Re-run analysis" button wired to /reprocess. - scripts/reprocess_csv.py: offline CLI to re-run extraction from a CSV (print or --write-db). - QUESTIONS_DIR / QUESTIONS_MASTER_CSV config, .gitignore, README, tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Dg6XWUwprmP5QCL18HxssY
2026-08-27 08:50:36 +00:00 · 2026-06-19 01:06:40 +00:00
parent c1daef6411
commit 866bd60225
10 changed files with 988 additions and 51 deletions
@@ -0,0 +1,117 @@
+"""Unit tests for the CSV answer store.
+
+These exercise the file I/O directly with a lightweight stand-in for the
+DiscoveryConversation/User ORM objects, so they need no database. The
+QUESTIONS_DIR / QUESTIONS_MASTER_CSV env vars are pointed at tmp_path.
+"""
+import csv
+import os
+from types import SimpleNamespace
+
+from app.services import answer_store
+
+
+def _conversation(conv_id="conv-1", user_id="user-1", **answers):
+    base = {
+        "id": conv_id,
+        "user_id": user_id,
+        "prompt_alive": "",
+        "prompt_friction": "",
+        "prompt_pull": "",
+        "prompt_recognition": "",
+        "prompt_future": "",
+        "prompt_goals_short": "",
+        "prompt_goals_long": "",
+    }
+    base.update(answers)
+    return SimpleNamespace(**base)
+
+
+def _point_env(monkeypatch, tmp_path):
+    qdir = tmp_path / "questions"
+    master = tmp_path / "questions_master.csv"
+    monkeypatch.setenv("QUESTIONS_DIR", str(qdir))
+    monkeypatch.setenv("QUESTIONS_MASTER_CSV", str(master))
+    return qdir, master
+
+
+def test_write_conversation_csv_round_trips(monkeypatch, tmp_path):
+    _point_env(monkeypatch, tmp_path)
+    conv = _conversation(prompt_alive="felt alive", prompt_goals_long="big plans")
+    user = SimpleNamespace(email="a@b.com")
+
+    assert answer_store.write_conversation_csv(conv, user, "responses_saved")
+
+    path = answer_store.conversation_csv_path("conv-1")
+    assert os.path.exists(path)
+
+    answers = answer_store.read_conversation_answers("conv-1")
+    assert answers["alive"] == "felt alive"
+    assert answers["goals_long"] == "big plans"
+    # Unanswered prompts round-trip as empty strings, all 7 present.
+    assert len(answers) == 7
+    assert answers["friction"] == ""
+
+
+def test_write_is_atomic_overwrite(monkeypatch, tmp_path):
+    _point_env(monkeypatch, tmp_path)
+    user = SimpleNamespace(email="a@b.com")
+
+    answer_store.write_conversation_csv(
+        _conversation(prompt_alive="first"), user, "responses_saved"
+    )
+    answer_store.write_conversation_csv(
+        _conversation(prompt_alive="second"), user, "completed"
+    )
+
+    answers = answer_store.read_conversation_answers("conv-1")
+    assert answers["alive"] == "second"
+    # No leftover temp file.
+    assert not os.path.exists(
+        answer_store.conversation_csv_path("conv-1") + ".tmp"
+    )
+
+
+def test_append_master_writes_header_once(monkeypatch, tmp_path):
+    _, master = _point_env(monkeypatch, tmp_path)
+    user = SimpleNamespace(email="a@b.com")
+
+    answer_store.append_master(
+        _conversation("c1", prompt_alive="x"), user, "responses_saved"
+    )
+    answer_store.append_master(
+        _conversation("c2", prompt_alive="y"), user, "completed"
+    )
+
+    with open(master, newline="", encoding="utf-8") as fh:
+        rows = list(csv.DictReader(fh))
+    # 7 prompts per event, two events.
+    assert len(rows) == 14
+    conv_ids = {r["conversation_id"] for r in rows}
+    assert conv_ids == {"c1", "c2"}
+    statuses = {r["status"] for r in rows}
+    assert statuses == {"responses_saved", "completed"}
+
+
+def test_read_missing_file_returns_empty(monkeypatch, tmp_path):
+    _point_env(monkeypatch, tmp_path)
+    assert answer_store.read_conversation_answers("nope") == {}
+
+
+def test_conversation_csv_text_from_db(monkeypatch, tmp_path):
+    _point_env(monkeypatch, tmp_path)
+    conv = _conversation(prompt_pull="natural pull")
+    user = SimpleNamespace(email="a@b.com")
+    text = answer_store.conversation_csv_text(conv, user, "completed")
+    assert "prompt_key" in text  # header present
+    assert "natural pull" in text
+    assert text.count("\n") >= 8  # header + 7 rows
+
+
+def test_save_answers_writes_both_artifacts(monkeypatch, tmp_path):
+    qdir, master = _point_env(monkeypatch, tmp_path)
+    conv = _conversation(prompt_alive="hi")
+    user = SimpleNamespace(email="a@b.com")
+    answer_store.save_answers(conv, user, "responses_saved")
+    assert os.path.exists(answer_store.conversation_csv_path("conv-1"))
+    assert os.path.exists(master)
@@ -0,0 +1,158 @@
+"""Tests for the discovery CSV/reprocess endpoints.
+
+Run under the X-API-Key admin identity so they don't depend on the Google
+OAuth flow. The DiscoveryExtractor is replaced with a fake so no network call
+or API key is needed. QUESTIONS_DIR / QUESTIONS_MASTER_CSV are pointed at
+tmp_path so the suite never writes into the repo's data dir.
+"""
+import os
+
+from app.services import answer_store
+
+API_KEY = {"X-API-Key": "test-api-key"}
+
+
+class FakeExtractor:
+    """Stand-in for DiscoveryExtractor: returns a fixed well-formed profile."""
+
+    def __init__(self, api_key=None, model=None):
+        pass
+
+    async def extract(self, responses):
+        return {
+            "triad": "gut",
+            "probable_type": 8,
+            "wing": 9,
+            "instinctual_variant": "sp",
+            "instinctual_stack": "sp/so/sx",
+            "love_summary": "love",
+            "strength_summary": "strength",
+            "mission_summary": "mission",
+            "vocation_summary": "vocation",
+            "overlap_narrative": "narrative",
+            "short_term_goals": "short",
+            "long_term_goals": "long",
+            "confidence": {
+                "triad": "high",
+                "type": "high",
+                "variant": "medium",
+                "ikigai": "high",
+            },
+            "extraction_notes": "ok",
+        }
+
+
+def _patch(monkeypatch, tmp_path):
+    monkeypatch.setenv("QUESTIONS_DIR", str(tmp_path / "questions"))
+    monkeypatch.setenv("QUESTIONS_MASTER_CSV", str(tmp_path / "master.csv"))
+    monkeypatch.setattr(
+        "app.routers.discovery.DiscoveryExtractor", FakeExtractor
+    )
+
+
+async def _start_and_respond(app_client):
+    start = await app_client.post("/discovery/start", headers=API_KEY)
+    assert start.status_code == 200
+    conv_id = start.json()["conversation_id"]
+    r = await app_client.put(
+        f"/discovery/{conv_id}/respond",
+        headers=API_KEY,
+        json={
+            "prompt_alive": "I felt alive building things",
+            "prompt_friction": "unfairness bothers me",
+            "prompt_goals_short": "ship the app",
+        },
+    )
+    assert r.status_code == 200
+    return conv_id
+
+
+async def test_respond_writes_csv(app_client, monkeypatch, tmp_path):
+    _patch(monkeypatch, tmp_path)
+    conv_id = await _start_and_respond(app_client)
+
+    assert os.path.exists(answer_store.conversation_csv_path(conv_id))
+    answers = answer_store.read_conversation_answers(conv_id)
+    assert answers["alive"] == "I felt alive building things"
+    assert os.path.exists(str(tmp_path / "master.csv"))
+
+
+async def test_get_answers(app_client, monkeypatch, tmp_path):
+    _patch(monkeypatch, tmp_path)
+    conv_id = await _start_and_respond(app_client)
+
+    r = await app_client.get(f"/discovery/{conv_id}/answers", headers=API_KEY)
+    assert r.status_code == 200
+    body = r.json()
+    assert body["conversation_id"] == conv_id
+    assert body["completed_at"] is None
+    by_key = {a["prompt_key"]: a["answer"] for a in body["answers"]}
+    assert by_key["friction"] == "unfairness bothers me"
+    assert len(body["answers"]) == 7
+
+
+async def test_download_answers_csv(app_client, monkeypatch, tmp_path):
+    _patch(monkeypatch, tmp_path)
+    conv_id = await _start_and_respond(app_client)
+
+    r = await app_client.get(
+        f"/discovery/{conv_id}/answers.csv", headers=API_KEY
+    )
+    assert r.status_code == 200
+    assert r.headers["content-type"].startswith("text/csv")
+    assert "attachment" in r.headers["content-disposition"]
+    assert "ship the app" in r.text
+
+
+async def test_reprocess_creates_profile(app_client, monkeypatch, tmp_path):
+    _patch(monkeypatch, tmp_path)
+    conv_id = await _start_and_respond(app_client)
+
+    r = await app_client.post(
+        f"/discovery/{conv_id}/reprocess", headers=API_KEY
+    )
+    assert r.status_code == 200
+    body = r.json()
+    assert body["conversation_id"] == conv_id
+    assert body["triad"] == "gut"
+    assert body["probable_type"] == 8
+
+    # A profile now exists for the user.
+    prof = await app_client.get("/discovery/profile/me", headers=API_KEY)
+    assert prof.status_code == 200
+
+
+async def test_reprocess_locked_returns_409(app_client, monkeypatch, tmp_path):
+    _patch(monkeypatch, tmp_path)
+    conv_id = await _start_and_respond(app_client)
+
+    # Complete to create a profile, then lock it via confirm.
+    complete = await app_client.post(
+        f"/discovery/{conv_id}/complete", headers=API_KEY
+    )
+    assert complete.status_code == 200
+    confirm = await app_client.put(
+        "/discovery/profile/me/confirm", headers=API_KEY
+    )
+    assert confirm.status_code == 200
+
+    r = await app_client.post(
+        f"/discovery/{conv_id}/reprocess", headers=API_KEY
+    )
+    assert r.status_code == 409
+
+
+async def test_complete_writes_completed_csv(app_client, monkeypatch, tmp_path):
+    _patch(monkeypatch, tmp_path)
+    conv_id = await _start_and_respond(app_client)
+
+    complete = await app_client.post(
+        f"/discovery/{conv_id}/complete", headers=API_KEY
+    )
+    assert complete.status_code == 200
+
+    # The answers endpoint now reports a completion timestamp.
+    answers = await app_client.get(
+        f"/discovery/{conv_id}/answers", headers=API_KEY
+    )
+    assert answers.json()["completed_at"] is not None