Persist discovery answers to CSV for reprocessing and recovery

Save each discovery conversation's prompts and answers to durable CSV files (per-conversation + append-only master log) on both save and completion, so answers survive an extraction error, can be re-fed to the AI, and can be reviewed/resumed by the user. - app/services/answer_store.py: canonical prompt list + atomic CSV writes, master append, and read-back helpers (DB stays system of record; CSV failures are logged, never fatal). - discovery router: write CSV on /respond and /complete; new endpoints GET /answers, GET /answers.csv, POST /reprocess (shared extraction helper; locked profiles return 409). - discovery.html: prefill/resume from saved answers after an error and a "Re-run analysis" button wired to /reprocess. - scripts/reprocess_csv.py: offline CLI to re-run extraction from a CSV (print or --write-db). - QUESTIONS_DIR / QUESTIONS_MASTER_CSV config, .gitignore, README, tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Dg6XWUwprmP5QCL18HxssY
2026-08-27 08:50:36 +00:00 · 2026-06-19 01:06:40 +00:00
parent c1daef6411
commit 866bd60225
10 changed files with 988 additions and 51 deletions
@@ -0,0 +1,117 @@
+"""Unit tests for the CSV answer store.
+
+These exercise the file I/O directly with a lightweight stand-in for the
+DiscoveryConversation/User ORM objects, so they need no database. The
+QUESTIONS_DIR / QUESTIONS_MASTER_CSV env vars are pointed at tmp_path.
+"""
+import csv
+import os
+from types import SimpleNamespace
+
+from app.services import answer_store
+
+
+def _conversation(conv_id="conv-1", user_id="user-1", **answers):
+    base = {
+        "id": conv_id,
+        "user_id": user_id,
+        "prompt_alive": "",
+        "prompt_friction": "",
+        "prompt_pull": "",
+        "prompt_recognition": "",
+        "prompt_future": "",
+        "prompt_goals_short": "",
+        "prompt_goals_long": "",
+    }
+    base.update(answers)
+    return SimpleNamespace(**base)
+
+
+def _point_env(monkeypatch, tmp_path):
+    qdir = tmp_path / "questions"
+    master = tmp_path / "questions_master.csv"
+    monkeypatch.setenv("QUESTIONS_DIR", str(qdir))
+    monkeypatch.setenv("QUESTIONS_MASTER_CSV", str(master))
+    return qdir, master
+
+
+def test_write_conversation_csv_round_trips(monkeypatch, tmp_path):
+    _point_env(monkeypatch, tmp_path)
+    conv = _conversation(prompt_alive="felt alive", prompt_goals_long="big plans")
+    user = SimpleNamespace(email="a@b.com")
+
+    assert answer_store.write_conversation_csv(conv, user, "responses_saved")
+
+    path = answer_store.conversation_csv_path("conv-1")
+    assert os.path.exists(path)
+
+    answers = answer_store.read_conversation_answers("conv-1")
+    assert answers["alive"] == "felt alive"
+    assert answers["goals_long"] == "big plans"
+    # Unanswered prompts round-trip as empty strings, all 7 present.
+    assert len(answers) == 7
+    assert answers["friction"] == ""
+
+
+def test_write_is_atomic_overwrite(monkeypatch, tmp_path):
+    _point_env(monkeypatch, tmp_path)
+    user = SimpleNamespace(email="a@b.com")
+
+    answer_store.write_conversation_csv(
+        _conversation(prompt_alive="first"), user, "responses_saved"
+    )
+    answer_store.write_conversation_csv(
+        _conversation(prompt_alive="second"), user, "completed"
+    )
+
+    answers = answer_store.read_conversation_answers("conv-1")
+    assert answers["alive"] == "second"
+    # No leftover temp file.
+    assert not os.path.exists(
+        answer_store.conversation_csv_path("conv-1") + ".tmp"
+    )
+
+
+def test_append_master_writes_header_once(monkeypatch, tmp_path):
+    _, master = _point_env(monkeypatch, tmp_path)
+    user = SimpleNamespace(email="a@b.com")
+
+    answer_store.append_master(
+        _conversation("c1", prompt_alive="x"), user, "responses_saved"
+    )
+    answer_store.append_master(
+        _conversation("c2", prompt_alive="y"), user, "completed"
+    )
+
+    with open(master, newline="", encoding="utf-8") as fh:
+        rows = list(csv.DictReader(fh))
+    # 7 prompts per event, two events.
+    assert len(rows) == 14
+    conv_ids = {r["conversation_id"] for r in rows}
+    assert conv_ids == {"c1", "c2"}
+    statuses = {r["status"] for r in rows}
+    assert statuses == {"responses_saved", "completed"}
+
+
+def test_read_missing_file_returns_empty(monkeypatch, tmp_path):
+    _point_env(monkeypatch, tmp_path)
+    assert answer_store.read_conversation_answers("nope") == {}
+
+
+def test_conversation_csv_text_from_db(monkeypatch, tmp_path):
+    _point_env(monkeypatch, tmp_path)
+    conv = _conversation(prompt_pull="natural pull")
+    user = SimpleNamespace(email="a@b.com")
+    text = answer_store.conversation_csv_text(conv, user, "completed")
+    assert "prompt_key" in text  # header present
+    assert "natural pull" in text
+    assert text.count("\n") >= 8  # header + 7 rows
+
+
+def test_save_answers_writes_both_artifacts(monkeypatch, tmp_path):
+    qdir, master = _point_env(monkeypatch, tmp_path)
+    conv = _conversation(prompt_alive="hi")
+    user = SimpleNamespace(email="a@b.com")
+    answer_store.save_answers(conv, user, "responses_saved")
+    assert os.path.exists(answer_store.conversation_csv_path("conv-1"))
+    assert os.path.exists(master)