mirror of
https://github.com/computerim/impactflow-discovery.git
synced 2026-08-27 08:50:36 +00:00
Persist discovery answers to CSV for reprocessing and recovery
Save each discovery conversation's prompts and answers to durable CSV files (per-conversation + append-only master log) on both save and completion, so answers survive an extraction error, can be re-fed to the AI, and can be reviewed/resumed by the user. - app/services/answer_store.py: canonical prompt list + atomic CSV writes, master append, and read-back helpers (DB stays system of record; CSV failures are logged, never fatal). - discovery router: write CSV on /respond and /complete; new endpoints GET /answers, GET /answers.csv, POST /reprocess (shared extraction helper; locked profiles return 409). - discovery.html: prefill/resume from saved answers after an error and a "Re-run analysis" button wired to /reprocess. - scripts/reprocess_csv.py: offline CLI to re-run extraction from a CSV (print or --write-db). - QUESTIONS_DIR / QUESTIONS_MASTER_CSV config, .gitignore, README, tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Dg6XWUwprmP5QCL18HxssY
This commit is contained in:
Claude
@@ -0,0 +1,223 @@
|
||||
"""Durable CSV record of discovery questions and answers.
|
||||
|
||||
The SQLite database remains the system of record. This module keeps a
|
||||
secondary, portable copy of every conversation's prompts and answers on disk
|
||||
so that:
|
||||
|
||||
* answers survive even if the AI extraction step errors (they are written
|
||||
on save, before extraction runs);
|
||||
* a saved conversation can be re-fed to the extractor (``reprocess``);
|
||||
* a person can review or resume from their original answers.
|
||||
|
||||
Two artifacts are written for every save:
|
||||
|
||||
* a *per-conversation* file (``data/questions/{conversation_id}.csv``) that
|
||||
is rewritten in full on each save — always the latest answers for that
|
||||
conversation, easy to hand to an AI or download; and
|
||||
* a *master* append-only log (``data/questions_master.csv``) that records
|
||||
every save/complete event across all conversations, for batch
|
||||
re-processing.
|
||||
|
||||
CSV writes must never break an API request: the caller's DB commit has
|
||||
already succeeded, so any I/O failure here is logged and swallowed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - typing only
|
||||
from app.models import DiscoveryConversation, User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Canonical list of discovery prompts: the short key used by the extractor and
|
||||
# CSV, the DB column on DiscoveryConversation, and the human-facing title.
|
||||
# This is the single source of truth shared by the store, the router, and the
|
||||
# reprocess paths.
|
||||
DISCOVERY_PROMPTS = [
|
||||
{"key": "alive", "column": "prompt_alive", "title": "The Alive Moment"},
|
||||
{"key": "friction", "column": "prompt_friction", "title": "The Friction Moment"},
|
||||
{"key": "pull", "column": "prompt_pull", "title": "The Natural Pull"},
|
||||
{"key": "recognition", "column": "prompt_recognition", "title": "The Recognition Moment"},
|
||||
{"key": "future", "column": "prompt_future", "title": "The Future Pull"},
|
||||
{"key": "goals_short", "column": "prompt_goals_short", "title": "Near-Term Goals (6–12 months)"},
|
||||
{"key": "goals_long", "column": "prompt_goals_long", "title": "Long-Term Goals (3–5 years)"},
|
||||
]
|
||||
|
||||
CSV_FIELDS = [
|
||||
"conversation_id",
|
||||
"user_id",
|
||||
"user_email",
|
||||
"status",
|
||||
"saved_at",
|
||||
"prompt_key",
|
||||
"prompt_title",
|
||||
"answer",
|
||||
]
|
||||
|
||||
|
||||
def _questions_dir() -> str:
|
||||
return os.getenv("QUESTIONS_DIR", "./data/questions")
|
||||
|
||||
|
||||
def _master_path() -> str:
|
||||
return os.getenv("QUESTIONS_MASTER_CSV", "./data/questions_master.csv")
|
||||
|
||||
|
||||
def conversation_csv_path(conversation_id: str) -> str:
|
||||
"""Filesystem path of the per-conversation CSV for ``conversation_id``."""
|
||||
return os.path.join(_questions_dir(), f"{conversation_id}.csv")
|
||||
|
||||
|
||||
def _rows_for(
|
||||
conversation: "DiscoveryConversation",
|
||||
user_email: str,
|
||||
status: str,
|
||||
saved_at: str,
|
||||
) -> list[dict]:
|
||||
"""One row per prompt, in canonical prompt order."""
|
||||
rows = []
|
||||
for prompt in DISCOVERY_PROMPTS:
|
||||
answer = getattr(conversation, prompt["column"], None) or ""
|
||||
rows.append(
|
||||
{
|
||||
"conversation_id": conversation.id,
|
||||
"user_id": conversation.user_id,
|
||||
"user_email": user_email,
|
||||
"status": status,
|
||||
"saved_at": saved_at,
|
||||
"prompt_key": prompt["key"],
|
||||
"prompt_title": prompt["title"],
|
||||
"answer": answer,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def write_conversation_csv(
|
||||
conversation: "DiscoveryConversation",
|
||||
user: "User | None",
|
||||
status: str,
|
||||
) -> bool:
|
||||
"""Rewrite the per-conversation CSV with the conversation's current answers.
|
||||
|
||||
The write is atomic (temp file + ``os.replace``) so a crash mid-write can
|
||||
never leave a half-written file. Returns True on success, False if the
|
||||
write failed (failures are logged, never raised).
|
||||
"""
|
||||
saved_at = datetime.now(timezone.utc).isoformat()
|
||||
email = getattr(user, "email", "") or ""
|
||||
rows = _rows_for(conversation, email, status, saved_at)
|
||||
path = conversation_csv_path(conversation.id)
|
||||
tmp_path = f"{path}.tmp"
|
||||
try:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(tmp_path, "w", newline="", encoding="utf-8") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=CSV_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
os.replace(tmp_path, path)
|
||||
return True
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to write conversation CSV for %s: %s", conversation.id, exc
|
||||
)
|
||||
try:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def append_master(
|
||||
conversation: "DiscoveryConversation",
|
||||
user: "User | None",
|
||||
status: str,
|
||||
) -> bool:
|
||||
"""Append this save/complete event's rows to the master log.
|
||||
|
||||
Writes the header row once, when the file is first created. Returns True on
|
||||
success, False on a logged (never raised) failure.
|
||||
"""
|
||||
saved_at = datetime.now(timezone.utc).isoformat()
|
||||
email = getattr(user, "email", "") or ""
|
||||
rows = _rows_for(conversation, email, status, saved_at)
|
||||
path = _master_path()
|
||||
try:
|
||||
directory = os.path.dirname(path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
is_new = not os.path.exists(path) or os.path.getsize(path) == 0
|
||||
with open(path, "a", newline="", encoding="utf-8") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=CSV_FIELDS)
|
||||
if is_new:
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
return True
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to append master CSV for %s: %s", conversation.id, exc
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def save_answers(
|
||||
conversation: "DiscoveryConversation",
|
||||
user: "User | None",
|
||||
status: str,
|
||||
) -> None:
|
||||
"""Persist both CSV artifacts for a conversation. Never raises."""
|
||||
write_conversation_csv(conversation, user, status)
|
||||
append_master(conversation, user, status)
|
||||
|
||||
|
||||
def conversation_csv_text(
|
||||
conversation: "DiscoveryConversation",
|
||||
user: "User | None",
|
||||
status: str = "responses_saved",
|
||||
) -> str:
|
||||
"""Render a conversation's answers as CSV text, built from the DB row.
|
||||
|
||||
Used by the download endpoint so it works even if the on-disk file was
|
||||
never written (DB stays the system of record).
|
||||
"""
|
||||
import io
|
||||
|
||||
saved_at = datetime.now(timezone.utc).isoformat()
|
||||
email = getattr(user, "email", "") or ""
|
||||
rows = _rows_for(conversation, email, status, saved_at)
|
||||
buf = io.StringIO()
|
||||
writer = csv.DictWriter(buf, fieldnames=CSV_FIELDS)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def read_conversation_answers(conversation_id: str) -> dict[str, str]:
|
||||
"""Read saved answers from the per-conversation CSV.
|
||||
|
||||
Returns a ``{prompt_key: answer}`` dict, or ``{}`` if the file is missing
|
||||
or unreadable. The per-conversation file is rewritten in full on each save,
|
||||
so it always holds the latest answers.
|
||||
"""
|
||||
path = conversation_csv_path(conversation_id)
|
||||
answers: dict[str, str] = {}
|
||||
try:
|
||||
with open(path, newline="", encoding="utf-8") as fh:
|
||||
for row in csv.DictReader(fh):
|
||||
key = row.get("prompt_key")
|
||||
if key:
|
||||
answers[key] = row.get("answer", "") or ""
|
||||
except FileNotFoundError:
|
||||
return {}
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Failed to read conversation CSV for %s: %s", conversation_id, exc
|
||||
)
|
||||
return {}
|
||||
return answers
|
||||
Reference in New Issue
Block a user