diff --git a/.github/workflows/azure-static-web-apps-jolly-ocean-0fcd5df0f.yml b/.github/workflows/azure-static-web-apps-jolly-ocean-0fcd5df0f.yml index 1061902..484b4fa 100644 --- a/.github/workflows/azure-static-web-apps-jolly-ocean-0fcd5df0f.yml +++ b/.github/workflows/azure-static-web-apps-jolly-ocean-0fcd5df0f.yml @@ -33,6 +33,8 @@ jobs: run: | echo "VITE_API_BASE=${{ secrets.VITE_API_BASE }}" >> .env echo "VITE_CLERK_PUBLISHABLE_KEY=${{ secrets.VITE_CLERK_PUBLISHABLE_KEY }}" >> .env + echo "VITE_POSTHOG_KEY=${{ secrets.VITE_POSTHOG_KEY }}" >> .env + echo "VITE_POSTHOG_HOST=${{ secrets.VITE_POSTHOG_HOST }}" >> .env - name: Install working-directory: packages/frontend diff --git a/packages/backend/.env.example b/packages/backend/.env.example index d76c12f..e81251a 100644 --- a/packages/backend/.env.example +++ b/packages/backend/.env.example @@ -9,3 +9,10 @@ INNGEST_SIGNING_KEY= CLERK_JWKS_URL=https://.clerk.accounts.dev/.well-known/jwks.json # Comma-separated list of allowed frontend origins (matches the `azp` JWT claim) CLERK_AUTHORIZED_PARTIES=["http://localhost:5173"] + +# PostHog analytics (optional — leave POSTHOG_ENABLED=false to disable entirely) +POSTHOG_API_KEY= +POSTHOG_HOST=https://us.i.posthog.com +POSTHOG_ENABLED=false + +POSTHOG_REDACT_PROMPTS=true diff --git a/packages/backend/app/ai/core/__init__.py b/packages/backend/app/ai/core/__init__.py index 5efebf8..1c3abd7 100644 --- a/packages/backend/app/ai/core/__init__.py +++ b/packages/backend/app/ai/core/__init__.py @@ -1,4 +1,4 @@ -from app.ai.core.client import DEFAULT_MODEL, GUARDRAIL_MODEL, client +from app.ai.core.client import DEFAULT_MODEL, GUARDRAIL_MODEL, client, posthog_client from app.ai.core.output_cleaner import ( clean_section_output, promote_h1_headings, @@ -18,6 +18,7 @@ __all__ = [ "client", + "posthog_client", "DEFAULT_MODEL", "GUARDRAIL_MODEL", "truncate_section", diff --git a/packages/backend/app/ai/core/client.py b/packages/backend/app/ai/core/client.py index 839c406..9b77eb4 100644 --- a/packages/backend/app/ai/core/client.py +++ b/packages/backend/app/ai/core/client.py @@ -1,10 +1,16 @@ -from openai import AsyncOpenAI +from posthog import Posthog +from posthog.ai.openai import AsyncOpenAI from app.config import settings +posthog_client: Posthog | None = None +if settings.POSTHOG_ENABLED and settings.POSTHOG_API_KEY: + posthog_client = Posthog(settings.POSTHOG_API_KEY, host=settings.POSTHOG_HOST) + client = AsyncOpenAI( base_url=settings.AZURE_OPENAI_ENDPOINT, - api_key=settings.AZURE_OPENAI_API_KEY + api_key=settings.AZURE_OPENAI_API_KEY, + posthog_client=posthog_client, ) DEFAULT_MODEL = settings.AZURE_OPENAI_DEPLOYMENT GUARDRAIL_MODEL = settings.AZURE_OPENAI_GUARDRAIL_DEPLOYMENT diff --git a/packages/backend/app/config.py b/packages/backend/app/config.py index ccfb216..d184318 100644 --- a/packages/backend/app/config.py +++ b/packages/backend/app/config.py @@ -30,6 +30,12 @@ class Settings(BaseSettings): GUIDED_DOCUMENT_COST: int = 3 EDITOR_DOCUMENT_COST: int = 1 + # PostHog analytics — all disabled by default; set POSTHOG_ENABLED=true to activate + POSTHOG_API_KEY: str = "" + POSTHOG_HOST: str = "https://us.i.posthog.com" + POSTHOG_ENABLED: bool = False + POSTHOG_REDACT_PROMPTS: bool = True + model_config = {"env_file": ENV_FILE, "extra": "ignore"} diff --git a/packages/backend/app/guardrails/input.py b/packages/backend/app/guardrails/input.py index 94540a9..9d0c961 100644 --- a/packages/backend/app/guardrails/input.py +++ b/packages/backend/app/guardrails/input.py @@ -103,7 +103,13 @@ def validate_refinement_message(message: str) -> ValidationError | None: } -async def validate_message_intent(message: str, section_type: str, section_summary: str | None = None) -> ValidationError | None: +async def validate_message_intent( + message: str, + section_type: str, + section_summary: str | None = None, + posthog_distinct_id: str | None = None, + posthog_properties: dict | None = None, +) -> ValidationError | None: """LLM-based intent classifier: rejects messages unrelated to document editing. Failures are swallowed — a guardrail error must never block the user. @@ -115,18 +121,23 @@ async def validate_message_intent(message: str, section_type: str, section_summa system_prompt = load_yaml_prompt("guardrails", "intent_classification", "system") try: - response = await client.chat.completions.create( - model=GUARDRAIL_MODEL, - messages=[ + create_kwargs: dict = { + "model": GUARDRAIL_MODEL, + "messages": [ {"role": "system", "content": system_prompt}, { "role": "user", "content": f"Section: {section_type}\nSummary: {section_summary}\nMessage: {message}", }, ], - response_format=_INTENT_SCHEMA, - temperature=0.0, - ) + "response_format": _INTENT_SCHEMA, + "temperature": 1.0, + } + if posthog_distinct_id is not None: + create_kwargs["posthog_distinct_id"] = posthog_distinct_id + if posthog_properties is not None: + create_kwargs["posthog_properties"] = posthog_properties + response = await client.chat.completions.create(**create_kwargs) result = json.loads(response.choices[0].message.content) if not result.get("is_valid", True): logger.warning( diff --git a/packages/backend/app/guardrails/output.py b/packages/backend/app/guardrails/output.py index f2919da..0612cf3 100644 --- a/packages/backend/app/guardrails/output.py +++ b/packages/backend/app/guardrails/output.py @@ -17,6 +17,8 @@ async def call_with_retry( temperature: float = 0.3, required_fields: list[str] | None = None, section_type: str | None = None, + posthog_distinct_id: str | None = None, + posthog_properties: dict | None = None, ) -> Any: """ Call the AI model and validate the structured response. @@ -37,6 +39,10 @@ async def call_with_retry( kwargs["tools"] = tools if tool_choice is not None: kwargs["tool_choice"] = tool_choice + if posthog_distinct_id is not None: + kwargs["posthog_distinct_id"] = posthog_distinct_id + if posthog_properties is not None: + kwargs["posthog_properties"] = posthog_properties response = await client.chat.completions.create(**kwargs) error = _validate_response(response, required_fields) diff --git a/packages/backend/app/phases/alignment/ai.py b/packages/backend/app/phases/alignment/ai.py index 1df76d1..dba2a07 100644 --- a/packages/backend/app/phases/alignment/ai.py +++ b/packages/backend/app/phases/alignment/ai.py @@ -60,6 +60,8 @@ async def generate_alignment( rejected_sections: list[dict] | None = None, db: AsyncSession | None = None, document_type_id: uuid.UUID | None = None, + posthog_distinct_id: str | None = None, + doc_id: str | None = None, ) -> dict: """Generate 1-2 sentence summaries for each section using per-section discovery contexts.""" logger.info( @@ -85,6 +87,11 @@ async def generate_alignment( response_format=ALIGNMENT_SCHEMA, temperature=0.3, required_fields=["summaries"], + posthog_distinct_id=posthog_distinct_id, + posthog_properties=( + {"$ai_trace_id": doc_id, "$ai_parent_id": f"alignment-{doc_id}"} + if doc_id else None + ), ) log_usage("alignment", response.usage) diff --git a/packages/backend/app/phases/alignment/contract.py b/packages/backend/app/phases/alignment/contract.py index 8f2a953..e4a01ae 100644 --- a/packages/backend/app/phases/alignment/contract.py +++ b/packages/backend/app/phases/alignment/contract.py @@ -54,6 +54,8 @@ async def extract_document_contract( user_preferences: str | None = None, db: AsyncSession | None = None, document_type_id: uuid.UUID | None = None, + posthog_distinct_id: str | None = None, + doc_id: str | None = None, ) -> dict: """Extract a structured document contract from the approved alignment summaries.""" logger.info("[AI:contract] extracting document contract") @@ -83,6 +85,11 @@ async def extract_document_contract( response_format=CONTRACT_SCHEMA, temperature=0.1, required_fields=["entities", "decisions", "terminology", "constraints"], + posthog_distinct_id=posthog_distinct_id, + posthog_properties=( + {"$ai_trace_id": doc_id, "$ai_parent_id": f"alignment-{doc_id}"} + if doc_id else None + ), ) log_usage("contract", response.usage) diff --git a/packages/backend/app/phases/alignment/function.py b/packages/backend/app/phases/alignment/function.py index 16ded85..70beff1 100644 --- a/packages/backend/app/phases/alignment/function.py +++ b/packages/backend/app/phases/alignment/function.py @@ -9,6 +9,7 @@ from app.phases._shared.concurrency import DOC_CONCURRENCY from app.phases._shared.failure import workflow_on_failure from app.services import db as db_service +from app.services.observability import capture_event, capture_span @inngest_client.create_function( @@ -21,6 +22,7 @@ async def function(ctx: inngest.Context): step = ctx.step doc_id = ctx.event.data["document_id"] document_type_id = ctx.event.data.get("document_type_id") + user_id = ctx.event.data.get("user_id", "") logger.info("[orchestrator] ALIGNMENT start | doc_id={}", doc_id) @@ -66,6 +68,8 @@ async def _generate_summaries(sc=section_contexts): None, db=db, document_type_id=uuid.UUID(document_type_id) if document_type_id else None, + posthog_distinct_id=user_id, + doc_id=doc_id, ) summaries = await step.run(f"generate-summaries-{iteration}", _generate_summaries) @@ -105,6 +109,8 @@ async def _extract_contract(s=summaries, sc=section_contexts): user_preferences=doc.user_preferences, db=db, document_type_id=uuid.UUID(document_type_id) if document_type_id else None, + posthog_distinct_id=user_id, + doc_id=doc_id, ) import json as _json raw = _json.dumps(contract) @@ -113,17 +119,35 @@ async def _extract_contract(s=summaries, sc=section_contexts): await step.run("extract-contract", _extract_contract) else: + rejected = approval_event.data.get("rejected", []) logger.info( "[orchestrator] alignment rejected sections={} | doc_id={}", - approval_event.data.get("rejected", []), + rejected, doc_id, ) + async def _capture_rejected(r=rejected, it=iteration): + for section_type in r: + capture_event(user_id, doc_id, "alignment_section_reopened", {"section_type": section_type, "iteration": it}) + + await step.run(f"capture-alignment-rejected-{iteration}", _capture_rejected) + + async def _capture_phase_span(): + capture_span( + user_id, doc_id, + span_id=f"alignment-{doc_id}", + span_name="alignment", + input_state="Section contexts reviewed", + output_state="Alignment approved", + ) + + await step.run("capture-span-alignment", _capture_phase_span) + logger.info("[orchestrator] emitting alignment.completed | doc_id={}", doc_id) await step.send_event( "emit-alignment-completed", inngest.Event( name="docforge/alignment.completed", - data={"document_id": doc_id, "document_type_id": document_type_id}, + data={"document_id": doc_id, "document_type_id": document_type_id, "user_id": user_id}, ), ) diff --git a/packages/backend/app/phases/audit/ai.py b/packages/backend/app/phases/audit/ai.py index c71e727..bdac65b 100644 --- a/packages/backend/app/phases/audit/ai.py +++ b/packages/backend/app/phases/audit/ai.py @@ -88,7 +88,7 @@ def build_audit_context( return "\n\n".join(parts) -async def run_audit(doc_id: str, document_type_id: str | None = None) -> dict: +async def run_audit(doc_id: str, document_type_id: str | None = None, posthog_distinct_id: str | None = None) -> dict: """Run audit on all 4 finalized sections.""" logger.info("[AI:audit] run_audit | doc_id={}", doc_id) from app.services import db as db_service @@ -129,6 +129,11 @@ async def run_audit(doc_id: str, document_type_id: str | None = None) -> dict: response_format=AUDIT_SCHEMA, temperature=0.2, required_fields=["has_problems", "problems"], + posthog_distinct_id=posthog_distinct_id, + posthog_properties=( + {"$ai_trace_id": doc_id, "$ai_parent_id": f"audit-{doc_id}"} + if posthog_distinct_id else None + ), ) log_usage("audit", response.usage) diff --git a/packages/backend/app/phases/audit/function.py b/packages/backend/app/phases/audit/function.py index 4e1022d..798a017 100644 --- a/packages/backend/app/phases/audit/function.py +++ b/packages/backend/app/phases/audit/function.py @@ -8,6 +8,7 @@ from app.phases._shared.concurrency import DOC_CONCURRENCY from app.phases._shared.failure import workflow_on_failure from app.services import db as db_service +from app.services.observability import capture_span @inngest_client.create_function( @@ -20,6 +21,7 @@ async def function(ctx: inngest.Context): step = ctx.step doc_id = ctx.event.data["document_id"] document_type_id = ctx.event.data.get("document_type_id") + user_id = ctx.event.data.get("user_id", "") logger.info("[orchestrator] AUDIT start | doc_id={}", doc_id) @@ -31,7 +33,7 @@ async def _set_audit(): async def _run_audit(): from app.phases.audit.ai import run_audit - return await run_audit(doc_id, document_type_id=document_type_id) + return await run_audit(doc_id, document_type_id=document_type_id, posthog_distinct_id=user_id) result = await step.run("run-audit", _run_audit) @@ -55,10 +57,23 @@ async def _save_findings(r=result): await step.run("save-findings", _save_findings) + async def _capture_phase_span(r=result): + problems = r.get("problems", []) + output = f"Audit complete. {len(problems)} issue(s) found." if problems else "Audit complete. No issues found." + capture_span( + user_id, doc_id, + span_id=f"audit-{doc_id}", + span_name="audit", + input_state="Document sections reviewed", + output_state=output, + ) + + await step.run("capture-span-audit", _capture_phase_span) + await step.send_event( "emit-audit-completed", inngest.Event( name="docforge/document.audit_completed", - data={"document_id": doc_id, "document_type_id": document_type_id}, + data={"document_id": doc_id, "document_type_id": document_type_id, "user_id": user_id}, ), ) diff --git a/packages/backend/app/phases/completion/function.py b/packages/backend/app/phases/completion/function.py index 7445401..48ae8df 100644 --- a/packages/backend/app/phases/completion/function.py +++ b/packages/backend/app/phases/completion/function.py @@ -6,6 +6,7 @@ from app.phases._shared.concurrency import DOC_CONCURRENCY from app.phases._shared.failure import workflow_on_failure from app.services import db as db_service +from app.services.observability import capture_trace @inngest_client.create_function( @@ -17,6 +18,7 @@ async def function(ctx: inngest.Context): step = ctx.step doc_id = ctx.event.data["document_id"] + user_id = ctx.event.data.get("user_id", "") logger.info("[orchestrator] COMPLETED | doc_id={}", doc_id) @@ -25,3 +27,8 @@ async def _set_completed(): await db_service.set_phase(db, doc_id, "completed") await step.run("set-phase-completed", _set_completed) + + async def _close_trace(): + capture_trace(user_id, doc_id, output_state="Document complete") + + await step.run("capture-trace-complete", _close_trace) diff --git a/packages/backend/app/phases/discovery/ai.py b/packages/backend/app/phases/discovery/ai.py index 2977b13..c8f5f9f 100644 --- a/packages/backend/app/phases/discovery/ai.py +++ b/packages/backend/app/phases/discovery/ai.py @@ -59,6 +59,8 @@ async def analyze_discovery( section_role: str, db: AsyncSession, document_type_id: uuid.UUID | None, + posthog_distinct_id: str | None = None, + doc_id: str | None = None, ) -> dict: """Analyze if context is sufficient for a specific section, or generate follow-up questions.""" logger.info( @@ -80,6 +82,11 @@ async def analyze_discovery( response_format=DISCOVERY_SCHEMA, temperature=0.3, required_fields=["is_sufficient", "follow_up_questions"], + posthog_distinct_id=posthog_distinct_id, + posthog_properties=( + {"$ai_trace_id": doc_id, "$ai_parent_id": f"discovery-{section_key}-{doc_id}"} + if doc_id else None + ), ) log_usage("discovery", response.usage) diff --git a/packages/backend/app/phases/discovery/function.py b/packages/backend/app/phases/discovery/function.py index 08aa8ab..77fbf7e 100644 --- a/packages/backend/app/phases/discovery/function.py +++ b/packages/backend/app/phases/discovery/function.py @@ -12,6 +12,7 @@ from app.phases._shared.concurrency import DOC_CONCURRENCY from app.phases._shared.failure import workflow_on_failure from app.services import db as db_service +from app.services.observability import capture_span @inngest_client.create_function( @@ -26,6 +27,7 @@ async def function(ctx: inngest.Context): document_context = ctx.event.data["document_context"] user_preferences = ctx.event.data["user_preferences"] document_type_id = ctx.event.data.get("document_type_id") + user_id = ctx.event.data.get("user_id", "") logger.info("[orchestrator] DISCOVERY start | doc_id={}", doc_id) @@ -104,6 +106,8 @@ async def _analyze(): section_role=sr, db=db, document_type_id=uuid.UUID(document_type_id) if document_type_id else None, + posthog_distinct_id=user_id, + doc_id=doc_id, ) return _analyze @@ -182,11 +186,34 @@ async def _save_context(sid=section_id_str, ctx_text=context_to_save): await step.run(f"save-context-{section_key}", _save_context) + async def _capture_section_span(sk=section_key, ctx_text=context_to_save): + capture_span( + user_id, doc_id, + span_id=f"discovery-{sk}-{doc_id}", + span_name=sk, + parent_id=f"discovery-{doc_id}", + input_state=document_context, + output_state=ctx_text[:1000] if ctx_text else "", + ) + + await step.run(f"capture-span-discovery-{section_key}", _capture_section_span) + + async def _capture_phase_span(): + capture_span( + user_id, doc_id, + span_id=f"discovery-{doc_id}", + span_name="discovery", + input_state=document_context, + output_state="Discovery complete", + ) + + await step.run("capture-span-discovery", _capture_phase_span) + logger.info("[orchestrator] emitting discovery.completed | doc_id={}", doc_id) await step.send_event( "emit-discovery-completed", inngest.Event( name="docforge/discovery.completed", - data={"document_id": doc_id, "document_type_id": document_type_id}, + data={"document_id": doc_id, "document_type_id": document_type_id, "user_id": user_id}, ), ) diff --git a/packages/backend/app/phases/generation/ai.py b/packages/backend/app/phases/generation/ai.py index 5df243d..71c4a4a 100644 --- a/packages/backend/app/phases/generation/ai.py +++ b/packages/backend/app/phases/generation/ai.py @@ -93,6 +93,8 @@ async def generate_section( document_contract: dict | None = None, db: AsyncSession | None = None, document_type_id: uuid.UUID | None = None, + posthog_distinct_id: str | None = None, + doc_id: str | None = None, ) -> str: """Generate full markdown content for a section.""" logger.info("[AI:generation] generating section | type={}", section_type) @@ -109,6 +111,11 @@ async def generate_section( {"role": "user", "content": user_content}, ], temperature=0.5, + posthog_distinct_id=posthog_distinct_id, + posthog_properties=( + {"$ai_trace_id": doc_id, "$ai_parent_id": f"generation-{section_type}-{doc_id}"} + if doc_id else None + ), ) log_usage("generation", response.usage, section_type=section_type) @@ -121,7 +128,7 @@ async def generate_section( return content -async def refine_cross_references(doc_id: str, document_type_id: str | None = None) -> None: +async def refine_cross_references(doc_id: str, document_type_id: str | None = None, posthog_distinct_id: str | None = None) -> None: """Run a coherence pass on all sections to ensure cross-references are accurate.""" from app.database import async_session from app.services import db as db_service @@ -175,6 +182,11 @@ async def refine_cross_references(doc_id: str, document_type_id: str | None = No section_type=section_type, messages=messages, temperature=0.2, + posthog_distinct_id=posthog_distinct_id, + posthog_properties=( + {"$ai_trace_id": doc_id, "$ai_parent_id": f"generation-{section_type}-{doc_id}"} + if posthog_distinct_id else None + ), ) log_usage("generation:coherence", response.usage, section_type=section_type) diff --git a/packages/backend/app/phases/generation/function.py b/packages/backend/app/phases/generation/function.py index 7405f82..69c81d4 100644 --- a/packages/backend/app/phases/generation/function.py +++ b/packages/backend/app/phases/generation/function.py @@ -9,6 +9,7 @@ from app.phases._shared.failure import workflow_on_failure from app.phases.generation.helpers import generate_section_root from app.services import db as db_service +from app.services.observability import capture_span @inngest_client.create_function( @@ -21,6 +22,7 @@ async def function(ctx: inngest.Context): step = ctx.step doc_id = ctx.event.data["document_id"] document_type_id = ctx.event.data.get("document_type_id") + user_id = ctx.event.data.get("user_id", "") logger.info("[orchestrator] GENERATION start | doc_id={}", doc_id) @@ -50,20 +52,39 @@ async def _get_section_order(): for section_key in section_order: async def _generate(sk=section_key): logger.info("[orchestrator] generating section {} | doc_id={}", sk, doc_id) - await generate_section_root(doc_id, sk) + content = await generate_section_root(doc_id, sk, posthog_distinct_id=user_id) + capture_span( + user_id, doc_id, + span_id=f"generation-{sk}-{doc_id}", + span_name=sk, + parent_id=f"generation-{doc_id}", + input_state=f"Generate section: {sk}", + output_state=content[:1000] if content else "", + ) await step.run(f"generate-{section_key}", _generate) async def _coherence_pass(): from app.phases.generation.ai import refine_cross_references - await refine_cross_references(doc_id, document_type_id=document_type_id) + await refine_cross_references(doc_id, document_type_id=document_type_id, posthog_distinct_id=user_id) await step.run("coherence-pass", _coherence_pass) + async def _capture_phase_span(): + capture_span( + user_id, doc_id, + span_id=f"generation-{doc_id}", + span_name="generation", + input_state="Sections generated", + output_state="Generation complete", + ) + + await step.run("capture-span-generation", _capture_phase_span) + logger.info("[orchestrator] emitting generation.completed | doc_id={}", doc_id) await step.send_event( "emit-generation-completed", inngest.Event( name="docforge/generation.completed", - data={"document_id": doc_id, "document_type_id": document_type_id}, + data={"document_id": doc_id, "document_type_id": document_type_id, "user_id": user_id}, ), ) diff --git a/packages/backend/app/phases/generation/helpers.py b/packages/backend/app/phases/generation/helpers.py index 0584225..6a1afc2 100644 --- a/packages/backend/app/phases/generation/helpers.py +++ b/packages/backend/app/phases/generation/helpers.py @@ -45,8 +45,12 @@ async def _build_cross_section_context_from_db(db, doc_id: uuid.UUID, target_typ return _build_cross_section_context(sections, target_type) -async def generate_section_root(doc_id: str, section_type: str) -> None: - """Generate the initial version of a section using AI.""" +async def generate_section_root( + doc_id: str, + section_type: str, + posthog_distinct_id: str | None = None, +) -> str: + """Generate the initial version of a section using AI. Returns the generated content.""" from app.phases.generation.ai import generate_section logger.info("[helpers] generate_section_root | doc_id={} type={}", doc_id, section_type) @@ -54,7 +58,7 @@ async def generate_section_root(doc_id: str, section_type: str) -> None: doc = await db_service.get_document_detail(db, uuid.UUID(doc_id)) section = next((s for s in doc.sections if s.section_type == section_type), None) if not section: - return + return "" cross_section_context = _build_cross_section_context(doc.sections, section_type) @@ -77,6 +81,8 @@ async def generate_section_root(doc_id: str, section_type: str) -> None: document_contract=contract_dict, db=db, document_type_id=doc.document_type_id, + posthog_distinct_id=posthog_distinct_id, + doc_id=doc_id, ) content = clean_section_output(content, section_type) @@ -92,3 +98,4 @@ async def generate_section_root(doc_id: str, section_type: str) -> None: section_type, len(content or ""), ) + return content or "" diff --git a/packages/backend/app/phases/refinement/ai.py b/packages/backend/app/phases/refinement/ai.py index eec6288..063eacc 100644 --- a/packages/backend/app/phases/refinement/ai.py +++ b/packages/backend/app/phases/refinement/ai.py @@ -154,6 +154,8 @@ async def refine_section( document_contract: dict | None = None, db: AsyncSession | None = None, document_type_id: uuid.UUID | None = None, + posthog_distinct_id: str | None = None, + doc_id: str | None = None, ) -> dict: """Process a refinement interaction using function calling.""" logger.info( @@ -184,6 +186,11 @@ async def refine_section( tools=REFINEMENT_TOOLS, tool_choice=tool_choice, temperature=0.4, + posthog_distinct_id=posthog_distinct_id, + posthog_properties=( + {"$ai_trace_id": doc_id, "$ai_parent_id": f"refinement-{section_type}-{doc_id}"} + if doc_id else None + ), ) log_usage("refinement", response.usage, section_type=section_type) diff --git a/packages/backend/app/phases/refinement/function.py b/packages/backend/app/phases/refinement/function.py index fdfd6b4..392bfd8 100644 --- a/packages/backend/app/phases/refinement/function.py +++ b/packages/backend/app/phases/refinement/function.py @@ -9,6 +9,7 @@ from app.phases._shared.failure import workflow_on_failure from app.phases.refinement.helpers import process_edit, process_question from app.services import db as db_service +from app.services.observability import capture_span @inngest_client.create_function( @@ -21,6 +22,7 @@ async def start_function(ctx: inngest.Context): """Set up refinement phase. Actions are handled by action_function.""" step = ctx.step doc_id = ctx.event.data["document_id"] + user_id = ctx.event.data.get("user_id", "") logger.info("[orchestrator] REFINEMENT start | doc_id={}", doc_id) @@ -31,6 +33,26 @@ async def _set_refinement(): await step.run("set-phase-refinement", _set_refinement) + async def _capture_refinement_spans(): + async with async_session() as db: + doc = await db_service.get_document_detail(db, doc_id) + capture_span( + user_id, doc_id, + span_id=f"refinement-{doc_id}", + span_name="refinement", + input_state="User refinement phase started", + ) + for section in doc.sections: + capture_span( + user_id, doc_id, + span_id=f"refinement-{section.section_type}-{doc_id}", + span_name=section.section_type, + parent_id=f"refinement-{doc_id}", + input_state="Awaiting user edits", + ) + + await step.run("capture-span-refinement", _capture_refinement_spans) + @inngest_client.create_function( fn_id="handle-section-action", @@ -50,6 +72,7 @@ async def action_function(ctx: inngest.Context): doc_id = ctx.event.data["document_id"] section_id = ctx.event.data["section_id"] action = ctx.event.data["action_type"] + user_id = ctx.event.data.get("user_id", "") logger.info( "[orchestrator] section action | doc_id={} section_id={} action={}", @@ -60,12 +83,12 @@ async def action_function(ctx: inngest.Context): if action == "request_edit": async def _apply_edit(): - await process_edit(section_id, ctx.event.data["prompt"]) + await process_edit(section_id, ctx.event.data["prompt"], posthog_distinct_id=user_id) await step.run("apply-edit", _apply_edit) elif action == "ask_question": async def _answer_question(): - await process_question(section_id, ctx.event.data["message"]) + await process_question(section_id, ctx.event.data["message"], posthog_distinct_id=user_id) await step.run("answer-question", _answer_question) elif action == "finalize": @@ -91,6 +114,6 @@ async def _check_finalized(): "emit-refinement-completed", inngest.Event( name="docforge/document.refinement_completed", - data={"document_id": doc_id, "document_type_id": doc_type_id}, + data={"document_id": doc_id, "document_type_id": doc_type_id, "user_id": user_id}, ), ) diff --git a/packages/backend/app/phases/refinement/helpers.py b/packages/backend/app/phases/refinement/helpers.py index e39d10b..347924b 100644 --- a/packages/backend/app/phases/refinement/helpers.py +++ b/packages/backend/app/phases/refinement/helpers.py @@ -78,7 +78,7 @@ async def _build_cross_section_context_from_db(db, doc_id: uuid.UUID, target_typ return _build_cross_section_context(sections, target_type) -async def process_edit(section_id: str, prompt: str) -> None: +async def process_edit(section_id: str, prompt: str, posthog_distinct_id: str | None = None) -> None: """Process an edit request via AI refinement.""" from app.phases.refinement.ai import refine_section @@ -95,6 +95,8 @@ async def process_edit(section_id: str, prompt: str) -> None: return doc = section.document + _phog_id = posthog_distinct_id or str(doc.user_id) + _doc_id = str(doc.id) cross_context = _build_cross_section_context_from_db(db, doc.id, section.section_type) chat_history = await _get_chat_history(db, section.id) contract_dict = await _get_contract_dict(db, doc.id) @@ -102,7 +104,15 @@ async def process_edit(section_id: str, prompt: str) -> None: await db_service.add_chat_message(db, doc.id, section.id, "user", prompt) from app.guardrails import validate_message_intent - intent_err = await validate_message_intent(prompt, section.section_type, section.summary) + intent_err = await validate_message_intent( + prompt, section.section_type, section.summary, + posthog_distinct_id=_phog_id, + posthog_properties={ + "$ai_trace_id": _doc_id, + "$ai_parent_id": f"refinement-{section.section_type}-{_doc_id}", + "$ai_span_name": "edit-guardrail", + }, + ) if intent_err: await db_service.add_chat_message(db, doc.id, section.id, "agent", intent_err.message) return @@ -120,6 +130,8 @@ async def process_edit(section_id: str, prompt: str) -> None: document_contract=contract_dict, db=db, document_type_id=doc.document_type_id, + posthog_distinct_id=_phog_id, + doc_id=_doc_id, ) except BadRequestError as exc: body = exc.body if isinstance(exc.body, dict) else {} @@ -155,7 +167,7 @@ async def process_edit(section_id: str, prompt: str) -> None: await db_service.add_chat_message(db, doc.id, section.id, "agent", ai_result["reply"]) -async def process_question(section_id: str, message: str) -> None: +async def process_question(section_id: str, message: str, posthog_distinct_id: str | None = None) -> None: """Process a question about a section via AI.""" from app.phases.refinement.ai import refine_section @@ -172,6 +184,8 @@ async def process_question(section_id: str, message: str) -> None: return doc = section.document + _phog_id = posthog_distinct_id or str(doc.user_id) + _doc_id = str(doc.id) cross_context = _build_cross_section_context_from_db(db, doc.id, section.section_type) chat_history = await _get_chat_history(db, section.id) contract_dict = await _get_contract_dict(db, doc.id) @@ -179,7 +193,15 @@ async def process_question(section_id: str, message: str) -> None: await db_service.add_chat_message(db, doc.id, section.id, "user", message) from app.guardrails import validate_message_intent - intent_err = await validate_message_intent(message, section.section_type, section.summary) + intent_err = await validate_message_intent( + message, section.section_type, section.summary, + posthog_distinct_id=_phog_id, + posthog_properties={ + "$ai_trace_id": _doc_id, + "$ai_parent_id": f"refinement-{section.section_type}-{_doc_id}", + "$ai_span_name": "question-guardrail", + }, + ) if intent_err: await db_service.add_chat_message(db, doc.id, section.id, "agent", intent_err.message) return @@ -197,6 +219,8 @@ async def process_question(section_id: str, message: str) -> None: document_contract=contract_dict, db=db, document_type_id=doc.document_type_id, + posthog_distinct_id=_phog_id, + doc_id=_doc_id, ) except BadRequestError as exc: body = exc.body if isinstance(exc.body, dict) else {} diff --git a/packages/backend/app/routers/documents.py b/packages/backend/app/routers/documents.py index 7b37892..093f940 100644 --- a/packages/backend/app/routers/documents.py +++ b/packages/backend/app/routers/documents.py @@ -44,6 +44,7 @@ from app.schemas.events import AnswerQuestionRequest, EventRequest from app.services import db as db_service from app.services import sse as sse_service +from app.services.observability import capture_event, capture_trace router = APIRouter(tags=["documents"]) @@ -256,6 +257,7 @@ async def create_document( await db.commit() # Dispatch Inngest event + capture_trace(str(current_user.id), str(doc.id), span_name=doc.title, input_state=payload.document_context) try: await inngest_client.send( inngest.Event( @@ -265,6 +267,7 @@ async def create_document( "document_context": payload.document_context, "user_preferences": payload.user_preferences or "", "document_type_id": str(doc_type.id), + "user_id": str(current_user.id), }, ) ) @@ -330,6 +333,8 @@ async def _create_editor_document( )) api_key_id = getattr(request.state, "api_key_id", None) + if api_key_id: + capture_event(str(current_user.id), str(doc.id), "mcp_document_created", {"has_context": bool(payload.user_preferences)}) await db_service.log_activity( db, doc.id, "document_created", description="Document created via Editor", @@ -474,7 +479,7 @@ async def dispatch_event( detail="Message limit reached for this section (10/10).", ) - event_data = {**payload.data, "document_id": str(document_id)} + event_data = {**payload.data, "document_id": str(document_id), "user_id": str(current_user.id)} try: await inngest_client.send( diff --git a/packages/backend/app/routers/sections.py b/packages/backend/app/routers/sections.py index ac5fef4..979d70e 100644 --- a/packages/backend/app/routers/sections.py +++ b/packages/backend/app/routers/sections.py @@ -18,6 +18,7 @@ VersionRestoreResponse, ) from app.services.db import create_version_snapshot, restore_version, update_section_content +from app.services.observability import capture_event router = APIRouter(tags=["sections"]) @@ -95,6 +96,12 @@ async def create_section_snapshot( except ValueError as e: raise HTTPException(status_code=409, detail=str(e)) + if api_key_id: + capture_event(str(current_user.id), str(section.document_id), "mcp_snapshot_saved", { + "has_name": bool(payload.version_name if payload else None), + "has_summary": bool(payload.change_summary if payload else None), + }) + return SectionVersionResponse.model_validate(version) @@ -152,6 +159,13 @@ async def update_section_content_endpoint( ) except ValueError as e: raise HTTPException(status_code=409, detail=str(e)) + + if api_key_id: + capture_event(str(current_user.id), str(section.document_id), "mcp_section_written", { + "content_length": len(payload.content), + "has_note": bool(note), + }) + return SectionVersionResponse.model_validate(version) @@ -182,4 +196,8 @@ async def restore_section_version( raise HTTPException(status_code=404, detail="Version not found for this section") restored = await restore_version(db, section_id, version_id, doc_id=section.document_id, api_key_id=api_key_id) + + if api_key_id: + capture_event(str(current_user.id), str(section.document_id), "mcp_version_activated", {}) + return VersionRestoreResponse(success=True, active_version_id=restored.id) diff --git a/packages/backend/app/services/observability.py b/packages/backend/app/services/observability.py new file mode 100644 index 0000000..d863572 --- /dev/null +++ b/packages/backend/app/services/observability.py @@ -0,0 +1,60 @@ +"""PostHog AI observability helpers — trace and span capture for document workflows.""" +from app.ai.core.client import posthog_client + + +def capture_trace( + distinct_id: str, + doc_id: str, + *, + span_name: str | None = None, + input_state: str | None = None, + output_state: str | None = None, +) -> None: + if posthog_client is None: + return + properties: dict = {"$ai_trace_id": doc_id} + if span_name is not None: + properties["$ai_span_name"] = span_name + if input_state is not None: + properties["$ai_input_state"] = input_state + if output_state is not None: + properties["$ai_output_state"] = output_state + posthog_client.capture(distinct_id=distinct_id, event="$ai_trace", properties=properties) + + +def capture_event( + distinct_id: str, + doc_id: str, + event_name: str, + properties: dict | None = None, +) -> None: + if posthog_client is None: + return + full_props: dict = {"$ai_trace_id": doc_id, **(properties or {})} + posthog_client.capture(distinct_id=distinct_id, event=event_name, properties=full_props) + + +def capture_span( + distinct_id: str, + doc_id: str, + span_id: str, + span_name: str, + *, + parent_id: str | None = None, + input_state: str | None = None, + output_state: str | None = None, +) -> None: + if posthog_client is None: + return + properties: dict = { + "$ai_trace_id": doc_id, + "$ai_span_id": span_id, + "$ai_span_name": span_name, + } + if parent_id is not None: + properties["$ai_parent_id"] = parent_id + if input_state is not None: + properties["$ai_input_state"] = input_state + if output_state is not None: + properties["$ai_output_state"] = output_state + posthog_client.capture(distinct_id=distinct_id, event="$ai_span", properties=properties) diff --git a/packages/backend/pyproject.toml b/packages/backend/pyproject.toml index 3a574d8..74c0e09 100644 --- a/packages/backend/pyproject.toml +++ b/packages/backend/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "python-jose[cryptography]>=3.3.0", "httpx>=0.27.0", "pyyaml>=6.0.0", + "posthog>=7.16.2", ] [project.optional-dependencies] diff --git a/packages/frontend/.env.example b/packages/frontend/.env.example index cb94f12..8e5fdaa 100644 --- a/packages/frontend/.env.example +++ b/packages/frontend/.env.example @@ -1,2 +1,6 @@ VITE_API_BASE=http://localhost:8000/api VITE_CLERK_PUBLISHABLE_KEY=pk_test_... + +# PostHog analytics (optional — leave blank to disable) +VITE_POSTHOG_KEY= +VITE_POSTHOG_HOST=https://us.i.posthog.com diff --git a/packages/frontend/package.json b/packages/frontend/package.json index ddb393f..79dd15a 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -24,6 +24,7 @@ "react-router-dom": "^7.14.2", "remark-gfm": "^4.0.1", "tailwind-merge": "^3.0.2", + "posthog-js": "^1.240.0", "zustand": "^5.0.12" }, "devDependencies": { diff --git a/packages/frontend/src/App.tsx b/packages/frontend/src/App.tsx index ee2fb38..1283cdf 100644 --- a/packages/frontend/src/App.tsx +++ b/packages/frontend/src/App.tsx @@ -1,7 +1,12 @@ +import { useState } from "react"; +import posthog from "posthog-js"; +import { PostHogProvider } from "posthog-js/react"; import { BrowserRouter, Routes, Route } from "react-router-dom"; import { ClerkProvider } from "@clerk/clerk-react"; import { HomePage, DocumentPage, LoadingPage, LandingPage, LegalPage, HowItWorksPage, PricingPage, McpPage, BillingPage } from "./pages"; import { ProtectedRoute } from "./components/shared"; +import CookieConsentBanner from "./components/shared/CookieConsentBanner"; +import { useClerkPosthogIdentity } from "./hooks/useClerkPosthogIdentity"; const CLERK_PUBLISHABLE_KEY = import.meta.env.VITE_CLERK_PUBLISHABLE_KEY as string; @@ -9,24 +14,49 @@ if (!CLERK_PUBLISHABLE_KEY) { throw new Error("VITE_CLERK_PUBLISHABLE_KEY is not set"); } +if (import.meta.env.VITE_POSTHOG_KEY) { + posthog.init(import.meta.env.VITE_POSTHOG_KEY as string, { + api_host: (import.meta.env.VITE_POSTHOG_HOST as string) ?? "https://us.i.posthog.com", + opt_out_capturing_by_default: true, + capture_pageview: true, + capture_pageleave: true, + session_recording: { + maskTextSelector: "[data-ph-mask], .editor-content, .markdown-preview", + }, + }); +} + +function AppInner() { + const [consentGiven, setConsentGiven] = useState(false); + useClerkPosthogIdentity(consentGiven); + return ( + <> + setConsentGiven(true)} /> + + } /> + } /> + } /> + } /> + }> + } /> + } /> + } /> + } /> + } /> + + + + ); +} + export default function App() { return ( - - - - } /> - } /> - } /> - } /> - }> - } /> - } /> - } /> - } /> - } /> - - - - + + + + + + + ); } diff --git a/packages/frontend/src/components/phases/alignment/AlignmentLayout.tsx b/packages/frontend/src/components/phases/alignment/AlignmentLayout.tsx index c942f51..6794f65 100644 --- a/packages/frontend/src/components/phases/alignment/AlignmentLayout.tsx +++ b/packages/frontend/src/components/phases/alignment/AlignmentLayout.tsx @@ -1,5 +1,6 @@ import { useState, useEffect } from "react"; import { useAuth } from "@clerk/clerk-react"; +import posthog from "posthog-js"; import type { Section, GuidedSectionType as SectionType } from "@/types"; import { apiSendEvent } from "@/utils/api"; import { ForgeLoader } from "@/components/shared"; @@ -102,6 +103,7 @@ export function AlignmentLayout({ documentId, sections }: AlignmentLayoutProps) const handleReopen = (type: SectionType) => { setCards((prev) => ({ ...prev, [type]: { ...prev[type], status: "pending" } })); + posthog.capture("alignment_section_reopened", { document_id: documentId, section_type: type }); }; return ( diff --git a/packages/frontend/src/components/phases/alignment/SummaryCard.tsx b/packages/frontend/src/components/phases/alignment/SummaryCard.tsx index f265d01..08f673d 100644 --- a/packages/frontend/src/components/phases/alignment/SummaryCard.tsx +++ b/packages/frontend/src/components/phases/alignment/SummaryCard.tsx @@ -139,6 +139,7 @@ export function SummaryCard({ ) : (

{problem.section} -

+

{problem.issue}

{problem.severity && ( diff --git a/packages/frontend/src/components/phases/completed/CompletedLayout.tsx b/packages/frontend/src/components/phases/completed/CompletedLayout.tsx index 5eeb4d4..81092bd 100644 --- a/packages/frontend/src/components/phases/completed/CompletedLayout.tsx +++ b/packages/frontend/src/components/phases/completed/CompletedLayout.tsx @@ -1,5 +1,6 @@ import { useEffect, useRef, useState } from "react"; import { useAuth } from "@clerk/clerk-react"; +import posthog from "posthog-js"; import type { Section, GuidedSectionType as SectionType, AuditFinding } from "@/types"; import { apiFetchAuditFindings, apiDismissAuditFinding } from "@/utils/api"; import { MarkdownRenderer } from "@/components/shared"; @@ -281,6 +282,7 @@ function SectionBlock({ sectionType, content, onEdit }: SectionBlockProps) { {isEditing ? (