diff --git a/.env.example b/.env.example index 999178ce..6bde5532 100644 --- a/.env.example +++ b/.env.example @@ -31,6 +31,15 @@ LLM_FLASH_MODEL=gemini-3.1-flash-lite LLM_FLASH_TEMPERATURE=0.4 LLM_FLASH_MAX_TOKENS=512 +# 음성 STT (Phase 2). auto=DEEPGRAM_API_KEY 있으면 deepgram, 없고 OPENAI_API_KEY 있으면 openai_whisper, 둘 다 없으면 mock. +STT_PROVIDER=auto +DEEPGRAM_API_KEY= +DEEPGRAM_MODEL=whisper-large +DEEPGRAM_LANGUAGE=ko +OPENAI_API_KEY= +WHISPER_MODEL=whisper-1 +WHISPER_LANGUAGE=ko + # RealTime server REALTIME_PORT=38020 REALTIME_LOG_LEVEL=info diff --git a/.github/workflows/deploy-app.yml b/.github/workflows/deploy-app.yml index 9193348d..eb9b7d7f 100644 --- a/.github/workflows/deploy-app.yml +++ b/.github/workflows/deploy-app.yml @@ -55,6 +55,8 @@ jobs: CORE_INTERNAL_API_KEY: ${{ secrets.CORE_INTERNAL_API_KEY }} GITHUB_OAUTH_CLIENT_ID: ${{ secrets.GH_OAUTH_CLIENT_ID }} GITHUB_OAUTH_CLIENT_SECRET: ${{ secrets.GH_OAUTH_CLIENT_SECRET }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }} run: | cd "$DEPLOY_DIR" if [ ! -f .env ]; then @@ -107,6 +109,13 @@ jobs: upsert_env "CORE_INTERNAL_API_KEY" "$CORE_INTERNAL_API_KEY" upsert_env "GITHUB_OAUTH_CLIENT_ID" "$GITHUB_OAUTH_CLIENT_ID" upsert_env "GITHUB_OAUTH_CLIENT_SECRET" "$GITHUB_OAUTH_CLIENT_SECRET" + upsert_env "OPENAI_API_KEY" "$OPENAI_API_KEY" + upsert_env "DEEPGRAM_API_KEY" "$DEEPGRAM_API_KEY" + # STT_PROVIDER 자동 선택. auto 로 두면 AI factory 가 보유 키 기준으로 선택 (deepgram > openai_whisper > mock). + # 키가 1개라도 있으면 auto 로 설정해두면 충분 — 명시적 전환 불필요. + if [ -n "$DEEPGRAM_API_KEY" ] || [ -n "$OPENAI_API_KEY" ]; then + upsert_env "STT_PROVIDER" "auto" + fi - name: Build and restart app services run: | diff --git a/ai/.env.example b/ai/.env.example index 65866f0f..c910b2b9 100644 --- a/ai/.env.example +++ b/ai/.env.example @@ -47,3 +47,18 @@ EMBEDDING_CHUNK_OVERLAP=200 EMBEDDING_BATCH_SIZE=32 GEMINI_API_KEY= + +# 음성 STT (Phase 2). auto=키 보유에 따라 deepgram > openai_whisper > mock 자동 선택. +STT_PROVIDER=auto +# Deepgram (한국어 정확도 우선 권장. 신규 가입 시 $200 크레딧). +DEEPGRAM_API_KEY= +DEEPGRAM_BASE_URL=https://api.deepgram.com/v1 +DEEPGRAM_MODEL=whisper-large +DEEPGRAM_LANGUAGE=ko +DEEPGRAM_TIMEOUT_SEC=60 +# OpenAI Whisper (Mindlogic 미지원, 직접 호출). +OPENAI_API_KEY= +OPENAI_BASE_URL=https://api.openai.com/v1 +WHISPER_MODEL=whisper-1 +WHISPER_LANGUAGE=ko +WHISPER_TIMEOUT_SEC=60 diff --git a/ai/src/ai_server/chain/feedback_generation_chain.py b/ai/src/ai_server/chain/feedback_generation_chain.py new file mode 100644 index 00000000..fc2ced3f --- /dev/null +++ b/ai/src/ai_server/chain/feedback_generation_chain.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from typing import Protocol + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import Runnable +from pydantic import BaseModel, Field + +from ai_server.chain.prompts.feedback_generation import HUMAN_PROMPT, SYSTEM_PROMPT +from ai_server.config.settings import Settings +from ai_server.core.client import CoreClient +from ai_server.observability.llm_logging_callback import CoreAiLogCallback + + +class FeedbackResult(BaseModel): + overall_score: float | None = Field(None, description="0~100") + technical_accuracy: float | None = Field(None, description="0~100") + logic_score: float | None = Field(None, description="0~100") + communication_score: float | None = Field(None, description="0~100") + strengths_summary: str | None = Field(None) + weaknesses_summary: str | None = Field(None) + improvement_keywords: list[str] = Field(default_factory=list) + + +class FeedbackGenerator(Protocol): + async def generate( + self, + *, + job_category: str, + interview_type: str, + total_question_count: int | None, + end_reason: str | None, + transcript: str, + rag_context: str, + ) -> FeedbackResult: ... + + +class LlmFeedbackGenerator: + def __init__(self, chain: Runnable) -> None: + self._chain = chain + + async def generate( + self, + *, + job_category: str, + interview_type: str, + total_question_count: int | None, + end_reason: str | None, + transcript: str, + rag_context: str, + ) -> FeedbackResult: + result = await self._chain.ainvoke( + { + "job_category": job_category, + "interview_type": interview_type, + "total_question_count": total_question_count or 0, + "end_reason": end_reason or "USER_REQUEST", + "transcript": transcript, + "rag_context": rag_context or "(none)", + } + ) + if not isinstance(result, FeedbackResult): + raise TypeError( + f"chain returned {type(result).__name__}, expected FeedbackResult" + ) + return result + + +def build_feedback_generation_chain( + settings: Settings, core_client: CoreClient | None = None +) -> Runnable: + from langchain_openai import ChatOpenAI + + parser = PydanticOutputParser(pydantic_object=FeedbackResult) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", SYSTEM_PROMPT), + ("human", HUMAN_PROMPT), + ] + ).partial(format_instructions=parser.get_format_instructions()) + + callbacks = [] + if core_client is not None: + callbacks.append( + CoreAiLogCallback( + core_client=core_client, + request_type="generate.feedback", + default_model=settings.llm_pro_model, + ) + ) + + llm = ChatOpenAI( + model=settings.llm_pro_model, + temperature=settings.llm_pro_temperature, + api_key=settings.llm_api_key or None, + base_url=settings.llm_base_url, + callbacks=callbacks, + ) + return prompt | llm | parser diff --git a/ai/src/ai_server/chain/prompts/feedback_generation.py b/ai/src/ai_server/chain/prompts/feedback_generation.py new file mode 100644 index 00000000..31a496ef --- /dev/null +++ b/ai/src/ai_server/chain/prompts/feedback_generation.py @@ -0,0 +1,33 @@ +# 종합 피드백 생성 (US-24) +# Pro 모델 + 세션 전체 메시지 시퀀스 + (옵션) RAG 컨텍스트 청크. +# 출력: 0~100 점수 4개 + 강점·약점 요약 + 개선 키워드 리스트. + +SYSTEM_PROMPT = ( + "당신은 IT 직군 면접 평가관입니다. 지원자의 모든 답변을 종합해 객관적이고 건설적인 피드백을 한국어로 작성합니다.\n" + "- 점수 (0~100 정수형, 산정 불가 시 null):\n" + " - overall_score: 종합 점수\n" + " - technical_accuracy: 기술 정확도\n" + " - logic_score: 논리·인과관계 명확성\n" + " - communication_score: 답변의 명료성·구조화\n" + "- 요약:\n" + " - strengths_summary: 가장 잘한 점 3가지 이내 (각 1~2문장).\n" + " - weaknesses_summary: 가장 부족한 점 3가지 이내 (각 1~2문장).\n" + " - improvement_keywords: 다음 면접에서 채울 키워드 5~10개 (짧은 명사구).\n" + "- 평가 원칙:\n" + " - 단일 답변보다 시퀀스 흐름을 우선 고려 (꼬리질문 대응의 깊이가 중요).\n" + " - 답변이 짧거나 비어 있으면 해당 점수는 낮게 또는 null.\n" + " - 컨텍스트 청크(분석 문서 일부) 가 있다면 사실 검증에만 활용 (직접 인용 X).\n" + "- 응답은 반드시 지정된 JSON 스키마를 따릅니다." +) + +HUMAN_PROMPT = ( + "직군: {job_category}\n" + "면접 유형: {interview_type}\n" + "총 질문 수: {total_question_count}\n" + "종료 사유: {end_reason}\n\n" + "=== 메시지 시퀀스 ===\n" + "{transcript}\n\n" + "=== RAG 컨텍스트 청크 (참고용, 직접 인용 금지) ===\n" + "{rag_context}\n\n" + "{format_instructions}" +) diff --git a/ai/src/ai_server/config/settings.py b/ai/src/ai_server/config/settings.py index 65f57120..4b54bb99 100644 --- a/ai/src/ai_server/config/settings.py +++ b/ai/src/ai_server/config/settings.py @@ -24,10 +24,30 @@ class Settings(BaseSettings): ai_queue_web: str = "ai.analyze.web" ai_queue_questions: str = "ai.generate.questions" ai_queue_followup: str = "ai.generate.followup" + ai_queue_feedback: str = "ai.generate.feedback" + ai_queue_voice: str = "ai.analyze.voice" ai_queue_prefetch: int = 10 ai_callback_exchange: str = "stackup.ai-to-core" ai_callback_routing_analysis: str = "callback.analysis" ai_callback_routing_questions: str = "callback.questions" + ai_callback_routing_feedback: str = "callback.feedback" + ai_callback_routing_voice: str = "callback.voice" + feedback_rag_top_k: int = 5 + + # STT (음성 답변). "auto" 면 deepgram > openai_whisper > mock 순으로 키 보유 여부에 따라 자동 선택. + stt_provider: Literal["auto", "mock", "openai_whisper", "deepgram"] = "auto" + openai_api_key: str = "" + openai_base_url: str = "https://api.openai.com/v1" + whisper_model: str = "whisper-1" + whisper_language: str = "ko" + whisper_timeout_sec: float = 60.0 + deepgram_api_key: str = "" + deepgram_base_url: str = "https://api.deepgram.com/v1" + deepgram_model: str = "whisper-large" # 한국어 정확도 우선; 저비용 우선 시 nova-2. + deepgram_language: str = "ko" + deepgram_timeout_sec: float = 60.0 + # 음성 분석 + voice_filler_pattern: str = r"(?:음+|어+|그+|아+)" ai_publisher_name: str = "ai-server" ai_idempotency_lru_size: int = 1024 diff --git a/ai/src/ai_server/core/client.py b/ai/src/ai_server/core/client.py index 9242ecab..98cacab1 100644 --- a/ai/src/ai_server/core/client.py +++ b/ai/src/ai_server/core/client.py @@ -32,6 +32,14 @@ class EmbeddingChunkPayload: embedding: list[float] +@dataclass(frozen=True) +class EmbeddingSearchHit: + document_id: int + chunk_index: int + chunk_text: str + distance: float + + # 코어 서버 API 호출용 class CoreClient(Protocol): async def fetch_github_token(self, user_id: int) -> str: ... @@ -45,6 +53,14 @@ async def upsert_embeddings( chunks: list[EmbeddingChunkPayload], ) -> int: ... + async def search_embeddings( + self, + *, + query_embedding: list[float], + document_ids: list[int] | None = None, + top_k: int = 5, + ) -> list[EmbeddingSearchHit]: ... + async def record_ai_log( self, *, @@ -234,6 +250,54 @@ async def _do_upsert( return len(body["chunks"]) return count + async def search_embeddings( + self, + *, + query_embedding: list[float], + document_ids: list[int] | None = None, + top_k: int = 5, + ) -> list[EmbeddingSearchHit]: + """pgvector cosine topK 검색. 실패 시 빈 리스트 반환 (RAG 보강용이므로 fatal 아님).""" + body = { + "queryEmbedding": query_embedding, + "documentIds": list(document_ids or []), + "topK": top_k, + } + path = "/api/internal/embeddings/search" + try: + if self._client is not None: + resp = await self._client.post(path, json=body) + else: + async with self._build_client() as client: + resp = await client.post(path, json=body) + except httpx.HTTPError as exc: + log.warn("core.embedding.search.failed", error=str(exc)) + return [] + + if resp.status_code >= 400: + log.warn( + "core.embedding.search.non_2xx", + status=resp.status_code, + body=resp.text[:200], + ) + return [] + try: + data = resp.json() + except ValueError: + return [] + hits = data.get("hits") if isinstance(data, dict) else None + if not isinstance(hits, list): + return [] + return [ + EmbeddingSearchHit( + document_id=int(h.get("documentId", 0)), + chunk_index=int(h.get("chunkIndex", 0)), + chunk_text=str(h.get("chunkText", "")), + distance=float(h.get("distance", 0.0)), + ) + for h in hits + ] + async def record_ai_log( self, *, diff --git a/ai/src/ai_server/messaging/consumers/feedback_consumer.py b/ai/src/ai_server/messaging/consumers/feedback_consumer.py new file mode 100644 index 00000000..f0f4d4df --- /dev/null +++ b/ai/src/ai_server/messaging/consumers/feedback_consumer.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +from typing import Protocol + +import structlog +from aio_pika.abc import AbstractIncomingMessage + +from ai_server.chain.feedback_generation_chain import FeedbackGenerator +from ai_server.core.client import CoreClient +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.messaging.publisher import CallbackPublisher +from ai_server.model.envelope import Envelope +from ai_server.model.messages.feedback import ( + FeedbackCallbackPayload, + FeedbackMessageItem, + GenerateFeedbackRequest, +) +from ai_server.rag.embedder import EmbeddingProvider + +log = structlog.get_logger(__name__) + + +class FeedbackConsumer: + """generate.feedback consumer (US-24). + + 흐름: + 1. envelope parse + 멱등 체크 + 2. transcript 텍스트 빌드 (시퀀스 순) + 3. (옵션) RAG: 마지막 답변을 쿼리로 임베딩 → Core /api/internal/embeddings/search → topK 청크 + 4. chain.generate → FeedbackResult + 5. callback.feedback 발행 + """ + + def __init__( + self, + *, + generator: FeedbackGenerator, + publisher: CallbackPublisher, + idempotency: LruIdempotencyStore, + callback_routing_key: str, + core_client: CoreClient, + embedder: EmbeddingProvider | None = None, + rag_top_k: int = 5, + ) -> None: + self._generator = generator + self._publisher = publisher + self._idempotency = idempotency + self._callback_routing_key = callback_routing_key + self._core = core_client + self._embedder = embedder + self._rag_top_k = rag_top_k + + async def handle(self, message: AbstractIncomingMessage) -> None: + async with message.process(requeue=False): + try: + envelope = Envelope[GenerateFeedbackRequest].model_validate_json( + message.body + ) + except Exception as exc: + log.error( + "feedback.parse.failed", + error=str(exc), + delivery_tag=message.delivery_tag, + ) + raise + + if self._idempotency.is_seen_then_mark(envelope.message_id): + log.info( + "feedback.idempotent.skip", + message_id=envelope.message_id, + trace_id=envelope.trace_id, + ) + return + + req = envelope.payload + log.info( + "feedback.generate.start", + message_id=envelope.message_id, + session_id=req.session_id, + msg_count=len(req.messages), + ctx_count=len(req.context_document_ids), + trace_id=envelope.trace_id, + ) + + transcript = _build_transcript(req.messages) + rag_context = await self._build_rag_context(req) + + result = await self._generator.generate( + job_category=req.job_category, + interview_type=req.interview_type, + total_question_count=req.total_question_count, + end_reason=req.end_reason, + transcript=transcript, + rag_context=rag_context, + ) + + payload = FeedbackCallbackPayload( + session_id=req.session_id, + overall_score=result.overall_score, + technical_accuracy=result.technical_accuracy, + logic_score=result.logic_score, + communication_score=result.communication_score, + strengths_summary=result.strengths_summary, + weaknesses_summary=result.weaknesses_summary, + improvement_keywords=result.improvement_keywords, + report_s3_key=None, + ) + + await self._publisher.publish( + routing_key=self._callback_routing_key, + message_type="callback.feedback", + payload=payload, + trace_id=envelope.trace_id, + correlation_id=envelope.message_id, + context=envelope.context, + ) + log.info( + "feedback.generate.done", + message_id=envelope.message_id, + session_id=req.session_id, + trace_id=envelope.trace_id, + ) + + async def _build_rag_context(self, req: GenerateFeedbackRequest) -> str: + if not self._embedder or not req.context_document_ids: + return "(none)" + last_answer = next( + (m.content for m in reversed(req.messages) if m.role == "INTERVIEWEE"), + None, + ) + if not last_answer: + return "(none)" + try: + query_vec = (await self._embedder.embed([last_answer]))[0] + hits = await self._core.search_embeddings( + query_embedding=query_vec, + document_ids=req.context_document_ids, + top_k=self._rag_top_k, + ) + except Exception as exc: + log.warn("feedback.rag.failed", error=str(exc), session_id=req.session_id) + return "(none)" + if not hits: + return "(none)" + return "\n---\n".join(f"[doc#{h.document_id} chunk#{h.chunk_index}] {h.chunk_text}" for h in hits) + + +def _build_transcript(messages: list[FeedbackMessageItem]) -> str: + if not messages: + return "(empty)" + lines: list[str] = [] + for m in messages: + speaker = "면접관" if m.role == "INTERVIEWER" else ("지원자" if m.role == "INTERVIEWEE" else m.role) + lines.append(f"[{m.sequence_number}] {speaker}: {m.content}") + return "\n".join(lines) diff --git a/ai/src/ai_server/messaging/consumers/voice_consumer.py b/ai/src/ai_server/messaging/consumers/voice_consumer.py new file mode 100644 index 00000000..b4adfb4f --- /dev/null +++ b/ai/src/ai_server/messaging/consumers/voice_consumer.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import structlog +from aio_pika.abc import AbstractIncomingMessage + +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.messaging.publisher import CallbackPublisher +from ai_server.model.envelope import Envelope +from ai_server.model.messages.voice import ( + AnalyzeVoiceRequest, + VoiceCallbackPayload, +) +from ai_server.storage.base import ObjectStorage +from ai_server.voice.analysis.metrics import analyze +from ai_server.voice.stt.base import SttError, SttProvider + +log = structlog.get_logger(__name__) + + +class VoiceConsumer: + """analyze.voice consumer (음성 답변 STT + 정량 분석). + + 흐름: + 1. envelope parse + 멱등 체크 + 2. S3 GET (audio_s3_key) + 3. STT → TranscriptionResult + 4. 음성 분석 → VoiceMetrics + 5. callback.voice 발행 (성공: transcript + metrics, 실패: error_code) + """ + + def __init__( + self, + *, + stt: SttProvider, + storage: ObjectStorage, + publisher: CallbackPublisher, + idempotency: LruIdempotencyStore, + callback_routing_key: str, + filler_pattern: str, + ) -> None: + self._stt = stt + self._storage = storage + self._publisher = publisher + self._idempotency = idempotency + self._callback_routing_key = callback_routing_key + self._filler_pattern = filler_pattern + + async def handle(self, message: AbstractIncomingMessage) -> None: + async with message.process(requeue=False): + try: + envelope = Envelope[AnalyzeVoiceRequest].model_validate_json(message.body) + except Exception as exc: + log.error("voice.parse.failed", error=str(exc), delivery_tag=message.delivery_tag) + raise + + if self._idempotency.is_seen_then_mark(envelope.message_id): + log.info("voice.idempotent.skip", message_id=envelope.message_id) + return + + req = envelope.payload + log.info( + "voice.analyze.start", + message_id=envelope.message_id, + session_id=req.session_id, + interview_message_id=req.message_id, + key=req.audio_s3_key, + trace_id=envelope.trace_id, + ) + + try: + audio_bytes = await self._storage.get_bytes(req.audio_s3_key) + except Exception as exc: + log.error("voice.storage.failed", error=str(exc), key=req.audio_s3_key) + await self._publish_failed(envelope, req, code="AUDIO_FETCH_FAILED") + return + + try: + result = await self._stt.transcribe( + audio_bytes=audio_bytes, + content_type=req.content_type, + hint=req.previous_question_text, + ) + except SttError as exc: + log.error( + "voice.stt.failed", + error=str(exc), + code=exc.code, + session_id=req.session_id, + ) + await self._publish_failed(envelope, req, code=exc.code) + return + except Exception as exc: + log.error("voice.stt.unexpected", error=str(exc), session_id=req.session_id) + await self._publish_failed(envelope, req, code="TRANSCRIPTION_FAILED") + return + + metrics = analyze(result, filler_pattern=self._filler_pattern) + + payload = VoiceCallbackPayload( + session_id=req.session_id, + interview_message_id=req.message_id, + transcript=result.text, + speaking_rate_wpm=metrics.speaking_rate_wpm, + silence_duration_sec=metrics.silence_duration_sec, + filler_word_counts=metrics.filler_word_counts, + pronunciation_accuracy=metrics.pronunciation_accuracy, + error_code=None, + ) + await self._publisher.publish( + routing_key=self._callback_routing_key, + message_type="callback.voice", + payload=payload, + trace_id=envelope.trace_id, + correlation_id=envelope.message_id, + context=envelope.context, + ) + log.info( + "voice.analyze.done", + message_id=envelope.message_id, + session_id=req.session_id, + interview_message_id=req.message_id, + wpm=metrics.speaking_rate_wpm, + trace_id=envelope.trace_id, + ) + + async def _publish_failed(self, envelope, req: AnalyzeVoiceRequest, *, code: str) -> None: + payload = VoiceCallbackPayload( + session_id=req.session_id, + interview_message_id=req.message_id, + transcript=None, + speaking_rate_wpm=None, + silence_duration_sec=None, + filler_word_counts={}, + pronunciation_accuracy=None, + error_code=code, + ) + await self._publisher.publish( + routing_key=self._callback_routing_key, + message_type="callback.voice", + payload=payload, + trace_id=envelope.trace_id, + correlation_id=envelope.message_id, + context=envelope.context, + ) diff --git a/ai/src/ai_server/messaging/runner.py b/ai/src/ai_server/messaging/runner.py index e18005e2..a17712a3 100644 --- a/ai/src/ai_server/messaging/runner.py +++ b/ai/src/ai_server/messaging/runner.py @@ -17,6 +17,10 @@ LlmFollowupGenerator, build_followup_generation_chain, ) +from ai_server.chain.feedback_generation_chain import ( + LlmFeedbackGenerator, + build_feedback_generation_chain, +) from ai_server.chain.question_generation_chain import ( LlmQuestionGenerator, build_question_generation_chain, @@ -26,8 +30,11 @@ from ai_server.messaging.connection import RabbitConnection from ai_server.rag.chunker import MarkdownChunker from ai_server.rag.embedder import build_embedding_provider +from ai_server.messaging.consumers.feedback_consumer import FeedbackConsumer from ai_server.messaging.consumers.followup_consumer import FollowupConsumer from ai_server.messaging.consumers.questions_consumer import QuestionsConsumer +from ai_server.messaging.consumers.voice_consumer import VoiceConsumer +from ai_server.voice.stt.factory import build_stt_provider from ai_server.messaging.consumers.repository_consumer import RepositoryConsumer from ai_server.messaging.consumers.resume_consumer import ResumeConsumer from ai_server.messaging.consumers.web_consumer import WebResumeConsumer @@ -159,6 +166,31 @@ def __init__(self, settings: Settings) -> None: callback_routing_key=settings.ai_callback_routing_questions, ) + # 종합 피드백 생성 (US-24) + feedback_generator = LlmFeedbackGenerator( + build_feedback_generation_chain(settings, core_client=core_client) + ) + self._feedback_consumer = FeedbackConsumer( + generator=feedback_generator, + publisher=self._publisher, + idempotency=self._idempotency, + callback_routing_key=settings.ai_callback_routing_feedback, + core_client=core_client, + embedder=embedder, + rag_top_k=settings.feedback_rag_top_k, + ) + + # 음성 답변 STT + 분석 (Phase 2) + stt = build_stt_provider(settings) + self._voice_consumer = VoiceConsumer( + stt=stt, + storage=storage, + publisher=self._publisher, + idempotency=self._idempotency, + callback_routing_key=settings.ai_callback_routing_voice, + filler_pattern=settings.voice_filler_pattern, + ) + self._consumers: list[tuple[AbstractRobustQueue, str]] = [] async def start(self) -> None: @@ -192,6 +224,16 @@ async def start(self) -> None: queue_name=self._settings.ai_queue_followup, handler=self._followup_consumer.handle, ) + await self._start_consumer( + channel, + queue_name=self._settings.ai_queue_feedback, + handler=self._feedback_consumer.handle, + ) + await self._start_consumer( + channel, + queue_name=self._settings.ai_queue_voice, + handler=self._voice_consumer.handle, + ) async def _start_consumer(self, channel, *, queue_name, handler) -> None: queue = await channel.declare_queue( diff --git a/ai/src/ai_server/model/messages/feedback.py b/ai/src/ai_server/model/messages/feedback.py new file mode 100644 index 00000000..a567899c --- /dev/null +++ b/ai/src/ai_server/model/messages/feedback.py @@ -0,0 +1,44 @@ +from typing import Literal + +from pydantic import BaseModel, Field + +from ai_server.model._config import camel_config + + +class FeedbackMessageItem(BaseModel): + """세션 시퀀스 한 줄 (Core 가 통째로 동봉).""" + model_config = camel_config() + + id: int + sequence_number: int + role: Literal["INTERVIEWER", "INTERVIEWEE", "SYSTEM"] + content: str + parent_message_id: int | None = None + + +class GenerateFeedbackRequest(BaseModel): + """Core 가 세션 COMPLETED commit 후 발행.""" + model_config = camel_config() + + session_id: int + interview_type: Literal["PERSONALITY", "TECHNICAL", "LIVE_CODING", "INTEGRATED"] + job_category: Literal["FRONTEND", "BACKEND", "INFRA", "DBA"] + total_question_count: int | None = None + end_reason: Literal["USER_REQUEST", "MAX_QUESTIONS_REACHED"] | None = None + messages: list[FeedbackMessageItem] = Field(default_factory=list) + context_document_ids: list[int] = Field(default_factory=list) + + +class FeedbackCallbackPayload(BaseModel): + """AI → Core 종합 피드백. 점수는 0~100 (NULL 허용).""" + model_config = camel_config() + + session_id: int + overall_score: float | None = None + technical_accuracy: float | None = None + logic_score: float | None = None + communication_score: float | None = None + strengths_summary: str | None = None + weaknesses_summary: str | None = None + improvement_keywords: list[str] = Field(default_factory=list) + report_s3_key: str | None = None diff --git a/ai/src/ai_server/model/messages/voice.py b/ai/src/ai_server/model/messages/voice.py new file mode 100644 index 00000000..e1842a99 --- /dev/null +++ b/ai/src/ai_server/model/messages/voice.py @@ -0,0 +1,33 @@ +from typing import Literal + +from pydantic import BaseModel, Field + +from ai_server.model._config import camel_config + + +class AnalyzeVoiceRequest(BaseModel): + """Core 가 음성 답변 업로드 commit 후 발행.""" + model_config = camel_config() + + session_id: int + message_id: int # interview_messages.id (placeholder, STT 후 content 채움) + parent_question_message_id: int | None = None + audio_s3_key: str + content_type: str + previous_question_text: str | None = None + interview_type: Literal["PERSONALITY", "TECHNICAL", "LIVE_CODING", "INTEGRATED"] + job_category: Literal["FRONTEND", "BACKEND", "INFRA", "DBA"] + + +class VoiceCallbackPayload(BaseModel): + """AI → Core. STT 결과 + 음성 지표.""" + model_config = camel_config() + + session_id: int + interview_message_id: int + transcript: str | None = None + speaking_rate_wpm: float | None = None + silence_duration_sec: float | None = None + filler_word_counts: dict[str, int] = Field(default_factory=dict) + pronunciation_accuracy: float | None = None + error_code: str | None = None # 실패 시 코드 (Core 가 message FAILED 마킹) diff --git a/ai/src/ai_server/voice/__init__.py b/ai/src/ai_server/voice/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/src/ai_server/voice/analysis/__init__.py b/ai/src/ai_server/voice/analysis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/src/ai_server/voice/analysis/metrics.py b/ai/src/ai_server/voice/analysis/metrics.py new file mode 100644 index 00000000..09edbf51 --- /dev/null +++ b/ai/src/ai_server/voice/analysis/metrics.py @@ -0,0 +1,96 @@ +"""음성 답변 정량 분석. + +입력: STT 결과 (TranscriptionResult). +출력: WPM, 간투어 카운트, 무음(silence) 합계, 평균 logprob(=pronunciation_accuracy 근사). + +- WPM: word_count / duration_minutes. 한국어는 공백 기반 토큰이 단어 단위가 아니므로 + 대화 답변에는 '어절(공백 단위)' 기준이 실용적. 영어/혼용도 같은 식이 자연스러움. +- 간투어: 한국어 정규식 (settings.voice_filler_pattern). 음/어/그/아 같은 반복형. +- 무음(silence_duration_sec): Whisper segment 들의 빈틈(start_(i+1) - end_i) 누적. + 앞/뒤 패딩(처음 segment 시작 전, 마지막 segment 종료 후 ~ duration_sec)도 포함. +- pronunciation_accuracy 근사: segment 평균 logprob → exp(logprob) 의 평균으로 0~1 근사. + logprob 가 비어있는 segment 는 제외. 모든 segment 가 비면 None. +""" + +from __future__ import annotations + +import math +import re +from collections import Counter +from dataclasses import dataclass + +from ai_server.voice.stt.base import TranscriptionResult + + +@dataclass(frozen=True) +class VoiceMetrics: + speaking_rate_wpm: float | None + silence_duration_sec: float | None + filler_word_counts: dict[str, int] + pronunciation_accuracy: float | None + + +def analyze(result: TranscriptionResult, *, filler_pattern: str) -> VoiceMetrics: + text = (result.text or "").strip() + duration = result.duration_sec + + word_count = _word_count(text) + wpm: float | None = None + if duration and duration > 0: + wpm = round((word_count / duration) * 60.0, 2) + + fillers = _count_fillers(text, filler_pattern) + silence = _silence_seconds(result) + accuracy = _pronunciation_accuracy(result) + + return VoiceMetrics( + speaking_rate_wpm=wpm, + silence_duration_sec=silence, + filler_word_counts=fillers, + pronunciation_accuracy=accuracy, + ) + + +def _word_count(text: str) -> int: + if not text: + return 0 + return len([t for t in text.split() if t]) + + +def _count_fillers(text: str, pattern: str) -> dict[str, int]: + if not text: + return {} + try: + regex = re.compile(pattern) + except re.error: + return {} + counter: Counter[str] = Counter() + for match in regex.findall(text): + # 반복형(예: 음음음) 을 키로 normalize: 첫 글자만 유지. + token = match[0] if match else match + counter[token] += 1 + return dict(counter) + + +def _silence_seconds(result: TranscriptionResult) -> float | None: + if not result.segments: + return None + silence = 0.0 + # 앞 패딩 + silence += max(0.0, result.segments[0].start_sec) + # 사이 갭 + for prev, nxt in zip(result.segments, result.segments[1:]): + silence += max(0.0, nxt.start_sec - prev.end_sec) + # 뒤 패딩 + if result.duration_sec is not None: + silence += max(0.0, result.duration_sec - result.segments[-1].end_sec) + return round(silence, 3) + + +def _pronunciation_accuracy(result: TranscriptionResult) -> float | None: + logprobs = [s.avg_logprob for s in result.segments if s.avg_logprob is not None] + if not logprobs: + return None + # exp(logprob) ∈ (0, 1], 평균. + probs = [math.exp(lp) for lp in logprobs] + return round(sum(probs) / len(probs), 4) diff --git a/ai/src/ai_server/voice/stt/__init__.py b/ai/src/ai_server/voice/stt/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai/src/ai_server/voice/stt/base.py b/ai/src/ai_server/voice/stt/base.py new file mode 100644 index 00000000..c380d52a --- /dev/null +++ b/ai/src/ai_server/voice/stt/base.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Protocol + + +@dataclass(frozen=True) +class TranscriptionSegment: + start_sec: float + end_sec: float + text: str + avg_logprob: float | None = None + + +@dataclass(frozen=True) +class TranscriptionResult: + text: str + language: str | None + duration_sec: float | None + segments: list[TranscriptionSegment] = field(default_factory=list) + + +class SttProvider(Protocol): + async def transcribe( + self, + *, + audio_bytes: bytes, + content_type: str, + hint: str | None = None, + ) -> TranscriptionResult: ... + + +class SttError(Exception): + def __init__(self, *, code: str, message: str, retriable: bool) -> None: + super().__init__(message) + self.code = code + self.message = message + self.retriable = retriable diff --git a/ai/src/ai_server/voice/stt/deepgram.py b/ai/src/ai_server/voice/stt/deepgram.py new file mode 100644 index 00000000..0a658f64 --- /dev/null +++ b/ai/src/ai_server/voice/stt/deepgram.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import httpx +import structlog + +from ai_server.voice.stt.base import ( + SttError, + TranscriptionResult, + TranscriptionSegment, +) + +log = structlog.get_logger(__name__) + + +class DeepgramSttProvider: + """Deepgram /v1/listen STT 호출. + + Mindlogic 게이트웨이가 STT 미지원이라 Deepgram 직접 호출. DEEPGRAM_API_KEY 필요. + 응답에서 transcript + utterances(=문장 단위 segment) + words 활용. + + 한국어 정확도 우선 시 model=whisper-large 권장 (whisper-1 동등). + 저비용 우선 시 nova-2 (한국어는 살짝 떨어짐). + """ + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.deepgram.com/v1", + model: str = "whisper-large", + language: str | None = "ko", + timeout_sec: float = 60.0, + client: httpx.AsyncClient | None = None, + ) -> None: + if not api_key: + raise ValueError("Deepgram API key 누락") + self._api_key = api_key + self._base_url = base_url.rstrip("/") + self._model = model + self._language = language + self._timeout_sec = timeout_sec + self._client = client + + async def transcribe( + self, + *, + audio_bytes: bytes, + content_type: str, + hint: str | None = None, + ) -> TranscriptionResult: + params: dict[str, str] = { + "model": self._model, + "smart_format": "true", + "punctuate": "true", + "utterances": "true", + } + if self._language: + params["language"] = self._language + if hint: + # Deepgram 의 keyword boost. 토큰 단위로 콜론+boost 지정 가능. 간단히 hint 텍스트 그대로. + params["keywords"] = hint[:200] + + headers = { + "Authorization": f"Token {self._api_key}", + "Content-Type": content_type or "application/octet-stream", + } + url = f"{self._base_url}/listen" + + try: + if self._client is not None: + resp = await self._client.post(url, params=params, headers=headers, content=audio_bytes) + else: + async with httpx.AsyncClient(timeout=self._timeout_sec) as client: + resp = await client.post(url, params=params, headers=headers, content=audio_bytes) + except httpx.HTTPError as exc: + raise SttError(code="STT_UNAVAILABLE", message=f"Deepgram 호출 실패: {exc}", retriable=True) from exc + + if resp.status_code in (401, 403): + raise SttError(code="STT_AUTH_FAILED", message=f"Deepgram 인증 실패: {resp.status_code}", retriable=False) + if resp.status_code >= 500: + raise SttError(code="STT_UNAVAILABLE", message=f"Deepgram 5xx: {resp.status_code}", retriable=True) + if resp.status_code >= 400: + raise SttError( + code="STT_BAD_REQUEST", + message=f"Deepgram {resp.status_code}: {resp.text[:200]}", + retriable=False, + ) + + try: + data = resp.json() + except ValueError as exc: + raise SttError(code="STT_BAD_RESPONSE", message=f"JSON 파싱 실패: {exc}", retriable=True) from exc + + metadata = data.get("metadata") or {} + results = data.get("results") or {} + channels = results.get("channels") or [] + if not channels: + return TranscriptionResult(text="", language=self._language, duration_sec=None, segments=[]) + + alt = (channels[0].get("alternatives") or [{}])[0] + text = str(alt.get("transcript") or "") + duration = metadata.get("duration") + + # utterances (문장/발화 단위) 를 우선 사용. 없으면 단일 segment fallback. + segments: list[TranscriptionSegment] = [] + for utt in results.get("utterances") or []: + segments.append( + TranscriptionSegment( + start_sec=float(utt.get("start", 0.0)), + end_sec=float(utt.get("end", 0.0)), + text=str(utt.get("transcript") or ""), + avg_logprob=_confidence_to_logprob(utt.get("confidence")), + ) + ) + if not segments and text: + segments.append( + TranscriptionSegment( + start_sec=0.0, + end_sec=float(duration) if duration is not None else 0.0, + text=text, + avg_logprob=_confidence_to_logprob(alt.get("confidence")), + ) + ) + + return TranscriptionResult( + text=text, + language=self._language, + duration_sec=(float(duration) if duration is not None else None), + segments=segments, + ) + + +def _confidence_to_logprob(confidence) -> float | None: + """Deepgram confidence(0~1) 를 logprob 으로 변환 — exp(logprob) ≈ confidence. + + metrics.analyze() 가 pronunciation_accuracy 를 exp(avg_logprob) 평균으로 산정하므로 + logprob = ln(confidence) 변환만 해주면 그대로 정확도 근사값으로 작동. + """ + if confidence is None: + return None + try: + c = float(confidence) + except (TypeError, ValueError): + return None + if c <= 0: + return None + import math + + return math.log(min(c, 1.0)) diff --git a/ai/src/ai_server/voice/stt/factory.py b/ai/src/ai_server/voice/stt/factory.py new file mode 100644 index 00000000..b10b0c4b --- /dev/null +++ b/ai/src/ai_server/voice/stt/factory.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import structlog + +from ai_server.config.settings import Settings +from ai_server.voice.stt.base import SttProvider +from ai_server.voice.stt.deepgram import DeepgramSttProvider +from ai_server.voice.stt.mock import MockSttProvider +from ai_server.voice.stt.openai_whisper import OpenAiWhisperSttProvider + +log = structlog.get_logger(__name__) + + +def build_stt_provider(settings: Settings) -> SttProvider: + """STT 공급자 선택. + + 명시적 우선순위: + 1. STT_PROVIDER 가 명시되면 그 값을 따름. + 2. "auto" 인 경우 DEEPGRAM_API_KEY → OPENAI_API_KEY → mock 순. + 3. 키 누락 시에는 안전한 mock fallback (서비스 부팅은 항상 성공). + """ + + provider = (settings.stt_provider or "auto").lower() + + if provider == "auto": + if settings.deepgram_api_key: + provider = "deepgram" + elif settings.openai_api_key: + provider = "openai_whisper" + else: + provider = "mock" + + if provider == "deepgram": + if not settings.deepgram_api_key: + log.warn("stt.fallback_to_mock", reason="DEEPGRAM_API_KEY 누락") + return MockSttProvider() + return DeepgramSttProvider( + api_key=settings.deepgram_api_key, + base_url=settings.deepgram_base_url, + model=settings.deepgram_model, + language=settings.deepgram_language or None, + timeout_sec=settings.deepgram_timeout_sec, + ) + + if provider == "openai_whisper": + if not settings.openai_api_key: + log.warn("stt.fallback_to_mock", reason="OPENAI_API_KEY 누락") + return MockSttProvider() + return OpenAiWhisperSttProvider( + api_key=settings.openai_api_key, + base_url=settings.openai_base_url, + model=settings.whisper_model, + language=settings.whisper_language or None, + timeout_sec=settings.whisper_timeout_sec, + ) + + if provider == "mock": + return MockSttProvider() + + log.warn("stt.unknown_provider_fallback_mock", provider=provider) + return MockSttProvider() diff --git a/ai/src/ai_server/voice/stt/mock.py b/ai/src/ai_server/voice/stt/mock.py new file mode 100644 index 00000000..b6c13655 --- /dev/null +++ b/ai/src/ai_server/voice/stt/mock.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import structlog + +from ai_server.voice.stt.base import TranscriptionResult, TranscriptionSegment + +log = structlog.get_logger(__name__) + + +class MockSttProvider: + """개발/테스트용. 실제 STT 안 함 — 더미 transcript + 단일 segment 반환.""" + + def __init__(self, *, default_text: str = "(mock transcript)") -> None: + self._default_text = default_text + + async def transcribe( + self, + *, + audio_bytes: bytes, + content_type: str, + hint: str | None = None, + ) -> TranscriptionResult: + # 매우 단순한 길이 추정: 1초당 16KB 가정. + estimated_duration = max(1.0, len(audio_bytes) / 16_000.0) + text = self._default_text + return TranscriptionResult( + text=text, + language="ko", + duration_sec=estimated_duration, + segments=[ + TranscriptionSegment( + start_sec=0.0, + end_sec=estimated_duration, + text=text, + avg_logprob=None, + ) + ], + ) diff --git a/ai/src/ai_server/voice/stt/openai_whisper.py b/ai/src/ai_server/voice/stt/openai_whisper.py new file mode 100644 index 00000000..7f95d1e2 --- /dev/null +++ b/ai/src/ai_server/voice/stt/openai_whisper.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import httpx +import structlog + +from ai_server.voice.stt.base import ( + SttError, + TranscriptionResult, + TranscriptionSegment, +) + +log = structlog.get_logger(__name__) + + +class OpenAiWhisperSttProvider: + """OpenAI Whisper API (`/audio/transcriptions`) 호출. + + Mindlogic 게이트웨이가 STT 미지원이라 OpenAI 직접 호출. OPENAI_API_KEY 필요. + response_format=verbose_json 으로 segment timestamps 와 logprob 회수. + """ + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.openai.com/v1", + model: str = "whisper-1", + language: str | None = "ko", + timeout_sec: float = 60.0, + client: httpx.AsyncClient | None = None, + ) -> None: + if not api_key: + raise ValueError("OpenAI API key 누락") + self._api_key = api_key + self._base_url = base_url.rstrip("/") + self._model = model + self._language = language + self._timeout_sec = timeout_sec + self._client = client + + async def transcribe( + self, + *, + audio_bytes: bytes, + content_type: str, + hint: str | None = None, + ) -> TranscriptionResult: + url = f"{self._base_url}/audio/transcriptions" + filename = "audio." + _ext_for(content_type) + files = {"file": (filename, audio_bytes, content_type)} + data: dict[str, str] = { + "model": self._model, + "response_format": "verbose_json", + } + if self._language: + data["language"] = self._language + if hint: + data["prompt"] = hint[:200] + headers = {"Authorization": f"Bearer {self._api_key}"} + + try: + if self._client is not None: + resp = await self._client.post(url, headers=headers, files=files, data=data) + else: + async with httpx.AsyncClient(timeout=self._timeout_sec) as client: + resp = await client.post(url, headers=headers, files=files, data=data) + except httpx.HTTPError as exc: + raise SttError(code="STT_UNAVAILABLE", message=f"OpenAI 호출 실패: {exc}", retriable=True) from exc + + if resp.status_code in (401, 403): + raise SttError(code="STT_AUTH_FAILED", message=f"OpenAI 인증 실패: {resp.status_code}", retriable=False) + if resp.status_code >= 500: + raise SttError(code="STT_UNAVAILABLE", message=f"OpenAI 5xx: {resp.status_code}", retriable=True) + if resp.status_code >= 400: + raise SttError( + code="STT_BAD_REQUEST", + message=f"OpenAI {resp.status_code}: {resp.text[:200]}", + retriable=False, + ) + + try: + data_resp = resp.json() + except ValueError as exc: + raise SttError(code="STT_BAD_RESPONSE", message=f"JSON 파싱 실패: {exc}", retriable=True) from exc + + segments = [] + for seg in data_resp.get("segments") or []: + segments.append( + TranscriptionSegment( + start_sec=float(seg.get("start", 0.0)), + end_sec=float(seg.get("end", 0.0)), + text=str(seg.get("text", "")), + avg_logprob=(float(seg["avg_logprob"]) if "avg_logprob" in seg else None), + ) + ) + return TranscriptionResult( + text=str(data_resp.get("text", "")), + language=data_resp.get("language"), + duration_sec=(float(data_resp["duration"]) if "duration" in data_resp else None), + segments=segments, + ) + + +def _ext_for(content_type: str) -> str: + ct = (content_type or "").lower() + if "webm" in ct: + return "webm" + if "ogg" in ct: + return "ogg" + if "mpeg" in ct or "mp3" in ct: + return "mp3" + if "wav" in ct: + return "wav" + if "mp4" in ct or "m4a" in ct: + return "m4a" + return "bin" diff --git a/ai/tests/test_deepgram_stt.py b/ai/tests/test_deepgram_stt.py new file mode 100644 index 00000000..5abd70fc --- /dev/null +++ b/ai/tests/test_deepgram_stt.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import math + +import httpx +import pytest + +from ai_server.voice.stt.base import SttError +from ai_server.voice.stt.deepgram import DeepgramSttProvider + + +def _client(*, status: int, body: dict | str) -> httpx.AsyncClient: + def handler(request: httpx.Request) -> httpx.Response: + if isinstance(body, dict): + return httpx.Response(status, json=body) + return httpx.Response(status, text=body) + + return httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + +@pytest.mark.asyncio +async def test_deepgram_returns_segments_and_logprob_from_utterances(): + response = { + "metadata": {"duration": 6.0}, + "results": { + "channels": [ + { + "alternatives": [ + { + "transcript": "안녕하세요 백엔드 지원자입니다", + "confidence": 0.9, + } + ] + } + ], + "utterances": [ + {"start": 0.0, "end": 2.5, "transcript": "안녕하세요", "confidence": 0.95}, + {"start": 3.0, "end": 6.0, "transcript": "백엔드 지원자입니다", "confidence": 0.88}, + ], + }, + } + async with _client(status=200, body=response) as client: + provider = DeepgramSttProvider(api_key="k", client=client) + result = await provider.transcribe(audio_bytes=b"fake", content_type="audio/webm") + + assert result.text.startswith("안녕하세요") + assert result.duration_sec == 6.0 + assert len(result.segments) == 2 + # confidence 0.95 → logprob ≈ ln(0.95) + assert abs(result.segments[0].avg_logprob - math.log(0.95)) < 1e-6 + + +@pytest.mark.asyncio +async def test_deepgram_fallback_segment_when_no_utterances(): + response = { + "metadata": {"duration": 4.2}, + "results": { + "channels": [ + { + "alternatives": [ + {"transcript": "한 줄 답변", "confidence": 0.82}, + ] + } + ], + }, + } + async with _client(status=200, body=response) as client: + provider = DeepgramSttProvider(api_key="k", client=client) + result = await provider.transcribe(audio_bytes=b"fake", content_type="audio/webm") + + assert result.text == "한 줄 답변" + assert len(result.segments) == 1 + assert result.segments[0].end_sec == 4.2 + + +@pytest.mark.asyncio +async def test_deepgram_auth_error_raises_non_retriable(): + async with _client(status=401, body="unauthorized") as client: + provider = DeepgramSttProvider(api_key="k", client=client) + with pytest.raises(SttError) as exc_info: + await provider.transcribe(audio_bytes=b"x", content_type="audio/webm") + assert exc_info.value.code == "STT_AUTH_FAILED" + assert exc_info.value.retriable is False + + +@pytest.mark.asyncio +async def test_deepgram_5xx_is_retriable(): + async with _client(status=502, body="bad gateway") as client: + provider = DeepgramSttProvider(api_key="k", client=client) + with pytest.raises(SttError) as exc_info: + await provider.transcribe(audio_bytes=b"x", content_type="audio/webm") + assert exc_info.value.code == "STT_UNAVAILABLE" + assert exc_info.value.retriable is True diff --git a/ai/tests/test_feedback_consumer.py b/ai/tests/test_feedback_consumer.py new file mode 100644 index 00000000..57d54596 --- /dev/null +++ b/ai/tests/test_feedback_consumer.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from ai_server.chain.feedback_generation_chain import FeedbackResult +from ai_server.core.client import EmbeddingSearchHit +from ai_server.messaging.consumers.feedback_consumer import FeedbackConsumer +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.model.messages.feedback import FeedbackCallbackPayload + + +class _StubMessage: + def __init__(self, body: bytes): + self.body = body + self.delivery_tag = 1 + + def process(self, requeue: bool = False): + return _NoopCtx() + + +class _NoopCtx: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + +def _envelope(*, context_documents: list[int] | None = None) -> bytes: + env = { + "messageId": "fb-1", + "messageType": "generate.feedback", + "version": "v1", + "traceId": "t-1", + "publishedAt": "2026-05-30T00:00:00Z", + "publisher": "core-server", + "payload": { + "sessionId": 50, + "interviewType": "TECHNICAL", + "jobCategory": "BACKEND", + "totalQuestionCount": 2, + "endReason": "MAX_QUESTIONS_REACHED", + "messages": [ + {"id": 100, "sequenceNumber": 1, "role": "INTERVIEWER", "content": "ACID?"}, + {"id": 101, "sequenceNumber": 2, "role": "INTERVIEWEE", + "content": "원자성·일관성·격리성·영속성", "parentMessageId": 100}, + ], + "contextDocumentIds": context_documents or [], + }, + "context": {"userId": 1, "sessionId": 50}, + } + return json.dumps(env).encode() + + +def _generator(): + g = MagicMock() + g.generate = AsyncMock( + return_value=FeedbackResult( + overall_score=85.0, + technical_accuracy=82.0, + logic_score=88.0, + communication_score=80.0, + strengths_summary="ACID 4요소를 명확히 답변.", + weaknesses_summary="구체적 사례 부족.", + improvement_keywords=["MVCC", "Repeatable Read"], + ) + ) + return g + + +@pytest.mark.asyncio +async def test_consumer_generates_feedback_and_publishes_callback(): + generator = _generator() + publisher = MagicMock() + publisher.publish = AsyncMock() + core = MagicMock() + + consumer = FeedbackConsumer( + generator=generator, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=10), + callback_routing_key="callback.feedback", + core_client=core, + embedder=None, + ) + await consumer.handle(_StubMessage(_envelope())) + + generator.generate.assert_awaited_once() + publisher.publish.assert_awaited_once() + payload: FeedbackCallbackPayload = publisher.publish.await_args.kwargs["payload"] + assert payload.session_id == 50 + assert payload.overall_score == 85.0 + assert publisher.publish.await_args.kwargs["message_type"] == "callback.feedback" + + +@pytest.mark.asyncio +async def test_consumer_calls_rag_when_documents_and_embedder_present(): + generator = _generator() + publisher = MagicMock() + publisher.publish = AsyncMock() + core = MagicMock() + core.search_embeddings = AsyncMock( + return_value=[ + EmbeddingSearchHit(document_id=7, chunk_index=2, chunk_text="JPA dirty checking", distance=0.12) + ] + ) + embedder = MagicMock() + embedder.embed = AsyncMock(return_value=[[0.1, 0.2, 0.3]]) + + consumer = FeedbackConsumer( + generator=generator, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=10), + callback_routing_key="callback.feedback", + core_client=core, + embedder=embedder, + rag_top_k=3, + ) + await consumer.handle(_StubMessage(_envelope(context_documents=[7]))) + + embedder.embed.assert_awaited_once() + core.search_embeddings.assert_awaited_once() + # rag_context 가 chain 호출 인자로 전달됐는지 확인 + invoked_kwargs = generator.generate.await_args.kwargs + assert "JPA dirty checking" in invoked_kwargs["rag_context"] + + +@pytest.mark.asyncio +async def test_consumer_idempotent_skip(): + generator = _generator() + publisher = MagicMock() + publisher.publish = AsyncMock() + idempotency = LruIdempotencyStore(max_size=10) + idempotency.is_seen_then_mark("fb-1") + + consumer = FeedbackConsumer( + generator=generator, + publisher=publisher, + idempotency=idempotency, + callback_routing_key="callback.feedback", + core_client=MagicMock(), + embedder=None, + ) + await consumer.handle(_StubMessage(_envelope())) + generator.generate.assert_not_awaited() + publisher.publish.assert_not_awaited() diff --git a/ai/tests/test_voice_consumer.py b/ai/tests/test_voice_consumer.py new file mode 100644 index 00000000..51debc84 --- /dev/null +++ b/ai/tests/test_voice_consumer.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from ai_server.messaging.consumers.voice_consumer import VoiceConsumer +from ai_server.messaging.idempotency import LruIdempotencyStore +from ai_server.model.messages.voice import VoiceCallbackPayload +from ai_server.voice.stt.base import ( + SttError, + TranscriptionResult, + TranscriptionSegment, +) + + +class _StubMessage: + def __init__(self, body: bytes): + self.body = body + self.delivery_tag = 1 + + def process(self, requeue: bool = False): + return _NoopCtx() + + +class _NoopCtx: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + +def _envelope() -> bytes: + env = { + "messageId": "voice-1", + "messageType": "analyze.voice", + "version": "v1", + "traceId": "t-1", + "publishedAt": "2026-05-30T00:00:00Z", + "publisher": "core-server", + "payload": { + "sessionId": 99, + "messageId": 501, + "parentQuestionMessageId": 500, + "audioS3Key": "interview/voice/raw/99/501.webm", + "contentType": "audio/webm", + "previousQuestionText": "ACID 설명해주세요", + "interviewType": "TECHNICAL", + "jobCategory": "BACKEND", + }, + "context": {"userId": 42, "sessionId": 99}, + } + return json.dumps(env).encode() + + +def _stt_ok(): + stt = MagicMock() + stt.transcribe = AsyncMock( + return_value=TranscriptionResult( + text="ACID는 원자성 일관성 격리성 영속성 입니다", + language="ko", + duration_sec=30.0, + segments=[ + TranscriptionSegment(start_sec=0.0, end_sec=30.0, + text="ACID는 원자성 일관성 격리성 영속성 입니다", + avg_logprob=-0.3) + ], + ) + ) + return stt + + +def _storage_ok(): + storage = MagicMock() + storage.get_bytes = AsyncMock(return_value=b"fake-audio") + return storage + + +@pytest.mark.asyncio +async def test_consumer_publishes_transcript_and_metrics(): + publisher = MagicMock() + publisher.publish = AsyncMock() + + consumer = VoiceConsumer( + stt=_stt_ok(), + storage=_storage_ok(), + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=10), + callback_routing_key="callback.voice", + filler_pattern=r"(?:음+|어+|그+|아+)", + ) + await consumer.handle(_StubMessage(_envelope())) + + publisher.publish.assert_awaited_once() + payload: VoiceCallbackPayload = publisher.publish.await_args.kwargs["payload"] + assert payload.session_id == 99 + assert payload.interview_message_id == 501 + assert payload.transcript.startswith("ACID") + assert payload.speaking_rate_wpm is not None + assert publisher.publish.await_args.kwargs["message_type"] == "callback.voice" + + +@pytest.mark.asyncio +async def test_consumer_publishes_error_code_on_stt_failure(): + stt = MagicMock() + stt.transcribe = AsyncMock( + side_effect=SttError(code="STT_AUTH_FAILED", message="bad key", retriable=False) + ) + publisher = MagicMock() + publisher.publish = AsyncMock() + + consumer = VoiceConsumer( + stt=stt, + storage=_storage_ok(), + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=10), + callback_routing_key="callback.voice", + filler_pattern=r"(?:음+|어+|그+|아+)", + ) + await consumer.handle(_StubMessage(_envelope())) + + payload: VoiceCallbackPayload = publisher.publish.await_args.kwargs["payload"] + assert payload.error_code == "STT_AUTH_FAILED" + assert payload.transcript is None + + +@pytest.mark.asyncio +async def test_consumer_publishes_error_code_on_storage_failure(): + storage = MagicMock() + storage.get_bytes = AsyncMock(side_effect=RuntimeError("s3 down")) + publisher = MagicMock() + publisher.publish = AsyncMock() + + consumer = VoiceConsumer( + stt=_stt_ok(), + storage=storage, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=10), + callback_routing_key="callback.voice", + filler_pattern=r"(?:음+|어+|그+|아+)", + ) + await consumer.handle(_StubMessage(_envelope())) + + payload: VoiceCallbackPayload = publisher.publish.await_args.kwargs["payload"] + assert payload.error_code == "AUDIO_FETCH_FAILED" + + +@pytest.mark.asyncio +async def test_consumer_idempotent_skip(): + publisher = MagicMock() + publisher.publish = AsyncMock() + idempotency = LruIdempotencyStore(max_size=10) + idempotency.is_seen_then_mark("voice-1") + + consumer = VoiceConsumer( + stt=_stt_ok(), + storage=_storage_ok(), + publisher=publisher, + idempotency=idempotency, + callback_routing_key="callback.voice", + filler_pattern=r"(?:음+|어+|그+|아+)", + ) + await consumer.handle(_StubMessage(_envelope())) + publisher.publish.assert_not_awaited() diff --git a/ai/tests/test_voice_metrics.py b/ai/tests/test_voice_metrics.py new file mode 100644 index 00000000..caf2bcd1 --- /dev/null +++ b/ai/tests/test_voice_metrics.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import math + +from ai_server.voice.analysis.metrics import analyze +from ai_server.voice.stt.base import TranscriptionResult, TranscriptionSegment + +FILLER_PATTERN = r"(?:음+|어+|그+|아+)" + + +def test_word_count_and_wpm(): + result = TranscriptionResult( + text="저는 백엔드 개발자 입니다", + language="ko", + duration_sec=60.0, + segments=[ + TranscriptionSegment(start_sec=0.0, end_sec=60.0, text="저는 백엔드 개발자 입니다", + avg_logprob=-0.2), + ], + ) + m = analyze(result, filler_pattern=FILLER_PATTERN) + # 4 words / 60s * 60 = 4 wpm (어절 기반) + assert m.speaking_rate_wpm == 4.0 + # logprob exp(-0.2) ≈ 0.8187 + assert m.pronunciation_accuracy is not None + assert abs(m.pronunciation_accuracy - math.exp(-0.2)) < 1e-3 + + +def test_filler_count_normalized(): + result = TranscriptionResult( + text="음 그게 어 음음 어어어 그러니까", + language="ko", + duration_sec=10.0, + segments=[], + ) + m = analyze(result, filler_pattern=FILLER_PATTERN) + # 음(첫 글자) 2회, 어 2회, 그 2회 — 매칭은 음/어/그/아 패턴 + assert m.filler_word_counts.get("음", 0) >= 1 + assert m.filler_word_counts.get("어", 0) >= 1 + + +def test_silence_includes_gaps_and_padding(): + result = TranscriptionResult( + text="안녕 반갑", + language="ko", + duration_sec=10.0, + segments=[ + TranscriptionSegment(start_sec=1.0, end_sec=3.0, text="안녕"), + TranscriptionSegment(start_sec=5.0, end_sec=7.0, text="반갑"), + ], + ) + m = analyze(result, filler_pattern=FILLER_PATTERN) + # 앞 1s + 중간 2s + 뒤 3s = 6s + assert m.silence_duration_sec == 6.0 + + +def test_empty_segments_returns_none_silence_and_accuracy(): + result = TranscriptionResult(text="", language=None, duration_sec=None, segments=[]) + m = analyze(result, filler_pattern=FILLER_PATTERN) + assert m.speaking_rate_wpm is None + assert m.silence_duration_sec is None + assert m.pronunciation_accuracy is None diff --git a/backend/src/main/java/com/stackup/stackup/common/config/properties/RabbitMqProperties.java b/backend/src/main/java/com/stackup/stackup/common/config/properties/RabbitMqProperties.java index 2fc95448..4ab130d8 100644 --- a/backend/src/main/java/com/stackup/stackup/common/config/properties/RabbitMqProperties.java +++ b/backend/src/main/java/com/stackup/stackup/common/config/properties/RabbitMqProperties.java @@ -67,8 +67,12 @@ public record Names( @NotBlank String aiAnalyzeRepository, @NotBlank String aiGenerateQuestions, @NotBlank String aiGenerateFollowup, + @NotBlank String aiGenerateFeedback, + @NotBlank String aiAnalyzeVoice, @NotBlank String coreCallbackAnalysis, - @NotBlank String coreCallbackQuestions + @NotBlank String coreCallbackQuestions, + @NotBlank String coreCallbackFeedback, + @NotBlank String coreCallbackVoice ) { } } @@ -78,8 +82,12 @@ public record RoutingKeyProperties( @NotBlank String analyzeRepository, @NotBlank String generateQuestions, @NotBlank String generateFollowup, + @NotBlank String generateFeedback, + @NotBlank String analyzeVoice, @NotBlank String callbackAnalysis, @NotBlank String callbackQuestions, + @NotBlank String callbackFeedback, + @NotBlank String callbackVoice, @NotBlank String realtimeSessionNotify ) { } diff --git a/backend/src/main/java/com/stackup/stackup/common/exception/ApiErrorCode.java b/backend/src/main/java/com/stackup/stackup/common/exception/ApiErrorCode.java index f21e728a..27cf4431 100644 --- a/backend/src/main/java/com/stackup/stackup/common/exception/ApiErrorCode.java +++ b/backend/src/main/java/com/stackup/stackup/common/exception/ApiErrorCode.java @@ -33,6 +33,9 @@ public enum ApiErrorCode { SESSION_MAX_REACHED(HttpStatus.UNPROCESSABLE_CONTENT, "세션 제한에 도달했습니다."), SESSION_NOT_FOUND(HttpStatus.NOT_FOUND, "세션을 찾을 수 없습니다."), SESSION_FORBIDDEN(HttpStatus.FORBIDDEN, "세션에 접근할 수 없습니다."), + FEEDBACK_NOT_READY(HttpStatus.NOT_FOUND, "피드백이 아직 생성되지 않았습니다."), + VOICE_INVALID_CONTENT_TYPE(HttpStatus.BAD_REQUEST, "지원하지 않는 음성 형식입니다."), + VOICE_MESSAGE_NOT_FOUND(HttpStatus.NOT_FOUND, "음성 메시지를 찾을 수 없습니다."), VALIDATION_ERROR(HttpStatus.BAD_REQUEST, "요청 값이 올바르지 않습니다."), ACCESS_DENIED(HttpStatus.FORBIDDEN, "접근 권한이 없습니다."), diff --git a/backend/src/main/java/com/stackup/stackup/common/messaging/RabbitMqConfig.java b/backend/src/main/java/com/stackup/stackup/common/messaging/RabbitMqConfig.java index 1c1ef4de..02484570 100644 --- a/backend/src/main/java/com/stackup/stackup/common/messaging/RabbitMqConfig.java +++ b/backend/src/main/java/com/stackup/stackup/common/messaging/RabbitMqConfig.java @@ -87,6 +87,16 @@ public Queue aiGenerateFollowupQueue() { return workQueue(properties.queues().names().aiGenerateFollowup()); } + @Bean + public Queue aiGenerateFeedbackQueue() { + return workQueue(properties.queues().names().aiGenerateFeedback()); + } + + @Bean + public Queue aiAnalyzeVoiceQueue() { + return workQueue(properties.queues().names().aiAnalyzeVoice()); + } + @Bean public Queue coreCallbackAnalysisQueue() { return workQueue(properties.queues().names().coreCallbackAnalysis()); @@ -97,6 +107,16 @@ public Queue coreCallbackQuestionsQueue() { return workQueue(properties.queues().names().coreCallbackQuestions()); } + @Bean + public Queue coreCallbackFeedbackQueue() { + return workQueue(properties.queues().names().coreCallbackFeedback()); + } + + @Bean + public Queue coreCallbackVoiceQueue() { + return workQueue(properties.queues().names().coreCallbackVoice()); + } + @Bean public Declarables rabbitDeclarables( TopicExchange coreToAiExchange, @@ -107,15 +127,23 @@ public Declarables rabbitDeclarables( Queue aiAnalyzeRepositoryQueue, Queue aiGenerateQuestionsQueue, Queue aiGenerateFollowupQueue, + Queue aiGenerateFeedbackQueue, + Queue aiAnalyzeVoiceQueue, Queue coreCallbackAnalysisQueue, - Queue coreCallbackQuestionsQueue + Queue coreCallbackQuestionsQueue, + Queue coreCallbackFeedbackQueue, + Queue coreCallbackVoiceQueue ) { Queue dlqAiAnalyzeResume = dlq(properties.queues().names().aiAnalyzeResume()); Queue dlqAiAnalyzeRepository = dlq(properties.queues().names().aiAnalyzeRepository()); Queue dlqAiGenerateQuestions = dlq(properties.queues().names().aiGenerateQuestions()); Queue dlqAiGenerateFollowup = dlq(properties.queues().names().aiGenerateFollowup()); + Queue dlqAiGenerateFeedback = dlq(properties.queues().names().aiGenerateFeedback()); + Queue dlqAiAnalyzeVoice = dlq(properties.queues().names().aiAnalyzeVoice()); Queue dlqCoreCallbackAnalysis = dlq(properties.queues().names().coreCallbackAnalysis()); Queue dlqCoreCallbackQuestions = dlq(properties.queues().names().coreCallbackQuestions()); + Queue dlqCoreCallbackFeedback = dlq(properties.queues().names().coreCallbackFeedback()); + Queue dlqCoreCallbackVoice = dlq(properties.queues().names().coreCallbackVoice()); return new Declarables( coreToAiExchange, @@ -126,26 +154,42 @@ public Declarables rabbitDeclarables( aiAnalyzeRepositoryQueue, aiGenerateQuestionsQueue, aiGenerateFollowupQueue, + aiGenerateFeedbackQueue, + aiAnalyzeVoiceQueue, coreCallbackAnalysisQueue, coreCallbackQuestionsQueue, + coreCallbackFeedbackQueue, + coreCallbackVoiceQueue, dlqAiAnalyzeResume, dlqAiAnalyzeRepository, dlqAiGenerateQuestions, dlqAiGenerateFollowup, + dlqAiGenerateFeedback, + dlqAiAnalyzeVoice, dlqCoreCallbackAnalysis, dlqCoreCallbackQuestions, + dlqCoreCallbackFeedback, + dlqCoreCallbackVoice, BindingBuilder.bind(aiAnalyzeResumeQueue).to(coreToAiExchange).with(properties.routingKeys().analyzeResume()), BindingBuilder.bind(aiAnalyzeRepositoryQueue).to(coreToAiExchange).with(properties.routingKeys().analyzeRepository()), BindingBuilder.bind(aiGenerateQuestionsQueue).to(coreToAiExchange).with(properties.routingKeys().generateQuestions()), BindingBuilder.bind(aiGenerateFollowupQueue).to(coreToAiExchange).with(properties.routingKeys().generateFollowup()), + BindingBuilder.bind(aiGenerateFeedbackQueue).to(coreToAiExchange).with(properties.routingKeys().generateFeedback()), + BindingBuilder.bind(aiAnalyzeVoiceQueue).to(coreToAiExchange).with(properties.routingKeys().analyzeVoice()), BindingBuilder.bind(coreCallbackAnalysisQueue).to(aiToCoreExchange).with(properties.routingKeys().callbackAnalysis()), BindingBuilder.bind(coreCallbackQuestionsQueue).to(aiToCoreExchange).with(properties.routingKeys().callbackQuestions()), + BindingBuilder.bind(coreCallbackFeedbackQueue).to(aiToCoreExchange).with(properties.routingKeys().callbackFeedback()), + BindingBuilder.bind(coreCallbackVoiceQueue).to(aiToCoreExchange).with(properties.routingKeys().callbackVoice()), BindingBuilder.bind(dlqAiAnalyzeResume).to(deadLetterExchange).with(dlqAiAnalyzeResume.getName()), BindingBuilder.bind(dlqAiAnalyzeRepository).to(deadLetterExchange).with(dlqAiAnalyzeRepository.getName()), BindingBuilder.bind(dlqAiGenerateQuestions).to(deadLetterExchange).with(dlqAiGenerateQuestions.getName()), BindingBuilder.bind(dlqAiGenerateFollowup).to(deadLetterExchange).with(dlqAiGenerateFollowup.getName()), + BindingBuilder.bind(dlqAiGenerateFeedback).to(deadLetterExchange).with(dlqAiGenerateFeedback.getName()), + BindingBuilder.bind(dlqAiAnalyzeVoice).to(deadLetterExchange).with(dlqAiAnalyzeVoice.getName()), BindingBuilder.bind(dlqCoreCallbackAnalysis).to(deadLetterExchange).with(dlqCoreCallbackAnalysis.getName()), - BindingBuilder.bind(dlqCoreCallbackQuestions).to(deadLetterExchange).with(dlqCoreCallbackQuestions.getName()) + BindingBuilder.bind(dlqCoreCallbackQuestions).to(deadLetterExchange).with(dlqCoreCallbackQuestions.getName()), + BindingBuilder.bind(dlqCoreCallbackFeedback).to(deadLetterExchange).with(dlqCoreCallbackFeedback.getName()), + BindingBuilder.bind(dlqCoreCallbackVoice).to(deadLetterExchange).with(dlqCoreCallbackVoice.getName()) // 주의: q.realtime.session.notify 큐 + binding (+ DLQ) 은 RealTime 서버 측에서 // infra/rabbitmq/definitions.json 으로 import 되므로 Core 는 exchange 만 declare. ); diff --git a/backend/src/main/java/com/stackup/stackup/document/application/DocumentEmbeddingService.java b/backend/src/main/java/com/stackup/stackup/document/application/DocumentEmbeddingService.java index 4057dc4a..c063c9da 100644 --- a/backend/src/main/java/com/stackup/stackup/document/application/DocumentEmbeddingService.java +++ b/backend/src/main/java/com/stackup/stackup/document/application/DocumentEmbeddingService.java @@ -29,4 +29,10 @@ public int upsert(EmbeddingUpsertCommand command) { .toList(); return embeddingRepository.upsertAll(command.documentId(), command.model(), mapped); } + + public List search( + float[] queryEmbedding, List documentIds, int topK + ) { + return embeddingRepository.search(queryEmbedding, documentIds, topK); + } } diff --git a/backend/src/main/java/com/stackup/stackup/document/domain/DocumentEmbeddingRepository.java b/backend/src/main/java/com/stackup/stackup/document/domain/DocumentEmbeddingRepository.java index fbb91d98..1b41b5d2 100644 --- a/backend/src/main/java/com/stackup/stackup/document/domain/DocumentEmbeddingRepository.java +++ b/backend/src/main/java/com/stackup/stackup/document/domain/DocumentEmbeddingRepository.java @@ -8,6 +8,12 @@ public interface DocumentEmbeddingRepository { int countByDocumentId(long documentId); + // pgvector cosine distance topK 검색. documentIds 가 비어 있으면 전체 대상. + List search(float[] queryEmbedding, List documentIds, int topK); + record EmbeddingChunk(int chunkIndex, String chunkText, float[] embedding) { } + + record SearchHit(long documentId, int chunkIndex, String chunkText, double distance) { + } } diff --git a/backend/src/main/java/com/stackup/stackup/document/infrastructure/JdbcDocumentEmbeddingRepository.java b/backend/src/main/java/com/stackup/stackup/document/infrastructure/JdbcDocumentEmbeddingRepository.java index e299ab06..fae61cb6 100644 --- a/backend/src/main/java/com/stackup/stackup/document/infrastructure/JdbcDocumentEmbeddingRepository.java +++ b/backend/src/main/java/com/stackup/stackup/document/infrastructure/JdbcDocumentEmbeddingRepository.java @@ -2,13 +2,16 @@ import com.stackup.stackup.document.domain.DocumentEmbeddingRepository; import java.sql.PreparedStatement; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.sql.DataSource; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; -// 네이티브 쿼리가 더 쉬워서 JPA 대신 씀 +// 네이티브 쿼리가 더 쉬워서 JPA 대신 씀 @Repository public class JdbcDocumentEmbeddingRepository implements DocumentEmbeddingRepository { @@ -26,9 +29,11 @@ ON CONFLICT (document_id, chunk_index) "SELECT count(*) FROM document_embeddings WHERE document_id = ?"; private final JdbcTemplate jdbc; + private final NamedParameterJdbcTemplate namedJdbc; public JdbcDocumentEmbeddingRepository(DataSource dataSource) { this.jdbc = new JdbcTemplate(dataSource); + this.namedJdbc = new NamedParameterJdbcTemplate(dataSource); } @Override @@ -59,6 +64,33 @@ public int countByDocumentId(long documentId) { return n == null ? 0 : n; } + @Override + public List search(float[] queryEmbedding, List documentIds, int topK) { + if (queryEmbedding == null || queryEmbedding.length == 0) { + return List.of(); + } + int limit = topK <= 0 ? 5 : topK; + boolean filterByDoc = documentIds != null && !documentIds.isEmpty(); + StringBuilder sql = new StringBuilder( + "SELECT document_id, chunk_index, chunk_text, (embedding <=> CAST(:qvec AS vector)) AS distance " + + "FROM document_embeddings "); + Map params = new HashMap<>(); + params.put("qvec", toVectorLiteral(queryEmbedding)); + if (filterByDoc) { + sql.append("WHERE document_id IN (:documentIds) "); + params.put("documentIds", documentIds); + } + sql.append("ORDER BY embedding <=> CAST(:qvec AS vector) LIMIT :limit"); + params.put("limit", limit); + + return namedJdbc.query(sql.toString(), params, (rs, rowNum) -> new SearchHit( + rs.getLong("document_id"), + rs.getInt("chunk_index"), + rs.getString("chunk_text"), + rs.getDouble("distance") + )); + } + private static String toVectorLiteral(float[] embedding) { StringBuilder sb = new StringBuilder(embedding.length * 8 + 2); sb.append('['); diff --git a/backend/src/main/java/com/stackup/stackup/document/presentation/InternalEmbeddingSearchController.java b/backend/src/main/java/com/stackup/stackup/document/presentation/InternalEmbeddingSearchController.java new file mode 100644 index 00000000..8f031832 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/document/presentation/InternalEmbeddingSearchController.java @@ -0,0 +1,68 @@ +package com.stackup.stackup.document.presentation; + +import com.stackup.stackup.document.application.DocumentEmbeddingService; +import com.stackup.stackup.document.domain.DocumentEmbeddingRepository.SearchHit; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Positive; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@Tag(name = "Internal: Embedding Search", + description = "X-Internal-API-Key 필요. AI 서버가 질문/피드백 생성 시 컨텍스트 청크 검색 (pgvector cosine).") +@RestController +@RequestMapping("/api/internal/embeddings") +@RequiredArgsConstructor +public class InternalEmbeddingSearchController { + + private final DocumentEmbeddingService embeddingService; + + @Operation( + operationId = "internalSearchEmbeddings", + summary = "pgvector cosine topK 검색", + description = "queryEmbedding 으로 가장 가까운 청크 topK 반환. documentIds 가 비어 있으면 전체." + ) + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "검색 결과"), + @ApiResponse(responseCode = "400", description = "queryEmbedding 누락"), + @ApiResponse(responseCode = "401", description = "X-Internal-API-Key 인증 실패") + }) + @PostMapping("/search") + public SearchResponse search(@Valid @RequestBody SearchRequest request) { + List hits = embeddingService.search( + request.queryEmbedding(), + request.documentIds() == null ? List.of() : request.documentIds(), + request.topK() == null ? 5 : request.topK() + ); + return new SearchResponse(hits.stream().map(SearchResponseHit::from).toList()); + } + + public record SearchRequest( + @NotNull float[] queryEmbedding, + List documentIds, + @Positive Integer topK + ) { + } + + public record SearchResponse(List hits) { + } + + public record SearchResponseHit( + long documentId, + int chunkIndex, + String chunkText, + double distance + ) { + public static SearchResponseHit from(SearchHit h) { + return new SearchResponseHit(h.documentId(), h.chunkIndex(), h.chunkText(), h.distance()); + } + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/FeedbackCallbackService.java b/backend/src/main/java/com/stackup/stackup/session/application/FeedbackCallbackService.java new file mode 100644 index 00000000..2888e46f --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/FeedbackCallbackService.java @@ -0,0 +1,128 @@ +package com.stackup.stackup.session.application; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.stackup.stackup.common.messaging.domain.ProcessedMessage; +import com.stackup.stackup.common.messaging.domain.ProcessedMessageRepository; +import com.stackup.stackup.common.sse.SseEventPublisher; +import com.stackup.stackup.common.sse.SseEventType; +import com.stackup.stackup.session.application.dto.FeedbackCallbackEnvelope; +import com.stackup.stackup.session.application.dto.FeedbackCallbackPayload; +import com.stackup.stackup.session.domain.InterviewSession; +import com.stackup.stackup.session.domain.InterviewSessionRepository; +import com.stackup.stackup.session.domain.SessionFeedback; +import com.stackup.stackup.session.domain.SessionFeedbackRepository; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +// callback.feedback 처리 — SessionFeedback INSERT + SSE session.feedback push. +// 멱등: processed_messages (envelope messageId) + session_feedbacks UNIQUE (session_id). +@Service +@RequiredArgsConstructor +@Transactional(readOnly = true) +public class FeedbackCallbackService { + + private static final Logger log = LoggerFactory.getLogger(FeedbackCallbackService.class); + private static final String CONSUMER_NAME = "core.callback.feedback"; + private static final ObjectMapper JSON = new ObjectMapper(); + + private final InterviewSessionRepository sessionRepository; + private final SessionFeedbackRepository feedbackRepository; + private final ProcessedMessageRepository processedMessageRepository; + private final SseEventPublisher sseEventPublisher; + + @Transactional + public void apply(FeedbackCallbackEnvelope envelope) { + if (envelope == null || envelope.payload() == null) { + log.warn("callback.feedback envelope or payload is null — skip"); + return; + } + if (isProcessed(envelope.messageId())) { + log.info("callback.feedback duplicate, skip. messageId={}", envelope.messageId()); + return; + } + FeedbackCallbackPayload payload = envelope.payload(); + Long sessionId = payload.sessionId(); + if (sessionId == null) { + log.warn("callback.feedback missing sessionId. messageId={}", envelope.messageId()); + markProcessed(envelope.messageId()); + return; + } + InterviewSession session = sessionRepository.findById(sessionId).orElse(null); + if (session == null || session.isDeleted()) { + log.warn("callback.feedback session not found or deleted. id={}, messageId={}", + sessionId, envelope.messageId()); + markProcessed(envelope.messageId()); + return; + } + if (feedbackRepository.existsBySession_Id(sessionId)) { + log.info("callback.feedback feedback already exists. sessionId={}", sessionId); + markProcessed(envelope.messageId()); + return; + } + + SessionFeedback feedback = SessionFeedback.of( + session, + payload.overallScore(), + payload.technicalAccuracy(), + payload.logicScore(), + payload.communicationScore(), + payload.strengthsSummary(), + payload.weaknessesSummary(), + keywordsToJson(payload.improvementKeywords()), + payload.reportS3Key() + ); + try { + feedback = feedbackRepository.save(feedback); + } catch (DataIntegrityViolationException race) { + log.info("callback.feedback unique race — feedback inserted concurrently. sessionId={}", sessionId); + markProcessed(envelope.messageId()); + return; + } + + sseEventPublisher.publishToSession(sessionId, SseEventType.FEEDBACK_READY, + new SessionFeedbackNotice(sessionId, feedback.getId())); + sseEventPublisher.publishToUser(session.getUser().getId(), SseEventType.FEEDBACK_READY, + new SessionFeedbackNotice(sessionId, feedback.getId())); + + markProcessed(envelope.messageId()); + log.info("callback.feedback processed. sessionId={}, feedbackId={}", sessionId, feedback.getId()); + } + + public record SessionFeedbackNotice(Long sessionId, Long feedbackId) { + } + + private String keywordsToJson(java.util.List keywords) { + if (keywords == null) { + return null; + } + try { + return JSON.writeValueAsString(keywords); + } catch (JsonProcessingException e) { + log.warn("improvement_keywords json serialize failed", e); + return "[]"; + } + } + + private boolean isProcessed(String messageId) { + if (messageId == null || messageId.isBlank()) { + return false; + } + return processedMessageRepository.existsById(messageId); + } + + private void markProcessed(String messageId) { + if (messageId == null || messageId.isBlank()) { + return; + } + try { + processedMessageRepository.save(ProcessedMessage.of(messageId, CONSUMER_NAME)); + } catch (DataIntegrityViolationException ignored) { + // race + } + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/QuestionsCallbackService.java b/backend/src/main/java/com/stackup/stackup/session/application/QuestionsCallbackService.java index 2f496d65..08615fa6 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/QuestionsCallbackService.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/QuestionsCallbackService.java @@ -7,6 +7,8 @@ import com.stackup.stackup.session.application.dto.QuestionsCallbackEnvelope; import com.stackup.stackup.session.application.dto.QuestionsCallbackPayload; import com.stackup.stackup.session.application.dto.QuestionsCallbackPayload.GeneratedQuestion; +import com.stackup.stackup.session.application.event.SessionEndedEvent; +import org.springframework.context.ApplicationEventPublisher; import com.stackup.stackup.session.domain.InterviewMessage; import com.stackup.stackup.session.domain.InterviewMessageRepository; import com.stackup.stackup.session.domain.InterviewSession; @@ -35,6 +37,7 @@ public class QuestionsCallbackService { private final InterviewMessageRepository messageRepository; private final ProcessedMessageRepository processedMessageRepository; private final SseEventPublisher sseEventPublisher; + private final ApplicationEventPublisher events; @Transactional public void apply(QuestionsCallbackEnvelope envelope) { @@ -127,6 +130,8 @@ private void applyFollowup(InterviewSession session, QuestionsCallbackPayload pa new SessionStateNotice(session.getId(), session.getStatus().name(), "MAX_QUESTIONS_REACHED")); sseEventPublisher.publishToUser(session.getUser().getId(), SseEventType.SESSION_STATE, new SessionStateNotice(session.getId(), session.getStatus().name(), "MAX_QUESTIONS_REACHED")); + events.publishEvent(new SessionEndedEvent( + session.getUser().getId(), session.getId(), "MAX_QUESTIONS_REACHED")); log.info("session auto-completed on max questions. sessionId={}, max={}", session.getId(), max); } catch (IllegalStateException e) { diff --git a/backend/src/main/java/com/stackup/stackup/session/application/SessionFeedbackQueryService.java b/backend/src/main/java/com/stackup/stackup/session/application/SessionFeedbackQueryService.java new file mode 100644 index 00000000..1f62eb4e --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/SessionFeedbackQueryService.java @@ -0,0 +1,28 @@ +package com.stackup.stackup.session.application; + +import com.stackup.stackup.common.exception.ApiErrorCode; +import com.stackup.stackup.common.exception.DomainException; +import com.stackup.stackup.session.application.dto.FeedbackResult; +import com.stackup.stackup.session.domain.InterviewSessionRepository; +import com.stackup.stackup.session.domain.SessionFeedback; +import com.stackup.stackup.session.domain.SessionFeedbackRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +@RequiredArgsConstructor +@Transactional(readOnly = true) +public class SessionFeedbackQueryService { + + private final InterviewSessionRepository sessionRepository; + private final SessionFeedbackRepository feedbackRepository; + + public FeedbackResult get(Long userId, Long sessionId) { + sessionRepository.findByIdAndUser_IdAndDeletedFalse(sessionId, userId) + .orElseThrow(() -> new DomainException(ApiErrorCode.SESSION_NOT_FOUND)); + SessionFeedback feedback = feedbackRepository.findBySession_Id(sessionId) + .orElseThrow(() -> new DomainException(ApiErrorCode.FEEDBACK_NOT_READY)); + return FeedbackResult.of(feedback); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/SessionFeedbackRequester.java b/backend/src/main/java/com/stackup/stackup/session/application/SessionFeedbackRequester.java new file mode 100644 index 00000000..00975030 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/SessionFeedbackRequester.java @@ -0,0 +1,91 @@ +package com.stackup.stackup.session.application; + +import com.stackup.stackup.common.config.properties.RabbitMqProperties; +import com.stackup.stackup.common.messaging.MessageContext; +import com.stackup.stackup.common.messaging.RabbitMessagePublisher; +import com.stackup.stackup.session.application.dto.GenerateFeedbackPayload; +import com.stackup.stackup.session.application.dto.GenerateFeedbackPayload.MessageItem; +import com.stackup.stackup.session.application.event.SessionEndedEvent; +import com.stackup.stackup.session.domain.InterviewMessage; +import com.stackup.stackup.session.domain.InterviewMessageRepository; +import com.stackup.stackup.session.domain.InterviewSession; +import com.stackup.stackup.session.domain.InterviewSessionRepository; +import com.stackup.stackup.session.domain.SessionContext; +import com.stackup.stackup.session.domain.SessionContextRepository; +import com.stackup.stackup.session.domain.SessionFeedbackRepository; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionPhase; +import org.springframework.transaction.event.TransactionalEventListener; + +// 세션 COMPLETED commit 후 발화 → generate.feedback 발행 (US-24). +// 멱등: session_feedbacks UNIQUE (session_id). 이미 피드백이 있으면 skip. +@Component +@RequiredArgsConstructor +public class SessionFeedbackRequester { + + private static final Logger log = LoggerFactory.getLogger(SessionFeedbackRequester.class); + + private final RabbitMessagePublisher publisher; + private final RabbitMqProperties properties; + private final InterviewSessionRepository sessionRepository; + private final InterviewMessageRepository messageRepository; + private final SessionContextRepository contextRepository; + private final SessionFeedbackRepository feedbackRepository; + + @Transactional(propagation = Propagation.REQUIRES_NEW) + @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT) + public void onSessionEnded(SessionEndedEvent event) { + if (feedbackRepository.existsBySession_Id(event.sessionId())) { + log.info("generate.feedback skipped — feedback already exists. sessionId={}", event.sessionId()); + return; + } + InterviewSession session = sessionRepository.findById(event.sessionId()).orElse(null); + if (session == null || session.isDeleted()) { + log.warn("generate.feedback skipped — session missing/deleted. sessionId={}", event.sessionId()); + return; + } + + List messages = messageRepository + .findBySession_IdOrderBySequenceNumberAsc(event.sessionId()).stream() + .map(this::toItem) + .toList(); + List contextDocumentIds = contextRepository.findBySession_Id(event.sessionId()).stream() + .map(c -> c.getDocument().getId()) + .toList(); + + GenerateFeedbackPayload payload = new GenerateFeedbackPayload( + session.getId(), + session.getInterviewType().name(), + session.getJobCategory().name(), + session.getTotalQuestionCount(), + event.reason(), + messages, + contextDocumentIds + ); + + publisher.publishToAi( + properties.routingKeys().generateFeedback(), + payload, + new MessageContext(event.userId(), session.getId(), null, null) + ); + log.info("generate.feedback published. sessionId={}, msgCount={}, ctx={}, reason={}", + session.getId(), messages.size(), contextDocumentIds.size(), event.reason()); + } + + private MessageItem toItem(InterviewMessage m) { + Long parentId = m.getParentMessage() == null ? null : m.getParentMessage().getId(); + return new MessageItem( + m.getId(), + m.getSequenceNumber(), + m.getRole().name(), + m.getContent(), + parentId + ); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/SessionService.java b/backend/src/main/java/com/stackup/stackup/session/application/SessionService.java index ef893d21..b3e78522 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/SessionService.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/SessionService.java @@ -8,6 +8,7 @@ import com.stackup.stackup.session.application.dto.SessionCreateCommand; import com.stackup.stackup.session.application.dto.SessionResult; import com.stackup.stackup.session.application.event.SessionCreatedEvent; +import com.stackup.stackup.session.application.event.SessionEndedEvent; import com.stackup.stackup.session.domain.InterviewSession; import com.stackup.stackup.session.domain.InterviewSessionRepository; import com.stackup.stackup.session.domain.SessionContext; @@ -94,6 +95,7 @@ public SessionResult end(Long userId, Long sessionId) { } catch (IllegalStateException e) { throw new DomainException(ApiErrorCode.SESSION_INVALID_STATE); } + events.publishEvent(new SessionEndedEvent(userId, sessionId, "USER_REQUEST")); return SessionResult.of(session, contextDocumentIds(sessionId)); } diff --git a/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java b/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java new file mode 100644 index 00000000..24d091a8 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/VoiceAnswerUploadService.java @@ -0,0 +1,118 @@ +package com.stackup.stackup.session.application; + +import com.stackup.stackup.common.config.properties.RabbitMqProperties; +import com.stackup.stackup.common.exception.ApiErrorCode; +import com.stackup.stackup.common.exception.DomainException; +import com.stackup.stackup.common.messaging.MessageContext; +import com.stackup.stackup.common.messaging.RabbitMessagePublisher; +import com.stackup.stackup.common.storage.ObjectStorageClient; +import com.stackup.stackup.session.application.dto.AnalyzeVoicePayload; +import com.stackup.stackup.session.application.dto.MessageResult; +import com.stackup.stackup.session.application.dto.VoiceAnswerUploadCommand; +import com.stackup.stackup.session.domain.InterviewMessage; +import com.stackup.stackup.session.domain.InterviewMessageRepository; +import com.stackup.stackup.session.domain.InterviewSession; +import com.stackup.stackup.session.domain.InterviewSessionRepository; +import com.stackup.stackup.session.domain.MessageRole; +import com.stackup.stackup.session.domain.SessionStatus; +import java.util.Set; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +// 음성 답변 업로드: S3 PUT → InterviewMessage INSERT (content=null, audio_file_path=key) → analyze.voice 발행. +// STT/분석 결과는 callback.voice 로 도착해서 InterviewMessage.content 채움 + voice metrics INSERT + followup 트리거. +@Service +@RequiredArgsConstructor +public class VoiceAnswerUploadService { + + private static final Logger log = LoggerFactory.getLogger(VoiceAnswerUploadService.class); + private static final Set ALLOWED_CONTENT_TYPES = Set.of( + "audio/webm", "audio/ogg", "audio/mpeg", "audio/mp4", "audio/wav", + "audio/x-wav", "audio/m4a", "audio/x-m4a" + ); + private static final long MAX_BYTES = 25L * 1024 * 1024; // Whisper API 25MB 제한 + + private final InterviewSessionRepository sessionRepository; + private final InterviewMessageRepository messageRepository; + private final ObjectStorageClient storage; + private final RabbitMessagePublisher publisher; + private final RabbitMqProperties properties; + + @Transactional + public MessageResult submit(Long userId, Long sessionId, VoiceAnswerUploadCommand cmd) { + validate(cmd); + InterviewSession session = sessionRepository.findByIdAndUser_IdAndDeletedFalse(sessionId, userId) + .orElseThrow(() -> new DomainException(ApiErrorCode.SESSION_NOT_FOUND)); + + if (cmd.idempotencyKey() != null && !cmd.idempotencyKey().isBlank()) { + var existing = messageRepository.findBySession_IdAndIdempotencyKey(sessionId, cmd.idempotencyKey()); + if (existing.isPresent()) { + return MessageResult.of(existing.get()); + } + } + + if (session.getStatus() != SessionStatus.IN_PROGRESS) { + throw new DomainException(ApiErrorCode.SESSION_INVALID_STATE); + } + InterviewMessage latest = messageRepository + .findFirstBySession_IdOrderBySequenceNumberDesc(sessionId) + .orElseThrow(() -> new DomainException(ApiErrorCode.SESSION_INVALID_STATE)); + if (latest.getRole() != MessageRole.INTERVIEWER) { + throw new DomainException(ApiErrorCode.SESSION_INVALID_STATE); + } + int nextSeq = latest.getSequenceNumber() + 1; + + // 메시지 ID 없이도 키를 만들어야 하므로 먼저 INSERT 후 key 갱신은 ID 의존. 단순화: 임시키로 PUT 후 INSERT. + // 더 안전한 패턴: messageRepository.save 먼저 (id 채번) → key 결정 → S3 PUT → audio_file_path 갱신. + InterviewMessage placeholder = messageRepository.save( + InterviewMessage.voiceInterviewee(session, nextSeq, latest, + cmd.idempotencyKey() != null && !cmd.idempotencyKey().isBlank() ? cmd.idempotencyKey() : null) + ); + String key = buildKey(sessionId, placeholder.getId(), cmd.contentType()); + storage.put(key, cmd.content(), cmd.size(), cmd.contentType()); + placeholder.attachAudio(key); + + AnalyzeVoicePayload payload = new AnalyzeVoicePayload( + sessionId, + placeholder.getId(), + latest.getId(), + key, + cmd.contentType(), + latest.getContent(), + session.getInterviewType().name(), + session.getJobCategory().name() + ); + publisher.publishToAi( + properties.routingKeys().analyzeVoice(), + payload, + new MessageContext(userId, sessionId, null, null) + ); + log.info("analyze.voice published. sessionId={}, messageId={}, key={}", + sessionId, placeholder.getId(), key); + return MessageResult.of(placeholder); + } + + private void validate(VoiceAnswerUploadCommand cmd) { + if (cmd.size() <= 0 || cmd.size() > MAX_BYTES) { + throw new DomainException(ApiErrorCode.RESUME_FILE_TOO_LARGE); + } + if (cmd.contentType() == null || !ALLOWED_CONTENT_TYPES.contains(cmd.contentType().toLowerCase())) { + throw new DomainException(ApiErrorCode.VOICE_INVALID_CONTENT_TYPE); + } + } + + private static String buildKey(Long sessionId, Long messageId, String contentType) { + String ext = switch (contentType.toLowerCase()) { + case "audio/webm" -> "webm"; + case "audio/ogg" -> "ogg"; + case "audio/mpeg" -> "mp3"; + case "audio/mp4", "audio/m4a", "audio/x-m4a" -> "m4a"; + case "audio/wav", "audio/x-wav" -> "wav"; + default -> "bin"; + }; + return "interview/voice/raw/%d/%d.%s".formatted(sessionId, messageId, ext); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/VoiceCallbackService.java b/backend/src/main/java/com/stackup/stackup/session/application/VoiceCallbackService.java new file mode 100644 index 00000000..407909f6 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/VoiceCallbackService.java @@ -0,0 +1,148 @@ +package com.stackup.stackup.session.application; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.stackup.stackup.common.messaging.domain.ProcessedMessage; +import com.stackup.stackup.common.messaging.domain.ProcessedMessageRepository; +import com.stackup.stackup.common.sse.SseEventPublisher; +import com.stackup.stackup.common.sse.SseEventType; +import com.stackup.stackup.session.application.dto.VoiceCallbackEnvelope; +import com.stackup.stackup.session.application.dto.VoiceCallbackPayload; +import com.stackup.stackup.session.application.event.AnswerSubmittedEvent; +import com.stackup.stackup.session.domain.InterviewMessage; +import com.stackup.stackup.session.domain.InterviewMessageRepository; +import com.stackup.stackup.session.domain.MessageStatus; +import com.stackup.stackup.session.domain.MessageVoiceAnalysis; +import com.stackup.stackup.session.domain.MessageVoiceAnalysisRepository; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +// callback.voice 처리 — InterviewMessage.content 채움 + MessageVoiceAnalysis INSERT + AnswerSubmittedEvent 발화. +// AnswerSubmittedEvent → SessionFollowupRequester 가 generate.followup 발행 (텍스트 답변과 동일 흐름). +// 멱등: processed_messages + message_voice_analyses UNIQUE(message_id). +@Service +@RequiredArgsConstructor +@Transactional(readOnly = true) +public class VoiceCallbackService { + + private static final Logger log = LoggerFactory.getLogger(VoiceCallbackService.class); + private static final String CONSUMER_NAME = "core.callback.voice"; + private static final ObjectMapper JSON = new ObjectMapper(); + + private final InterviewMessageRepository messageRepository; + private final MessageVoiceAnalysisRepository voiceAnalysisRepository; + private final ProcessedMessageRepository processedMessageRepository; + private final SseEventPublisher sseEventPublisher; + private final ApplicationEventPublisher events; + + @Transactional + public void apply(VoiceCallbackEnvelope envelope) { + if (envelope == null || envelope.payload() == null) { + log.warn("callback.voice envelope or payload is null — skip"); + return; + } + if (isProcessed(envelope.messageId())) { + log.info("callback.voice duplicate, skip. messageId={}", envelope.messageId()); + return; + } + VoiceCallbackPayload p = envelope.payload(); + if (p.interviewMessageId() == null) { + log.warn("callback.voice missing interviewMessageId. messageId={}", envelope.messageId()); + markProcessed(envelope.messageId()); + return; + } + InterviewMessage message = messageRepository.findById(p.interviewMessageId()).orElse(null); + if (message == null) { + log.warn("callback.voice message not found. id={}", p.interviewMessageId()); + markProcessed(envelope.messageId()); + return; + } + + if (p.errorCode() != null && !p.errorCode().isBlank()) { + message.markStatus(MessageStatus.FAILED); + sseEventPublisher.publishToSession(p.sessionId(), SseEventType.SESSION_MESSAGE, + new VoiceFailedNotice(p.sessionId(), message.getId(), p.errorCode())); + markProcessed(envelope.messageId()); + log.warn("callback.voice STT failed. sessionId={}, msg={}, code={}", + p.sessionId(), message.getId(), p.errorCode()); + return; + } + + message.completeWithTranscript(p.transcript()); + + if (!voiceAnalysisRepository.existsByMessage_Id(message.getId())) { + try { + voiceAnalysisRepository.save(MessageVoiceAnalysis.of( + message, + p.speakingRateWpm(), + p.silenceDurationSec(), + fillerToJson(p.fillerWordCounts()), + p.pronunciationAccuracy() + )); + } catch (DataIntegrityViolationException ignored) { + // race + } + } + + sseEventPublisher.publishToSession(p.sessionId(), SseEventType.SESSION_MESSAGE, + new VoiceTranscribedNotice(p.sessionId(), message.getId(), p.transcript())); + sseEventPublisher.publishToUser(message.getSession().getUser().getId(), + SseEventType.SESSION_MESSAGE, + new VoiceTranscribedNotice(p.sessionId(), message.getId(), p.transcript())); + + // 텍스트 답변과 동일 흐름으로 followup 트리거. + Long parentId = message.getParentMessage() == null ? null : message.getParentMessage().getId(); + events.publishEvent(new AnswerSubmittedEvent( + message.getSession().getUser().getId(), + p.sessionId(), + parentId, + message.getId() + )); + + markProcessed(envelope.messageId()); + log.info("callback.voice processed. sessionId={}, msg={}, wpm={}, filler={}", + p.sessionId(), message.getId(), p.speakingRateWpm(), + p.fillerWordCounts() == null ? 0 : p.fillerWordCounts().size()); + } + + public record VoiceTranscribedNotice(Long sessionId, Long messageId, String transcript) { + } + + public record VoiceFailedNotice(Long sessionId, Long messageId, String errorCode) { + } + + private String fillerToJson(java.util.Map map) { + if (map == null || map.isEmpty()) { + return null; + } + try { + return JSON.writeValueAsString(map); + } catch (JsonProcessingException e) { + log.warn("filler_word_counts json serialize failed", e); + return null; + } + } + + private boolean isProcessed(String messageId) { + if (messageId == null || messageId.isBlank()) { + return false; + } + return processedMessageRepository.existsById(messageId); + } + + private void markProcessed(String messageId) { + if (messageId == null || messageId.isBlank()) { + return; + } + try { + processedMessageRepository.save(ProcessedMessage.of(messageId, CONSUMER_NAME)); + } catch (DataIntegrityViolationException ignored) { + // race + } + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/AnalyzeVoicePayload.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/AnalyzeVoicePayload.java new file mode 100644 index 00000000..7da809b5 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/AnalyzeVoicePayload.java @@ -0,0 +1,14 @@ +package com.stackup.stackup.session.application.dto; + +// analyze.voice envelope payload (Core → AI). AI 가 S3 GET → STT → 분석 후 callback.voice 발행. +public record AnalyzeVoicePayload( + Long sessionId, + Long messageId, + Long parentQuestionMessageId, // 직전 INTERVIEWER 메시지 (꼬리질문 트리거용 reference) + String audioS3Key, + String contentType, + String previousQuestionText, // STT 정확도 향상용 hint (optional) + String interviewType, + String jobCategory +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackCallbackEnvelope.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackCallbackEnvelope.java new file mode 100644 index 00000000..07c513ca --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackCallbackEnvelope.java @@ -0,0 +1,16 @@ +package com.stackup.stackup.session.application.dto; + +import com.stackup.stackup.common.messaging.MessageContext; +import java.time.Instant; + +public record FeedbackCallbackEnvelope( + String messageId, + String messageType, + String version, + String traceId, + Instant publishedAt, + String publisher, + FeedbackCallbackPayload payload, + MessageContext context +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackCallbackPayload.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackCallbackPayload.java new file mode 100644 index 00000000..840e3cc5 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackCallbackPayload.java @@ -0,0 +1,19 @@ +package com.stackup.stackup.session.application.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.List; + +// callback.feedback (AI → Core). 점수는 0~100, NULL 허용 (LLM 산정 불가 시). +@JsonInclude(JsonInclude.Include.NON_NULL) +public record FeedbackCallbackPayload( + Long sessionId, + Double overallScore, + Double technicalAccuracy, + Double logicScore, + Double communicationScore, + String strengthsSummary, + String weaknessesSummary, + List improvementKeywords, + String reportS3Key +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackResult.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackResult.java new file mode 100644 index 00000000..18fe6885 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/FeedbackResult.java @@ -0,0 +1,54 @@ +package com.stackup.stackup.session.application.dto; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.stackup.stackup.session.domain.SessionFeedback; +import java.time.Instant; +import java.util.Collections; +import java.util.List; + +public record FeedbackResult( + Long id, + Long sessionId, + Double overallScore, + Double technicalAccuracy, + Double logicScore, + Double communicationScore, + String strengthsSummary, + String weaknessesSummary, + List improvementKeywords, + String reportFilePath, + Instant createdAt +) { + + private static final ObjectMapper JSON = new ObjectMapper(); + + public static FeedbackResult of(SessionFeedback f) { + return new FeedbackResult( + f.getId(), + f.getSession().getId(), + f.getOverallScore(), + f.getTechnicalAccuracy(), + f.getLogicScore(), + f.getCommunicationScore(), + f.getStrengthsSummary(), + f.getWeaknessesSummary(), + parseKeywords(f.getImprovementKeywords()), + f.getReportFilePath(), + f.getCreatedAt() + ); + } + + @SuppressWarnings("unchecked") + private static List parseKeywords(String json) { + if (json == null || json.isBlank()) { + return List.of(); + } + try { + List parsed = JSON.readValue(json, List.class); + return parsed.stream().map(Object::toString).toList(); + } catch (JsonProcessingException e) { + return Collections.emptyList(); + } + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/GenerateFeedbackPayload.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/GenerateFeedbackPayload.java new file mode 100644 index 00000000..590b22fb --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/GenerateFeedbackPayload.java @@ -0,0 +1,25 @@ +package com.stackup.stackup.session.application.dto; + +import java.util.List; + +// generate.feedback envelope payload (Core → AI). +// AI 가 세션 메시지/평가/분석문서 컨텍스트 기반으로 종합 피드백을 생성. +public record GenerateFeedbackPayload( + Long sessionId, + String interviewType, + String jobCategory, + Integer totalQuestionCount, + String endReason, // USER_REQUEST | MAX_QUESTIONS_REACHED + List messages, + List contextDocumentIds // pgvector RAG 검색용 — AI 가 search API 호출 시 사용 +) { + + public record MessageItem( + Long id, + Integer sequenceNumber, + String role, // INTERVIEWER | INTERVIEWEE + String content, + Long parentMessageId + ) { + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceAnswerUploadCommand.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceAnswerUploadCommand.java new file mode 100644 index 00000000..aab19f20 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceAnswerUploadCommand.java @@ -0,0 +1,12 @@ +package com.stackup.stackup.session.application.dto; + +import java.io.InputStream; + +public record VoiceAnswerUploadCommand( + InputStream content, + long size, + String contentType, + String originalFilename, + String idempotencyKey +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceCallbackEnvelope.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceCallbackEnvelope.java new file mode 100644 index 00000000..c0b930b3 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceCallbackEnvelope.java @@ -0,0 +1,16 @@ +package com.stackup.stackup.session.application.dto; + +import com.stackup.stackup.common.messaging.MessageContext; +import java.time.Instant; + +public record VoiceCallbackEnvelope( + String messageId, + String messageType, + String version, + String traceId, + Instant publishedAt, + String publisher, + VoiceCallbackPayload payload, + MessageContext context +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceCallbackPayload.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceCallbackPayload.java new file mode 100644 index 00000000..378dc2db --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/VoiceCallbackPayload.java @@ -0,0 +1,18 @@ +package com.stackup.stackup.session.application.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.Map; + +// callback.voice (AI → Core). STT 결과 transcript + 음성 분석 지표. +@JsonInclude(JsonInclude.Include.NON_NULL) +public record VoiceCallbackPayload( + Long sessionId, + Long interviewMessageId, + String transcript, + Double speakingRateWpm, + Double silenceDurationSec, + Map fillerWordCounts, // {"음":3, "어":5, "그":2} + Double pronunciationAccuracy, // 0~1 (Whisper logprob 기반, null 허용) + String errorCode // STT 실패 시 코드 (TRANSCRIPTION_FAILED 등) +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/session/application/event/SessionEndedEvent.java b/backend/src/main/java/com/stackup/stackup/session/application/event/SessionEndedEvent.java new file mode 100644 index 00000000..55498bb0 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/application/event/SessionEndedEvent.java @@ -0,0 +1,10 @@ +package com.stackup.stackup.session.application.event; + +// 세션 종료(COMPLETED 전이) commit 후 발화. FeedbackRequester 가 수신. +// 사유: USER_REQUEST | MAX_QUESTIONS_REACHED +public record SessionEndedEvent( + Long userId, + Long sessionId, + String reason +) { +} diff --git a/backend/src/main/java/com/stackup/stackup/session/domain/InterviewMessage.java b/backend/src/main/java/com/stackup/stackup/session/domain/InterviewMessage.java index 266d3f22..93f25ec3 100644 --- a/backend/src/main/java/com/stackup/stackup/session/domain/InterviewMessage.java +++ b/backend/src/main/java/com/stackup/stackup/session/domain/InterviewMessage.java @@ -103,9 +103,28 @@ public static InterviewMessage interviewee(InterviewSession session, int seq, St MessageStatus.COMPLETED, idempotencyKey); } + // 음성 답변 placeholder. content 는 STT 완료 후 채움. 생성 시 CHECK 제약 만족을 위해 임시 공백 + audio_file_path 후속 갱신. + public static InterviewMessage voiceInterviewee(InterviewSession session, int seq, + InterviewMessage parent, String idempotencyKey) { + InterviewMessage m = new InterviewMessage(session, seq, MessageRole.INTERVIEWEE, "(transcribing)", + parent, MessageStatus.CREATED, idempotencyKey); + return m; + } + public void markStatus(MessageStatus newStatus) { if (newStatus != null) { this.status = newStatus; } } + + public void attachAudio(String audioFilePath) { + this.audioFilePath = audioFilePath; + } + + public void completeWithTranscript(String transcript) { + if (transcript != null && !transcript.isBlank()) { + this.content = transcript; + } + this.status = MessageStatus.COMPLETED; + } } diff --git a/backend/src/main/java/com/stackup/stackup/session/domain/MessageVoiceAnalysis.java b/backend/src/main/java/com/stackup/stackup/session/domain/MessageVoiceAnalysis.java index 3c12527a..e36a0cb5 100644 --- a/backend/src/main/java/com/stackup/stackup/session/domain/MessageVoiceAnalysis.java +++ b/backend/src/main/java/com/stackup/stackup/session/domain/MessageVoiceAnalysis.java @@ -11,6 +11,8 @@ import jakarta.persistence.JoinColumn; import jakarta.persistence.OneToOne; import jakarta.persistence.Table; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import lombok.AccessLevel; import lombok.Getter; import lombok.NoArgsConstructor; @@ -40,9 +42,30 @@ public class MessageVoiceAnalysis extends BaseTimeEntity { @Column(name = "silence_duration_sec") private Double silenceDurationSec; + @JdbcTypeCode(SqlTypes.JSON) @Column(name = "filler_word_counts", columnDefinition = "jsonb") private String fillerWordCounts; @Column(name = "pronunciation_accuracy") private Double pronunciationAccuracy; + + private MessageVoiceAnalysis(InterviewMessage message, Double speakingRateWpm, + Double silenceDurationSec, String fillerWordCountsJson, + Double pronunciationAccuracy) { + if (message == null) { + throw new IllegalArgumentException("message must not be null"); + } + this.message = message; + this.speakingRateWpm = speakingRateWpm; + this.silenceDurationSec = silenceDurationSec; + this.fillerWordCounts = fillerWordCountsJson; + this.pronunciationAccuracy = pronunciationAccuracy; + } + + public static MessageVoiceAnalysis of(InterviewMessage message, Double speakingRateWpm, + Double silenceDurationSec, String fillerWordCountsJson, + Double pronunciationAccuracy) { + return new MessageVoiceAnalysis(message, speakingRateWpm, silenceDurationSec, + fillerWordCountsJson, pronunciationAccuracy); + } } diff --git a/backend/src/main/java/com/stackup/stackup/session/domain/MessageVoiceAnalysisRepository.java b/backend/src/main/java/com/stackup/stackup/session/domain/MessageVoiceAnalysisRepository.java index 09303ba0..de840eca 100644 --- a/backend/src/main/java/com/stackup/stackup/session/domain/MessageVoiceAnalysisRepository.java +++ b/backend/src/main/java/com/stackup/stackup/session/domain/MessageVoiceAnalysisRepository.java @@ -6,4 +6,6 @@ public interface MessageVoiceAnalysisRepository extends JpaRepository { Optional findByMessage_Id(Long messageId); + + boolean existsByMessage_Id(Long messageId); } diff --git a/backend/src/main/java/com/stackup/stackup/session/domain/SessionFeedback.java b/backend/src/main/java/com/stackup/stackup/session/domain/SessionFeedback.java index 3f56c8dc..a1d2d6d2 100644 --- a/backend/src/main/java/com/stackup/stackup/session/domain/SessionFeedback.java +++ b/backend/src/main/java/com/stackup/stackup/session/domain/SessionFeedback.java @@ -10,6 +10,8 @@ import jakarta.persistence.JoinColumn; import jakarta.persistence.OneToOne; import jakarta.persistence.Table; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import lombok.AccessLevel; import lombok.Getter; import lombok.NoArgsConstructor; @@ -46,9 +48,38 @@ public class SessionFeedback extends BaseSoftDeleteEntity { @Column(name = "weaknesses_summary", columnDefinition = "text") private String weaknessesSummary; + @JdbcTypeCode(SqlTypes.JSON) @Column(name = "improvement_keywords", columnDefinition = "jsonb") private String improvementKeywords; @Column(name = "report_file_path", length = 1000) private String reportFilePath; + + private SessionFeedback(InterviewSession session, Double overallScore, Double technicalAccuracy, + Double logicScore, Double communicationScore, + String strengthsSummary, String weaknessesSummary, + String improvementKeywordsJson, String reportFilePath) { + if (session == null) { + throw new IllegalArgumentException("session must not be null"); + } + this.session = session; + this.overallScore = overallScore; + this.technicalAccuracy = technicalAccuracy; + this.logicScore = logicScore; + this.communicationScore = communicationScore; + this.strengthsSummary = strengthsSummary; + this.weaknessesSummary = weaknessesSummary; + this.improvementKeywords = improvementKeywordsJson; + this.reportFilePath = reportFilePath; + } + + public static SessionFeedback of(InterviewSession session, Double overallScore, + Double technicalAccuracy, Double logicScore, + Double communicationScore, + String strengthsSummary, String weaknessesSummary, + String improvementKeywordsJson, String reportFilePath) { + return new SessionFeedback(session, overallScore, technicalAccuracy, logicScore, + communicationScore, strengthsSummary, weaknessesSummary, + improvementKeywordsJson, reportFilePath); + } } diff --git a/backend/src/main/java/com/stackup/stackup/session/domain/SessionFeedbackRepository.java b/backend/src/main/java/com/stackup/stackup/session/domain/SessionFeedbackRepository.java index 516b8289..09a44fd7 100644 --- a/backend/src/main/java/com/stackup/stackup/session/domain/SessionFeedbackRepository.java +++ b/backend/src/main/java/com/stackup/stackup/session/domain/SessionFeedbackRepository.java @@ -11,6 +11,8 @@ public interface SessionFeedbackRepository extends JpaRepository findBySession_Id(Long sessionId); + boolean existsBySession_Id(Long sessionId); + @Query(""" SELECT f FROM SessionFeedback f WHERE f.session.user.id = :userId diff --git a/backend/src/main/java/com/stackup/stackup/session/infrastructure/FeedbackCallbackHandler.java b/backend/src/main/java/com/stackup/stackup/session/infrastructure/FeedbackCallbackHandler.java new file mode 100644 index 00000000..9ff878d6 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/infrastructure/FeedbackCallbackHandler.java @@ -0,0 +1,30 @@ +package com.stackup.stackup.session.infrastructure; + +import com.stackup.stackup.session.application.FeedbackCallbackService; +import com.stackup.stackup.session.application.dto.FeedbackCallbackEnvelope; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Component; + +// stackup.ai-to-core / callback.feedback 큐 consumer (US-24). +@Component +@RequiredArgsConstructor +public class FeedbackCallbackHandler { + + private static final Logger log = LoggerFactory.getLogger(FeedbackCallbackHandler.class); + + private final FeedbackCallbackService callbackService; + + @RabbitListener(queues = "${app.messaging.rabbitmq.queues.names.core-callback-feedback}") + public void handle(FeedbackCallbackEnvelope envelope) { + try { + callbackService.apply(envelope); + } catch (RuntimeException e) { + log.error("callback.feedback processing failed. messageId={}", + envelope == null ? null : envelope.messageId(), e); + throw e; + } + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/infrastructure/VoiceCallbackHandler.java b/backend/src/main/java/com/stackup/stackup/session/infrastructure/VoiceCallbackHandler.java new file mode 100644 index 00000000..0ecb605d --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/infrastructure/VoiceCallbackHandler.java @@ -0,0 +1,30 @@ +package com.stackup.stackup.session.infrastructure; + +import com.stackup.stackup.session.application.VoiceCallbackService; +import com.stackup.stackup.session.application.dto.VoiceCallbackEnvelope; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.amqp.rabbit.annotation.RabbitListener; +import org.springframework.stereotype.Component; + +// stackup.ai-to-core / callback.voice 큐 consumer (음성 답변 STT/분석 결과). +@Component +@RequiredArgsConstructor +public class VoiceCallbackHandler { + + private static final Logger log = LoggerFactory.getLogger(VoiceCallbackHandler.class); + + private final VoiceCallbackService callbackService; + + @RabbitListener(queues = "${app.messaging.rabbitmq.queues.names.core-callback-voice}") + public void handle(VoiceCallbackEnvelope envelope) { + try { + callbackService.apply(envelope); + } catch (RuntimeException e) { + log.error("callback.voice processing failed. messageId={}", + envelope == null ? null : envelope.messageId(), e); + throw e; + } + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/presentation/SessionFeedbackController.java b/backend/src/main/java/com/stackup/stackup/session/presentation/SessionFeedbackController.java new file mode 100644 index 00000000..b0513e94 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/presentation/SessionFeedbackController.java @@ -0,0 +1,38 @@ +package com.stackup.stackup.session.presentation; + +import com.stackup.stackup.common.security.UserPrincipal; +import com.stackup.stackup.session.application.SessionFeedbackQueryService; +import com.stackup.stackup.session.presentation.dto.FeedbackResponse; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@Tag(name = "Session Feedback", description = "세션 종합 피드백 (US-24)") +@RestController +@RequestMapping("/api/sessions/{sessionId}/feedback") +@RequiredArgsConstructor +public class SessionFeedbackController { + + private final SessionFeedbackQueryService queryService; + + @Operation(operationId = "getSessionFeedback", summary = "세션 종합 피드백 조회") + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "피드백"), + @ApiResponse(responseCode = "401", description = "인증 실패"), + @ApiResponse(responseCode = "404", description = "세션 또는 피드백 없음") + }) + @GetMapping + public FeedbackResponse get( + @AuthenticationPrincipal UserPrincipal principal, + @PathVariable Long sessionId + ) { + return FeedbackResponse.from(queryService.get(principal.userId(), sessionId)); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/presentation/VoiceAnswerController.java b/backend/src/main/java/com/stackup/stackup/session/presentation/VoiceAnswerController.java new file mode 100644 index 00000000..13c8ccd1 --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/presentation/VoiceAnswerController.java @@ -0,0 +1,64 @@ +package com.stackup.stackup.session.presentation; + +import com.stackup.stackup.common.security.UserPrincipal; +import com.stackup.stackup.session.application.VoiceAnswerUploadService; +import com.stackup.stackup.session.application.dto.MessageResult; +import com.stackup.stackup.session.application.dto.VoiceAnswerUploadCommand; +import com.stackup.stackup.session.presentation.dto.MessageResponse; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import java.io.IOException; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestPart; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +@Tag(name = "Voice Answer", description = "음성 답변 업로드 (Phase 2 - 음성 면접)") +@RestController +@RequestMapping("/api/sessions/{sessionId}/messages/voice") +@RequiredArgsConstructor +public class VoiceAnswerController { + + private final VoiceAnswerUploadService uploadService; + + @Operation( + operationId = "submitVoiceAnswer", + summary = "음성 답변 업로드 + STT/분석 트리거", + description = "multipart/form-data 로 audio 파일 업로드. 응답은 transcribing 상태 메시지. " + + "STT 완료 후 SESSION_MESSAGE SSE 로 transcript 가 도착합니다." + ) + @ApiResponses({ + @ApiResponse(responseCode = "201", description = "메시지 INSERT + analyze.voice 발행"), + @ApiResponse(responseCode = "400", description = "파일 형식/크기 오류"), + @ApiResponse(responseCode = "401", description = "인증 실패"), + @ApiResponse(responseCode = "404", description = "세션 없음"), + @ApiResponse(responseCode = "422", description = "세션이 IN_PROGRESS 아니거나 직전 메시지가 질문 아님") + }) + @PostMapping(consumes = "multipart/form-data") + @ResponseStatus(HttpStatus.CREATED) + public MessageResponse submit( + @AuthenticationPrincipal UserPrincipal principal, + @PathVariable Long sessionId, + @RequestHeader(value = "Idempotency-Key", required = false) String idempotencyKey, + @RequestPart("audio") MultipartFile audio + ) throws IOException { + VoiceAnswerUploadCommand cmd = new VoiceAnswerUploadCommand( + audio.getInputStream(), + audio.getSize(), + audio.getContentType(), + audio.getOriginalFilename(), + idempotencyKey + ); + MessageResult result = uploadService.submit(principal.userId(), sessionId, cmd); + return MessageResponse.from(result); + } +} diff --git a/backend/src/main/java/com/stackup/stackup/session/presentation/dto/FeedbackResponse.java b/backend/src/main/java/com/stackup/stackup/session/presentation/dto/FeedbackResponse.java new file mode 100644 index 00000000..f833d00f --- /dev/null +++ b/backend/src/main/java/com/stackup/stackup/session/presentation/dto/FeedbackResponse.java @@ -0,0 +1,29 @@ +package com.stackup.stackup.session.presentation.dto; + +import com.stackup.stackup.session.application.dto.FeedbackResult; +import java.time.Instant; +import java.util.List; + +public record FeedbackResponse( + Long id, + Long sessionId, + Double overallScore, + Double technicalAccuracy, + Double logicScore, + Double communicationScore, + String strengthsSummary, + String weaknessesSummary, + List improvementKeywords, + String reportFilePath, + Instant createdAt +) { + + public static FeedbackResponse from(FeedbackResult r) { + return new FeedbackResponse( + r.id(), r.sessionId(), + r.overallScore(), r.technicalAccuracy(), r.logicScore(), r.communicationScore(), + r.strengthsSummary(), r.weaknessesSummary(), + r.improvementKeywords(), r.reportFilePath(), r.createdAt() + ); + } +} diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index 564f9d24..b543758f 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -94,15 +94,23 @@ app: ai-analyze-repository: ai.analyze.repository ai-generate-questions: ai.generate.questions ai-generate-followup: ai.generate.followup + ai-generate-feedback: ai.generate.feedback + ai-analyze-voice: ai.analyze.voice core-callback-analysis: core.callback.analysis core-callback-questions: core.callback.questions + core-callback-feedback: core.callback.feedback + core-callback-voice: core.callback.voice routing-keys: analyze-resume: analyze.resume analyze-repository: analyze.repository generate-questions: generate.questions generate-followup: generate.followup + generate-feedback: generate.feedback + analyze-voice: analyze.voice callback-analysis: callback.analysis callback-questions: callback.questions + callback-feedback: callback.feedback + callback-voice: callback.voice realtime-session-notify: realtime.session.notify dead-letter: exchange: stackup.dlx diff --git a/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMessagePublisherTest.java b/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMessagePublisherTest.java index e8c1bc43..93ae12ff 100644 --- a/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMessagePublisherTest.java +++ b/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMessagePublisherTest.java @@ -92,8 +92,12 @@ private RabbitMqProperties rabbitMqProperties() { "ai.analyze.repository", "ai.generate.questions", "ai.generate.followup", + "ai.generate.feedback", + "ai.analyze.voice", "core.callback.analysis", - "core.callback.questions" + "core.callback.questions", + "core.callback.feedback", + "core.callback.voice" ) ), new RabbitMqProperties.RoutingKeyProperties( @@ -101,8 +105,12 @@ private RabbitMqProperties rabbitMqProperties() { "analyze.repository", "generate.questions", "generate.followup", + "generate.feedback", + "analyze.voice", "callback.analysis", "callback.questions", + "callback.feedback", + "callback.voice", "realtime.session.notify" ), new RabbitMqProperties.DeadLetter("stackup.dlx", "dlq."), diff --git a/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMqConfigTest.java b/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMqConfigTest.java index f58514b4..e7f0246d 100644 --- a/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMqConfigTest.java +++ b/backend/src/test/java/com/stackup/stackup/common/messaging/RabbitMqConfigTest.java @@ -59,8 +59,12 @@ private RabbitMqProperties rabbitMqProperties() { "ai.analyze.repository", "ai.generate.questions", "ai.generate.followup", + "ai.generate.feedback", + "ai.analyze.voice", "core.callback.analysis", - "core.callback.questions" + "core.callback.questions", + "core.callback.feedback", + "core.callback.voice" ) ), new RabbitMqProperties.RoutingKeyProperties( @@ -68,8 +72,12 @@ private RabbitMqProperties rabbitMqProperties() { "analyze.repository", "generate.questions", "generate.followup", + "generate.feedback", + "analyze.voice", "callback.analysis", "callback.questions", + "callback.feedback", + "callback.voice", "realtime.session.notify" ), new RabbitMqProperties.DeadLetter("stackup.dlx", "dlq."), diff --git a/backend/src/test/java/com/stackup/stackup/session/application/FeedbackCallbackServiceTest.java b/backend/src/test/java/com/stackup/stackup/session/application/FeedbackCallbackServiceTest.java new file mode 100644 index 00000000..0758fe36 --- /dev/null +++ b/backend/src/test/java/com/stackup/stackup/session/application/FeedbackCallbackServiceTest.java @@ -0,0 +1,104 @@ +package com.stackup.stackup.session.application; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.stackup.stackup.common.messaging.domain.ProcessedMessageRepository; +import com.stackup.stackup.common.sse.SseEventPublisher; +import com.stackup.stackup.common.sse.SseEventType; +import com.stackup.stackup.session.application.dto.FeedbackCallbackEnvelope; +import com.stackup.stackup.session.application.dto.FeedbackCallbackPayload; +import com.stackup.stackup.session.domain.InterviewSession; +import com.stackup.stackup.session.domain.InterviewSessionRepository; +import com.stackup.stackup.session.domain.InterviewType; +import com.stackup.stackup.session.domain.JobCategory; +import com.stackup.stackup.session.domain.SessionFeedback; +import com.stackup.stackup.session.domain.SessionFeedbackRepository; +import com.stackup.stackup.session.domain.SessionMode; +import com.stackup.stackup.user.domain.User; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +@ExtendWith(MockitoExtension.class) +class FeedbackCallbackServiceTest { + + @Mock InterviewSessionRepository sessionRepository; + @Mock SessionFeedbackRepository feedbackRepository; + @Mock ProcessedMessageRepository processedMessageRepository; + @Mock SseEventPublisher sseEventPublisher; + @InjectMocks FeedbackCallbackService service; + + @Test + void apply_insertsFeedbackAndPushesSse() { + InterviewSession session = sessionFixture(50L); + FeedbackCallbackEnvelope env = envelope(50L, "fb-1", + new FeedbackCallbackPayload(50L, 85.0, 80.0, 90.0, 75.0, + "강점 요약", "약점 요약", List.of("Spring", "JPA"), null)); + + when(processedMessageRepository.existsById("fb-1")).thenReturn(false); + when(feedbackRepository.existsBySession_Id(50L)).thenReturn(false); + when(sessionRepository.findById(50L)).thenReturn(Optional.of(session)); + when(feedbackRepository.save(any(SessionFeedback.class))).thenAnswer(inv -> { + SessionFeedback f = inv.getArgument(0); + ReflectionTestUtils.setField(f, "id", 700L); + return f; + }); + + service.apply(env); + + ArgumentCaptor cap = ArgumentCaptor.forClass(SessionFeedback.class); + verify(feedbackRepository).save(cap.capture()); + assertThat(cap.getValue().getOverallScore()).isEqualTo(85.0); + assertThat(cap.getValue().getImprovementKeywords()).contains("Spring"); + verify(sseEventPublisher).publishToSession(eq(50L), eq(SseEventType.FEEDBACK_READY), any()); + } + + @Test + void apply_skipsWhenDuplicateMessage() { + FeedbackCallbackEnvelope env = envelope(50L, "dup", + new FeedbackCallbackPayload(50L, 80.0, null, null, null, null, null, List.of(), null)); + when(processedMessageRepository.existsById("dup")).thenReturn(true); + + service.apply(env); + + verify(feedbackRepository, never()).save(any(SessionFeedback.class)); + } + + @Test + void apply_skipsWhenFeedbackAlreadyExists() { + InterviewSession session = sessionFixture(50L); + FeedbackCallbackEnvelope env = envelope(50L, "fb-2", + new FeedbackCallbackPayload(50L, 80.0, null, null, null, null, null, List.of(), null)); + when(processedMessageRepository.existsById("fb-2")).thenReturn(false); + when(sessionRepository.findById(50L)).thenReturn(Optional.of(session)); + when(feedbackRepository.existsBySession_Id(50L)).thenReturn(true); + + service.apply(env); + + verify(feedbackRepository, never()).save(any(SessionFeedback.class)); + } + + private FeedbackCallbackEnvelope envelope(Long sessionId, String messageId, FeedbackCallbackPayload payload) { + return new FeedbackCallbackEnvelope(messageId, "callback.feedback", "1", "t", null, "ai", payload, null); + } + + private InterviewSession sessionFixture(Long id) { + User user = User.createGithubUser(1L, "u", null, null, "t"); + ReflectionTestUtils.setField(user, "id", 1L); + InterviewSession s = InterviewSession.create(user, "t", null, SessionMode.ONLINE, + InterviewType.TECHNICAL, JobCategory.BACKEND, 5, 30); + ReflectionTestUtils.setField(s, "id", id); + return s; + } +} diff --git a/backend/src/test/java/com/stackup/stackup/session/application/QuestionsCallbackServiceTest.java b/backend/src/test/java/com/stackup/stackup/session/application/QuestionsCallbackServiceTest.java index 95e3ecfb..acb8af0b 100644 --- a/backend/src/test/java/com/stackup/stackup/session/application/QuestionsCallbackServiceTest.java +++ b/backend/src/test/java/com/stackup/stackup/session/application/QuestionsCallbackServiceTest.java @@ -39,6 +39,7 @@ class QuestionsCallbackServiceTest { @Mock InterviewMessageRepository messageRepository; @Mock ProcessedMessageRepository processedMessageRepository; @Mock SseEventPublisher sseEventPublisher; + @Mock org.springframework.context.ApplicationEventPublisher events; @InjectMocks QuestionsCallbackService service; @Test diff --git a/backend/src/test/java/com/stackup/stackup/session/application/SessionFeedbackRequesterTest.java b/backend/src/test/java/com/stackup/stackup/session/application/SessionFeedbackRequesterTest.java new file mode 100644 index 00000000..28d7d286 --- /dev/null +++ b/backend/src/test/java/com/stackup/stackup/session/application/SessionFeedbackRequesterTest.java @@ -0,0 +1,91 @@ +package com.stackup.stackup.session.application; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.stackup.stackup.common.config.properties.RabbitMqProperties; +import com.stackup.stackup.common.messaging.MessageContext; +import com.stackup.stackup.common.messaging.RabbitMessagePublisher; +import com.stackup.stackup.session.application.dto.GenerateFeedbackPayload; +import com.stackup.stackup.session.application.event.SessionEndedEvent; +import com.stackup.stackup.session.domain.InterviewMessage; +import com.stackup.stackup.session.domain.InterviewMessageRepository; +import com.stackup.stackup.session.domain.InterviewSession; +import com.stackup.stackup.session.domain.InterviewSessionRepository; +import com.stackup.stackup.session.domain.InterviewType; +import com.stackup.stackup.session.domain.JobCategory; +import com.stackup.stackup.session.domain.SessionContextRepository; +import com.stackup.stackup.session.domain.SessionFeedbackRepository; +import com.stackup.stackup.session.domain.SessionMode; +import com.stackup.stackup.user.domain.User; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; + +@ExtendWith(MockitoExtension.class) +class SessionFeedbackRequesterTest { + + @Mock RabbitMessagePublisher publisher; + @Mock RabbitMqProperties properties; + @Mock InterviewSessionRepository sessionRepository; + @Mock InterviewMessageRepository messageRepository; + @Mock SessionContextRepository contextRepository; + @Mock SessionFeedbackRepository feedbackRepository; + @InjectMocks SessionFeedbackRequester requester; + + @Test + void onSessionEnded_publishesGenerateFeedback() { + InterviewSession session = sessionFixture(50L); + InterviewMessage q = InterviewMessage.interviewer(session, 1, "Q"); + InterviewMessage a = InterviewMessage.interviewee(session, 2, "A", q, null); + ReflectionTestUtils.setField(q, "id", 100L); + ReflectionTestUtils.setField(a, "id", 101L); + + when(feedbackRepository.existsBySession_Id(50L)).thenReturn(false); + when(sessionRepository.findById(50L)).thenReturn(Optional.of(session)); + when(messageRepository.findBySession_IdOrderBySequenceNumberAsc(50L)).thenReturn(List.of(q, a)); + when(contextRepository.findBySession_Id(50L)).thenReturn(List.of()); + RabbitMqProperties.RoutingKeyProperties rk = mockRoutingKeys(); + when(properties.routingKeys()).thenReturn(rk); + + requester.onSessionEnded(new SessionEndedEvent(1L, 50L, "MAX_QUESTIONS_REACHED")); + + verify(publisher).publishToAi(eq("generate.feedback"), any(GenerateFeedbackPayload.class), + any(MessageContext.class)); + } + + @Test + void onSessionEnded_skipsWhenFeedbackExists() { + when(feedbackRepository.existsBySession_Id(50L)).thenReturn(true); + + requester.onSessionEnded(new SessionEndedEvent(1L, 50L, "USER_REQUEST")); + + verify(publisher, never()).publishToAi(any(), any(), any()); + } + + private InterviewSession sessionFixture(Long id) { + User user = User.createGithubUser(1L, "u", null, null, "t"); + ReflectionTestUtils.setField(user, "id", 1L); + InterviewSession s = InterviewSession.create(user, "t", null, SessionMode.ONLINE, + InterviewType.TECHNICAL, JobCategory.BACKEND, 5, 30); + ReflectionTestUtils.setField(s, "id", id); + return s; + } + + private RabbitMqProperties.RoutingKeyProperties mockRoutingKeys() { + return new RabbitMqProperties.RoutingKeyProperties( + "analyze.resume", "analyze.repository", + "generate.questions", "generate.followup", "generate.feedback", "analyze.voice", + "callback.analysis", "callback.questions", "callback.feedback", "callback.voice", + "realtime.session.notify"); + } +} diff --git a/backend/src/test/java/com/stackup/stackup/session/application/VoiceCallbackServiceTest.java b/backend/src/test/java/com/stackup/stackup/session/application/VoiceCallbackServiceTest.java new file mode 100644 index 00000000..8e1a02c5 --- /dev/null +++ b/backend/src/test/java/com/stackup/stackup/session/application/VoiceCallbackServiceTest.java @@ -0,0 +1,115 @@ +package com.stackup.stackup.session.application; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.stackup.stackup.common.messaging.domain.ProcessedMessageRepository; +import com.stackup.stackup.common.sse.SseEventPublisher; +import com.stackup.stackup.common.sse.SseEventType; +import com.stackup.stackup.session.application.dto.VoiceCallbackEnvelope; +import com.stackup.stackup.session.application.dto.VoiceCallbackPayload; +import com.stackup.stackup.session.application.event.AnswerSubmittedEvent; +import com.stackup.stackup.session.domain.InterviewMessage; +import com.stackup.stackup.session.domain.InterviewMessageRepository; +import com.stackup.stackup.session.domain.InterviewSession; +import com.stackup.stackup.session.domain.InterviewType; +import com.stackup.stackup.session.domain.JobCategory; +import com.stackup.stackup.session.domain.MessageStatus; +import com.stackup.stackup.session.domain.MessageVoiceAnalysis; +import com.stackup.stackup.session.domain.MessageVoiceAnalysisRepository; +import com.stackup.stackup.session.domain.SessionMode; +import com.stackup.stackup.user.domain.User; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.test.util.ReflectionTestUtils; + +@ExtendWith(MockitoExtension.class) +class VoiceCallbackServiceTest { + + @Mock InterviewMessageRepository messageRepository; + @Mock MessageVoiceAnalysisRepository voiceAnalysisRepository; + @Mock ProcessedMessageRepository processedMessageRepository; + @Mock SseEventPublisher sseEventPublisher; + @Mock ApplicationEventPublisher events; + @InjectMocks VoiceCallbackService service; + + @Test + void apply_setsTranscriptAndPublishesAnswerSubmittedEvent() { + InterviewSession session = sessionFixture(50L); + InterviewMessage question = InterviewMessage.interviewer(session, 1, "ACID?"); + ReflectionTestUtils.setField(question, "id", 100L); + InterviewMessage voiceMsg = InterviewMessage.voiceInterviewee(session, 2, question, "k1"); + ReflectionTestUtils.setField(voiceMsg, "id", 200L); + + VoiceCallbackPayload payload = new VoiceCallbackPayload(50L, 200L, + "원자성 일관성 격리성 영속성", + 120.0, 1.5, Map.of("음", 2), 0.85, null); + VoiceCallbackEnvelope env = new VoiceCallbackEnvelope("vc-1", "callback.voice", "1", "t", + null, "ai", payload, null); + + when(processedMessageRepository.existsById("vc-1")).thenReturn(false); + when(messageRepository.findById(200L)).thenReturn(Optional.of(voiceMsg)); + when(voiceAnalysisRepository.existsByMessage_Id(200L)).thenReturn(false); + + service.apply(env); + + assertThat(voiceMsg.getContent()).isEqualTo("원자성 일관성 격리성 영속성"); + assertThat(voiceMsg.getStatus()).isEqualTo(MessageStatus.COMPLETED); + verify(voiceAnalysisRepository).save(any(MessageVoiceAnalysis.class)); + verify(events).publishEvent(any(AnswerSubmittedEvent.class)); + verify(sseEventPublisher).publishToSession(eq(50L), eq(SseEventType.SESSION_MESSAGE), any()); + } + + @Test + void apply_marksMessageFailedOnErrorCode() { + InterviewSession session = sessionFixture(50L); + InterviewMessage voiceMsg = InterviewMessage.voiceInterviewee(session, 2, null, null); + ReflectionTestUtils.setField(voiceMsg, "id", 200L); + + VoiceCallbackPayload payload = new VoiceCallbackPayload(50L, 200L, null, null, null, + Map.of(), null, "STT_AUTH_FAILED"); + VoiceCallbackEnvelope env = new VoiceCallbackEnvelope("vc-fail", "callback.voice", "1", "t", + null, "ai", payload, null); + + when(processedMessageRepository.existsById("vc-fail")).thenReturn(false); + when(messageRepository.findById(200L)).thenReturn(Optional.of(voiceMsg)); + + service.apply(env); + + assertThat(voiceMsg.getStatus()).isEqualTo(MessageStatus.FAILED); + verify(voiceAnalysisRepository, never()).save(any(MessageVoiceAnalysis.class)); + verify(events, never()).publishEvent(any(AnswerSubmittedEvent.class)); + } + + @Test + void apply_skipsDuplicateMessageId() { + VoiceCallbackPayload payload = new VoiceCallbackPayload(50L, 200L, "t", null, null, + Map.of(), null, null); + VoiceCallbackEnvelope env = new VoiceCallbackEnvelope("dup", "callback.voice", "1", "t", + null, "ai", payload, null); + when(processedMessageRepository.existsById("dup")).thenReturn(true); + + service.apply(env); + + verify(messageRepository, never()).findById(any()); + } + + private InterviewSession sessionFixture(Long id) { + User user = User.createGithubUser(1L, "u", null, null, "t"); + ReflectionTestUtils.setField(user, "id", 1L); + InterviewSession s = InterviewSession.create(user, "t", null, SessionMode.ONLINE, + InterviewType.TECHNICAL, JobCategory.BACKEND, 5, 30); + ReflectionTestUtils.setField(s, "id", id); + return s; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 44742b60..ac5f2228 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,6 +90,15 @@ services: LLM_FLASH_MODEL: ${LLM_FLASH_MODEL:-gemini-3.1-flash-lite} LLM_FLASH_TEMPERATURE: ${LLM_FLASH_TEMPERATURE:-0.4} LLM_FLASH_MAX_TOKENS: ${LLM_FLASH_MAX_TOKENS:-512} + STT_PROVIDER: ${STT_PROVIDER:-auto} + DEEPGRAM_API_KEY: ${DEEPGRAM_API_KEY:-} + DEEPGRAM_BASE_URL: ${DEEPGRAM_BASE_URL:-https://api.deepgram.com/v1} + DEEPGRAM_MODEL: ${DEEPGRAM_MODEL:-whisper-large} + DEEPGRAM_LANGUAGE: ${DEEPGRAM_LANGUAGE:-ko} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OPENAI_BASE_URL: ${OPENAI_BASE_URL:-https://api.openai.com/v1} + WHISPER_MODEL: ${WHISPER_MODEL:-whisper-1} + WHISPER_LANGUAGE: ${WHISPER_LANGUAGE:-ko} # AI → Core 내부 API. 같은 compose 네트워크의 backend 서비스로 직접 라우팅. CORE_INTERNAL_BASE_URL: ${CORE_INTERNAL_BASE_URL:-http://backend:38010} CORE_INTERNAL_API_KEY: ${CORE_INTERNAL_API_KEY:-local-development-internal-api-key} @@ -107,6 +116,10 @@ services: condition: service_healthy minio-init: condition: service_completed_successfully + backend: + # Spring RabbitMqConfig 가 ai.generate.*/ai.analyze.* 큐를 declare 한 뒤에야 + # AI runner.py(passive=true 큐 검색)가 부팅 가능. backend healthy 이후 시작. + condition: service_healthy healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:38030/health')"] interval: 10s diff --git a/infra/rabbitmq/definitions.json b/infra/rabbitmq/definitions.json index ad8a95fa..778441f1 100644 --- a/infra/rabbitmq/definitions.json +++ b/infra/rabbitmq/definitions.json @@ -100,6 +100,26 @@ "x-dead-letter-routing-key": "dlq.ai.generate.followup" } }, + { + "name": "ai.generate.feedback", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + "x-dead-letter-exchange": "stackup.dlx", + "x-dead-letter-routing-key": "dlq.ai.generate.feedback" + } + }, + { + "name": "ai.analyze.voice", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + "x-dead-letter-exchange": "stackup.dlx", + "x-dead-letter-routing-key": "dlq.ai.analyze.voice" + } + }, { "name": "core.callback.analysis", "vhost": "/", @@ -120,6 +140,26 @@ "x-dead-letter-routing-key": "dlq.core.callback.questions" } }, + { + "name": "core.callback.feedback", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + "x-dead-letter-exchange": "stackup.dlx", + "x-dead-letter-routing-key": "dlq.core.callback.feedback" + } + }, + { + "name": "core.callback.voice", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": { + "x-dead-letter-exchange": "stackup.dlx", + "x-dead-letter-routing-key": "dlq.core.callback.voice" + } + }, { "name": "q.realtime.session.notify", "vhost": "/", @@ -165,6 +205,20 @@ "auto_delete": false, "arguments": {} }, + { + "name": "dlq.ai.generate.feedback", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "dlq.ai.analyze.voice", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, { "name": "dlq.core.callback.analysis", "vhost": "/", @@ -179,6 +233,20 @@ "auto_delete": false, "arguments": {} }, + { + "name": "dlq.core.callback.feedback", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "dlq.core.callback.voice", + "vhost": "/", + "durable": true, + "auto_delete": false, + "arguments": {} + }, { "name": "dlq.q.realtime.session.notify", "vhost": "/", @@ -223,6 +291,20 @@ "destination_type": "queue", "routing_key": "generate.followup" }, + { + "source": "stackup.core-to-ai", + "vhost": "/", + "destination": "ai.generate.feedback", + "destination_type": "queue", + "routing_key": "generate.feedback" + }, + { + "source": "stackup.core-to-ai", + "vhost": "/", + "destination": "ai.analyze.voice", + "destination_type": "queue", + "routing_key": "analyze.voice" + }, { "source": "stackup.ai-to-core", "vhost": "/", @@ -237,6 +319,20 @@ "destination_type": "queue", "routing_key": "callback.questions" }, + { + "source": "stackup.ai-to-core", + "vhost": "/", + "destination": "core.callback.feedback", + "destination_type": "queue", + "routing_key": "callback.feedback" + }, + { + "source": "stackup.ai-to-core", + "vhost": "/", + "destination": "core.callback.voice", + "destination_type": "queue", + "routing_key": "callback.voice" + }, { "source": "stackup.realtime", "vhost": "/", @@ -279,6 +375,20 @@ "destination_type": "queue", "routing_key": "dlq.ai.generate.followup" }, + { + "source": "stackup.dlx", + "vhost": "/", + "destination": "dlq.ai.generate.feedback", + "destination_type": "queue", + "routing_key": "dlq.ai.generate.feedback" + }, + { + "source": "stackup.dlx", + "vhost": "/", + "destination": "dlq.ai.analyze.voice", + "destination_type": "queue", + "routing_key": "dlq.ai.analyze.voice" + }, { "source": "stackup.dlx", "vhost": "/", @@ -293,6 +403,20 @@ "destination_type": "queue", "routing_key": "dlq.core.callback.questions" }, + { + "source": "stackup.dlx", + "vhost": "/", + "destination": "dlq.core.callback.feedback", + "destination_type": "queue", + "routing_key": "dlq.core.callback.feedback" + }, + { + "source": "stackup.dlx", + "vhost": "/", + "destination": "dlq.core.callback.voice", + "destination_type": "queue", + "routing_key": "dlq.core.callback.voice" + }, { "source": "stackup.dlx", "vhost": "/",