Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ai/src/ai_server/chain/feedback_generation_chain.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ async def generate(
end_reason: str | None,
transcript: str,
rag_context: str,
voice_analysis_summary: str,
) -> FeedbackResult: ...


Expand All @@ -49,6 +50,7 @@ async def generate(
end_reason: str | None,
transcript: str,
rag_context: str,
voice_analysis_summary: str = "",
) -> FeedbackResult:
result = await self._chain.ainvoke(
{
Expand All @@ -58,6 +60,8 @@ async def generate(
"end_reason": end_reason or "USER_REQUEST",
"transcript": transcript,
"rag_context": rag_context or "(none)",
"voice_analysis_summary": voice_analysis_summary
or "No voice analysis summary was provided.",
}
)
if not isinstance(result, FeedbackResult):
Expand Down
9 changes: 9 additions & 0 deletions ai/src/ai_server/chain/prompts/feedback_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@
"- 응답은 반드시 지정된 JSON 스키마를 따릅니다."
)

SYSTEM_PROMPT += (
"\n- Voice analysis guidance:\n"
" - Use speaking rate, silence duration, and filler word counts when judging communication_score.\n"
" - Mention notable pacing, long pauses, or repeated filler words in strengths_summary or weaknesses_summary when relevant.\n"
" - If voice analysis is absent or sparse, do not invent voice-related findings.\n"
)

HUMAN_PROMPT = (
"직군: {job_category}\n"
"면접 모드: {mode}\n"
Expand All @@ -29,5 +36,7 @@
"{transcript}\n\n"
"=== RAG 컨텍스트 청크 (참고용, 직접 인용 금지) ===\n"
"{rag_context}\n\n"
"=== Voice Analysis Summary ===\n"
"{voice_analysis_summary}\n\n"
"{format_instructions}"
)
40 changes: 38 additions & 2 deletions ai/src/ai_server/messaging/consumers/feedback_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
FeedbackCallbackPayload,
FeedbackMessageItem,
GenerateFeedbackRequest,
VoiceAnalysisSummary,
)
from ai_server.rag.embedder import EmbeddingProvider

Expand Down Expand Up @@ -84,6 +85,9 @@ async def handle(self, message: AbstractIncomingMessage) -> None:

transcript = _build_transcript(req.messages)
rag_context = await self._build_rag_context(req)
voice_analysis_summary = _build_voice_analysis_summary(
req.voice_analysis_summary
)

result = await self._generator.generate(
job_category=req.job_category,
Expand All @@ -92,6 +96,7 @@ async def handle(self, message: AbstractIncomingMessage) -> None:
end_reason=req.end_reason,
transcript=transcript,
rag_context=rag_context,
voice_analysis_summary=voice_analysis_summary,
)

payload = FeedbackCallbackPayload(
Expand Down Expand Up @@ -142,14 +147,45 @@ async def _build_rag_context(self, req: GenerateFeedbackRequest) -> str:
return "(none)"
if not hits:
return "(none)"
return "\n---\n".join(f"[doc#{h.document_id} chunk#{h.chunk_index}] {h.chunk_text}" for h in hits)
return "\n---\n".join(
f"[doc#{h.document_id} chunk#{h.chunk_index}] {h.chunk_text}" for h in hits
)


def _build_transcript(messages: list[FeedbackMessageItem]) -> str:
if not messages:
return "(empty)"
lines: list[str] = []
for m in messages:
speaker = "면접관" if m.role == "INTERVIEWER" else ("지원자" if m.role == "INTERVIEWEE" else m.role)
speaker = (
"면접관"
if m.role == "INTERVIEWER"
else ("지원자" if m.role == "INTERVIEWEE" else m.role)
)
lines.append(f"[{m.sequence_number}] {speaker}: {m.content}")
return "\n".join(lines)


def _build_voice_analysis_summary(summary: VoiceAnalysisSummary | None) -> str:
if summary is None:
return "No voice analysis summary was provided."

lines: list[str] = []
if summary.analyzed_message_count is not None:
lines.append(f"Analyzed answer messages: {summary.analyzed_message_count}")
if summary.average_speaking_rate_wpm is not None:
lines.append(
f"Average speaking rate: {summary.average_speaking_rate_wpm:g} WPM"
)
if summary.total_silence_duration_sec is not None:
lines.append(
f"Total silence duration: {summary.total_silence_duration_sec:g} seconds"
)
if summary.filler_word_counts:
filler_words = ", ".join(
f"{word}: {count}"
for word, count in sorted(summary.filler_word_counts.items())
)
lines.append(f"Filler word counts: {filler_words}")

return "\n".join(lines) if lines else "No voice analysis summary was provided."
15 changes: 15 additions & 0 deletions ai/src/ai_server/model/messages/feedback.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

class FeedbackMessageItem(BaseModel):
"""세션 시퀀스 한 줄 (Core 가 통째로 동봉)."""

model_config = camel_config()

id: int
Expand All @@ -18,8 +19,20 @@ class FeedbackMessageItem(BaseModel):
parent_message_id: int | None = None


class VoiceAnalysisSummary(BaseModel):
"""Aggregated voice metrics supplied by Core in generate.feedback."""

model_config = camel_config()

analyzed_message_count: int | None = None
average_speaking_rate_wpm: float | None = None
total_silence_duration_sec: float | None = None
filler_word_counts: dict[str, int] = Field(default_factory=dict)


class GenerateFeedbackRequest(BaseModel):
"""Core 가 세션 COMPLETED commit 후 발행."""

model_config = camel_config()

session_id: int
Expand All @@ -29,10 +42,12 @@ class GenerateFeedbackRequest(BaseModel):
end_reason: Literal["USER_REQUEST", "MAX_QUESTIONS_REACHED"] | None = None
messages: list[FeedbackMessageItem] = Field(default_factory=list)
context_document_ids: list[int] = Field(default_factory=list)
voice_analysis_summary: VoiceAnalysisSummary | None = None


class FeedbackCallbackPayload(BaseModel):
"""AI → Core 종합 피드백. 점수는 0~100 (NULL 허용)."""

model_config = camel_config()

session_id: int
Expand Down
99 changes: 93 additions & 6 deletions ai/tests/test_feedback_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,23 @@

import pytest

from ai_server.chain.feedback_generation_chain import FeedbackResult
from ai_server.chain.feedback_generation_chain import (
FeedbackResult,
LlmFeedbackGenerator,
)
from ai_server.chain.prompts.feedback_generation import HUMAN_PROMPT
from ai_server.core.client import EmbeddingSearchHit
from ai_server.messaging.consumers.feedback_consumer import FeedbackConsumer
from ai_server.messaging.idempotency import LruIdempotencyStore
from ai_server.model.messages.feedback import FeedbackCallbackPayload

VOICE_SUMMARY = {
"analyzedMessageCount": 2,
"averageSpeakingRateWpm": 132.5,
"totalSilenceDurationSec": 4.2,
"fillerWordCounts": {"um": 3, "like": 1},
}


class _StubMessage:
def __init__(self, body: bytes):
Expand All @@ -29,7 +40,11 @@ async def __aexit__(self, exc_type, exc, tb):
return False


def _envelope(*, context_documents: list[int] | None = None) -> bytes:
def _envelope(
*,
context_documents: list[int] | None = None,
voice_analysis_summary: dict | None = None,
) -> bytes:
env = {
"messageId": "fb-1",
"messageType": "generate.feedback",
Expand All @@ -44,14 +59,26 @@ def _envelope(*, context_documents: list[int] | None = None) -> bytes:
"totalQuestionCount": 2,
"endReason": "MAX_QUESTIONS_REACHED",
"messages": [
{"id": 100, "sequenceNumber": 1, "role": "INTERVIEWER", "content": "ACID?"},
{"id": 101, "sequenceNumber": 2, "role": "INTERVIEWEE",
"content": "원자성·일관성·격리성·영속성", "parentMessageId": 100},
{
"id": 100,
"sequenceNumber": 1,
"role": "INTERVIEWER",
"content": "ACID?",
},
{
"id": 101,
"sequenceNumber": 2,
"role": "INTERVIEWEE",
"content": "원자성·일관성·격리성·영속성",
"parentMessageId": 100,
},
],
"contextDocumentIds": context_documents or [],
},
"context": {"userId": 1, "sessionId": 50},
}
if voice_analysis_summary is not None:
env["payload"]["voiceAnalysisSummary"] = voice_analysis_summary
return json.dumps(env).encode()


Expand Down Expand Up @@ -104,7 +131,12 @@ async def test_consumer_calls_rag_when_documents_and_embedder_present():
core = MagicMock()
core.search_embeddings = AsyncMock(
return_value=[
EmbeddingSearchHit(document_id=7, chunk_index=2, chunk_text="JPA dirty checking", distance=0.12)
EmbeddingSearchHit(
document_id=7,
chunk_index=2,
chunk_text="JPA dirty checking",
distance=0.12,
)
]
)
embedder = MagicMock()
Expand All @@ -128,6 +160,61 @@ async def test_consumer_calls_rag_when_documents_and_embedder_present():
assert "JPA dirty checking" in invoked_kwargs["rag_context"]


@pytest.mark.asyncio
async def test_consumer_accepts_voice_summary_and_passes_it_to_generator():
generator = _generator()
publisher = MagicMock()
publisher.publish = AsyncMock()

consumer = FeedbackConsumer(
generator=generator,
publisher=publisher,
idempotency=LruIdempotencyStore(max_size=10),
callback_routing_key="callback.feedback",
core_client=MagicMock(),
embedder=None,
)
await consumer.handle(_StubMessage(_envelope(voice_analysis_summary=VOICE_SUMMARY)))

invoked_kwargs = generator.generate.await_args.kwargs
voice_context = invoked_kwargs["voice_analysis_summary"]
assert "Analyzed answer messages: 2" in voice_context
assert "Average speaking rate: 132.5 WPM" in voice_context
assert "Total silence duration: 4.2 seconds" in voice_context
assert "like: 1" in voice_context
assert "um: 3" in voice_context

payload: FeedbackCallbackPayload = publisher.publish.await_args.kwargs["payload"]
assert not hasattr(payload, "voice_analysis_summary")


@pytest.mark.asyncio
async def test_llm_feedback_generator_includes_voice_summary_in_chain_input():
class _FakeChain:
def __init__(self):
self.input = None

async def ainvoke(self, value):
self.input = value
return FeedbackResult(overall_score=70.0)

chain = _FakeChain()
generator = LlmFeedbackGenerator(chain)

await generator.generate(
job_category="BACKEND",
mode="TECHNICAL",
total_question_count=1,
end_reason="USER_REQUEST",
transcript="answer",
rag_context="(none)",
voice_analysis_summary="Average speaking rate: 132.5 WPM",
)

assert chain.input["voice_analysis_summary"] == "Average speaking rate: 132.5 WPM"
assert "voice_analysis_summary" in HUMAN_PROMPT


@pytest.mark.asyncio
async def test_consumer_idempotent_skip():
generator = _generator()
Expand Down