From 6906a6858939c42f304a8ff960537efdefb44e56 Mon Sep 17 00:00:00 2001 From: jmj Date: Wed, 3 Jun 2026 00:45:24 +0900 Subject: [PATCH] =?UTF-8?q?feat(followup):=20RAG=20=EB=B6=80=ED=99=9C=20+?= =?UTF-8?q?=20correctness=20=ED=8F=89=EA=B0=80=20+=20=EC=B9=B4=ED=85=8C?= =?UTF-8?q?=EA=B3=A0=EB=A6=AC=20=EB=A3=A8=EB=B8=8C=EB=A6=AD=20+=20?= =?UTF-8?q?=EB=8C=80=ED=99=94=20=ED=9E=88=EC=8A=A4=ED=86=A0=EB=A6=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 꼬리질문이 자료 근거 없이 답변 텍스트만 보고 생성되던 문제(Core가 context_document_ids 미발행 → RAG dead)를 해결하고, 평가/질문 품질을 끌어올린다. Core: - GenerateFollowupPayload 에 contextDocumentIds/parentCategory/history 추가 - SessionFollowupRequester: SessionContextRepository 로 doc ids(피드백 발행부 패턴), 최근 메시지 6개 history, 부모 질문 category 를 함께 발행 - QuestionsCallbackPayload.AnswerEvaluation 에 correctness 추가(대칭) AI: - GenerateFollowupRequest +parent_category/history (context_document_ids 는 기존) - AnswerEvaluation +correctness(0~5, RAG 컨텍스트 없으면 null) - 프롬프트: correctness 축 + 카테고리별 루브릭(STAR 는 인성형, 기술형은 정확성·깊이) + 히스토리 중복 회피 - 체인/컨슈머: parent_category·history 전달. RAG 는 doc ids 수신 시 자동 활성(하이브리드+리랭크) 테스트: AI 208 passed, backend compile + QuestionsCallbackServiceTest BUILD SUCCESSFUL. DB 변경 없음(평가 영속/피드백 재사용은 다음 작업). 프론트 변경 없음(내부 MQ 계약만). Co-Authored-By: Claude Opus 4.8 --- .../chain/followup_generation_chain.py | 22 ++++-- .../chain/prompts/followup_generation.py | 21 ++++-- .../messaging/consumers/followup_consumer.py | 11 +++ ai/src/ai_server/model/messages/followup.py | 25 +++++-- ai/tests/test_followup_consumer.py | 73 +++++++++++++++++++ .../application/SessionFollowupRequester.java | 25 ++++++- .../dto/GenerateFollowupPayload.java | 8 +- .../dto/QuestionsCallbackPayload.java | 3 +- 8 files changed, 167 insertions(+), 21 deletions(-) diff --git a/ai/src/ai_server/chain/followup_generation_chain.py b/ai/src/ai_server/chain/followup_generation_chain.py index 132d8fa3..1b4f5af9 100644 --- a/ai/src/ai_server/chain/followup_generation_chain.py +++ b/ai/src/ai_server/chain/followup_generation_chain.py @@ -28,6 +28,8 @@ async def generate( previous_question: str, answer_text: str, context: str = "(none)", + parent_category: str = "UNKNOWN", + history: str = "(none)", ) -> FollowupResult: ... @@ -43,6 +45,8 @@ async def generate( previous_question: str, answer_text: str, context: str = "(none)", + parent_category: str = "UNKNOWN", + history: str = "(none)", ) -> FollowupResult: result = await self._chain.ainvoke( { @@ -51,6 +55,8 @@ async def generate( "previous_question": previous_question, "answer_text": answer_text, "context": context, + "parent_category": parent_category, + "history": history, } ) if not isinstance(result, FollowupResult): @@ -60,7 +66,9 @@ async def generate( return result -def build_followup_generation_chain(settings: Settings, core_client: CoreClient | None = None) -> Runnable: +def build_followup_generation_chain( + settings: Settings, core_client: CoreClient | None = None +) -> Runnable: from langchain_openai import ChatOpenAI parser = PydanticOutputParser(pydantic_object=FollowupResult) @@ -73,11 +81,13 @@ def build_followup_generation_chain(settings: Settings, core_client: CoreClient callbacks = [] if core_client is not None: - callbacks.append(CoreAiLogCallback( - core_client=core_client, - request_type="generate.followup", - default_model=settings.llm_flash_model, - )) + callbacks.append( + CoreAiLogCallback( + core_client=core_client, + request_type="generate.followup", + default_model=settings.llm_flash_model, + ) + ) llm = ChatOpenAI( model=settings.llm_flash_model, diff --git a/ai/src/ai_server/chain/prompts/followup_generation.py b/ai/src/ai_server/chain/prompts/followup_generation.py index e863776d..091d58a1 100644 --- a/ai/src/ai_server/chain/prompts/followup_generation.py +++ b/ai/src/ai_server/chain/prompts/followup_generation.py @@ -1,27 +1,34 @@ # 꼬리질문 생성 + 답변 평가 (US-19) -# Flash 모델 + 저지연 (< 3s). 사용자 답변의 specificity·logic·structure 채점 + 부족 부분 파고드는 꼬리질문 1개. +# Flash 모델 + 저지연 (< 3s). 답변을 4축으로 평가하고 가장 약한 축을 파고드는 꼬리질문 1개. SYSTEM_PROMPT = ( "당신은 IT 직군 면접관입니다. 직전 질문에 대한 지원자의 답변을 평가하고, " - "부족한 부분(구체성·논리·구조)을 파고드는 꼬리질문 1개를 한국어로 만드세요.\n" + "가장 약한 부분을 파고드는 꼬리질문 1개를 한국어로 만드세요.\n" "- 평가 항목:\n" " - specificity (0~5): 답변에 구체적 수치/사례/기술 선택 근거가 있는가.\n" " - logic (0~5): 인과관계와 trade-off 가 명확한가.\n" " - structure: STAR (Situation-Task-Action-Result) 구조 측면.\n" - " - FULL_STAR: 네 요소 모두 명확.\n" - " - PARTIAL_STAR: 일부 요소 누락.\n" - " - NONE: 구조 부재.\n" - "- 꼬리질문은 답변에서 가장 약한 축 (예: 구체성 낮음 → 수치/사례 요구) 을 겨냥합니다.\n" + " - FULL_STAR / PARTIAL_STAR / NONE.\n" + " - 단, STAR 는 **행동/경험형 답변에 적합**한 기준이다. 카테고리가 " + "BEHAVIORAL/PERSONALITY 면 STAR 를 중시하고, CS_FUNDAMENTAL/TECH_CHOICE/" + "PROJECT_DEEP_DIVE 같은 기술형이면 structure 는 참고만 하고 정확성·깊이를 우선한다.\n" + " - correctness (0~5): 답변이 '검색 문서 컨텍스트'의 사실과 일치하는가. " + "**컨텍스트가 '(none)' 이면 판단 불가 → correctness 는 null**. 추측으로 채우지 마세요.\n" + "- 꼬리질문은 가장 약한 축을 겨냥합니다 (예: 구체성 낮음→수치/사례 요구, " + "correctness 의심→자료와의 불일치 확인).\n" + "- '이미 나눈 대화'에서 다룬 내용을 그대로 반복하지 말고 새로운 각도로 파고드세요.\n" "- 응답은 반드시 지정된 JSON 스키마를 따릅니다." ) HUMAN_PROMPT = ( "직군: {job_category}\n" - "면접 모드: {mode}\n\n" + "면접 모드: {mode}\n" + "직전 질문 카테고리: {parent_category}\n\n" "모드별 지침:\n" "- TECHNICAL: 기술 역량, 프로젝트 경험, 문제 해결을 중심으로 파고듭니다.\n" "- PERSONALITY: 협업, 갈등 해결, 성장 경험을 중심으로 파고듭니다.\n" "- INTEGRATED: 기술 질문과 인성 질문의 관점을 균형 있게 반영합니다.\n\n" + "이미 나눈 대화 (반복 금지):\n{history}\n\n" "직전 질문:\n{previous_question}\n\n" "지원자 답변:\n{answer_text}\n\n" "검색 문서 컨텍스트:\n---\n{context}\n---\n\n" diff --git a/ai/src/ai_server/messaging/consumers/followup_consumer.py b/ai/src/ai_server/messaging/consumers/followup_consumer.py index c6c03c80..121e5cc8 100644 --- a/ai/src/ai_server/messaging/consumers/followup_consumer.py +++ b/ai/src/ai_server/messaging/consumers/followup_consumer.py @@ -79,6 +79,8 @@ async def handle(self, message: AbstractIncomingMessage) -> None: previous_question=req.previous_question, answer_text=req.answer_text, context=await self._build_rag_context(req), + parent_category=req.parent_category or "UNKNOWN", + history=_format_history(req.history), ) payload = FollowupCallbackPayload( @@ -131,3 +133,12 @@ async def _build_rag_context(self, req: GenerateFollowupRequest) -> str: return "\n---\n".join( f"[doc#{h.document_id} chunk#{h.chunk_index}] {h.chunk_text}" for h in hits ) + + +def _format_history(history: list) -> str: + """대화 히스토리를 '화자: 내용' 라인으로. 비면 '(none)'.""" + if not history: + return "(none)" + speaker = {"INTERVIEWER": "면접관", "INTERVIEWEE": "지원자"} + lines = [f"{speaker.get(h.role, h.role)}: {h.content}" for h in history] + return "\n".join(lines) diff --git a/ai/src/ai_server/model/messages/followup.py b/ai/src/ai_server/model/messages/followup.py index c2fb7b1b..a5e44dd8 100644 --- a/ai/src/ai_server/model/messages/followup.py +++ b/ai/src/ai_server/model/messages/followup.py @@ -7,27 +7,42 @@ InterviewMode = Literal["PERSONALITY", "TECHNICAL", "INTEGRATED"] +class HistoryItem(BaseModel): + """대화 히스토리 한 줄 (중복 질문 회피용).""" + + model_config = camel_config() + + role: str # INTERVIEWER | INTERVIEWEE | SYSTEM + content: str + + class GenerateFollowupRequest(BaseModel): """Core 가 답변 commit 후 발행.""" + model_config = camel_config() session_id: int - parent_message_id: int # 직전 질문 메시지 ID - answer_message_id: int # 답변 메시지 ID + parent_message_id: int # 직전 질문 메시지 ID + answer_message_id: int # 답변 메시지 ID previous_question: str answer_text: str mode: InterviewMode job_category: Literal["FRONTEND", "BACKEND", "INFRA", "DBA"] context_document_ids: list[int] = Field(default_factory=list) + parent_category: str | None = None # 직전 질문 카테고리 (루브릭 선택) + history: list[HistoryItem] = Field(default_factory=list) # 최근 대화 (중복 회피) class AnswerEvaluation(BaseModel): - """답변 평가 (US-19). LLM 이 specificity/logic/structure 채움.""" + """답변 평가 (US-19). LLM 이 specificity/logic/structure/correctness 채움.""" + model_config = camel_config() - specificity: float # 0~5 - logic: float # 0~5 + specificity: float # 0~5 + logic: float # 0~5 structure: Literal["FULL_STAR", "PARTIAL_STAR", "NONE"] + # 답변이 자료(RAG)와 사실적으로 맞는지. 검색 컨텍스트 없으면 null. + correctness: float | None = None # 0~5 class FollowupCallbackPayload(BaseModel): diff --git a/ai/tests/test_followup_consumer.py b/ai/tests/test_followup_consumer.py index 72ab45a0..f8b10a69 100644 --- a/ai/tests/test_followup_consumer.py +++ b/ai/tests/test_followup_consumer.py @@ -185,3 +185,76 @@ async def test_consumer_idempotent_skip(): await consumer.handle(_StubMessage(_envelope())) generator.generate.assert_not_awaited() publisher.publish.assert_not_awaited() + + +def test_format_history_formats_and_empty(): + from ai_server.messaging.consumers.followup_consumer import _format_history + from ai_server.model.messages.followup import HistoryItem + + assert _format_history([]) == "(none)" + out = _format_history( + [ + HistoryItem(role="INTERVIEWER", content="Q1?"), + HistoryItem(role="INTERVIEWEE", content="A1"), + ] + ) + assert out == "면접관: Q1?\n지원자: A1" + + +def test_answer_evaluation_correctness_defaults_none_and_parses(): + e1 = AnswerEvaluation(specificity=1.0, logic=2.0, structure="NONE") + assert e1.correctness is None + e2 = AnswerEvaluation.model_validate( + {"specificity": 4, "logic": 4, "structure": "FULL_STAR", "correctness": 3.5} + ) + assert e2.correctness == 3.5 + + +@pytest.mark.asyncio +async def test_consumer_passes_parent_category_and_history_to_generator(): + generator = MagicMock() + generator.generate = AsyncMock( + return_value=FollowupResult( + followup_question="새 각도 질문", + answer_evaluation=AnswerEvaluation( + specificity=2.0, logic=2.0, structure="NONE" + ), + ) + ) + publisher = MagicMock() + publisher.publish = AsyncMock() + consumer = FollowupConsumer( + generator=generator, + publisher=publisher, + idempotency=LruIdempotencyStore(max_size=10), + callback_routing_key="callback.questions", + ) + env = { + "messageId": "m-9", + "messageType": "generate.followup", + "version": "v1", + "traceId": "t-9", + "publishedAt": "2026-05-29T00:00:00Z", + "publisher": "core-server", + "payload": { + "sessionId": 99, + "parentMessageId": 501, + "answerMessageId": 502, + "previousQuestion": "Q?", + "answerText": "A.", + "mode": "TECHNICAL", + "jobCategory": "BACKEND", + "parentCategory": "PROJECT_DEEP_DIVE", + "history": [ + {"role": "INTERVIEWER", "content": "이전 질문"}, + {"role": "INTERVIEWEE", "content": "이전 답변"}, + ], + }, + "context": {"userId": 42, "sessionId": 99}, + } + await consumer.handle(_StubMessage(json.dumps(env).encode())) + + kwargs = generator.generate.await_args.kwargs + assert kwargs["parent_category"] == "PROJECT_DEEP_DIVE" + assert "이전 질문" in kwargs["history"] + assert "면접관:" in kwargs["history"] diff --git a/backend/src/main/java/com/stackup/stackup/session/application/SessionFollowupRequester.java b/backend/src/main/java/com/stackup/stackup/session/application/SessionFollowupRequester.java index e216d152..596032f2 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/SessionFollowupRequester.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/SessionFollowupRequester.java @@ -4,10 +4,13 @@ import com.stackup.stackup.common.messaging.MessageContext; import com.stackup.stackup.common.messaging.RabbitMessagePublisher; import com.stackup.stackup.session.application.dto.GenerateFollowupPayload; +import com.stackup.stackup.session.application.dto.GenerateFollowupPayload.HistoryItem; import com.stackup.stackup.session.application.event.AnswerSubmittedEvent; import com.stackup.stackup.session.domain.InterviewMessage; import com.stackup.stackup.session.domain.InterviewMessageRepository; import com.stackup.stackup.session.domain.InterviewSession; +import com.stackup.stackup.session.domain.SessionContextRepository; +import java.util.List; import lombok.RequiredArgsConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,9 +27,12 @@ public class SessionFollowupRequester { private static final Logger log = LoggerFactory.getLogger(SessionFollowupRequester.class); + private static final int HISTORY_MAX = 6; + private final RabbitMessagePublisher publisher; private final RabbitMqProperties properties; private final InterviewMessageRepository messageRepository; + private final SessionContextRepository contextRepository; @Transactional(propagation = Propagation.REQUIRES_NEW) @TransactionalEventListener(phase = TransactionPhase.AFTER_COMMIT) @@ -39,6 +45,20 @@ public void onAnswerSubmitted(AnswerSubmittedEvent event) { return; } InterviewSession session = parent.getSession(); + + // RAG(자료 근거/correctness) 용 세션 컨텍스트 문서 — 피드백 발행부와 동일 패턴. + List contextDocumentIds = contextRepository.findBySession_Id(session.getId()).stream() + .map(c -> c.getDocument().getId()) + .toList(); + + // 최근 대화 히스토리 (중복 질문 회피). 시퀀스 순 → 마지막 N개. + List ordered = + messageRepository.findBySession_IdOrderBySequenceNumberAsc(session.getId()); + List history = ordered.stream() + .skip(Math.max(0, ordered.size() - HISTORY_MAX)) + .map(m -> new HistoryItem(m.getRole().name(), m.getContent())) + .toList(); + GenerateFollowupPayload payload = new GenerateFollowupPayload( session.getId(), parent.getId(), @@ -46,7 +66,10 @@ public void onAnswerSubmitted(AnswerSubmittedEvent event) { parent.getContent(), answer.getContent(), session.getMode(), - session.getJobCategory() + session.getJobCategory(), + contextDocumentIds, + parent.getCategory(), + history ); publisher.publishToAi( properties.routingKeys().generateFollowup(), diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/GenerateFollowupPayload.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/GenerateFollowupPayload.java index 0b5b4469..273a0333 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/dto/GenerateFollowupPayload.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/GenerateFollowupPayload.java @@ -2,6 +2,7 @@ import com.stackup.stackup.session.domain.JobCategory; import com.stackup.stackup.session.domain.SessionMode; +import java.util.List; public record GenerateFollowupPayload( Long sessionId, @@ -10,6 +11,11 @@ public record GenerateFollowupPayload( String previousQuestion, String answerText, SessionMode mode, - JobCategory jobCategory + JobCategory jobCategory, + List contextDocumentIds, // RAG(자료 근거/correctness) 용 + String parentCategory, // 직전 질문 카테고리 (루브릭 선택) + List history // 최근 대화 (중복 회피) ) { + public record HistoryItem(String role, String content) { + } } diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/QuestionsCallbackPayload.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/QuestionsCallbackPayload.java index 978afa79..87412535 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/dto/QuestionsCallbackPayload.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/QuestionsCallbackPayload.java @@ -31,7 +31,8 @@ public record GeneratedQuestion( public record AnswerEvaluation( Double specificity, Double logic, - String structure + String structure, + Double correctness // 자료 근거 사실성(0~5). RAG 컨텍스트 없으면 null. ) { } }