Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ai/src/ai_server/chain/prompts/feedback_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
" - improvement_keywords: 다음 면접에서 채울 키워드 5~10개 (짧은 명사구).\n"
"- 평가 원칙:\n"
" - 단일 답변보다 시퀀스 흐름을 우선 고려 (꼬리질문 대응의 깊이가 중요).\n"
" - 전사에 답변별 '답변평가'(specificity/logic/structure/correctness)가 붙어 있으면 "
"이를 **근거로 종합**하세요. 무시하고 처음부터 재평가하지 말고 그 점수들을 누적·일관되게 반영합니다.\n"
" - 점수를 매기기 **전에** 강점/약점의 근거를 먼저 정리한 뒤 점수를 산정하세요(즉흥 점수 금지).\n"
" - strengths_summary·weaknesses_summary 에는 **어느 질문/순간 때문인지 구체적으로** 적으세요 "
"(예: 'Q3에서 동시성 처리를 RDB 락으로만 설명해 분산 환경 고려가 빠짐').\n"
" - 답변이 짧거나 비어 있으면 해당 점수는 낮게 또는 null.\n"
" - 컨텍스트 청크(분석 문서 일부) 가 있다면 사실 검증에만 활용 (직접 인용 X).\n"
"- 응답은 반드시 지정된 JSON 스키마를 따릅니다."
Expand Down
18 changes: 17 additions & 1 deletion ai/src/ai_server/messaging/consumers/feedback_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,10 +171,26 @@ def _build_transcript(messages: list[FeedbackMessageItem]) -> str:
if m.role == "INTERVIEWER"
else ("지원자" if m.role == "INTERVIEWEE" else m.role)
)
lines.append(f"[{m.sequence_number}] {speaker}: {m.content}")
line = f"[{m.sequence_number}] {speaker}: {m.content}"
if m.role == "INTERVIEWEE" and m.evaluation is not None:
line += f"\n └ 답변평가: {_format_evaluation(m.evaluation)}"
lines.append(line)
return "\n".join(lines)


def _format_evaluation(e) -> str:
parts: list[str] = []
if e.specificity is not None:
parts.append(f"specificity={e.specificity:g}")
if e.logic is not None:
parts.append(f"logic={e.logic:g}")
if e.structure:
parts.append(f"structure={e.structure}")
if e.correctness is not None:
parts.append(f"correctness={e.correctness:g}")
return ", ".join(parts) if parts else "(없음)"


def _build_voice_analysis_summary(summary: VoiceAnalysisSummary | None) -> str:
if summary is None:
return "No voice analysis summary was provided."
Expand Down
1 change: 1 addition & 0 deletions ai/src/ai_server/messaging/consumers/followup_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ async def handle(self, message: AbstractIncomingMessage) -> None:
session_id=req.session_id,
kind="FOLLOWUP",
parent_message_id=req.parent_message_id,
answer_message_id=req.answer_message_id,
followup_question=result.followup_question,
answer_evaluation=result.answer_evaluation,
)
Expand Down
13 changes: 13 additions & 0 deletions ai/src/ai_server/model/messages/feedback.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,17 @@
InterviewMode = Literal["PERSONALITY", "TECHNICAL", "INTEGRATED"]


class MessageEvaluation(BaseModel):
"""답변별 평가 (꼬리질문 단계에서 채점·영속된 값). 피드백에서 롤업 재활용."""

model_config = camel_config()

specificity: float | None = None
logic: float | None = None
structure: str | None = None
correctness: float | None = None


class FeedbackMessageItem(BaseModel):
"""세션 시퀀스 한 줄 (Core 가 통째로 동봉)."""

Expand All @@ -17,6 +28,8 @@ class FeedbackMessageItem(BaseModel):
role: Literal["INTERVIEWER", "INTERVIEWEE", "SYSTEM"]
content: str
parent_message_id: int | None = None
# 답변(INTERVIEWEE) 메시지에만 채워짐. 피드백 종합 채점의 근거.
evaluation: MessageEvaluation | None = None


class VoiceAnalysisSummary(BaseModel):
Expand Down
1 change: 1 addition & 0 deletions ai/src/ai_server/model/messages/followup.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,6 @@ class FollowupCallbackPayload(BaseModel):
session_id: int
kind: Literal["FOLLOWUP"] = "FOLLOWUP"
parent_message_id: int
answer_message_id: int # 평가가 달릴 답변 메시지 (Core 가 평가 영속에 사용)
followup_question: str
answer_evaluation: AnswerEvaluation | None = None
26 changes: 26 additions & 0 deletions ai/tests/test_feedback_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,3 +234,29 @@ async def test_consumer_idempotent_skip():
await consumer.handle(_StubMessage(_envelope()))
generator.generate.assert_not_awaited()
publisher.publish.assert_not_awaited()


def test_build_transcript_annotates_interviewee_evaluation():
from ai_server.messaging.consumers.feedback_consumer import _build_transcript
from ai_server.model.messages.feedback import FeedbackMessageItem, MessageEvaluation

msgs = [
FeedbackMessageItem(
id=1, sequence_number=1, role="INTERVIEWER", content="질문?"
),
FeedbackMessageItem(
id=2,
sequence_number=2,
role="INTERVIEWEE",
content="답변.",
evaluation=MessageEvaluation(
specificity=2.0, logic=3.0, structure="PARTIAL_STAR", correctness=1.0
),
),
]
out = _build_transcript(msgs)
assert "답변평가:" in out
assert "specificity=2" in out
assert "correctness=1" in out
# 면접관 줄엔 평가 주석 없음
assert out.splitlines()[0].endswith("질문?")
24 changes: 24 additions & 0 deletions ai/tests/test_followup_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,3 +258,27 @@ async def test_consumer_passes_parent_category_and_history_to_generator():
assert kwargs["parent_category"] == "PROJECT_DEEP_DIVE"
assert "이전 질문" in kwargs["history"]
assert "면접관:" in kwargs["history"]


@pytest.mark.asyncio
async def test_callback_includes_answer_message_id():
generator = MagicMock()
generator.generate = AsyncMock(
return_value=FollowupResult(
followup_question="Q",
answer_evaluation=AnswerEvaluation(
specificity=2.0, logic=2.0, structure="NONE"
),
)
)
publisher = MagicMock()
publisher.publish = AsyncMock()
consumer = FollowupConsumer(
generator=generator,
publisher=publisher,
idempotency=LruIdempotencyStore(max_size=10),
callback_routing_key="callback.questions",
)
await consumer.handle(_StubMessage(_envelope()))
payload = publisher.publish.await_args.kwargs["payload"]
assert payload.answer_message_id == 502 # _envelope 의 answerMessageId
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,9 @@ private void applyFollowup(InterviewSession session, QuestionsCallbackPayload pa
? null
: messageRepository.findById(payload.parentMessageId()).orElse(null);

// 답변 평가를 해당 답변 메시지에 영속 (피드백 롤업 재활용). 평가/대상 없으면 skip.
recordAnswerEvaluation(payload);

long currentMsgs = messageRepository.countBySession_Id(session.getId());
int nextSeq = (int) currentMsgs + 1;

Expand Down Expand Up @@ -159,6 +162,18 @@ private void applyFollowup(InterviewSession session, QuestionsCallbackPayload pa
public record SessionStateNotice(Long sessionId, String status, String reason) {
}

private void recordAnswerEvaluation(QuestionsCallbackPayload payload) {
QuestionsCallbackPayload.AnswerEvaluation eval = payload.answerEvaluation();
if (eval == null || payload.answerMessageId() == null) {
return;
}
messageRepository.findById(payload.answerMessageId()).ifPresent(answer -> {
answer.recordAnswerEvaluation(
eval.specificity(), eval.logic(), eval.structure(), eval.correctness());
messageRepository.save(answer);
});
}

private boolean isProcessed(String messageId) {
if (messageId == null || messageId.isBlank()) {
return false;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.stackup.stackup.common.messaging.MessageContext;
import com.stackup.stackup.common.messaging.RabbitMessagePublisher;
import com.stackup.stackup.session.application.dto.GenerateFeedbackPayload;
import com.stackup.stackup.session.application.dto.GenerateFeedbackPayload.MessageEvaluation;
import com.stackup.stackup.session.application.dto.GenerateFeedbackPayload.MessageItem;
import com.stackup.stackup.session.application.dto.GenerateFeedbackPayload.VoiceAnalysisSummary;
import com.stackup.stackup.session.application.event.SessionEndedEvent;
Expand Down Expand Up @@ -89,12 +90,24 @@ public void onSessionEnded(SessionEndedEvent event) {

private MessageItem toItem(InterviewMessage m) {
Long parentId = m.getParentMessage() == null ? null : m.getParentMessage().getId();
MessageEvaluation evaluation = null;
boolean hasEval = m.getAnswerSpecificity() != null || m.getAnswerLogic() != null
|| m.getAnswerStructure() != null || m.getAnswerCorrectness() != null;
if (m.getRole() == com.stackup.stackup.session.domain.MessageRole.INTERVIEWEE && hasEval) {
evaluation = new MessageEvaluation(
m.getAnswerSpecificity(),
m.getAnswerLogic(),
m.getAnswerStructure(),
m.getAnswerCorrectness()
);
}
return new MessageItem(
m.getId(),
m.getSequenceNumber(),
m.getRole().name(),
m.getContent(),
parentId
parentId,
evaluation
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,16 @@ public record MessageItem(
Integer sequenceNumber,
String role,
String content,
Long parentMessageId
Long parentMessageId,
MessageEvaluation evaluation // INTERVIEWEE 답변에만(없으면 null)
) {
}

public record MessageEvaluation(
Double specificity,
Double logic,
String structure,
Double correctness
) {
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ public record QuestionsCallbackPayload(
Long sessionId,
String kind, // POOL | FOLLOWUP
List<GeneratedQuestion> questions, // POOL legacy kind: initial question result
Long parentMessageId, // FOLLOWUP 시 사용
Long parentMessageId, // FOLLOWUP 시 사용 (직전 질문)
Long answerMessageId, // FOLLOWUP 시 평가 대상 답변 메시지
String followupQuestion, // FOLLOWUP 시 사용
AnswerEvaluation answerEvaluation // FOLLOWUP 시 옵션
) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,19 @@ public class InterviewMessage extends BaseTimeEntity {
@Column(name = "expected_signal", columnDefinition = "text")
private String expectedSignal;

// 답변 평가 (INTERVIEWEE 메시지에만 채워짐). 꼬리질문 단계 채점값 → 피드백 롤업에 재사용.
@Column(name = "answer_specificity")
private Double answerSpecificity;

@Column(name = "answer_logic")
private Double answerLogic;

@Column(name = "answer_structure", length = 20)
private String answerStructure;

@Column(name = "answer_correctness")
private Double answerCorrectness;

private InterviewMessage(InterviewSession session, Integer sequenceNumber, MessageRole role,
String content, InterviewMessage parentMessage,
MessageStatus initialStatus, String idempotencyKey) {
Expand Down Expand Up @@ -155,6 +168,15 @@ public void markStatus(MessageStatus newStatus) {
}
}

// 꼬리질문 콜백의 답변 평가를 이 답변 메시지에 기록 (피드백 재활용용).
public void recordAnswerEvaluation(Double specificity, Double logic,
String structure, Double correctness) {
this.answerSpecificity = specificity;
this.answerLogic = logic;
this.answerStructure = structure;
this.answerCorrectness = correctness;
}

public void attachAudio(String audioFilePath) {
this.audioFilePath = audioFilePath;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
-- =============================================================================
-- 답변 평가 영속 (꼬리질문 단계 채점값) — 피드백에서 롤업 재활용
-- =============================================================================
-- 답변(INTERVIEWEE) 메시지에만 채워진다. 모두 nullable.
-- specificity/logic/correctness: 0~5, structure: STAR enum 문자열.
ALTER TABLE interview_messages
ADD COLUMN answer_specificity DOUBLE PRECISION,
ADD COLUMN answer_logic DOUBLE PRECISION,
ADD COLUMN answer_structure VARCHAR(20),
ADD COLUMN answer_correctness DOUBLE PRECISION;
Original file line number Diff line number Diff line change
Expand Up @@ -123,16 +123,46 @@ void apply_skipsDuplicateMessageId() {
verify(messageRepository, never()).save(any(InterviewMessage.class));
}

@Test
void apply_followupPersistsAnswerEvaluationOntoAnswerMessage() {
InterviewSession session = sessionFixture(11L, SessionStatus.IN_PROGRESS);
InterviewMessage answer =
InterviewMessage.interviewee(session, 2, "내 답변", null, "idem-1");
ReflectionTestUtils.setField(answer, "id", 600L);

QuestionsCallbackPayload payload = new QuestionsCallbackPayload(
11L, "FOLLOWUP", null, 500L, 600L, "다음 질문?",
new QuestionsCallbackPayload.AnswerEvaluation(2.0, 3.0, "PARTIAL_STAR", 1.5)
);
QuestionsCallbackEnvelope env = new QuestionsCallbackEnvelope(
"m-eval", "callback.questions", "1", "t", null, "ai", payload, null);

when(processedMessageRepository.existsById("m-eval")).thenReturn(false);
when(sessionRepository.findById(11L)).thenReturn(Optional.of(session));
when(messageRepository.findById(500L)).thenReturn(Optional.empty());
when(messageRepository.findById(600L)).thenReturn(Optional.of(answer));
when(messageRepository.countBySession_Id(11L)).thenReturn(2L);
when(messageRepository.save(any(InterviewMessage.class)))
.thenAnswer(inv -> inv.getArgument(0));

service.apply(env);

assertThat(answer.getAnswerSpecificity()).isEqualTo(2.0);
assertThat(answer.getAnswerLogic()).isEqualTo(3.0);
assertThat(answer.getAnswerStructure()).isEqualTo("PARTIAL_STAR");
assertThat(answer.getAnswerCorrectness()).isEqualTo(1.5);
}

private QuestionsCallbackEnvelope poolEnvelope(Long sessionId, List<GeneratedQuestion> questions) {
QuestionsCallbackPayload payload = new QuestionsCallbackPayload(
sessionId, "POOL", questions, null, null, null
sessionId, "POOL", questions, null, null, null, null
);
return new QuestionsCallbackEnvelope("m-1", "callback.questions", "1", "t", null, "ai", payload, null);
}

private QuestionsCallbackEnvelope followupEnvelope(Long sessionId, Long parentId, String followup) {
QuestionsCallbackPayload payload = new QuestionsCallbackPayload(
sessionId, "FOLLOWUP", null, parentId, followup, null
sessionId, "FOLLOWUP", null, parentId, null, followup, null
);
return new QuestionsCallbackEnvelope("m-2", "callback.questions", "1", "t", null, "ai", payload, null);
}
Expand Down
9 changes: 9 additions & 0 deletions frontend/src/features/feedback/api/feedbackApi.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { apiClient } from '@/shared/api'
import type { components } from '@/shared/api/generated'

type S = components['schemas']
export type Feedback = S['FeedbackResponse']

export async function getFeedback(sessionId: number): Promise<Feedback> {
return (await apiClient.get<Feedback>(`/api/sessions/${sessionId}/feedback`)).data
}
3 changes: 3 additions & 0 deletions frontend/src/features/feedback/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export { FeedbackReport } from './ui/FeedbackReport'
export { useFeedback, isFeedbackPending, feedbackKeys } from './model/useFeedback'
export type { Feedback } from './api/feedbackApi'
23 changes: 23 additions & 0 deletions frontend/src/features/feedback/model/useFeedback.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { useQuery } from '@tanstack/react-query'
import { isApiError } from '@/shared/api'
import { getFeedback } from '../api/feedbackApi'

export const feedbackKeys = {
all: ['feedback'] as const,
detail: (sessionId: number) => [...feedbackKeys.all, sessionId] as const,
}

// 세션 종료 직후엔 피드백이 비동기 생성 중이라 아직 없음(FEEDBACK_NOT_READY/404).
// 이 경우는 에러가 아니라 "생성 중"이므로 일정 횟수까지 polling 한다.
export function isFeedbackPending(err: unknown): boolean {
return isApiError(err) && (err.code === 'FEEDBACK_NOT_READY' || err.status === 404)
}

export function useFeedback(sessionId: number) {
return useQuery({
queryKey: feedbackKeys.detail(sessionId),
queryFn: () => getFeedback(sessionId),
retry: (count, err) => isFeedbackPending(err) && count < 40,
retryDelay: 3000,
})
}
Loading