diff --git a/backend/openapi.json b/backend/openapi.json index ca31a921..4772afe0 100644 --- a/backend/openapi.json +++ b/backend/openapi.json @@ -2613,6 +2613,12 @@ }, "targetEvidence" : { "type" : "string" + }, + "ttsAudioUrl" : { + "type" : "string" + }, + "audioFileUrl" : { + "type" : "string" } } }, diff --git a/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java b/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java index 58e9944e..16bb145b 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/InterviewMessageService.java @@ -2,6 +2,7 @@ import com.stackup.stackup.common.exception.ApiErrorCode; import com.stackup.stackup.common.exception.DomainException; +import com.stackup.stackup.common.storage.ObjectStorageClient; import com.stackup.stackup.session.application.dto.MessageResult; import com.stackup.stackup.session.application.event.AnswerSubmittedEvent; import com.stackup.stackup.session.domain.InterviewMessage; @@ -11,8 +12,13 @@ import com.stackup.stackup.session.domain.MessageRole; import com.stackup.stackup.session.domain.MessageStatus; import com.stackup.stackup.session.domain.SessionStatus; +import com.stackup.stackup.session.domain.TtsStatus; +import java.net.URI; +import java.time.Duration; import java.util.List; import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -24,17 +30,43 @@ @Transactional(readOnly = true) public class InterviewMessageService { + private static final Logger log = LoggerFactory.getLogger(InterviewMessageService.class); + private static final Duration AUDIO_URL_TTL = Duration.ofMinutes(30); + private final InterviewSessionRepository sessionRepository; private final InterviewMessageRepository messageRepository; + private final ObjectStorageClient storage; private final ApplicationEventPublisher events; public List list(Long userId, Long sessionId) { ownedSession(userId, sessionId); return messageRepository.findBySession_IdOrderBySequenceNumberAsc(sessionId).stream() - .map(MessageResult::of) + .map(this::toResultWithAudioUrls) .toList(); } + // 재생용 presigned URL 동봉: 질문 TTS(SUCCEEDED) + 음성 답변 원본. + // presign 실패가 메시지 조회 전체를 깨뜨리지 않도록 개별 try/catch. + private MessageResult toResultWithAudioUrls(InterviewMessage m) { + String ttsUrl = m.getTtsStatus() == TtsStatus.SUCCEEDED + ? presign(m.getTtsAudioPath()) : null; + String audioUrl = presign(m.getAudioFilePath()); + return MessageResult.of(m, ttsUrl, audioUrl); + } + + private String presign(String key) { + if (key == null || key.isBlank()) { + return null; + } + try { + URI url = storage.createPresignedGetUrl(key, AUDIO_URL_TTL); + return url == null ? null : url.toString(); + } catch (RuntimeException e) { + log.warn("presigned audio URL creation failed. key={}", key, e); + return null; + } + } + @Transactional public MessageResult submitAnswer(Long userId, Long sessionId, String content, String idempotencyKey) { InterviewSession session = ownedSession(userId, sessionId); diff --git a/backend/src/main/java/com/stackup/stackup/session/application/dto/MessageResult.java b/backend/src/main/java/com/stackup/stackup/session/application/dto/MessageResult.java index a4ef63cd..8a199e03 100644 --- a/backend/src/main/java/com/stackup/stackup/session/application/dto/MessageResult.java +++ b/backend/src/main/java/com/stackup/stackup/session/application/dto/MessageResult.java @@ -20,10 +20,17 @@ public record MessageResult( MessageStatus status, Instant createdAt, String category, - String targetEvidence + String targetEvidence, + // 재생용 presigned URL. 조회(list) 경로에서만 채우고, 그 외에는 null. + String ttsAudioUrl, + String audioFileUrl // expectedSignal 은 의도적으로 제외 — 정답 유출 방지(라이브 비노출). ) { public static MessageResult of(InterviewMessage m) { + return of(m, null, null); + } + + public static MessageResult of(InterviewMessage m, String ttsAudioUrl, String audioFileUrl) { return new MessageResult( m.getId(), m.getSession().getId(), @@ -38,7 +45,9 @@ public static MessageResult of(InterviewMessage m) { m.getStatus(), m.getCreatedAt(), m.getCategory(), - m.getTargetEvidence() + m.getTargetEvidence(), + ttsAudioUrl, + audioFileUrl ); } } diff --git a/backend/src/main/java/com/stackup/stackup/session/presentation/dto/MessageResponse.java b/backend/src/main/java/com/stackup/stackup/session/presentation/dto/MessageResponse.java index b2a9f9e5..67bbd03a 100644 --- a/backend/src/main/java/com/stackup/stackup/session/presentation/dto/MessageResponse.java +++ b/backend/src/main/java/com/stackup/stackup/session/presentation/dto/MessageResponse.java @@ -20,7 +20,10 @@ public record MessageResponse( MessageStatus status, Instant createdAt, String category, - String targetEvidence + String targetEvidence, + // 재생용 presigned URL (질문 TTS / 음성 답변 원본). 조회 응답에서만 채움. + String ttsAudioUrl, + String audioFileUrl // expectedSignal 은 노출하지 않음 — 자기연습 시 정답 유출 방지. ) { public static MessageResponse from(MessageResult r) { @@ -38,7 +41,9 @@ public static MessageResponse from(MessageResult r) { r.status(), r.createdAt(), r.category(), - r.targetEvidence() + r.targetEvidence(), + r.ttsAudioUrl(), + r.audioFileUrl() ); } } diff --git a/frontend/src/domain/session/index.ts b/frontend/src/domain/session/index.ts index bd0488de..8b359c08 100644 --- a/frontend/src/domain/session/index.ts +++ b/frontend/src/domain/session/index.ts @@ -7,5 +7,13 @@ export type { JobCategory, MessageRole, } from './model/types' -export { isQuestion, isAnswer, currentTurn, canSubmitAnswer, sessionProgress } from './lib/turn' +export { + isQuestion, + isAnswer, + isTranscribing, + VOICE_TRANSCRIBING_TEXT, + currentTurn, + canSubmitAnswer, + sessionProgress, +} from './lib/turn' export type { Turn } from './lib/turn' diff --git a/frontend/src/domain/session/lib/turn.ts b/frontend/src/domain/session/lib/turn.ts index f5e340f6..f3f455b0 100644 --- a/frontend/src/domain/session/lib/turn.ts +++ b/frontend/src/domain/session/lib/turn.ts @@ -10,6 +10,17 @@ export function isAnswer(message: Message): boolean { return message.role === 'INTERVIEWEE' } +// 음성 답변 placeholder 의 임시 content (백엔드 InterviewMessage.VOICE_TRANSCRIPTION_PENDING_TEXT 와 동일). +// STT 완료 전까지 이 값이 들어 있고, 완료되면 실제 transcript 로 교체된다. +export const VOICE_TRANSCRIBING_TEXT = '(transcribing)' + +// STT 대기 중(transcript 아직 없음)인 음성 답변인지. +export function isTranscribing(message: Message): boolean { + if (!isAnswer(message)) return false + const c = (message.content ?? '').trim() + return c === '' || c === VOICE_TRANSCRIBING_TEXT +} + export function currentTurn(messages: Message[]): Turn { const last = messages[messages.length - 1] // 마지막이 '내용이 있는' 면접관 질문일 때만 답변 차례. 질문 생성 중(빈 content) diff --git a/frontend/src/features/interview/api/messageApi.ts b/frontend/src/features/interview/api/messageApi.ts index 3a32cd95..a6785975 100644 --- a/frontend/src/features/interview/api/messageApi.ts +++ b/frontend/src/features/interview/api/messageApi.ts @@ -6,3 +6,26 @@ type S = components['schemas'] export async function listMessages(sessionId: number): Promise { return (await apiClient.get(`/api/sessions/${sessionId}/messages`)).data } + +// 음성 답변 업로드(multipart). 응답은 transcribing placeholder 메시지. +// STT 완료 후 SESSION_MESSAGE SSE 로 transcript 가 도착한다. +export async function submitVoiceAnswer( + sessionId: number, + audio: Blob, + idempotencyKey?: string, +): Promise { + const form = new FormData() + const ext = audio.type.includes('ogg') ? 'ogg' : 'webm' + form.append('audio', audio, `answer.${ext}`) + const { data } = await apiClient.post( + `/api/sessions/${sessionId}/messages/voice`, + form, + { + headers: { + 'Content-Type': 'multipart/form-data', + ...(idempotencyKey ? { 'Idempotency-Key': idempotencyKey } : {}), + }, + }, + ) + return data +} diff --git a/frontend/src/features/interview/lib/media/useVoiceRecorder.ts b/frontend/src/features/interview/lib/media/useVoiceRecorder.ts new file mode 100644 index 00000000..f4a1bde2 --- /dev/null +++ b/frontend/src/features/interview/lib/media/useVoiceRecorder.ts @@ -0,0 +1,90 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +export type RecorderStatus = 'idle' | 'requesting' | 'recording' | 'denied' | 'unsupported' + +// MediaRecorder 가 만들 수 있는 후보. 백엔드 허용 목록(webm/ogg/mpeg/mp4/wav)과 교집합. +const PREFERRED_MIME = ['audio/webm;codecs=opus', 'audio/webm', 'audio/ogg;codecs=opus', 'audio/ogg'] + +function pickMimeType(): string | undefined { + if (typeof MediaRecorder === 'undefined') return undefined + return PREFERRED_MIME.find((t) => MediaRecorder.isTypeSupported(t)) +} + +// 라이브 면접 음성 답변 녹음. start → 마이크 권한 요청 + 녹음, stop → 오디오 Blob 반환. +// 권한 거부/미지원 시 status 로 알려서 호출부가 텍스트 입력으로 fallback 한다. +export function useVoiceRecorder() { + const [status, setStatus] = useState(() => + typeof MediaRecorder === 'undefined' || !navigator.mediaDevices?.getUserMedia + ? 'unsupported' + : 'idle', + ) + + const recorderRef = useRef(null) + const streamRef = useRef(null) + const chunksRef = useRef([]) + + const cleanup = useCallback(() => { + streamRef.current?.getTracks().forEach((t) => t.stop()) + streamRef.current = null + recorderRef.current = null + chunksRef.current = [] + }, []) + + useEffect(() => cleanup, [cleanup]) + + const start = useCallback(async (): Promise => { + if (status === 'unsupported' || status === 'recording') return false + setStatus('requesting') + let stream: MediaStream + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch { + setStatus('denied') + return false + } + const mimeType = pickMimeType() + const recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined) + chunksRef.current = [] + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunksRef.current.push(e.data) + } + recorder.start() + streamRef.current = stream + recorderRef.current = recorder + setStatus('recording') + return true + }, [status]) + + // 녹음 중지 후 Blob 반환. 녹음 중이 아니면 null. + const stop = useCallback((): Promise => { + const recorder = recorderRef.current + if (!recorder || recorder.state === 'inactive') { + cleanup() + setStatus('idle') + return Promise.resolve(null) + } + return new Promise((resolve) => { + recorder.onstop = () => { + const type = recorder.mimeType || chunksRef.current[0]?.type || 'audio/webm' + const blob = new Blob(chunksRef.current, { type }) + cleanup() + setStatus('idle') + resolve(blob.size > 0 ? blob : null) + } + recorder.stop() + }) + }, [cleanup]) + + // 녹음 폐기(업로드 안 함). + const cancel = useCallback(() => { + const recorder = recorderRef.current + if (recorder && recorder.state !== 'inactive') { + recorder.onstop = null + recorder.stop() + } + cleanup() + setStatus('idle') + }, [cleanup]) + + return { status, start, stop, cancel } +} diff --git a/frontend/src/features/interview/model/useLiveInterview.ts b/frontend/src/features/interview/model/useLiveInterview.ts index 9c635b00..101f5108 100644 --- a/frontend/src/features/interview/model/useLiveInterview.ts +++ b/frontend/src/features/interview/model/useLiveInterview.ts @@ -1,8 +1,9 @@ import { useCallback, useEffect, useState } from 'react' import { useNavigate } from 'react-router-dom' -import { useQueryClient } from '@tanstack/react-query' +import { useMutation, useQueryClient } from '@tanstack/react-query' import { currentTurn } from '@/domain/session' import type { Message } from '@/domain/session' +import { submitVoiceAnswer } from '../api/messageApi' import { sessionKeys, useSession } from './useSession' import { messageKeys, useSessionMessages } from './useSessionMessages' import { useSessionLifecycle } from './useSessionLifecycle' @@ -68,6 +69,20 @@ export function useLiveInterview(sessionId: number) { [socketSubmit], ) + // 음성 답변은 REST 업로드(multipart). 성공 시 placeholder 메시지를 + // 받으므로 목록을 무효화해 "음성 인식 중…" 버블을 띄운다. + // 이후 STT 완료는 callback.voice → SESSION_MESSAGE SSE 로 갱신된다. + const voiceMutation = useMutation({ + mutationFn: (audio: Blob) => submitVoiceAnswer(sessionId, audio, crypto.randomUUID()), + onSuccess: () => + queryClient.invalidateQueries({ queryKey: messageKeys.list(sessionId) }), + }) + + const submitVoice = useCallback( + (audio: Blob) => voiceMutation.mutate(audio), + [voiceMutation], + ) + return { session: sessionQuery.data, status, @@ -75,6 +90,9 @@ export function useLiveInterview(sessionId: number) { turn: currentTurn(items), connection, submitAnswer, + submitVoice, + voiceUploading: voiceMutation.isPending, + voiceError: voiceMutation.isError, endSession: () => end.mutate(), isLoading: sessionQuery.isLoading, } diff --git a/frontend/src/features/interview/ui/live/AnswerBubble.tsx b/frontend/src/features/interview/ui/live/AnswerBubble.tsx index e0a49966..407ecb1f 100644 --- a/frontend/src/features/interview/ui/live/AnswerBubble.tsx +++ b/frontend/src/features/interview/ui/live/AnswerBubble.tsx @@ -1,11 +1,31 @@ +import { isTranscribing } from '@/domain/session' import type { Message } from '@/domain/session' export function AnswerBubble({ message }: { message: Message }) { + const transcribing = isTranscribing(message) + const failed = transcribing && message.status === 'FAILED' + return ( -
+
- {message.content} + {!transcribing ? ( + message.content + ) : ( + + {failed ? ( + '음성 인식에 실패했어요. 다시 답변해 주세요.' + ) : ( + <> + + 음성 인식 중… + + )} + + )}
+ {message.audioFileUrl && ( +
) } diff --git a/frontend/src/features/interview/ui/live/AnswerComposer.tsx b/frontend/src/features/interview/ui/live/AnswerComposer.tsx index b2864eaf..a93a3856 100644 --- a/frontend/src/features/interview/ui/live/AnswerComposer.tsx +++ b/frontend/src/features/interview/ui/live/AnswerComposer.tsx @@ -2,15 +2,42 @@ import { useState } from 'react' import type { KeyboardEvent } from 'react' import { TextArea } from '@/shared/ui/TextArea' import { Button } from '@/shared/ui/Button' +import { useVoiceRecorder } from '../../lib/media/useVoiceRecorder' + +function MicIcon() { + return ( + + + + + ) +} export function AnswerComposer({ disabled = false, onSubmit, + onSubmitVoice, + voiceUploading = false, }: { disabled?: boolean onSubmit: (content: string) => void + onSubmitVoice?: (audio: Blob) => void + voiceUploading?: boolean }) { const [value, setValue] = useState('') + const { status: recStatus, start, stop, cancel } = useVoiceRecorder() + const recording = recStatus === 'recording' || recStatus === 'requesting' + const voiceSupported = recStatus !== 'unsupported' && Boolean(onSubmitVoice) const submit = () => { const trimmed = value.trim() @@ -26,20 +53,71 @@ export function AnswerComposer({ } } + const finishRecording = async () => { + const blob = await stop() + if (blob && onSubmitVoice) onSubmitVoice(blob) + } + + // 녹음/업로드 중에는 전용 바를 보여준다. + if (recording || voiceUploading) { + return ( +
+ + {voiceUploading ? ( + '음성 답변 업로드 중…' + ) : ( + <> + + 녹음 중… 답변이 끝나면 전송을 누르세요 + + )} + + {!voiceUploading && ( +
+ + +
+ )} +
+ ) + } + return ( -
-