Feature/analysis integration - 수신 처리 인프라 통합

ai/.env.example

@@ -18,3 +18,29 @@ LLM_API_KEY=
 LLM_BASE_URL=https://factchat-cloud.mindlogic.ai/v1/gateway
 LLM_PRO_MODEL=gemini-3.1-pro-preview
 LLM_PRO_TEMPERATURE=0.2
+
+# Github 관련
+CORE_INTERNAL_BASE_URL=http://localhost:38010
+CORE_INTERNAL_API_KEY=
+CORE_INTERNAL_TIMEOUT_SEC=10
+
+GITHUB_FALLBACK_TOKEN=
+GITHUB_API_BASE_URL=https://api.github.com
+REPO_MAX_SOURCE_FILES=8
+REPO_MAX_SOURCE_FILE_BYTES=50000
+REPO_FETCH_TIMEOUT_SEC=30
+
+# 웹 이력서 관련
+WEB_FETCH_TIMEOUT_SEC=20
+WEB_MAX_HTML_BYTES=2000000
+
+# 개발 및 테스트는 mock 
+# 운영은 gemini
+EMBEDDING_PROVIDER=gemini
+EMBEDDING_MODEL=gemini-embedding-001
+EMBEDDING_DIM=1536
+EMBEDDING_CHUNK_SIZE=1000
+EMBEDDING_CHUNK_OVERLAP=200
+EMBEDDING_BATCH_SIZE=32
+
+GEMINI_API_KEY=

ai/CLAUDE.md

@@ -88,11 +88,13 @@ ai/
 
 본 서버는 RabbitMQ **consumer**로 작동.
 
-| Queue | Bind |
-|-------|------|
-| `q.ai.resume` | `ai.request.resume.*` |
-| `q.ai.repo` | `ai.request.repo.*` |
-| `q.ai.session` | `ai.request.session.*` |
+| Queue | Routing key | 상태 |
+|-------|-------------|------|
+| `ai.analyze.resume` | `analyze.resume` | 본 구현 (PDF → MD) |
+| `ai.analyze.repository` | `analyze.repository` | 본 구현 (GitHub README + tree + 소스 sampling) |
+| `ai.analyze.web` | `analyze.web` | 본 구현 (URL → trafilatura) |
+| `ai.generate.questions` | `generate.questions` | 큐만, 코드 미구현 |
+| `ai.generate.followup` | `generate.followup` | 큐만, 코드 미구현 |
 
 콜백 발행: `ai.callback.{type}` 익스체인지.
 상세 envelope/스키마/재시도: [`/docs/messaging.md`](../docs/messaging.md).
@@ -162,11 +164,14 @@ chain = prompt | llm | PydanticOutputParser(pydantic_object=...)
 
 ## 7. RAG 파이프라인
 
-### 7.1 인제스트
-1. 마크다운 입력
-2. 청킹 (LangChain `RecursiveCharacterTextSplitter`, chunk_size=1000, overlap=200)
-3. 임베딩 생성 (Gemini `text-embedding-004` 또는 OpenAI `text-embedding-3-small`)
-4. Core API 호출 → pgvector INSERT
+### 7.1 인제스트 (본 구현)
+1. 마크다운 입력 (`analyzer/_embedding_step.chunk_embed_and_upsert`)
+2. 청킹 — `rag/chunker.MarkdownChunker` (`RecursiveCharacterTextSplitter`, 기본 1000/200)
+3. 임베딩 생성 — `rag/embedder.EmbeddingProvider`. 현재 구현체:
+   - `MockEmbeddingProvider` (default) — 차원 결정 보류, e2e 흐름 검증용
+   - `openai` / `ollama` 구현체는 후속 PR
+4. Core API 호출 — `CoreClient.upsert_embeddings(document_id, model, dim, chunks)` →
+   `PUT /api/internal/documents/{id}/embeddings` (idempotent upsert)
 
 ### 7.2 검색
 1. 쿼리 텍스트 → 임베딩

ai/pyproject.toml

@@ -16,6 +16,9 @@ dependencies = [
     "boto3>=1.42.77",
     "aiofiles>=24.1.0",
     "pypdf>=5.1.0",
+    "trafilatura>=2.0.0",
+    "langchain-text-splitters>=0.3.0",
+    "google-genai>=1.0.0",
     "langchain>=1.2.13",
     "langchain-core>=1.2.22",
     "langchain-community>=0.4.1",

ai/src/ai_server/analyzer/_embedding_step.py

@@ -0,0 +1,86 @@
+# 공통 임베딩 모듈 
+from __future__ import annotations
+
+import structlog
+
+from ai_server.core.client import (
+    CoreClient,
+    CoreEmbeddingUpsertError,
+    EmbeddingChunkPayload,
+)
+from ai_server.rag.chunker import MarkdownChunker
+from ai_server.rag.embedder import EmbeddingError, EmbeddingProvider
+
+log = structlog.get_logger(__name__)
+
+
+class EmbeddingStepError(Exception):
+    def __init__(self, *, code: str, message: str, retriable: bool) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.retriable = retriable
+
+
+async def chunk_embed_and_upsert(
+    *,
+    document_id: int,
+    markdown: str,
+    chunker: MarkdownChunker,
+    embedder: EmbeddingProvider,
+    core_client: CoreClient,
+    log_prefix: str = "analyze",
+) -> int:
+    chunks = chunker.split(markdown)
+    log.info(
+        f"{log_prefix}.chunk.done",
+        document_id=document_id,
+        chunk_count=len(chunks),
+    )
+
+    if not chunks:
+        return 0
+
+    try:
+        vectors = await embedder.embed([c.text for c in chunks])
+    except EmbeddingError as err:
+        raise EmbeddingStepError(
+            code=err.code, message=err.message, retriable=err.retriable
+        ) from err
+
+    if len(vectors) != len(chunks):
+        raise EmbeddingStepError(
+            code="EMBED_COUNT_MISMATCH",
+            message=(f"embedder가 chunk {len(chunks)}개 중 {len(vectors)}개만 반환"),
+            retriable=True,
+        )
+
+    payloads = [
+        EmbeddingChunkPayload(
+            chunk_index=chunks[i].index,
+            chunk_text=chunks[i].text,
+            embedding=vectors[i],
+        )
+        for i in range(len(chunks))
+    ]
+
+    try:
+        upserted = await core_client.upsert_embeddings(
+            document_id=document_id,
+            model=embedder.model,
+            dim=embedder.dim,
+            chunks=payloads,
+        )
+    except CoreEmbeddingUpsertError as err:
+        raise EmbeddingStepError(
+            code=err.code, message=err.message, retriable=err.retriable
+        ) from err
+
+    log.info(
+        f"{log_prefix}.embed.upserted",
+        document_id=document_id,
+        chunk_count=upserted,
+        model=embedder.model,
+        dim=embedder.dim,
+    )
+    return upserted

ai/src/ai_server/analyzer/repository_analyzer.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import structlog
+
+from ai_server.analyzer._embedding_step import (
+    EmbeddingStepError,
+    chunk_embed_and_upsert,
+)
+from ai_server.analyzer.sources.github_repo import (
+    GitHubRepoSourceExtractor,
+    RepositoryFetchError,
+)
+from ai_server.chain.document_analysis_chain import DocumentAnalyzer
+from ai_server.core.client import CoreClient, CoreTokenError
+from ai_server.rag.chunker import MarkdownChunker
+from ai_server.rag.embedder import EmbeddingProvider
+from ai_server.storage.base import ObjectStorage
+
+log = structlog.get_logger(__name__)
+
+
+class RepositoryAnalyzeError(Exception):
+    def __init__(self, *, code: str, message: str, retriable: bool) -> None:
+        super().__init__(message)
+        self.code = code
+        self.message = message
+        self.retriable = retriable
+
+
+@dataclass(frozen=True)
+class RepositoryAnalysisResult:
+    summary: str
+    tech_stack: list[str]
+    document_path: str
+    embedding_chunk_count: int
+
+
+# Core에서 사용자별 GitHub token 수령 → 레포 fetch → LLM 분석 → 마크다운 저장 → 청킹·임베딩
+class RepositoryAnalyzer:
+    def __init__(
+        self,
+        *,
+        extractor: GitHubRepoSourceExtractor,
+        core_client: CoreClient,
+        chain: DocumentAnalyzer,
+        storage: ObjectStorage,
+        chunker: MarkdownChunker,
+        embedder: EmbeddingProvider,
+        analyzed_key_template: str,
+    ) -> None:
+        self._extractor = extractor
+        self._core_client = core_client
+        self._chain = chain
+        self._storage = storage
+        self._chunker = chunker
+        self._embedder = embedder
+        self._analyzed_key_template = analyzed_key_template
+
+    async def analyze(
+        self,
+        *,
+        repository_id: int,
+        repo_full_name: str,
+        default_branch: str = "main",
+        user_id: int | None,
+        analyzed_document_id: int,
+    ) -> RepositoryAnalysisResult:
+        if user_id is None:
+            raise RepositoryAnalyzeError(
+                code="MISSING_USER_ID",
+                message="envelope.context.userId 없이는 GitHub token을 가져올 수 없음",
+                retriable=False,
+            )
+
+        log.info(
+            "repository.token.fetch",
+            user_id=user_id,
+            repository_id=repository_id,
+        )
+        try:
+            access_token = await self._core_client.fetch_github_token(user_id)
+        except CoreTokenError as err:
+            raise RepositoryAnalyzeError(
+                code=err.code, message=err.message, retriable=err.retriable
+            ) from err
+
+        log.info(
+            "repository.extract.start",
+            repository_id=repository_id,
+            repo_full_name=repo_full_name,
+            default_branch=default_branch,
+        )
+        try:
+            source = await self._extractor.extract(
+                repo_full_name,
+                access_token=access_token,
+            )
+        except RepositoryFetchError as err:
+            raise RepositoryAnalyzeError(
+                code=err.code, message=err.message, retriable=err.retriable
+            ) from err
+
+        if not source.text.strip():
+            raise RepositoryAnalyzeError(
+                code="EMPTY_REPO_CONTENT",
+                message="레포에서 추출된 텍스트가 비어 있음",
+                retriable=False,
+            )
+
+        log.info(
+            "repository.llm.start",
+            repository_id=repository_id,
+            text_chars=len(source.text),
+        )
+        analysis = await self._chain.analyze(
+            text=source.text,
+            source_type=source.source_type,
+        )
+
+        out_key = self._analyzed_key_template.format(repository_id=repository_id)
+        await self._storage.put_text(out_key, analysis.markdown)
+        log.info(
+            "repository.markdown.saved",
+            repository_id=repository_id,
+            key=out_key,
+            md_chars=len(analysis.markdown),
+        )
+
+        try:
+            chunk_count = await chunk_embed_and_upsert(
+                document_id=analyzed_document_id,
+                markdown=analysis.markdown,
+                chunker=self._chunker,
+                embedder=self._embedder,
+                core_client=self._core_client,
+                log_prefix="repository",
+            )
+        except EmbeddingStepError as err:
+            raise RepositoryAnalyzeError(
+                code=err.code, message=err.message, retriable=err.retriable
+            ) from err
+
+        return RepositoryAnalysisResult(
+            summary=analysis.summary,
+            tech_stack=list(analysis.tech_stack),
+            document_path=out_key,
+            embedding_chunk_count=chunk_count,
+        )