[S2-01] Implementar pipeline assíncrono de documento com estados pendente/processando/processado/falha, motivo e nova tentativa idempotente

.env.example

@@ -27,8 +27,25 @@ OLLAMA_PORT=11434
 OLLAMA_BASE_URL=http://localhost:11434
 # Modelos recomendados no PRD (Qwen 2.5 / Llama 3.1 para LLM, bge-m3 para embeddings)
 OLLAMA_LLM_MODEL=qwen2.5:1.5b
+OLLAMA_NUM_PREDICT=1024
+OLLAMA_NUM_CTX=8192
+OLLAMA_FILE_NUM_PREDICT=512
+OLLAMA_SYNTHESIS_NUM_PREDICT=3072
 OLLAMA_EMBEDDING_MODEL=bge-m3
 OLLAMA_KEEP_ALIVE=24h
+# URL do serviço RepoAnalyzer quando o backend roda em Docker Compose.
+REPO_ANALYZER_URL=http://ai-service:8000
+# Limite de arquivos priorizados nos perfis do RepoAnalyzer (Complete usa MAX_FILES).
+ANALYZER_QUICK_FILES=8
+ANALYZER_BALANCED_FILES=80
+# Limite rígido de arquivos elegíveis inventariados pelo RepoAnalyzer.
+MAX_FILES=1000
+ANALYZER_MAX_ACTIVE_RUNS=2
+ANALYZER_MAX_QUEUED_RUNS=6
+ANALYZER_WORKSPACE_QUOTA_MB=2048
+MAX_REPO_SIZE_MB=300
+ANALYZER_CLONE_MEMORY_MB=512
+ANALYZER_CLONE_TIMEOUT_SECONDS=300
 
 # --- Documentos enviados (S1-19/S1-22) ---
 # Limite de tamanho por arquivo, aplicado no backend e informado ao cliente

.github/workflows/ci.yml

@@ -55,7 +55,7 @@ jobs:
       - run: npm run test:seed
         working-directory: backend
       - name: Test archive cascade on PostgreSQL
-        run: node --import tsx --test src/modules/projects/projects.archive.db.test.ts src/modules/projects/hierarchy-archive.db.test.ts src/modules/projects/projects.backlog-tree.db.test.ts src/database/migration-010.db.test.ts src/database/migration-011.db.test.ts src/database/migration-012.db.test.ts src/modules/documents/documents.repository.db.test.ts src/modules/chat/chat.repository.db.test.ts src/modules/backlog-search/backlog-search.db.test.ts src/modules/decisions/decisions.db.test.ts src/database/migration-013.db.test.ts src/modules/priority-fixes.db.test.ts src/modules/documents/documents.ingestion.db.test.ts src/modules/projects/project-access.db.test.ts
+        run: node --import tsx --test src/modules/projects/projects.archive.db.test.ts src/modules/projects/hierarchy-archive.db.test.ts src/modules/projects/projects.backlog-tree.db.test.ts src/database/migration-010.db.test.ts src/database/migration-011.db.test.ts src/database/migration-012.db.test.ts src/modules/documents/documents.repository.db.test.ts src/modules/chat/chat.repository.db.test.ts src/modules/backlog-search/backlog-search.db.test.ts src/modules/decisions/decisions.db.test.ts src/database/migration-013.db.test.ts src/modules/priority-fixes.db.test.ts src/modules/documents/documents.ingestion.db.test.ts src/modules/projects/project-access.db.test.ts src/modules/repo-analyses/repo-analyses.dispatch.db.test.ts
         working-directory: backend
         env:
           ARCHIVE_TEST_DATABASE_URL: postgresql://seed_test:seed_test_only@localhost:5432/sinapse_seed_test

.github/workflows/e2e.yml

@@ -45,6 +45,9 @@ jobs:
       POSTGRES_USER: e2e
       POSTGRES_PASSWORD: e2e_only
       POSTGRES_DB: sinapse_e2e_test
+      E2E_DATABASE_URL: postgresql://e2e:e2e_only@localhost:5432/sinapse_e2e_test
+      AI_SERVICE_URL: http://127.0.0.1:9
+      REPO_ANALYZER_URL: http://127.0.0.1:9
       E2E_API_URL: http://localhost:3001/api/v1
       E2E_APP_URL: http://localhost:5173
       E2E_CHROME_PATH: /usr/bin/google-chrome
@@ -65,7 +68,7 @@ jobs:
         run: |
           export DOCUMENT_STORAGE_DIR="$RUNNER_TEMP/documents"
           mkdir -p "$DOCUMENT_STORAGE_DIR"
-          (cd backend && nohup npx tsx src/index.ts > "$RUNNER_TEMP/backend.log" 2>&1 &)
+          (cd backend && nohup env NODE_ENV=test node --import tsx ../e2e/server.mts > "$RUNNER_TEMP/backend.log" 2>&1 &)
           (cd frontend && nohup npx vite --port 5173 --strictPort > "$RUNNER_TEMP/frontend.log" 2>&1 &)
           for i in $(seq 1 30); do
             curl -fs http://localhost:3001/health > /dev/null && curl -fs http://localhost:5173 > /dev/null && exit 0

ai-service/Dockerfile

@@ -2,9 +2,10 @@ FROM python:3.11-slim
 
 WORKDIR /app
 
-# Install system dependencies
+# Git is required by the repository analyzer to clone public GitHub projects.
 RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
+    git \
     && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt .

ai-service/analyzer/config.py

@@ -15,13 +15,24 @@ class AnalyzerSettings:
     max_file_size_kb: int = int(os.getenv("MAX_FILE_SIZE_KB", "500"))
     max_chunk_chars: int = int(os.getenv("MAX_CHUNK_CHARS", "10000"))
     max_files: int = int(os.getenv("MAX_FILES", "1000"))
+    quick_profile_files: int = int(os.getenv("ANALYZER_QUICK_FILES", "8"))
+    balanced_profile_files: int = int(os.getenv("ANALYZER_BALANCED_FILES", "80"))
     workspace_dir: Path = Path(os.getenv("WORKSPACE_DIR", "workspace_analyzer"))
     request_timeout_seconds: int = int(os.getenv("REQUEST_TIMEOUT_SECONDS", "300"))
     ollama_keep_alive: str = os.getenv("OLLAMA_KEEP_ALIVE", "5m")
+    ollama_num_predict: int = max(128, int(os.getenv("OLLAMA_NUM_PREDICT", "1024")))
+    ollama_num_ctx: int = max(2048, int(os.getenv("OLLAMA_NUM_CTX", "8192")))
+    ollama_file_num_predict: int = max(128, int(os.getenv("OLLAMA_FILE_NUM_PREDICT", "512")))
+    ollama_synthesis_num_predict: int = max(128, int(os.getenv("OLLAMA_SYNTHESIS_NUM_PREDICT", "3072")))
     ollama_think: bool = False
     ollama_max_retries: int = int(os.getenv("OLLAMA_MAX_RETRIES", "2"))
     ollama_retry_backoff_seconds: float = float(os.getenv("OLLAMA_RETRY_BACKOFF_SECONDS", "3"))
     max_synthesis_chars: int = int(os.getenv("MAX_SYNTHESIS_CHARS", "60000"))
+    max_active_runs: int = max(1, int(os.getenv("ANALYZER_MAX_ACTIVE_RUNS", "2")))
+    max_queued_runs: int = max(0, int(os.getenv("ANALYZER_MAX_QUEUED_RUNS", "6")))
+    workspace_quota_mb: int = max(1, int(os.getenv("ANALYZER_WORKSPACE_QUOTA_MB", "2048")))
+    clone_timeout_seconds: int = max(1, int(os.getenv("ANALYZER_CLONE_TIMEOUT_SECONDS", "300")))
+    clone_memory_mb: int = max(128, int(os.getenv("ANALYZER_CLONE_MEMORY_MB", "512")))
     ollama_concurrency: int = max(1, int(os.getenv("OLLAMA_CONCURRENCY", "1")))
 
 

ai-service/analyzer/limited_git.py

@@ -0,0 +1,11 @@
+"""Exec git with per-process bounds without unsafe preexec_fn in threaded workers."""
+import os
+import resource
+import sys
+
+if __name__ == "__main__":
+    disk_mb, memory_mb, cpu_seconds = map(int, sys.argv[1:4])
+    resource.setrlimit(resource.RLIMIT_FSIZE, (disk_mb * 1024**2,) * 2)
+    resource.setrlimit(resource.RLIMIT_AS, (memory_mb * 1024**2,) * 2)
+    resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds,) * 2)
+    os.execvp("git", ["git", "-c", "core.hooksPath=/dev/null", *sys.argv[4:]])

ai-service/analyzer/models.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass, field
+import threading
 from typing import Any
 
 
@@ -20,6 +21,7 @@ class RunState:
     run_id: str
     url: str
     status: str = "created"
+    profile: str = "quick"
     stage: str = ""
     message: str = ""
     error: str = ""
@@ -32,7 +34,16 @@ class RunState:
     files_total: int = 0
     files_processed: int = 0
     files_ignored: int = 0
+    files_candidates: int = 0
+    files_skipped_by_scope: int = 0
     language_counts: dict[str, int] = field(default_factory=dict)
     current_files: list[str] = field(default_factory=list)
+    selected_paths: list[str] = field(default_factory=list)
+    completed_summaries: dict[str, str] = field(default_factory=dict)
+    partial_chunk_summaries: dict[str, list[str]] = field(default_factory=dict)
+    project_summary: str = ""
+    pause_event: Any = field(default_factory=threading.Event, repr=False)
+    cancel_event: Any = field(default_factory=threading.Event, repr=False)
+    worker_thread: Any = field(default=None, repr=False)
 
     stats: dict[str, Any] = field(default_factory=dict)

ai-service/analyzer/ollama_client.py

@@ -1,5 +1,7 @@
 import re
 import time
+import json
+from collections.abc import Callable
 
 import httpx
 
@@ -30,6 +32,8 @@ def __init__(
         think: bool = False,
         max_retries: int = 2,
         retry_backoff_seconds: float = 3.0,
+        num_predict: int = 1024,
+        num_ctx: int = 8192,
     ):
         self.base_url = base_url.rstrip("/")
         self.model = model
@@ -38,40 +42,85 @@ def __init__(
         self.think = False
         self.max_retries = max(0, max_retries)
         self.retry_backoff_seconds = retry_backoff_seconds
+        self.num_predict = max(128, num_predict)
+        self.num_ctx = max(2048, num_ctx)
         self._client = httpx.Client(timeout=timeout)
 
     def close(self) -> None:
         self._client.close()
 
-    def chat(self, system: str, user: str, temperature: float = 0.1) -> str:
+    def chat(
+        self,
+        system: str,
+        user: str,
+        temperature: float = 0.1,
+        should_cancel: Callable[[], None] | None = None,
+        num_predict: int | None = None,
+        accept_truncated: bool = False,
+    ) -> str:
         if not system.strip().startswith(("/nothink", "/no_think")):
             system = f"/nothink\n{system}"
 
+        output_limit = max(128, num_predict or self.num_predict)
         payload = {
             "model": self.model,
             "messages": [
                 {"role": "system", "content": system},
                 {"role": "user", "content": user},
             ],
-            "stream": False,
+            "stream": True,
             "keep_alive": self.keep_alive,
             "think": False,
             "options": {
-                "temperature": temperature
+                "temperature": temperature,
+                "num_predict": output_limit,
+                "num_ctx": self.num_ctx,
             }
         }
 
         last_error: Exception | None = None
-        for attempt in range(self.max_retries + 1):
+        output_limit_retries = 0
+        http_retries = 0
+        while True:
             try:
-                response = self._client.post(f"{self.base_url}/api/chat", json=payload)
-                response.raise_for_status()
-                data = response.json()
+                content_parts: list[str] = []
+                done_reason = ""
+                with self._client.stream("POST", f"{self.base_url}/api/chat", json=payload) as response:
+                    response.raise_for_status()
+                    for line in response.iter_lines():
+                        if should_cancel:
+                            should_cancel()
+                        if not line:
+                            continue
+                        data = json.loads(line)
+                        content = data.get("message", {}).get("content", "")
+                        if content:
+                            content_parts.append(content)
+                        if data.get("done"):
+                            done_reason = data.get("done_reason", "")
+                            break
+                if done_reason == "length":
+                    if accept_truncated:
+                        data = {"message": {"content": "".join(content_parts) +
+                            "\n\n[Análise resumida limitada pelo teto de tokens; trate este trecho como parcial.]"}}
+                        break
+                    if output_limit_retries < 4 and output_limit < 8192:
+                        output_limit = min(8192, output_limit * 2)
+                        output_limit_retries += 1
+                        http_retries = 0
+                        payload["options"]["num_predict"] = output_limit
+                        continue
+                    raise OllamaError(
+                        f"A resposta do modelo atingiu o limite de {output_limit} tokens. "
+                        "Aumente OLLAMA_FILE_NUM_PREDICT ou OLLAMA_SYNTHESIS_NUM_PREDICT."
+                    )
+                data = {"message": {"content": "".join(content_parts)}}
                 break
             except httpx.HTTPError as exc:
                 last_error = exc
-                if attempt < self.max_retries:
-                    time.sleep(self.retry_backoff_seconds * (attempt + 1))
+                if http_retries < self.max_retries:
+                    http_retries += 1
+                    time.sleep(self.retry_backoff_seconds * http_retries)
                     continue
                 raise OllamaError(
                     f"Não foi possível acessar o Ollama em {self.base_url} "