From 8eda505d3089057c1a42d43d6d3fb0766cf1ef02 Mon Sep 17 00:00:00 2001 From: LoadCG Date: Tue, 29 Sep 2026 14:51:36 -0300 Subject: [PATCH 1/9] feat: improve resumable repository analysis --- .env.example | 10 + ai-service/Dockerfile | 3 +- ai-service/analyzer/config.py | 5 + ai-service/analyzer/models.py | 11 + ai-service/analyzer/ollama_client.py | 48 ++- ai-service/analyzer/pipeline.py | 336 +++++++++++++++--- ai-service/analyzer/prompts.py | 65 +++- ai-service/analyzer/scanner.py | 32 ++ ai-service/main.py | 31 +- ai-service/tests/test_analyzer.py | 124 ++++++- ai-service/tests/test_ollama_client.py | 84 +++++ .../repo-analyses/repo-analyses.repository.ts | 20 +- .../repo-analyses.routes.test.ts | 34 +- .../repo-analyses/repo-analyses.routes.ts | 25 +- .../repo-analyses.service.test.ts | 37 +- .../repo-analyses/repo-analyses.service.ts | 57 ++- .../repo-analyses/repo-analyses.types.ts | 3 +- database/init.sql | 2 +- database/migrations/007_repo_analysis.sql | 2 +- docker-compose.yml | 11 + docs/Architecture/README.md | 4 + docs/api/openapi.yaml | 47 +++ frontend/src/assets/styles/repo-analyzer.css | 1 + frontend/src/models/repoAnalyzer.ts | 11 +- .../src/projects/repo-analyzer.api.test.ts | 13 +- frontend/src/projects/repo-analyzer.api.ts | 17 +- .../views/projects/RepoAnalyzerView.test.tsx | 54 ++- .../src/views/projects/RepoAnalyzerView.tsx | 78 +++- 28 files changed, 1060 insertions(+), 105 deletions(-) create mode 100644 ai-service/tests/test_ollama_client.py diff --git a/.env.example b/.env.example index b06bf70..dc69fcf 100644 --- a/.env.example +++ b/.env.example @@ -27,8 +27,18 @@ OLLAMA_PORT=11434 OLLAMA_BASE_URL=http://localhost:11434 # Modelos recomendados no PRD (Qwen 2.5 / Llama 3.1 para LLM, bge-m3 para embeddings) OLLAMA_LLM_MODEL=qwen2.5:1.5b +OLLAMA_NUM_PREDICT=1024 +OLLAMA_FILE_NUM_PREDICT=512 +OLLAMA_SYNTHESIS_NUM_PREDICT=3072 OLLAMA_EMBEDDING_MODEL=bge-m3 OLLAMA_KEEP_ALIVE=24h +# URL do serviço RepoAnalyzer quando o backend roda em Docker Compose. +REPO_ANALYZER_URL=http://ai-service:8000 +# Limite de arquivos priorizados nos perfis do RepoAnalyzer (Complete usa MAX_FILES). +ANALYZER_QUICK_FILES=8 +ANALYZER_BALANCED_FILES=80 +# Limite rígido de arquivos elegíveis inventariados pelo RepoAnalyzer. +MAX_FILES=1000 # --- Documentos enviados (S1-19/S1-22) --- # Limite de tamanho por arquivo, aplicado no backend e informado ao cliente diff --git a/ai-service/Dockerfile b/ai-service/Dockerfile index 449f246..53f16ba 100644 --- a/ai-service/Dockerfile +++ b/ai-service/Dockerfile @@ -2,9 +2,10 @@ FROM python:3.11-slim WORKDIR /app -# Install system dependencies +# Git is required by the repository analyzer to clone public GitHub projects. RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ + git \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . diff --git a/ai-service/analyzer/config.py b/ai-service/analyzer/config.py index a8f3dc8..052df38 100644 --- a/ai-service/analyzer/config.py +++ b/ai-service/analyzer/config.py @@ -15,9 +15,14 @@ class AnalyzerSettings: max_file_size_kb: int = int(os.getenv("MAX_FILE_SIZE_KB", "500")) max_chunk_chars: int = int(os.getenv("MAX_CHUNK_CHARS", "10000")) max_files: int = int(os.getenv("MAX_FILES", "1000")) + quick_profile_files: int = int(os.getenv("ANALYZER_QUICK_FILES", "8")) + balanced_profile_files: int = int(os.getenv("ANALYZER_BALANCED_FILES", "80")) workspace_dir: Path = Path(os.getenv("WORKSPACE_DIR", "workspace_analyzer")) request_timeout_seconds: int = int(os.getenv("REQUEST_TIMEOUT_SECONDS", "300")) ollama_keep_alive: str = os.getenv("OLLAMA_KEEP_ALIVE", "5m") + ollama_num_predict: int = max(128, int(os.getenv("OLLAMA_NUM_PREDICT", "1024"))) + ollama_file_num_predict: int = max(128, int(os.getenv("OLLAMA_FILE_NUM_PREDICT", "512"))) + ollama_synthesis_num_predict: int = max(128, int(os.getenv("OLLAMA_SYNTHESIS_NUM_PREDICT", "3072"))) ollama_think: bool = False ollama_max_retries: int = int(os.getenv("OLLAMA_MAX_RETRIES", "2")) ollama_retry_backoff_seconds: float = float(os.getenv("OLLAMA_RETRY_BACKOFF_SECONDS", "3")) diff --git a/ai-service/analyzer/models.py b/ai-service/analyzer/models.py index 723a1d7..ad494bf 100644 --- a/ai-service/analyzer/models.py +++ b/ai-service/analyzer/models.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +import threading from typing import Any @@ -20,6 +21,7 @@ class RunState: run_id: str url: str status: str = "created" + profile: str = "quick" stage: str = "" message: str = "" error: str = "" @@ -32,7 +34,16 @@ class RunState: files_total: int = 0 files_processed: int = 0 files_ignored: int = 0 + files_candidates: int = 0 + files_skipped_by_scope: int = 0 language_counts: dict[str, int] = field(default_factory=dict) current_files: list[str] = field(default_factory=list) + selected_paths: list[str] = field(default_factory=list) + completed_summaries: dict[str, str] = field(default_factory=dict) + partial_chunk_summaries: dict[str, list[str]] = field(default_factory=dict) + project_summary: str = "" + pause_event: Any = field(default_factory=threading.Event, repr=False) + cancel_event: Any = field(default_factory=threading.Event, repr=False) + worker_thread: Any = field(default=None, repr=False) stats: dict[str, Any] = field(default_factory=dict) diff --git a/ai-service/analyzer/ollama_client.py b/ai-service/analyzer/ollama_client.py index 157cf1c..a16e738 100644 --- a/ai-service/analyzer/ollama_client.py +++ b/ai-service/analyzer/ollama_client.py @@ -1,5 +1,7 @@ import re import time +import json +from collections.abc import Callable import httpx @@ -30,6 +32,7 @@ def __init__( think: bool = False, max_retries: int = 2, retry_backoff_seconds: float = 3.0, + num_predict: int = 1024, ): self.base_url = base_url.rstrip("/") self.model = model @@ -38,35 +41,68 @@ def __init__( self.think = False self.max_retries = max(0, max_retries) self.retry_backoff_seconds = retry_backoff_seconds + self.num_predict = max(128, num_predict) self._client = httpx.Client(timeout=timeout) def close(self) -> None: self._client.close() - def chat(self, system: str, user: str, temperature: float = 0.1) -> str: + def chat( + self, + system: str, + user: str, + temperature: float = 0.1, + should_cancel: Callable[[], None] | None = None, + num_predict: int | None = None, + ) -> str: if not system.strip().startswith(("/nothink", "/no_think")): system = f"/nothink\n{system}" + output_limit = max(128, num_predict or self.num_predict) payload = { "model": self.model, "messages": [ {"role": "system", "content": system}, {"role": "user", "content": user}, ], - "stream": False, + "stream": True, "keep_alive": self.keep_alive, "think": False, "options": { - "temperature": temperature + "temperature": temperature, + "num_predict": output_limit, } } last_error: Exception | None = None for attempt in range(self.max_retries + 1): try: - response = self._client.post(f"{self.base_url}/api/chat", json=payload) - response.raise_for_status() - data = response.json() + content_parts: list[str] = [] + done_reason = "" + with self._client.stream("POST", f"{self.base_url}/api/chat", json=payload) as response: + response.raise_for_status() + for line in response.iter_lines(): + if should_cancel: + should_cancel() + if not line: + continue + data = json.loads(line) + content = data.get("message", {}).get("content", "") + if content: + content_parts.append(content) + if data.get("done"): + done_reason = data.get("done_reason", "") + break + if done_reason == "length": + if attempt < self.max_retries and output_limit < 8192: + output_limit = min(8192, output_limit * 2) + payload["options"]["num_predict"] = output_limit + continue + raise OllamaError( + f"A resposta do modelo atingiu o limite de {output_limit} tokens. " + "Aumente OLLAMA_FILE_NUM_PREDICT ou OLLAMA_SYNTHESIS_NUM_PREDICT." + ) + data = {"message": {"content": "".join(content_parts)}} break except httpx.HTTPError as exc: last_error = exc diff --git a/ai-service/analyzer/pipeline.py b/ai-service/analyzer/pipeline.py index a03ce75..c68967c 100644 --- a/ai-service/analyzer/pipeline.py +++ b/ai-service/analyzer/pipeline.py @@ -21,7 +21,7 @@ project_synthesis_prompt, single_chunk_prompt, ) -from .scanner import scan_repository +from .scanner import scan_repository, select_analysis_files class AnalysisError(RuntimeError): @@ -41,6 +41,14 @@ class AnalysisError(RuntimeError): STAGE_LABELS["error"] = "Falha na análise" +class AnalysisPaused(Exception): + pass + + +class AnalysisCancelled(Exception): + pass + + class Analyzer: def __init__(self, settings: AnalyzerSettings): self.settings = settings @@ -52,11 +60,14 @@ def __init__(self, settings: AnalyzerSettings): think=settings.ollama_think, max_retries=settings.ollama_max_retries, retry_backoff_seconds=settings.ollama_retry_backoff_seconds, + num_predict=settings.ollama_num_predict, ) self.runs: dict[str, RunState] = {} self.lock = threading.Lock() + self.checkpoint_lock = threading.Lock() + self._restore_runs() - def start(self, url: str) -> str: + def start(self, url: str, profile: str = "quick") -> str: normalized_url = self._normalize_github_url(url) if not normalized_url: raise AnalysisError( @@ -64,20 +75,136 @@ def start(self, url: str) -> str: "(ex.: https://github.com/usuario/repositorio)." ) + if profile not in {"quick", "balanced", "complete"}: + raise AnalysisError("Perfil inválido. Use quick, balanced ou complete.") + run_id = uuid.uuid4().hex[:12] - state = RunState(run_id=run_id, url=normalized_url, status="queued") + state = RunState(run_id=run_id, url=normalized_url, status="queued", profile=profile) with self.lock: self.runs[run_id] = state + self._persist_checkpoint(state) + self._start_worker(state) + return run_id - thread = threading.Thread( - target=self._run, - args=(run_id,), - daemon=True, - ) + def _start_worker(self, state: RunState) -> None: + thread = threading.Thread(target=self._run, args=(state.run_id,), daemon=True) + state.worker_thread = thread thread.start() - return run_id + + def pause(self, run_id: str) -> dict: + state = self._get_run(run_id) + if state.status not in {"queued", "running"}: + raise AnalysisError("Só é possível pausar uma análise em andamento.") + self._push(run_id, status="pausing", message="Aguardando concluir a etapa atual para salvar o progresso…") + state.pause_event.set() + return self.status(run_id) + + def resume(self, run_id: str) -> dict: + state = self._get_run(run_id) + if state.status != "paused": + raise AnalysisError("A análise não está pausada.") + state.pause_event.clear() + state.cancel_event.clear() + self._push(run_id, status="queued", message="Retomando do último ponto salvo…") + self._start_worker(state) + return self.status(run_id) + + def cancel(self, run_id: str) -> dict: + state = self._get_run(run_id) + if state.status not in {"queued", "running", "pausing", "paused"}: + raise AnalysisError("Só é possível cancelar uma análise não finalizada.") + was_paused = state.status == "paused" + state.pause_event.clear() + self._push(run_id, status="cancelling", message="Cancelamento solicitado. A chamada atual ao modelo será concluída.") + state.cancel_event.set() + if was_paused: + self._push(run_id, status="cancelled", message="Análise cancelada pelo usuário.", current_files=[]) + return self.status(run_id) + + def _get_run(self, run_id: str) -> RunState: + state = self.runs.get(run_id) or self._load_persisted_run(run_id) + if not state: + raise AnalysisError("Execução não encontrada.") + return state + + def _restore_runs(self) -> None: + runs_dir = self.settings.workspace_dir / "runs" + if not runs_dir.exists(): + return + for run_dir in runs_dir.iterdir(): + checkpoint_path = run_dir / "checkpoint.json" + if not checkpoint_path.exists(): + continue + try: + data = json.loads(checkpoint_path.read_text(encoding="utf-8")) + state = RunState( + run_id=data["run_id"], + url=data["url"], + profile=data.get("profile", "quick"), + status=data.get("status", "paused"), + stage=data.get("stage", ""), + message=data.get("message", ""), + error=data.get("error", ""), + report_path=data.get("report_path", ""), + started_at=data.get("started_at", 0), + files_total=data.get("files_total", 0), + files_processed=data.get("files_processed", 0), + files_ignored=data.get("files_ignored", 0), + files_candidates=data.get("files_candidates", 0), + files_skipped_by_scope=data.get("files_skipped_by_scope", 0), + language_counts=data.get("language_counts", {}), + selected_paths=data.get("selected_paths", []), + completed_summaries=data.get("completed_summaries", {}), + partial_chunk_summaries=data.get("partial_chunk_summaries", {}), + project_summary=data.get("project_summary", ""), + llm_calls_done=data.get("llm_calls_done", 0), + llm_calls_estimated=data.get("llm_calls_estimated", 0), + ) + if state.status == "cancelling": + state.status = "cancelled" + state.message = "Análise cancelada durante o encerramento do serviço." + elif state.status in {"queued", "running", "pausing"}: + state.status = "paused" + state.message = "Serviço reiniciado. O progresso foi salvo; retome para continuar." + state.stats = self._snapshot(state) + self.runs[state.run_id] = state + self._persist_checkpoint(state) + except (OSError, ValueError, KeyError, TypeError): + continue def _load_persisted_run(self, run_id: str) -> RunState | None: + checkpoint_path = self.settings.workspace_dir / "runs" / run_id / "checkpoint.json" + if checkpoint_path.exists(): + try: + data = json.loads(checkpoint_path.read_text(encoding="utf-8")) + state = RunState( + run_id=data["run_id"], url=data["url"], profile=data.get("profile", "quick"), + status=data.get("status", "paused"), stage=data.get("stage", ""), + message=data.get("message", ""), error=data.get("error", ""), + report_path=data.get("report_path", ""), started_at=data.get("started_at", 0), + files_total=data.get("files_total", 0), files_processed=data.get("files_processed", 0), + files_ignored=data.get("files_ignored", 0), files_candidates=data.get("files_candidates", 0), + files_skipped_by_scope=data.get("files_skipped_by_scope", 0), + language_counts=data.get("language_counts", {}), selected_paths=data.get("selected_paths", []), + completed_summaries=data.get("completed_summaries", {}), + partial_chunk_summaries=data.get("partial_chunk_summaries", {}), + project_summary=data.get("project_summary", ""), + llm_calls_done=data.get("llm_calls_done", 0), llm_calls_estimated=data.get("llm_calls_estimated", 0), + ) + if state.status == "cancelling": + state.status = "cancelled" + state.message = "Análise cancelada durante o encerramento do serviço." + elif state.status in {"queued", "running", "pausing"}: + state.status = "paused" + state.message = "Serviço reiniciado. O progresso foi salvo; retome para continuar." + state.stats = self._snapshot(state) + with self.lock: + self.runs[run_id] = state + self._persist_checkpoint(state) + return state + except (OSError, ValueError, KeyError, TypeError): + pass + report_dir = self.settings.workspace_dir / "reports" / run_id report_path = report_dir / "report.md" if not report_path.exists(): @@ -143,6 +270,9 @@ def status(self, run_id: str) -> dict: "message": state.message, "error": state.error, "report_path": state.report_path, + "profile": state.profile, + "can_resume": state.status == "paused", + "can_cancel": state.status in {"queued", "running", "pausing", "paused"}, "stats": state.stats, } @@ -209,6 +339,10 @@ def _snapshot(self, state: RunState) -> dict: "files_total": state.files_total, "files_processed": state.files_processed, "files_ignored": state.files_ignored, + "files_candidates": state.files_candidates, + "files_selected": state.files_total, + "files_skipped_by_scope": state.files_skipped_by_scope, + "profile": state.profile, "files_progress_percent": files_progress_percent, "language_counts": state.language_counts, "current_files": list(state.current_files), @@ -226,6 +360,30 @@ def _push(self, run_id: str, **field_updates): for key, value in field_updates.items(): setattr(state, key, value) state.stats = self._snapshot(state) + self._persist_checkpoint(state) + + def _persist_checkpoint(self, state: RunState) -> None: + run_dir = self.settings.workspace_dir / "runs" / state.run_id + run_dir.mkdir(parents=True, exist_ok=True) + checkpoint = run_dir / "checkpoint.json" + temporary = checkpoint.with_suffix(".json.tmp") + with self.checkpoint_lock: + with self.lock: + payload = { + "run_id": state.run_id, "url": state.url, "profile": state.profile, + "status": state.status, "stage": state.stage, "message": state.message, + "error": state.error, "report_path": state.report_path, "started_at": state.started_at, + "files_total": state.files_total, "files_processed": state.files_processed, + "files_ignored": state.files_ignored, "files_candidates": state.files_candidates, + "files_skipped_by_scope": state.files_skipped_by_scope, + "language_counts": dict(state.language_counts), "selected_paths": list(state.selected_paths), + "completed_summaries": dict(state.completed_summaries), + "partial_chunk_summaries": {path: list(summaries) for path, summaries in state.partial_chunk_summaries.items()}, + "project_summary": state.project_summary, + "llm_calls_done": state.llm_calls_done, "llm_calls_estimated": state.llm_calls_estimated, + } + temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + temporary.replace(checkpoint) def _note_llm_call(self, run_id: str): with self.lock: @@ -241,13 +399,15 @@ def _file_started(self, run_id: str, path: str): state.current_files.append(path) state.stats = self._snapshot(state) - def _file_finished(self, run_id: str, path: str): + def _file_finished(self, run_id: str, path: str, succeeded: bool): with self.lock: state = self.runs[run_id] if path in state.current_files: state.current_files.remove(path) - state.files_processed += 1 + if succeeded: + state.files_processed += 1 state.stats = self._snapshot(state) + self._persist_checkpoint(state) def _estimate_llm_calls(self, files) -> int: max_chunk_chars = max(1, self.settings.max_chunk_chars) @@ -257,6 +417,13 @@ def _estimate_llm_calls(self, files) -> int: total_calls += 1 if chunks_est <= 1 else chunks_est + 1 return total_calls + def _check_control(self, run_id: str) -> None: + state = self.runs[run_id] + if state.cancel_event.is_set(): + raise AnalysisCancelled() + if state.pause_event.is_set(): + raise AnalysisPaused() + def _run(self, run_id: str): state = self.runs[run_id] run_dir = self.settings.workspace_dir / "runs" / run_id @@ -265,19 +432,18 @@ def _run(self, run_id: str): report_dir.mkdir(parents=True, exist_ok=True) try: - self._push( - run_id, - status="running", - stage="ollama", - message="Verificando se o Ollama está no ar e o modelo instalado...", - started_at=time.time(), - ) + self._push(run_id, status="running", stage=state.stage or "ollama", + message="Verificando se o Ollama está no ar e o modelo instalado...", + started_at=state.started_at or time.time()) + self._check_control(run_id) self.client.check() run_dir.mkdir(parents=True, exist_ok=True) - self._push(run_id, stage="clone", message="Clonando repositório (raso, 1 commit)...") - self._clone(state.url, repo_dir) + self._check_control(run_id) + if not repo_dir.exists(): + self._push(run_id, stage="clone", message="Clonando repositório (raso, 1 commit)...") + self._clone(state.url, repo_dir) size_mb = self._directory_size(repo_dir) / (1024 * 1024) if size_mb > self.settings.max_repo_size_mb: @@ -285,6 +451,7 @@ def _run(self, run_id: str): f"Repositório excede o limite de {self.settings.max_repo_size_mb} MB." ) + self._check_control(run_id) self._push(run_id, stage="scan", message="Inventariando arquivos do repositório...") inventory = scan_repository( repo_dir, @@ -292,29 +459,44 @@ def _run(self, run_id: str): self.settings.max_files, ) - files = inventory["files"] + candidates = inventory["files"] + files = select_analysis_files( + candidates, + state.profile, + self.settings.quick_profile_files, + self.settings.balanced_profile_files, + ) + state.selected_paths = [item.path for item in files] total = len(files) - llm_calls_estimated = self._estimate_llm_calls(files) + state.files_candidates = len(candidates) + state.files_skipped_by_scope = max(0, len(candidates) - total) + state.files_ignored = inventory["ignored_files"] + state.language_counts = inventory["language_counts"] + state.files_total = total + state.files_processed = sum(1 for item in files if item.path in state.completed_summaries) + llm_calls_estimated = self._estimate_llm_calls([item for item in files if item.path not in state.completed_summaries]) self._push( run_id, stage="files", message=( - f"Analisando {total} arquivo(s) com o modelo " + f"Analisando {total} de {len(candidates)} arquivo(s) elegíveis " + f"(perfil {state.profile}) com o modelo " f"{self.settings.ollama_model}..." ), files_total=total, - files_processed=0, + files_candidates=len(candidates), + files_skipped_by_scope=max(0, len(candidates) - total), + files_processed=state.files_processed, files_ignored=inventory["ignored_files"], language_counts=inventory["language_counts"], llm_calls_estimated=llm_calls_estimated, - llm_calls_done=0, - llm_start_time=0.0, current_files=[], ) file_summaries = self._analyze_files(run_id, repo_dir, files) + self._check_control(run_id) self._push( run_id, stage="synthesis", @@ -328,18 +510,32 @@ def _run(self, run_id: str): "ignored_files": inventory["ignored_files"], "language_counts": inventory["language_counts"], "ignored_examples": inventory["ignored"][:100], - "tree": [item.path for item in files], + "selected_files": [item.path for item in files], + "profile": state.profile, }, ensure_ascii=False, indent=2) synthesis_input, omitted_count = self._budget_summaries( file_summaries, self.settings.max_synthesis_chars ) - final = self.client.chat( - SYSTEM, - project_synthesis_prompt(inventory_text, synthesis_input), - ) - self._note_llm_call(run_id) + final = state.project_summary + if not final: + self._check_control(run_id) + final = self.client.chat( + SYSTEM, + project_synthesis_prompt( + inventory_text, + synthesis_input, + compact=state.profile == "quick", + ), + should_cancel=lambda: self._check_control(run_id), + num_predict=self.settings.ollama_synthesis_num_predict, + ) + self._note_llm_call(run_id) + with self.lock: + state.project_summary = final + self._persist_checkpoint(state) + self._check_control(run_id) if omitted_count: final += ( @@ -349,10 +545,7 @@ def _run(self, run_id: str): "detalhados individualmente na seção 'Arquivos analisados')." ) - coverage = ( - (inventory["analyzed_candidates"] / inventory["total_files"] * 100) - if inventory["total_files"] else 100 - ) + coverage = (total / len(candidates) * 100) if candidates else 100 report = self._build_report( state.url, @@ -360,6 +553,9 @@ def _run(self, run_id: str): final, file_summaries, coverage, + state.profile, + total, + state.files_skipped_by_scope, ) report_path = report_dir / "report.md" @@ -370,6 +566,9 @@ def _run(self, run_id: str): "url": state.url, "coverage_percent": coverage, "total_files": inventory["total_files"], + "selected_files": total, + "skipped_by_scope": state.files_skipped_by_scope, + "profile": state.profile, "ignored_files": inventory["ignored"], "language_counts": inventory["language_counts"], }, ensure_ascii=False, indent=2), @@ -385,6 +584,11 @@ def _run(self, run_id: str): files_processed=total, ) + except AnalysisPaused: + self._push(run_id, status="paused", message="Progresso salvo. Retome a análise quando quiser.", current_files=[]) + except AnalysisCancelled: + self._push(run_id, status="cancelled", message="Análise cancelada pelo usuário.", current_files=[]) + except Exception as exc: self._push( run_id, @@ -399,10 +603,15 @@ def _analyze_files(self, run_id: str, repo_dir: Path, files) -> list[str]: max_workers = max(1, self.settings.ollama_concurrency) file_summaries: list[str | None] = [None] * len(files) + for idx, info in enumerate(files): + if info.path in self.runs[run_id].completed_summaries: + file_summaries[idx] = self.runs[run_id].completed_summaries[info.path] + with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = { executor.submit(self._process_file, run_id, repo_dir, info): idx for idx, info in enumerate(files) + if file_summaries[idx] is None } try: for future in as_completed(futures): @@ -412,25 +621,37 @@ def _analyze_files(self, run_id: str, repo_dir: Path, files) -> list[str]: executor.shutdown(wait=False, cancel_futures=True) raise - return file_summaries + return [summary or "" for summary in file_summaries] def _process_file(self, run_id: str, repo_dir: Path, info) -> str: + self._check_control(run_id) self._file_started(run_id, info.path) + succeeded = False try: path = repo_dir / info.path text = path.read_text(encoding="utf-8", errors="replace") chunks = self._chunks(text, self.settings.max_chunk_chars) symbols_json = json.dumps(info.symbols, ensure_ascii=False) + state = self.runs[run_id] if len(chunks) == 1: synthesis = self.client.chat( SYSTEM, - single_chunk_prompt(info.path, info.language, symbols_json, chunks[0]), + single_chunk_prompt( + info.path, + info.language, + symbols_json, + chunks[0], + compact=state.profile == "quick", + ), + should_cancel=lambda: self._check_control(run_id), + num_predict=self.settings.ollama_file_num_predict, ) self._note_llm_call(run_id) else: - chunk_summaries = [] - for chunk_index, chunk in enumerate(chunks, start=1): + chunk_summaries = list(state.partial_chunk_summaries.get(info.path, [])) + for chunk_index, chunk in enumerate(chunks[len(chunk_summaries):], start=len(chunk_summaries) + 1): + self._check_control(run_id) prompt = chunk_prompt( info.path, info.language, @@ -438,10 +659,20 @@ def _process_file(self, run_id: str, repo_dir: Path, info) -> str: len(chunks), chunk, symbols_json, + compact=state.profile == "quick", ) - chunk_summaries.append(self.client.chat(SYSTEM, prompt)) + chunk_summaries.append(self.client.chat( + SYSTEM, + prompt, + should_cancel=lambda: self._check_control(run_id), + num_predict=self.settings.ollama_file_num_predict, + )) self._note_llm_call(run_id) + with self.lock: + state.partial_chunk_summaries[info.path] = list(chunk_summaries) + self._persist_checkpoint(state) + self._check_control(run_id) budgeted_chunk_summaries, omitted_chunks = self._budget_summaries( chunk_summaries, self.settings.max_synthesis_chars ) @@ -453,7 +684,10 @@ def _process_file(self, run_id: str, repo_dir: Path, info) -> str: info.language, symbols_json, budgeted_chunk_summaries, + compact=state.profile == "quick", ), + should_cancel=lambda: self._check_control(run_id), + num_predict=self.settings.ollama_file_num_predict, ) self._note_llm_call(run_id) @@ -466,15 +700,20 @@ def _process_file(self, run_id: str, repo_dir: Path, info) -> str: info.analyzed = True info.chunks = len(chunks) info.summary = synthesis - - return ( + summary = ( f"ARQUIVO: {info.path}\n" f"LINGUAGEM: {info.language}\n" f"SÍMBOLOS: {symbols_json}\n" f"ANÁLISE:\n{synthesis}" ) + state = self.runs[run_id] + with self.lock: + state.completed_summaries[info.path] = summary + state.partial_chunk_summaries.pop(info.path, None) + succeeded = True + return summary finally: - self._file_finished(run_id, info.path) + self._file_finished(run_id, info.path, succeeded) @staticmethod def _normalize_github_url(url: str) -> str | None: @@ -589,7 +828,9 @@ def _chunks(text: str, max_chars: int) -> list[str]: return chunks @staticmethod - def _build_report(url, inventory, final, file_summaries, coverage): + def _build_report(url, inventory, final, file_summaries, coverage, profile="complete", + selected_count=None, skipped_by_scope=0): + selected_count = len(file_summaries) if selected_count is None else selected_count lines = [ "# Repository Intelligence Report", "", @@ -598,7 +839,10 @@ def _build_report(url, inventory, final, file_summaries, coverage): "## Cobertura da análise", "", f"- Arquivos de texto elegíveis: **{inventory['total_files']}**", - f"- Arquivos analisados: **{inventory['analyzed_candidates']}**", + f"- Arquivos elegíveis encontrados: **{inventory['analyzed_candidates']}**", + f"- Arquivos selecionados pelo perfil `{profile}`: **{selected_count}**", + f"- Arquivos fora do escopo selecionado: **{skipped_by_scope}**", + f"- Arquivos analisados: **{len(file_summaries)}**", f"- Arquivos ignorados: **{inventory['ignored_files']}**", f"- Cobertura de arquivos elegíveis: **{coverage:.2f}%**", "", diff --git a/ai-service/analyzer/prompts.py b/ai-service/analyzer/prompts.py index d906f5a..2d8b9c8 100644 --- a/ai-service/analyzer/prompts.py +++ b/ai-service/analyzer/prompts.py @@ -16,7 +16,18 @@ """ -def chunk_prompt(path, language, chunk_index, total_chunks, content, symbols): +def chunk_prompt(path, language, chunk_index, total_chunks, content, symbols, compact=False): + if compact: + return f"""Analise o bloco {chunk_index}/{total_chunks} do arquivo `{path}` ({language}). + +Símbolos conhecidos: {symbols} +Conteúdo: +```text +{content} +``` + +Em português, resuma propósito e fatos principais em até 180 palavras. Liste só funções/dependências relevantes e um possível problema se houver evidência. Não repita o conteúdo nem invente fatos.""" + return f"""Analise o bloco {chunk_index}/{total_chunks} do arquivo `{path}`. Linguagem: {language} @@ -41,11 +52,24 @@ def chunk_prompt(path, language, chunk_index, total_chunks, content, symbols): Não invente informações ausentes.""" -def single_chunk_prompt(path, language, symbols, content): +def single_chunk_prompt(path, language, symbols, content, compact=False): """Usado quando o arquivo inteiro cabe em um único bloco: produz diretamente a análise consolidada do arquivo em UMA chamada ao modelo, em vez de uma chamada por bloco seguida de uma chamada de consolidação. """ + if compact: + return f"""Analise o arquivo `{path}` ({language}) por completo. + +Símbolos conhecidos: +{symbols} + +Conteúdo: +```text +{content} +``` + +Em português, descreva finalidade, funcionamento e evidências principais em até 250 palavras. Cite funções/imports relevantes e possíveis problemas apenas quando houver evidência. Não invente fatos.""" + return f"""Analise o arquivo `{path}` ({language}) por completo - ele cabe integralmente em um único bloco, então esta é a análise final e consolidada do arquivo (não apenas de um trecho). @@ -71,8 +95,17 @@ def single_chunk_prompt(path, language, symbols, content): Não invente informações ausentes.""" -def file_synthesis_prompt(path, language, symbols, chunk_summaries): +def file_synthesis_prompt(path, language, symbols, chunk_summaries, compact=False): joined = "\n\n--- BLOCO ---\n\n".join(chunk_summaries) + if compact: + return f"""Consolide em até 300 palavras a análise de `{path}` ({language}). + +Estrutura estática: {symbols} +Resumo dos blocos: +{joined} + +Use seções curtas para finalidade, responsabilidades, funcionamento e evidências. Identifique possíveis problemas só se houver evidência; declare quando algo não puder ser determinado.""" + return f"""Consolide a análise do arquivo `{path}` ({language}). Estrutura estática: @@ -94,9 +127,29 @@ def file_synthesis_prompt(path, language, symbols, chunk_summaries): Não introduza fatos que não apareçam nas evidências.""" -def project_synthesis_prompt(inventory, file_summaries): +def project_synthesis_prompt(inventory, file_summaries, compact=False): joined = "\n\n===== ARQUIVO =====\n\n".join(file_summaries) + if compact: + return f"""Você é o analista principal do repositório. + +INVENTÁRIO: +{inventory} + +ANÁLISES DOS ARQUIVOS: +{joined} + +Escreva uma síntese concisa em Markdown, com estas seções e nesta ordem: +# Visão Geral +# Objetivo Inferido +# Stack Tecnológica +# Arquitetura e Estrutura +# Funcionalidades Observadas +# Qualidade e Possíveis Problemas +# Limitações da Análise + +Use no máximo 2 frases por seção e até 700 palavras no total. Não repita listas de arquivos nem invente fatos. Separe fatos observados de recomendações. Quando faltar evidência, diga isso claramente.""" + return f"""Você é o analista principal do repositório. INVENTÁRIO: @@ -127,4 +180,6 @@ def project_synthesis_prompt(inventory, file_summaries): # Evidências Concretas Se alguma seção não puder ser determinada, declare explicitamente isso. -Não invente tecnologias, funcionalidades ou arquitetura.""" +Não invente tecnologias, funcionalidades ou arquitetura. +Mantenha cada seção objetiva e evite repetir a lista de arquivos do inventário. +Use apenas evidências que aparecem nas análises fornecidas.""" diff --git a/ai-service/analyzer/scanner.py b/ai-service/analyzer/scanner.py index 317e84d..07c3306 100644 --- a/ai-service/analyzer/scanner.py +++ b/ai-service/analyzer/scanner.py @@ -65,6 +65,38 @@ ".env.example", ".gitignore" } +def select_analysis_files(files: list[FileInfo], profile: str, quick_limit: int = 8, + balanced_limit: int = 80) -> list[FileInfo]: + """Pick the most useful files first while keeping selection deterministic.""" + limits = {"quick": max(1, quick_limit), "balanced": max(1, balanced_limit), "complete": None} + if profile not in limits: + raise ValueError("Perfil de análise inválido. Use quick, balanced ou complete.") + + def priority(info: FileInfo) -> tuple[int, str]: + name = Path(info.path).name + lower = info.path.lower() + score = 0 + if name.upper().startswith("README"): + score += 100 + if name in {"package.json", "pyproject.toml", "requirements.txt", "Cargo.toml", "go.mod", + "pom.xml", "build.gradle", "Dockerfile", "docker-compose.yml", "docker-compose.yaml"}: + score += 80 + if info.language not in {"Unknown", "Markdown", "Text", "JSON", "YAML", "TOML", "XML"}: + score += 50 + if any(part in lower for part in ("/test/", "/tests/", "_test.", ".test.", ".spec.")): + score -= 15 + if info.kind == "documentation": + score -= 20 + if name.lower() in {"package-lock.json", "yarn.lock", "pnpm-lock.yaml"}: + score -= 40 + if lower.startswith(("examples/", "fixtures/", "samples/")): + score -= 20 + return score, info.path + + ranked = sorted(files, key=lambda info: (-priority(info)[0], priority(info)[1])) + limit = limits[profile] + return ranked if limit is None else ranked[:limit] + def detect_language(path: Path) -> str: if path.name == "Dockerfile": diff --git a/ai-service/main.py b/ai-service/main.py index c3626bd..c6c028e 100644 --- a/ai-service/main.py +++ b/ai-service/main.py @@ -1,5 +1,5 @@ from contextlib import asynccontextmanager -from typing import Any +from typing import Any, Literal from fastapi import FastAPI, HTTPException, status from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import PlainTextResponse @@ -182,13 +182,16 @@ async def query_rag(req: RagQueryRequest): class AnalyzeRequest(BaseModel): url: str = Field(..., description="URL pública do repositório GitHub") + profile: Literal["quick", "balanced", "complete"] = Field( + "quick", description="Quantidade e prioridade dos arquivos enviados ao modelo local" + ) @app.post("/api/analyze", status_code=status.HTTP_200_OK) def analyze_repository(req: AnalyzeRequest): """Inicia a análise assíncrona de um repositório GitHub.""" try: - run_id = analyzer.start(str(req.url)) + run_id = analyzer.start(str(req.url), req.profile) return {"run_id": run_id, "status": "started"} except AnalysisError as exc: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) @@ -196,6 +199,30 @@ def analyze_repository(req: AnalyzeRequest): raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(exc)) +@app.post("/api/runs/{run_id}/pause") +def pause_analysis(run_id: str): + try: + return analyzer.pause(run_id) + except AnalysisError as exc: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) + + +@app.post("/api/runs/{run_id}/resume") +def resume_analysis(run_id: str): + try: + return analyzer.resume(run_id) + except AnalysisError as exc: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) + + +@app.post("/api/runs/{run_id}/cancel") +def cancel_analysis(run_id: str): + try: + return analyzer.cancel(run_id) + except AnalysisError as exc: + raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) + + @app.get("/api/runs") def list_analysis_runs(): """Lista as execuções de análise ativas e persistidas.""" diff --git a/ai-service/tests/test_analyzer.py b/ai-service/tests/test_analyzer.py index 752ddb8..a8fc1e8 100644 --- a/ai-service/tests/test_analyzer.py +++ b/ai-service/tests/test_analyzer.py @@ -1,9 +1,10 @@ import unittest from pathlib import Path +from tempfile import TemporaryDirectory from analyzer.pipeline import Analyzer, STAGE_KEYS, STAGE_LABELS from analyzer.config import AnalyzerSettings -from analyzer.scanner import detect_language, python_symbols, is_probably_binary -from analyzer.models import RunState +from analyzer.scanner import detect_language, python_symbols, is_probably_binary, select_analysis_files +from analyzer.models import FileInfo, RunState class TestAnalyzer(unittest.TestCase): @@ -85,6 +86,125 @@ def test_stages_consistency(self): for key in STAGE_KEYS: self.assertIn(key, STAGE_LABELS) + def test_analysis_profiles_prioritize_core_files_and_bound_llm_scope(self): + files = [ + FileInfo("src/service.py", 100, ".py", "Python", "source/config"), + FileInfo("README.md", 100, ".md", "Markdown", "documentation"), + FileInfo("tests/test_service.py", 100, ".py", "Python", "source/config"), + FileInfo("package-lock.json", 100, ".json", "JSON", "source/config"), + ] + quick = select_analysis_files(files, "quick", quick_limit=2) + self.assertEqual([item.path for item in quick], ["README.md", "src/service.py"]) + self.assertEqual(len(select_analysis_files(files, "balanced", balanced_limit=3)), 3) + self.assertEqual(len(select_analysis_files(files, "complete")), len(files)) + with self.assertRaises(ValueError): + select_analysis_files(files, "unknown") + + def test_quick_profile_defaults_to_eight_priority_files(self): + files = [ + FileInfo(f"src/module_{index}.py", 100, ".py", "Python", "source/config") + for index in range(12) + ] + self.assertEqual(len(select_analysis_files(files, "quick")), 8) + + def test_checkpoint_restores_running_analysis_as_resumable_and_keeps_finished_summaries(self): + with TemporaryDirectory() as workspace: + settings = AnalyzerSettings(workspace_dir=Path(workspace)) + original = Analyzer(settings) + state = RunState( + run_id="checkpoint1", url="https://github.com/acme/api", status="running", + stage="files", profile="quick", files_total=2, files_processed=1, + selected_paths=["README.md", "src/app.py"], + completed_summaries={"README.md": "resumo salvo"}, + partial_chunk_summaries={"src/app.py": ["primeiro bloco salvo"]}, + ) + original.runs[state.run_id] = state + original._persist_checkpoint(state) + + restored = Analyzer(settings) + saved = restored.status(state.run_id) + self.assertEqual(saved["status"], "paused") + self.assertTrue(saved["can_resume"]) + self.assertEqual(saved["stats"]["files_processed"], 1) + self.assertEqual(restored.runs[state.run_id].completed_summaries, {"README.md": "resumo salvo"}) + self.assertEqual(restored.runs[state.run_id].partial_chunk_summaries, {"src/app.py": ["primeiro bloco salvo"]}) + original.client.close() + restored.client.close() + + def test_cancel_from_paused_state_is_terminal(self): + with TemporaryDirectory() as workspace: + analyzer = Analyzer(AnalyzerSettings(workspace_dir=Path(workspace))) + state = RunState(run_id="paused123", url="https://github.com/acme/api", status="paused", stage="files") + analyzer.runs[state.run_id] = state + result = analyzer.cancel(state.run_id) + self.assertEqual(result["status"], "cancelled") + self.assertFalse(result["can_resume"]) + analyzer.client.close() + + def test_service_restart_does_not_resume_an_interrupted_cancellation(self): + with TemporaryDirectory() as workspace: + settings = AnalyzerSettings(workspace_dir=Path(workspace)) + original = Analyzer(settings) + original.runs["cancel123"] = RunState( + run_id="cancel123", url="https://github.com/acme/api", status="cancelling", stage="files" + ) + original._persist_checkpoint(original.runs["cancel123"]) + restored = Analyzer(settings) + self.assertEqual(restored.status("cancel123")["status"], "cancelled") + self.assertFalse(restored.status("cancel123")["can_resume"]) + original.client.close() + restored.client.close() + + def test_pause_and_resume_reuses_completed_file_summary(self): + with TemporaryDirectory() as workspace: + settings = AnalyzerSettings(workspace_dir=Path(workspace), quick_profile_files=1, ollama_max_retries=0) + analyzer = Analyzer(settings) + state = RunState(run_id="resume123", url="https://github.com/acme/api", status="queued", profile="quick") + analyzer.runs[state.run_id] = state + + def clone(_url, destination): + (destination / "src").mkdir(parents=True, exist_ok=True) + (destination / "README.md").write_text("Projeto de teste", encoding="utf-8") + (destination / "src" / "app.py").write_text("def main():\n return 1\n", encoding="utf-8") + + analyzer._clone = clone + + class FakeClient: + def __init__(self): + self.calls = [] + self.pause_on_first_call = True + + def check(self): + return None + + def chat(self, _system, prompt, should_cancel=None, num_predict=None): + self.calls.append(prompt) + if self.pause_on_first_call: + self.pause_on_first_call = False + analyzer.pause(state.run_id) + if should_cancel: + should_cancel() + return f"resposta-{len(self.calls)}" + + def close(self): + return None + + client = FakeClient() + analyzer.client = client + analyzer._start_worker(state) + state.worker_thread.join(timeout=5) + self.assertFalse(state.worker_thread.is_alive()) + self.assertEqual(analyzer.status(state.run_id)["status"], "paused") + self.assertEqual(state.files_processed, 0) + self.assertNotIn("README.md", state.completed_summaries) + + resumed = analyzer.resume(state.run_id) + self.assertIn(resumed["status"], {"queued", "running"}) + state.worker_thread.join(timeout=5) + self.assertFalse(state.worker_thread.is_alive()) + self.assertEqual(analyzer.status(state.run_id)["status"], "completed") + self.assertEqual(len(client.calls), 3) # Refaz a chamada interrompida e depois sintetiza o projeto. + if __name__ == "__main__": unittest.main() diff --git a/ai-service/tests/test_ollama_client.py b/ai-service/tests/test_ollama_client.py new file mode 100644 index 0000000..be48475 --- /dev/null +++ b/ai-service/tests/test_ollama_client.py @@ -0,0 +1,84 @@ +from copy import deepcopy +from unittest import TestCase +from unittest.mock import MagicMock, Mock, patch + +from analyzer.ollama_client import OllamaClient + + +class TestOllamaClient(TestCase): + @patch("analyzer.ollama_client.httpx.Client") + def test_chat_caps_generated_tokens(self, client_factory): + client = client_factory.return_value + response = Mock() + response.iter_lines.return_value = [ + '{"message":{"content":"resumo"},"done":true}' + ] + client.stream.return_value.__enter__.return_value = response + + ollama = OllamaClient( + "http://ollama:11434", "qwen2.5:1.5b", 300, "5m", num_predict=700 + ) + self.assertEqual(ollama.chat("sistema", "prompt"), "resumo") + + payload = client.stream.call_args.kwargs["json"] + self.assertEqual(payload["options"]["num_predict"], 700) + self.assertTrue(payload["stream"]) + + @patch("analyzer.ollama_client.httpx.Client") + def test_chat_checks_cancellation_while_streaming(self, client_factory): + client = client_factory.return_value + response = Mock() + response.iter_lines.return_value = [ + '{"message":{"content":"parcial"},"done":false}', + '{"message":{"content":"resposta"},"done":true}', + ] + client.stream.return_value.__enter__.return_value = response + checks = 0 + + def cancel_after_first_chunk(): + nonlocal checks + checks += 1 + if checks == 2: + raise RuntimeError("cancelado") + + ollama = OllamaClient("http://ollama:11434", "qwen", 300, "5m") + with self.assertRaisesRegex(RuntimeError, "cancelado"): + ollama.chat("sistema", "prompt", should_cancel=cancel_after_first_chunk) + + @patch("analyzer.ollama_client.httpx.Client") + def test_chat_retries_when_model_stops_at_output_limit(self, client_factory): + client = client_factory.return_value + first_response = Mock() + first_response.iter_lines.return_value = [ + '{"message":{"content":"cortada"},"done":true,"done_reason":"length"}' + ] + second_response = Mock() + second_response.iter_lines.return_value = [ + '{"message":{"content":"completa"},"done":true,"done_reason":"stop"}' + ] + first_stream, second_stream = MagicMock(), MagicMock() + first_stream.__enter__.return_value = first_response + second_stream.__enter__.return_value = second_response + streams = [first_stream, second_stream] + payloads = [] + + def stream(*_args, **kwargs): + payloads.append(deepcopy(kwargs["json"])) + return streams.pop(0) + + client.stream.side_effect = stream + + ollama = OllamaClient("http://ollama:11434", "qwen", 300, "5m", max_retries=1, num_predict=128) + self.assertEqual(ollama.chat("sistema", "prompt"), "completa") + limits = [payload["options"]["num_predict"] for payload in payloads] + self.assertEqual(limits, [128, 256]) + + def test_chat_uses_safe_default_token_limit(self): + with patch("analyzer.ollama_client.httpx.Client"): + ollama = OllamaClient("http://ollama:11434", "qwen", 300, "5m") + self.assertEqual(ollama.num_predict, 1024) + + def test_chat_enforces_minimum_token_limit(self): + with patch("analyzer.ollama_client.httpx.Client"): + ollama = OllamaClient("http://ollama:11434", "qwen", 300, "5m", num_predict=1) + self.assertEqual(ollama.num_predict, 128) diff --git a/backend/src/modules/repo-analyses/repo-analyses.repository.ts b/backend/src/modules/repo-analyses/repo-analyses.repository.ts index 26cbd6c..d0325dd 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.repository.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.repository.ts @@ -56,17 +56,17 @@ export class RepoAnalysesRepository { ): Promise { const query = ` UPDATE analise_repositorio - SET status = $1, - etapa = $2, - etapa_label = $3, - progresso = $4, - mensagem = $5, - erro = $6, - relatorio_markdown = COALESCE($7, relatorio_markdown), - metadados = COALESCE($8, metadados), + SET status = $1::varchar, + etapa = $2::varchar, + etapa_label = $3::varchar, + progresso = $4::integer, + mensagem = $5::text, + erro = $6::text, + relatorio_markdown = COALESCE($7::text, relatorio_markdown), + metadados = COALESCE($8::jsonb, metadados), updated_at = NOW(), - concluido_em = CASE WHEN $1 IN ('concluido', 'falha') THEN NOW() ELSE concluido_em END - WHERE run_id = $9; + concluido_em = CASE WHEN $1::varchar IN ('concluido', 'falha', 'cancelada') THEN NOW() ELSE concluido_em END + WHERE run_id = $9::varchar; `; await pool.query(query, [ statusData.status, diff --git a/backend/src/modules/repo-analyses/repo-analyses.routes.test.ts b/backend/src/modules/repo-analyses/repo-analyses.routes.test.ts index 969ee3d..1537d3e 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.routes.test.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.routes.test.ts @@ -17,21 +17,27 @@ const ANALYSIS = 'e0000000-0000-4000-8000-000000000001'; const record = { id: ANALYSIS, projeto_id: ACTIVE, status: 'iniciado', run_id: 'r1' } as RepoAnalysisRecord; class FakeService extends RepoAnalysesService { - public started: Array<{ projectId: string; userId: string; url: string }> = []; + public started: Array<{ projectId: string; userId: string; url: string; profile: string }> = []; public lookups: Array<{ projectId: string; id: string }> = []; + public controls: Array<{ projectId: string; id: string; action: string }> = []; public engineDown = false; constructor() { super({} as never, 'http://localhost:0'); } - async startAnalysis(projectId: string, userId: string, url: string): Promise { + async startAnalysis(projectId: string, userId: string, url: string, profile = 'quick'): Promise { if (!/^https:\/\/github\.com\/[\w.-]+\/[\w.-]+$/.test(url)) throw new ValidationError('URL do repositório GitHub inválida.'); if (this.engineDown) throw new AppError('Falha ao iniciar análise no motor de IA: serviço indisponível', 503, 'ANALYZER_UNAVAILABLE'); - this.started.push({ projectId, userId, url }); + this.started.push({ projectId, userId, url, profile }); return { ...record, projeto_id: projectId }; } + async controlAnalysis(projectId: string, id: string, action: 'pause' | 'resume' | 'cancel'): Promise { + this.controls.push({ projectId, id, action }); + return projectId === ACTIVE && id === ANALYSIS ? record : null; + } + async listByProject(projectId: string): Promise { return projectId === ACTIVE ? [record] : []; } @@ -94,7 +100,14 @@ test('id de projeto malformado retorna 400 e projeto inexistente retorna 404 ant test('inicia análise no projeto ativo com o usuário autenticado', async () => { const response = await call(`/${ACTIVE}/repo-analyses`, { method: 'POST', json: { repositorio_url: 'https://github.com/acme/api' } }); assert.equal(response.status, 201); - assert.deepEqual(service.started.at(-1), { projectId: ACTIVE, userId: 'b0000000-0000-4000-8000-000000000001', url: 'https://github.com/acme/api' }); + assert.deepEqual(service.started.at(-1), { projectId: ACTIVE, userId: 'b0000000-0000-4000-8000-000000000001', url: 'https://github.com/acme/api', profile: 'quick' }); +}); + +test('aceita perfil configurável e rejeita perfil desconhecido', async () => { + const response = await call(`/${ACTIVE}/repo-analyses`, { method: 'POST', json: { repositorio_url: 'https://github.com/acme/api', perfil: 'quick' } }); + assert.equal(response.status, 201); + assert.equal(service.started.at(-1)?.profile, 'quick'); + assert.equal((await call(`/${ACTIVE}/repo-analyses`, { method: 'POST', json: { repositorio_url: 'https://github.com/acme/api', perfil: 'turbo' } })).status, 400); }); test('URL inválida retorna 400 e falha do motor retorna 503 com mensagem legível', async () => { @@ -122,3 +135,16 @@ test('consulta por id respeita o projeto da rota e valida o formato', async () = assert.equal((await call(`/${ACTIVE}/repo-analyses/abc`)).status, 400); assert.deepEqual(await (await call(`/${OTHER}/repo-analyses`)).json(), []); }); + +test('pausa, retoma e cancela uma análise autenticada e vinculada ao projeto', async () => { + for (const action of ['pause', 'resume', 'cancel'] as const) { + assert.equal((await call(`/${ACTIVE}/repo-analyses/${ANALYSIS}/${action}`, { method: 'POST', json: {} })).status, 200); + } + assert.deepEqual(service.controls.slice(-3), [ + { projectId: ACTIVE, id: ANALYSIS, action: 'pause' }, + { projectId: ACTIVE, id: ANALYSIS, action: 'resume' }, + { projectId: ACTIVE, id: ANALYSIS, action: 'cancel' }, + ]); + assert.equal((await call(`/${OTHER}/repo-analyses/${ANALYSIS}/cancel`, { method: 'POST', json: {} })).status, 404); + assert.equal((await call(`/${ARCHIVED}/repo-analyses/${ANALYSIS}/resume`, { method: 'POST', json: {} })).status, 409); +}); diff --git a/backend/src/modules/repo-analyses/repo-analyses.routes.ts b/backend/src/modules/repo-analyses/repo-analyses.routes.ts index e18b71c..1ca099d 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.routes.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.routes.ts @@ -58,13 +58,36 @@ export function createRepoAnalysesRouter( } const url = req.body?.repositorio_url; if (typeof url !== 'string') throw new ValidationError('Informe a URL do repositório.'); - const analysis = await service.startAnalysis(String(req.params.projectId), req.auth?.id ?? '', url); + const profile = req.body?.perfil ?? 'quick'; + if (!['quick', 'balanced', 'complete'].includes(profile)) { + throw new ValidationError('Perfil inválido. Use quick, balanced ou complete.'); + } + const analysis = await service.startAnalysis(String(req.params.projectId), req.auth?.id ?? '', url, profile); res.status(201).json(analysis); } catch (error) { next(error); } }); + const control = (action: 'pause' | 'resume' | 'cancel') => async (req: Request, res: Response, next: NextFunction) => { + try { + const id = String(req.params.id); + validateUuid(id, 'ID da análise'); + if (action === 'resume' && res.locals.projectStatus === 'arquivado') { + throw new ArchiveConflict('Projeto arquivado é somente leitura e não retoma análises.'); + } + const analysis = await service.controlAnalysis(String(req.params.projectId), id, action); + if (!analysis) throw new NotFoundError('Análise não encontrada.'); + res.json(analysis); + } catch (error) { + next(error); + } + }; + + router.post('/:id/pause', control('pause')); + router.post('/:id/resume', control('resume')); + router.post('/:id/cancel', control('cancel')); + /** * @swagger * /api/v1/projects/{projectId}/repo-analyses: diff --git a/backend/src/modules/repo-analyses/repo-analyses.service.test.ts b/backend/src/modules/repo-analyses/repo-analyses.service.test.ts index 6e94b48..1c0cabf 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.service.test.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.service.test.ts @@ -3,10 +3,15 @@ import assert from 'node:assert/strict'; import { mapAnalyzerProgress, mapAnalyzerStatus } from './repo-analyses.service'; import { RepoAnalysesService } from './repo-analyses.service'; import { RepoAnalysesRepository } from './repo-analyses.repository'; +import axios from 'axios'; test('mapeia os estados reais do Analyzer para o contrato persistido pela API', () => { assert.equal(mapAnalyzerStatus('queued'), 'iniciado'); assert.equal(mapAnalyzerStatus('running'), 'em_execucao'); + assert.equal(mapAnalyzerStatus('pausing'), 'pausando'); + assert.equal(mapAnalyzerStatus('paused'), 'pausada'); + assert.equal(mapAnalyzerStatus('cancelling'), 'cancelando'); + assert.equal(mapAnalyzerStatus('cancelled'), 'cancelada'); assert.equal(mapAnalyzerStatus('completed'), 'concluido'); assert.equal(mapAnalyzerStatus('failed'), 'falha'); assert.throws(() => mapAnalyzerStatus('unexpected'), /Status desconhecido/); @@ -25,8 +30,38 @@ test('consulta análise vinculada ao projeto da rota, não apenas por ID', async test('calcula progresso limitado usando os campos stage_index/stage_count do Analyzer', () => { assert.equal(mapAnalyzerProgress(0, 6, 'iniciado'), 0); - assert.equal(mapAnalyzerProgress(3, 6, 'em_execucao'), 50); + assert.equal(mapAnalyzerProgress(3, 6, 'em_execucao'), 33); + assert.equal(mapAnalyzerProgress(4, 6, 'em_execucao', { calls_progress_percent: 50 }), 58); + assert.equal(mapAnalyzerProgress(4, 6, 'em_execucao', { calls_progress_percent: 0 }), 50); assert.equal(mapAnalyzerProgress(6, 6, 'concluido'), 100); assert.equal(mapAnalyzerProgress(99, 6, 'em_execucao'), 99); assert.equal(mapAnalyzerProgress(2, 0, 'em_execucao'), 0); }); + +test('marca como falha uma execução ativa ausente no motor, em vez de deixá-la ativa para sempre', async () => { + const record = { id: 'analysis-1', projeto_id: 'project-a', status: 'em_execucao', run_id: 'run-gone' }; + const updates: Array<{ status: string; etapa: string; mensagem?: string }> = []; + const repository = { + findByProjectId: async () => [record], + updateStatus: async (_runId: string, update: { status: string; etapa: string; mensagem?: string }) => { + updates.push(update); + Object.assign(record, update); + }, + } as unknown as RepoAnalysesRepository; + const originalGet = axios.get; + axios.get = Object.assign(async () => { + throw Object.assign(new Error('not found'), { isAxiosError: true, response: { status: 404 } }); + }, { ...originalGet }); + + try { + const service = new RepoAnalysesService(repository, 'http://localhost:8000'); + const [result] = await service.listByProject('project-a'); + + assert.equal(result.status, 'falha'); + assert.equal(result.etapa, 'error'); + assert.match(result.mensagem ?? '', /Inicie uma nova análise/); + assert.equal(updates.length, 1); + } finally { + axios.get = originalGet; + } +}); diff --git a/backend/src/modules/repo-analyses/repo-analyses.service.ts b/backend/src/modules/repo-analyses/repo-analyses.service.ts index 06f1388..32ae703 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.service.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.service.ts @@ -7,20 +7,35 @@ import { RepoAnalysisRecord, RepoAnalysisStatus, RepoAnalysisStep } from './repo const ANALYZER_STATUS: Record = { queued: 'iniciado', running: 'em_execucao', + pausing: 'pausando', + paused: 'pausada', + cancelling: 'cancelando', + cancelled: 'cancelada', completed: 'concluido', failed: 'falha', }; +const ACTIVE_STATUSES: RepoAnalysisStatus[] = ['iniciado', 'em_execucao', 'pausando', 'cancelando']; + export function mapAnalyzerStatus(status: string): RepoAnalysisStatus { const mapped = ANALYZER_STATUS[status]; if (!mapped) throw new Error(`Status desconhecido recebido do serviço de análise: ${status}`); return mapped; } -export function mapAnalyzerProgress(stageIndex: number, stageCount: number, status: RepoAnalysisStatus): number { +export function mapAnalyzerProgress( + stageIndex: number, + stageCount: number, + status: RepoAnalysisStatus, + stats?: Record, +): number { if (status === 'concluido') return 100; if (!Number.isFinite(stageIndex) || !Number.isFinite(stageCount) || stageCount <= 0) return 0; - return Math.max(0, Math.min(99, Math.round((stageIndex / stageCount) * 100))); + const completedStages = Math.max(0, Math.min(stageCount, stageIndex - 1)); + const stageProgress = stageIndex === 4 && typeof stats?.calls_progress_percent === 'number' + ? Math.max(0, Math.min(100, stats.calls_progress_percent)) / 100 + : 0; + return Math.max(0, Math.min(99, Math.round(((completedStages + stageProgress) / stageCount) * 100))); } export class RepoAnalysesService { @@ -34,14 +49,14 @@ export class RepoAnalysesService { return githubRegex.test(url.trim()); } - async startAnalysis(projetoId: string, usuarioId: string, repositorioUrl: string): Promise { + async startAnalysis(projetoId: string, usuarioId: string, repositorioUrl: string, profile: 'quick' | 'balanced' | 'complete' = 'quick'): Promise { if (!this.validateGithubUrl(repositorioUrl)) { throw new ValidationError('URL do repositório GitHub inválida. Utilize o formato https://github.com/usuario/repositorio'); } // Dispara a execução no ai-service / RepoAnalyzer try { - const response = await axios.post(`${this.analyzerBaseUrl}/api/analyze`, { url: repositorioUrl }, { timeout: 15_000 }); + const response = await axios.post(`${this.analyzerBaseUrl}/api/analyze`, { url: repositorioUrl, profile }, { timeout: 15_000 }); const { run_id } = response.data; @@ -64,7 +79,7 @@ export class RepoAnalysesService { // Opcional: Atualizar status das análises em andamento consultando o ai-service await Promise.all(analyses - .filter((analysis) => analysis.status === 'iniciado' || analysis.status === 'em_execucao') + .filter((analysis) => ACTIVE_STATUSES.includes(analysis.status)) .map((analysis) => this.syncAnalysisStatus(analysis.run_id))); return await this.repository.findByProjectId(projetoId); @@ -72,13 +87,30 @@ export class RepoAnalysesService { async getById(projetoId: string, id: string): Promise { const analysis = await this.repository.findById(id, projetoId); - if (analysis && (analysis.status === 'iniciado' || analysis.status === 'em_execucao')) { + if (analysis && ACTIVE_STATUSES.includes(analysis.status)) { await this.syncAnalysisStatus(analysis.run_id); return await this.repository.findById(id, projetoId); } return analysis; } + async controlAnalysis(projetoId: string, id: string, action: 'pause' | 'resume' | 'cancel'): Promise { + const analysis = await this.repository.findById(id, projetoId); + if (!analysis) return null; + if (!analysis.run_id) throw new AppError('Esta análise não possui execução retomável.', 409, 'ANALYSIS_NOT_RESUMABLE'); + try { + await axios.post(`${this.analyzerBaseUrl}/api/runs/${encodeURIComponent(analysis.run_id)}/${action}`, {}, { timeout: 15_000 }); + await this.syncAnalysisStatus(analysis.run_id); + return await this.repository.findById(id, projetoId); + } catch (error: unknown) { + const status = axios.isAxiosError(error) ? error.response?.status : undefined; + const detail = axios.isAxiosError(error) && typeof error.response?.data?.detail === 'string' + ? error.response.data.detail : 'serviço indisponível'; + if (status === 409) throw new AppError(detail, 409, 'ANALYSIS_CONTROL_CONFLICT'); + throw new AppError(`Falha ao ${action === 'pause' ? 'pausar' : action === 'resume' ? 'retomar' : 'cancelar'} análise: ${detail}`, 503, 'ANALYZER_UNAVAILABLE'); + } + } + private async syncAnalysisStatus(runId: string): Promise { try { const response = await axios.get(`${this.analyzerBaseUrl}/api/runs/${runId}`, { timeout: 15_000 }); @@ -101,13 +133,24 @@ export class RepoAnalysesService { status, etapa, etapaLabel: data.stage_label || etapa, - progresso: mapAnalyzerProgress(Number(data.stage_index), Number(data.stage_count), status), + progresso: mapAnalyzerProgress(Number(data.stage_index), Number(data.stage_count), status, stats), mensagem: data.message, erro: data.error, relatorioMarkdown, metadados: stats, }); } catch (error) { + if (axios.isAxiosError(error) && error.response?.status === 404) { + await this.repository.updateStatus(runId, { + status: 'falha', + etapa: 'error', + etapaLabel: 'Execução indisponível', + progresso: 0, + mensagem: 'O motor de análise não encontrou esta execução. Inicie uma nova análise para continuar.', + erro: 'Execução não encontrada no serviço de análise.', + }); + return; + } console.warn('[RepoAnalyzer] Falha ao sincronizar o status de uma análise; nova tentativa na próxima consulta.'); } } diff --git a/backend/src/modules/repo-analyses/repo-analyses.types.ts b/backend/src/modules/repo-analyses/repo-analyses.types.ts index 7f387ab..d628d32 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.types.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.types.ts @@ -1,4 +1,4 @@ -export type RepoAnalysisStatus = 'iniciado' | 'em_execucao' | 'concluido' | 'falha'; +export type RepoAnalysisStatus = 'iniciado' | 'em_execucao' | 'pausando' | 'pausada' | 'cancelando' | 'cancelada' | 'concluido' | 'falha'; export type RepoAnalysisStep = 'ollama' | 'clone' | 'scan' | 'files' | 'synthesis' | 'done' | 'error' | 'queued'; export interface RepoAnalysisRecord { @@ -24,4 +24,5 @@ export interface RepoAnalysisRecord { export interface StartAnalysisDTO { repositorio_url: string; + perfil?: 'quick' | 'balanced' | 'complete'; } diff --git a/database/init.sql b/database/init.sql index 3c01dec..c372611 100644 --- a/database/init.sql +++ b/database/init.sql @@ -202,7 +202,7 @@ CREATE TABLE IF NOT EXISTS analise_repositorio ( usuario_id UUID NOT NULL REFERENCES usuario(id) ON DELETE CASCADE, repositorio_url VARCHAR(500) NOT NULL, run_id VARCHAR(100), - status VARCHAR(50) NOT NULL DEFAULT 'iniciado', -- 'iniciado', 'em_execucao', 'concluido', 'falha' + status VARCHAR(50) NOT NULL DEFAULT 'iniciado', -- iniciado, em_execucao, pausando, pausada, cancelando, cancelada, concluido, falha etapa VARCHAR(50) DEFAULT 'queued', etapa_label VARCHAR(100) DEFAULT 'Na fila', progresso INT DEFAULT 0, diff --git a/database/migrations/007_repo_analysis.sql b/database/migrations/007_repo_analysis.sql index dd8ffae..87b5061 100644 --- a/database/migrations/007_repo_analysis.sql +++ b/database/migrations/007_repo_analysis.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS analise_repositorio ( usuario_id UUID NOT NULL REFERENCES usuario(id) ON DELETE CASCADE, repositorio_url VARCHAR(500) NOT NULL, run_id VARCHAR(100), - status VARCHAR(50) NOT NULL DEFAULT 'iniciado', -- 'iniciado', 'em_execucao', 'concluido', 'falha' + status VARCHAR(50) NOT NULL DEFAULT 'iniciado', -- iniciado, em_execucao, pausando, pausada, cancelando, cancelada, concluido, falha etapa VARCHAR(50) DEFAULT 'queued', etapa_label VARCHAR(100) DEFAULT 'Na fila', progresso INT DEFAULT 0, diff --git a/docker-compose.yml b/docker-compose.yml index 7a4c79a..9c2019a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,7 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-sinapse_dev_password} - POSTGRES_DB=${POSTGRES_DB:-sinapse} - AI_SERVICE_URL=http://ai-service:8000 + - REPO_ANALYZER_URL=${REPO_ANALYZER_URL:-http://ai-service:8000} - DOCUMENT_MAX_SIZE_MB=${DOCUMENT_MAX_SIZE_MB:-20} - DOCUMENT_STORAGE_DIR=/data/documents - DOCUMENT_EVENTS_WEBHOOK_URL=${DOCUMENT_EVENTS_WEBHOOK_URL:-} @@ -107,12 +108,20 @@ services: restart: unless-stopped ports: - "${AI_SERVICE_PORT:-8000}:8000" + volumes: + - repo_analysis_data:/app/workspace_analyzer environment: - AI_SERVICE_PORT=8000 - AI_SERVICE_HOST=0.0.0.0 - OLLAMA_BASE_URL=http://ollama:11434 - OLLAMA_LLM_MODEL=${OLLAMA_LLM_MODEL:-qwen2.5:1.5b} + - OLLAMA_NUM_PREDICT=${OLLAMA_NUM_PREDICT:-1024} + - OLLAMA_FILE_NUM_PREDICT=${OLLAMA_FILE_NUM_PREDICT:-512} + - OLLAMA_SYNTHESIS_NUM_PREDICT=${OLLAMA_SYNTHESIS_NUM_PREDICT:-3072} - OLLAMA_EMBEDDING_MODEL=${OLLAMA_EMBEDDING_MODEL:-bge-m3} + - MAX_FILES=${MAX_FILES:-1000} + - ANALYZER_QUICK_FILES=${ANALYZER_QUICK_FILES:-8} + - ANALYZER_BALANCED_FILES=${ANALYZER_BALANCED_FILES:-80} - POSTGRES_HOST=postgres - POSTGRES_PORT=5432 - POSTGRES_USER=${POSTGRES_USER:-sinapse} @@ -146,6 +155,8 @@ volumes: name: sinapse_ollama_data documents_data: name: sinapse_documents_data + repo_analysis_data: + name: sinapse_repo_analysis_data networks: sinapse-network: diff --git a/docs/Architecture/README.md b/docs/Architecture/README.md index 2444799..4edc1cc 100644 --- a/docs/Architecture/README.md +++ b/docs/Architecture/README.md @@ -72,6 +72,10 @@ O backend guarda conversas e mensagens por usuário, valida a posse da conversa O backend valida o projeto e a URL do GitHub, pede ao FastAPI para iniciar uma execução e persiste o identificador recebido. Consultas seguintes sincronizam estágio, progresso e relatório. O acesso ao projeto e o estado de arquivamento são revalidados nas rotas. +O escopo é escolhido na tela: **Rápida** (padrão) prioriza até 8 arquivos, **Equilibrada** até 80 e **Completa** analisa todos os arquivos elegíveis, respeitando o limite global configurado. A varredura ainda inventaria o repositório para relatar cobertura; somente os arquivos selecionados seguem para inferência local. README, manifestos de dependências e código de produção recebem prioridade sobre documentação extensa, exemplos e arquivos de lock. + +Os resumos de arquivos concluídos e os blocos já processados de arquivos grandes são salvos em `WORKSPACE_DIR/runs//checkpoint.json`. Pausar aguarda a chamada atual ao modelo e salva o checkpoint; retomar continua sem repetir blocos ou arquivos já concluídos. Esses checkpoints e os clones ficam no volume `repo_analysis_data` do serviço Python e sobrevivem à recriação do container. Cancelar encerra a execução em um ponto seguro e mantém o progresso salvo para consulta, mas não permite retomada. + ## Dados e evolução do schema - `database/init.sql` é o baseline aplicado quando um volume PostgreSQL é inicializado pela primeira vez. diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index df38dd8..e8d4862 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1681,6 +1681,11 @@ paths: required: [repositorio_url] properties: repositorio_url: {type: string, example: "https://github.com/usuario/repositorio"} + perfil: + type: string + enum: [quick, balanced, complete] + default: quick + description: Limite priorizado de arquivos para análise; quick e balanced reduzem chamadas ao modelo local. responses: '201': description: Análise iniciada @@ -1713,6 +1718,48 @@ paths: '404': description: Projeto ou análise não encontrados + /api/v1/projects/{projectId}/repo-analyses/{id}/pause: + post: + summary: Pausar após concluir o arquivo atual e salvar checkpoint + operationId: pauseRepoAnalysis + security: [{cookieAuth: []}] + parameters: + - {name: projectId, in: path, required: true, schema: {type: string, format: uuid}} + - {name: id, in: path, required: true, schema: {type: string, format: uuid}} + responses: + '200': {description: Análise pausada ou aguardando o ponto seguro de pausa} + '401': {$ref: '#/components/responses/Unauthorized'} + '404': {description: Projeto ou análise não encontrados} + '409': {description: Estado da análise não permite pausa} + + /api/v1/projects/{projectId}/repo-analyses/{id}/resume: + post: + summary: Retomar análise a partir do checkpoint salvo + operationId: resumeRepoAnalysis + security: [{cookieAuth: []}] + parameters: + - {name: projectId, in: path, required: true, schema: {type: string, format: uuid}} + - {name: id, in: path, required: true, schema: {type: string, format: uuid}} + responses: + '200': {description: Análise retomada} + '401': {$ref: '#/components/responses/Unauthorized'} + '404': {description: Projeto ou análise não encontrados} + '409': {description: Estado da análise não permite retomada} + + /api/v1/projects/{projectId}/repo-analyses/{id}/cancel: + post: + summary: Cancelar definitivamente a análise + operationId: cancelRepoAnalysis + security: [{cookieAuth: []}] + parameters: + - {name: projectId, in: path, required: true, schema: {type: string, format: uuid}} + - {name: id, in: path, required: true, schema: {type: string, format: uuid}} + responses: + '200': {description: Análise cancelada ou aguardando o ponto seguro de cancelamento} + '401': {$ref: '#/components/responses/Unauthorized'} + '404': {description: Projeto ou análise não encontrados} + '409': {description: Estado da análise não permite cancelamento} + /api/v1/search: get: summary: Busca textual no acervo indexado (chunks), com escopo opcional por projeto diff --git a/frontend/src/assets/styles/repo-analyzer.css b/frontend/src/assets/styles/repo-analyzer.css index 9062655..b9b376e 100644 --- a/frontend/src/assets/styles/repo-analyzer.css +++ b/frontend/src/assets/styles/repo-analyzer.css @@ -34,6 +34,7 @@ .analysis-stepper li[data-state="failed"] .analysis-step-marker { border-color: var(--red); background: color-mix(in srgb, var(--red) 16%, transparent); color: var(--red); } .analysis-progress { display: grid; gap: 8px; padding: 14px; background: var(--bg); border: 1px solid var(--line); border-radius: var(--r); } +.analysis-controls { display: flex; flex-wrap: wrap; gap: 8px; margin-top: 4px; } .analysis-progress-line { display: flex; justify-content: space-between; gap: 12px; font-size: 13px; font-weight: 600; } .analysis-languages { list-style: none; margin: 0; padding: 0; display: flex; flex-wrap: wrap; gap: 8px; } .analysis-languages li { padding: 3px 10px; border: 1px solid var(--line); border-radius: 999px; font-size: 12px; color: var(--muted); } diff --git a/frontend/src/models/repoAnalyzer.ts b/frontend/src/models/repoAnalyzer.ts index 0ae7b38..148882d 100644 --- a/frontend/src/models/repoAnalyzer.ts +++ b/frontend/src/models/repoAnalyzer.ts @@ -21,15 +21,19 @@ export function stageStates(analysis: Pick): S }); } -export const STATUS_VIEW: Record = { +export const STATUS_VIEW: Record = { iniciado: { label: "Na fila", tone: "info" }, em_execucao: { label: "Em execução", tone: "brand" }, + pausando: { label: "Pausando", tone: "warning" }, + pausada: { label: "Pausada", tone: "warning" }, + cancelando: { label: "Cancelando", tone: "warning" }, + cancelada: { label: "Cancelada", tone: "danger" }, concluido: { label: "Concluída", tone: "success" }, falha: { label: "Falhou", tone: "danger" }, }; export function isActive(analysis: Pick): boolean { - return analysis.status === "iniciado" || analysis.status === "em_execucao"; + return ["iniciado", "em_execucao", "pausando", "cancelando"].includes(analysis.status); } export function repositoryLabel(url: string): string { @@ -56,7 +60,8 @@ export function formatDuration(seconds: number): string { export function analysisDuration(analysis: Pick, now: number = Date.now()): number | null { const start = Date.parse(analysis.created_at); if (Number.isNaN(start)) return null; - const finishedAt = analysis.concluido_em ? Date.parse(analysis.concluido_em) : analysis.status === "falha" ? Date.parse(analysis.updated_at) : now; + const terminal = ["falha", "pausada", "cancelada"].includes(analysis.status); + const finishedAt = analysis.concluido_em ? Date.parse(analysis.concluido_em) : terminal ? Date.parse(analysis.updated_at) : now; if (Number.isNaN(finishedAt)) return null; return Math.max(0, (finishedAt - start) / 1000); } diff --git a/frontend/src/projects/repo-analyzer.api.test.ts b/frontend/src/projects/repo-analyzer.api.test.ts index 56ae534..71f8f7c 100644 --- a/frontend/src/projects/repo-analyzer.api.test.ts +++ b/frontend/src/projects/repo-analyzer.api.test.ts @@ -1,5 +1,5 @@ import { afterEach, expect, it, vi } from 'vitest'; -import { startRepoAnalysis } from './repo-analyzer.api'; +import { controlRepoAnalysis, startRepoAnalysis } from './repo-analyzer.api'; afterEach(() => vi.unstubAllGlobals()); @@ -13,5 +13,14 @@ it('envia a URL no campo repositorio_url esperado pela rota autenticada do backe const [url, options] = request.mock.calls[0]; expect(url).toBe('/api/v1/projects/project-1/repo-analyses'); expect(options?.method).toBe('POST'); - expect(JSON.parse(String(options?.body))).toEqual({ repositorio_url: 'https://github.com/owner/repo' }); + expect(JSON.parse(String(options?.body))).toEqual({ repositorio_url: 'https://github.com/owner/repo', perfil: 'quick' }); +}); + +it('envia controle de pausa para a rota autenticada da análise', async () => { + const request = vi.fn(async (_input: RequestInfo | URL, _init?: RequestInit) => new Response(JSON.stringify({ id: 'analysis-1', status: 'pausada' }), { status: 200 })); + vi.stubGlobal('fetch', request); + await controlRepoAnalysis('project-1', 'analysis-1', 'pause'); + const [url, options] = request.mock.calls[0]; + expect(url).toBe('/api/v1/projects/project-1/repo-analyses/analysis-1/pause'); + expect(options?.method).toBe('POST'); }); diff --git a/frontend/src/projects/repo-analyzer.api.ts b/frontend/src/projects/repo-analyzer.api.ts index bbc3279..22ef2a4 100644 --- a/frontend/src/projects/repo-analyzer.api.ts +++ b/frontend/src/projects/repo-analyzer.api.ts @@ -1,6 +1,8 @@ import { apiRequest } from "../api/api_auth"; -export type RepoAnalysisStatus = "iniciado" | "em_execucao" | "concluido" | "falha"; +export type RepoAnalysisStatus = "iniciado" | "em_execucao" | "pausando" | "pausada" | "cancelando" | "cancelada" | "concluido" | "falha"; +export type RepoAnalysisProfile = "quick" | "balanced" | "complete"; +export type RepoAnalysisAction = "pause" | "resume" | "cancel"; export interface RepoAnalysis { id: string; @@ -22,10 +24,19 @@ export interface RepoAnalysis { autor_email?: string | null; } -export async function startRepoAnalysis(projectId: string, repositoryUrl: string, signal?: AbortSignal): Promise { +export async function startRepoAnalysis(projectId: string, repositoryUrl: string, profile: RepoAnalysisProfile = "quick", signal?: AbortSignal): Promise { const response = await apiRequest(`/projects/${encodeURIComponent(projectId)}/repo-analyses`, { method: "POST", - body: JSON.stringify({ repositorio_url: repositoryUrl }), + body: JSON.stringify({ repositorio_url: repositoryUrl, perfil: profile }), + signal, + }); + return await response.json(); +} + +export async function controlRepoAnalysis(projectId: string, analysisId: string, action: RepoAnalysisAction, signal?: AbortSignal): Promise { + const response = await apiRequest(`/projects/${encodeURIComponent(projectId)}/repo-analyses/${encodeURIComponent(analysisId)}/${action}`, { + method: "POST", + body: JSON.stringify({}), signal, }); return await response.json(); diff --git a/frontend/src/views/projects/RepoAnalyzerView.test.tsx b/frontend/src/views/projects/RepoAnalyzerView.test.tsx index a6929ce..938320a 100644 --- a/frontend/src/views/projects/RepoAnalyzerView.test.tsx +++ b/frontend/src/views/projects/RepoAnalyzerView.test.tsx @@ -122,7 +122,7 @@ it("inicia a análise, seleciona a nova e exibe a mensagem do servidor quando re await waitFor(() => expect(screen.getByLabelText(/URL do repositório/)).toHaveValue("")); const [url, init] = request.mock.calls[2] as [string, RequestInit]; expect(url).toBe("/api/v1/projects/p-1/repo-analyses"); - expect(JSON.parse(String(init.body))).toEqual({ repositorio_url: "https://github.com/acme/novo" }); + expect(JSON.parse(String(init.body))).toEqual({ repositorio_url: "https://github.com/acme/novo", perfil: "quick" }); }); it("análise com falha mostra o motivo, permite repetir e alterna pelo histórico", async () => { @@ -146,7 +146,7 @@ it("análise com falha mostra o motivo, permite repetir e alterna pelo históric fireEvent.click(await screen.findByRole("button", { name: "Analisar novamente" })); await waitFor(() => expect(request).toHaveBeenCalledTimes(2)); const [, init] = request.mock.calls[1] as [string, RequestInit]; - expect(JSON.parse(String(init.body))).toEqual({ repositorio_url: "https://github.com/acme/api" }); + expect(JSON.parse(String(init.body))).toEqual({ repositorio_url: "https://github.com/acme/api", perfil: "quick" }); }); it("perfil sem permissão apenas consulta", async () => { @@ -155,3 +155,53 @@ it("perfil sem permissão apenas consulta", async () => { expect(await screen.findByText(/não pode iniciar novas/)).toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Iniciar análise" })).toBeNull(); }); + +it("envia perfil rápido ao iniciar uma análise", async () => { + const created = analysis({ status: "iniciado", etapa: "queued", etapa_label: "Na fila", progresso: 0 }); + const request = vi.fn() + .mockImplementationOnce(() => json([])) + .mockImplementationOnce(() => json(created, 201)) + .mockImplementation(() => json([created])); + vi.stubGlobal("fetch", request); + render(); + await screen.findByText("Nenhuma análise realizada neste projeto."); + + fireEvent.change(screen.getByDisplayValue("Rápida · até 8 arquivos prioritários"), { target: { value: "balanced" } }); + fireEvent.change(screen.getByLabelText(/URL do repositório/), { target: { value: "https://github.com/acme/api" } }); + fireEvent.click(screen.getByRole("button", { name: "Iniciar análise" })); + await waitFor(() => expect(request).toHaveBeenCalledTimes(3)); + const [, options] = request.mock.calls[1] as [string, RequestInit]; + expect(JSON.parse(String(options.body))).toEqual({ repositorio_url: "https://github.com/acme/api", perfil: "balanced" }); +}); + +it("pausa uma análise e oferece retomada pelo checkpoint", async () => { + const paused = analysis({ status: "pausada", etapa: "files", etapa_label: "Analisando arquivos", progresso: 42, mensagem: "Progresso salvo." }); + const request = vi.fn() + .mockImplementationOnce(() => json([analysis()])) + .mockImplementationOnce(() => json(paused)) + .mockImplementationOnce(() => json([paused])); + vi.stubGlobal("fetch", request); + render(); + await screen.findByRole("button", { name: "Pausar" }); + fireEvent.click(screen.getByRole("button", { name: "Pausar" })); + + expect(await screen.findByRole("button", { name: "Retomar análise" })).toBeInTheDocument(); + expect(screen.getAllByText("Pausada").length).toBeGreaterThan(0); + expect(request.mock.calls[1][0]).toBe("/api/v1/projects/p-1/repo-analyses/a-1/pause"); +}); + +it("pede confirmação antes de cancelar e registra o estado cancelado", async () => { + const cancelled = analysis({ status: "cancelada", etapa: "files", etapa_label: "Analisando arquivos", progresso: 30, mensagem: "Análise cancelada pelo usuário." }); + const request = vi.fn() + .mockImplementationOnce(() => json([analysis()])) + .mockImplementationOnce(() => json(cancelled)) + .mockImplementationOnce(() => json([cancelled])); + vi.stubGlobal("fetch", request); + vi.spyOn(window, "confirm").mockReturnValue(true); + render(); + fireEvent.click(await screen.findByRole("button", { name: "Cancelar análise" })); + + expect(window.confirm).toHaveBeenCalled(); + expect((await screen.findAllByText("Cancelada")).length).toBeGreaterThan(0); + expect(request.mock.calls[1][0]).toBe("/api/v1/projects/p-1/repo-analyses/a-1/cancel"); +}); diff --git a/frontend/src/views/projects/RepoAnalyzerView.tsx b/frontend/src/views/projects/RepoAnalyzerView.tsx index 042bf5b..e94e0c2 100644 --- a/frontend/src/views/projects/RepoAnalyzerView.tsx +++ b/frontend/src/views/projects/RepoAnalyzerView.tsx @@ -13,7 +13,7 @@ import { stageStates, validateRepositoryUrl, } from "../../models/repoAnalyzer"; -import { listRepoAnalyses, startRepoAnalysis, type RepoAnalysis } from "../../projects/repo-analyzer.api"; +import { controlRepoAnalysis, listRepoAnalyses, startRepoAnalysis, type RepoAnalysis, type RepoAnalysisAction, type RepoAnalysisProfile } from "../../projects/repo-analyzer.api"; import { Alert, Badge, Button, EmptyState, Field, Progress } from "../common/ui"; import { Markdown } from "../common/Markdown"; import "../../assets/styles/repo-analyzer.css"; @@ -63,11 +63,14 @@ export function RepoAnalyzerView({ projectId, canStart = true }: { projectId: st const [analyses, setAnalyses] = useState([]); const [selectedId, setSelectedId] = useState(null); const [repoUrl, setRepoUrl] = useState(""); + const [profile, setProfile] = useState("quick"); const [urlError, setUrlError] = useState(""); const [startError, setStartError] = useState(""); const [starting, setStarting] = useState(false); const [pollFailed, setPollFailed] = useState(false); const [refreshing, setRefreshing] = useState(false); + const [controlBusy, setControlBusy] = useState(false); + const [controlError, setControlError] = useState(""); const startingRef = useRef(false); const mounted = useRef(true); const requestId = useRef(0); @@ -110,13 +113,13 @@ export function RepoAnalyzerView({ projectId, canStart = true }: { projectId: st return () => window.clearInterval(timer); }, [hasActive, refresh]); - const launch = useCallback(async (url: string) => { + const launch = useCallback(async (url: string, selectedProfile: RepoAnalysisProfile = profile) => { if (startingRef.current) return; startingRef.current = true; setStarting(true); setStartError(""); try { - const created = await startRepoAnalysis(projectId, url); + const created = await startRepoAnalysis(projectId, url, selectedProfile); if (!mounted.current) return; setRepoUrl(""); setAnalyses((previous) => [created, ...previous.filter((item) => item.id !== created.id)]); @@ -129,7 +132,25 @@ export function RepoAnalyzerView({ projectId, canStart = true }: { projectId: st startingRef.current = false; if (mounted.current) setStarting(false); } - }, [projectId, refresh]); + }, [projectId, profile, refresh]); + + const control = useCallback(async (analysisId: string, action: RepoAnalysisAction) => { + if (controlBusy) return; + if (action === "cancel" && !window.confirm("Cancelar esta análise? O progresso salvo será mantido, mas ela não poderá ser retomada.")) return; + setControlBusy(true); + setControlError(""); + try { + const updated = await controlRepoAnalysis(projectId, analysisId, action); + if (mounted.current) { + setAnalyses((previous) => previous.map((item) => item.id === updated.id ? updated : item)); + void refresh(true); + } + } catch (error) { + if (mounted.current) setControlError(error instanceof Error ? error.message : "Não foi possível controlar a análise. Tente novamente."); + } finally { + if (mounted.current) setControlBusy(false); + } + }, [controlBusy, projectId, refresh]); const submit = (event: React.FormEvent) => { event.preventDefault(); @@ -168,6 +189,13 @@ export function RepoAnalyzerView({ projectId, canStart = true }: { projectId: st onChange={(event) => { setRepoUrl(event.target.value); setUrlError(""); setStartError(""); }} /> + + + ) : ( @@ -223,7 +251,16 @@ export function RepoAnalyzerView({ projectId, canStart = true }: { projectId: st
{selected ? ( - void launch(selected.repositorio_url)} /> + void launch(selected.repositorio_url)} + onControl={(action) => void control(selected.id, action)} + /> ) : ( void }) { +function AnalysisDetails({ analysis, now, canRetry, canControl, controlBusy, controlError, onRetry, onControl }: { + analysis: RepoAnalysis; + now: number; + canRetry: boolean; + canControl: boolean; + controlBusy: boolean; + controlError: string; + onRetry: () => void; + onControl: (action: RepoAnalysisAction) => void; +}) { const view = STATUS_VIEW[analysis.status] ?? STATUS_VIEW.iniciado; const active = isActive(analysis); const progress = analysis.progresso ?? 0; @@ -288,7 +334,7 @@ function AnalysisDetails({ analysis, now, canRetry, onRetry }: { analysis: RepoA ))} - {active && ( + {((active) || analysis.status === "pausada" || analysis.status === "cancelada") && (
{analysis.etapa_label || "Na fila"} @@ -299,11 +345,29 @@ function AnalysisDetails({ analysis, now, canRetry, onRetry }: { analysis: RepoA {stats.filesTotal !== null && stats.filesProcessed !== null && (

{stats.filesProcessed} de {stats.filesTotal} arquivos analisados + {typeof analysis.metadados?.files_candidates === "number" && analysis.metadados.files_candidates > stats.filesTotal + ? ` · ${analysis.metadados.files_candidates - stats.filesTotal} fora do escopo` : ""} {stats.etaSeconds ? ` · restante estimado ${formatDuration(stats.etaSeconds)}` : ""}

)} + {canControl && (active || analysis.status === "pausada") && ( +
+ {(analysis.status === "em_execucao" || analysis.status === "iniciado") && ( + + )} + {analysis.status === "pausada" && ( + + )} + {analysis.status !== "pausando" && analysis.status !== "cancelando" && ( + + )} +
+ )} + {analysis.status === "pausando" &&

A pausa será aplicada após a chamada atual ao modelo.

} + {analysis.status === "cancelando" &&

O cancelamento será aplicado após a chamada atual ao modelo.

}
)} + {controlError && {controlError}} {stats.languages.length > 0 && (
    From c7004cbf04b119cc4349daa66a0757474a7966bf Mon Sep 17 00:00:00 2001 From: LoadCG Date: Tue, 29 Sep 2026 14:56:47 -0300 Subject: [PATCH 2/9] test: relax analyzer report toggle timing --- frontend/src/views/projects/RepoAnalyzerView.test.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/views/projects/RepoAnalyzerView.test.tsx b/frontend/src/views/projects/RepoAnalyzerView.test.tsx index 938320a..3ba2f2e 100644 --- a/frontend/src/views/projects/RepoAnalyzerView.test.tsx +++ b/frontend/src/views/projects/RepoAnalyzerView.test.tsx @@ -58,7 +58,10 @@ it("alterna o relatório entre leitura e Markdown bruto", async () => { await screen.findByRole("heading", { name: "Título" }); fireEvent.click(screen.getByRole("button", { name: "Markdown" })); - await waitFor(() => expect(screen.getByRole("button", { name: "Markdown" })).toHaveAttribute("aria-pressed", "true")); + await waitFor( + () => expect(screen.getByRole("button", { name: "Markdown" })).toHaveAttribute("aria-pressed", "true"), + { timeout: 3000 }, + ); expect(screen.queryByRole("heading", { name: "Título" })).toBeNull(); expect(screen.getByText(/# Título/)).toBeInTheDocument(); expect(screen.getByRole("button", { name: "Markdown" })).toHaveAttribute("aria-pressed", "true"); From c7d4d982ede92a6dc8c5936050317ff0f70ef99d Mon Sep 17 00:00:00 2001 From: LoadCG Date: Tue, 29 Sep 2026 14:58:50 -0300 Subject: [PATCH 3/9] fix: retry truncated local model responses --- ai-service/analyzer/ollama_client.py | 13 ++++++++---- ai-service/tests/test_ollama_client.py | 29 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/ai-service/analyzer/ollama_client.py b/ai-service/analyzer/ollama_client.py index a16e738..4ffae47 100644 --- a/ai-service/analyzer/ollama_client.py +++ b/ai-service/analyzer/ollama_client.py @@ -75,7 +75,9 @@ def chat( } last_error: Exception | None = None - for attempt in range(self.max_retries + 1): + output_limit_retries = 0 + http_retries = 0 + while True: try: content_parts: list[str] = [] done_reason = "" @@ -94,8 +96,10 @@ def chat( done_reason = data.get("done_reason", "") break if done_reason == "length": - if attempt < self.max_retries and output_limit < 8192: + if output_limit_retries < 4 and output_limit < 8192: output_limit = min(8192, output_limit * 2) + output_limit_retries += 1 + http_retries = 0 payload["options"]["num_predict"] = output_limit continue raise OllamaError( @@ -106,8 +110,9 @@ def chat( break except httpx.HTTPError as exc: last_error = exc - if attempt < self.max_retries: - time.sleep(self.retry_backoff_seconds * (attempt + 1)) + if http_retries < self.max_retries: + http_retries += 1 + time.sleep(self.retry_backoff_seconds * http_retries) continue raise OllamaError( f"Não foi possível acessar o Ollama em {self.base_url} " diff --git a/ai-service/tests/test_ollama_client.py b/ai-service/tests/test_ollama_client.py index be48475..5a2190e 100644 --- a/ai-service/tests/test_ollama_client.py +++ b/ai-service/tests/test_ollama_client.py @@ -73,6 +73,35 @@ def stream(*_args, **kwargs): limits = [payload["options"]["num_predict"] for payload in payloads] self.assertEqual(limits, [128, 256]) + @patch("analyzer.ollama_client.httpx.Client") + def test_chat_keeps_expanding_truncated_output_until_safe_ceiling(self, client_factory): + client = client_factory.return_value + limits = [] + streams = [] + for index in range(5): + response = Mock() + if index < 4: + response.iter_lines.return_value = [ + '{"message":{"content":"parcial"},"done":true,"done_reason":"length"}' + ] + else: + response.iter_lines.return_value = [ + '{"message":{"content":"completa"},"done":true,"done_reason":"stop"}' + ] + stream = MagicMock() + stream.__enter__.return_value = response + streams.append(stream) + + def stream(*_args, **kwargs): + limits.append(kwargs["json"]["options"]["num_predict"]) + return streams.pop(0) + + client.stream.side_effect = stream + ollama = OllamaClient("http://ollama:11434", "qwen", 300, "5m", max_retries=0, num_predict=512) + + self.assertEqual(ollama.chat("sistema", "prompt"), "completa") + self.assertEqual(limits, [512, 1024, 2048, 4096, 8192]) + def test_chat_uses_safe_default_token_limit(self): with patch("analyzer.ollama_client.httpx.Client"): ollama = OllamaClient("http://ollama:11434", "qwen", 300, "5m") From 8e1d462cc91b137fe1a5998c600cb8d22ecf3519 Mon Sep 17 00:00:00 2001 From: LoadCG Date: Tue, 29 Sep 2026 15:27:05 -0300 Subject: [PATCH 4/9] fix: keep quick analyses resumable under local context limits --- .env.example | 1 + ai-service/analyzer/config.py | 1 + ai-service/analyzer/ollama_client.py | 8 +++++++ ai-service/analyzer/pipeline.py | 18 +++++++++++---- ai-service/tests/test_analyzer.py | 22 ++++++++++++++++++- ai-service/tests/test_ollama_client.py | 17 ++++++++++++++ docker-compose.yml | 1 + docs/Architecture/README.md | 4 ++-- .../views/projects/RepoAnalyzerView.test.tsx | 15 +++++++++++++ .../src/views/projects/RepoAnalyzerView.tsx | 12 +++++----- 10 files changed, 87 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index dc69fcf..b03007b 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,7 @@ OLLAMA_BASE_URL=http://localhost:11434 # Modelos recomendados no PRD (Qwen 2.5 / Llama 3.1 para LLM, bge-m3 para embeddings) OLLAMA_LLM_MODEL=qwen2.5:1.5b OLLAMA_NUM_PREDICT=1024 +OLLAMA_NUM_CTX=8192 OLLAMA_FILE_NUM_PREDICT=512 OLLAMA_SYNTHESIS_NUM_PREDICT=3072 OLLAMA_EMBEDDING_MODEL=bge-m3 diff --git a/ai-service/analyzer/config.py b/ai-service/analyzer/config.py index 052df38..6be0a0a 100644 --- a/ai-service/analyzer/config.py +++ b/ai-service/analyzer/config.py @@ -21,6 +21,7 @@ class AnalyzerSettings: request_timeout_seconds: int = int(os.getenv("REQUEST_TIMEOUT_SECONDS", "300")) ollama_keep_alive: str = os.getenv("OLLAMA_KEEP_ALIVE", "5m") ollama_num_predict: int = max(128, int(os.getenv("OLLAMA_NUM_PREDICT", "1024"))) + ollama_num_ctx: int = max(2048, int(os.getenv("OLLAMA_NUM_CTX", "8192"))) ollama_file_num_predict: int = max(128, int(os.getenv("OLLAMA_FILE_NUM_PREDICT", "512"))) ollama_synthesis_num_predict: int = max(128, int(os.getenv("OLLAMA_SYNTHESIS_NUM_PREDICT", "3072"))) ollama_think: bool = False diff --git a/ai-service/analyzer/ollama_client.py b/ai-service/analyzer/ollama_client.py index 4ffae47..53f67f2 100644 --- a/ai-service/analyzer/ollama_client.py +++ b/ai-service/analyzer/ollama_client.py @@ -33,6 +33,7 @@ def __init__( max_retries: int = 2, retry_backoff_seconds: float = 3.0, num_predict: int = 1024, + num_ctx: int = 8192, ): self.base_url = base_url.rstrip("/") self.model = model @@ -42,6 +43,7 @@ def __init__( self.max_retries = max(0, max_retries) self.retry_backoff_seconds = retry_backoff_seconds self.num_predict = max(128, num_predict) + self.num_ctx = max(2048, num_ctx) self._client = httpx.Client(timeout=timeout) def close(self) -> None: @@ -54,6 +56,7 @@ def chat( temperature: float = 0.1, should_cancel: Callable[[], None] | None = None, num_predict: int | None = None, + accept_truncated: bool = False, ) -> str: if not system.strip().startswith(("/nothink", "/no_think")): system = f"/nothink\n{system}" @@ -71,6 +74,7 @@ def chat( "options": { "temperature": temperature, "num_predict": output_limit, + "num_ctx": self.num_ctx, } } @@ -96,6 +100,10 @@ def chat( done_reason = data.get("done_reason", "") break if done_reason == "length": + if accept_truncated: + data = {"message": {"content": "".join(content_parts) + + "\n\n[Análise resumida limitada pelo teto de tokens; trate este trecho como parcial.]"}} + break if output_limit_retries < 4 and output_limit < 8192: output_limit = min(8192, output_limit * 2) output_limit_retries += 1 diff --git a/ai-service/analyzer/pipeline.py b/ai-service/analyzer/pipeline.py index c68967c..cd4f659 100644 --- a/ai-service/analyzer/pipeline.py +++ b/ai-service/analyzer/pipeline.py @@ -61,6 +61,7 @@ def __init__(self, settings: AnalyzerSettings): max_retries=settings.ollama_max_retries, retry_backoff_seconds=settings.ollama_retry_backoff_seconds, num_predict=settings.ollama_num_predict, + num_ctx=settings.ollama_num_ctx, ) self.runs: dict[str, RunState] = {} self.lock = threading.Lock() @@ -101,11 +102,12 @@ def pause(self, run_id: str) -> dict: def resume(self, run_id: str) -> dict: state = self._get_run(run_id) - if state.status != "paused": - raise AnalysisError("A análise não está pausada.") + has_progress = bool(state.completed_summaries or state.partial_chunk_summaries or state.project_summary) + if state.status != "paused" and not (state.status == "failed" and has_progress): + raise AnalysisError("A análise não está pausada ou não possui progresso recuperável.") state.pause_event.clear() state.cancel_event.clear() - self._push(run_id, status="queued", message="Retomando do último ponto salvo…") + self._push(run_id, status="queued", error="", message="Retomando do último ponto salvo…") self._start_worker(state) return self.status(run_id) @@ -271,7 +273,9 @@ def status(self, run_id: str) -> dict: "error": state.error, "report_path": state.report_path, "profile": state.profile, - "can_resume": state.status == "paused", + "can_resume": state.status == "paused" or ( + state.status == "failed" and bool(state.completed_summaries or state.partial_chunk_summaries or state.project_summary) + ), "can_cancel": state.status in {"queued", "running", "pausing", "paused"}, "stats": state.stats, } @@ -343,6 +347,9 @@ def _snapshot(self, state: RunState) -> dict: "files_selected": state.files_total, "files_skipped_by_scope": state.files_skipped_by_scope, "profile": state.profile, + "can_resume": state.status == "paused" or ( + state.status == "failed" and bool(state.completed_summaries or state.partial_chunk_summaries or state.project_summary) + ), "files_progress_percent": files_progress_percent, "language_counts": state.language_counts, "current_files": list(state.current_files), @@ -646,6 +653,7 @@ def _process_file(self, run_id: str, repo_dir: Path, info) -> str: ), should_cancel=lambda: self._check_control(run_id), num_predict=self.settings.ollama_file_num_predict, + accept_truncated=state.profile == "quick", ) self._note_llm_call(run_id) else: @@ -666,6 +674,7 @@ def _process_file(self, run_id: str, repo_dir: Path, info) -> str: prompt, should_cancel=lambda: self._check_control(run_id), num_predict=self.settings.ollama_file_num_predict, + accept_truncated=state.profile == "quick", )) self._note_llm_call(run_id) with self.lock: @@ -688,6 +697,7 @@ def _process_file(self, run_id: str, repo_dir: Path, info) -> str: ), should_cancel=lambda: self._check_control(run_id), num_predict=self.settings.ollama_file_num_predict, + accept_truncated=state.profile == "quick", ) self._note_llm_call(run_id) diff --git a/ai-service/tests/test_analyzer.py b/ai-service/tests/test_analyzer.py index a8fc1e8..61a54d9 100644 --- a/ai-service/tests/test_analyzer.py +++ b/ai-service/tests/test_analyzer.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch from pathlib import Path from tempfile import TemporaryDirectory from analyzer.pipeline import Analyzer, STAGE_KEYS, STAGE_LABELS @@ -107,6 +108,25 @@ def test_quick_profile_defaults_to_eight_priority_files(self): ] self.assertEqual(len(select_analysis_files(files, "quick")), 8) + def test_failed_analysis_with_saved_summaries_can_resume_without_repeating_them(self): + state = RunState( + run_id="failedresume1", url="https://github.com/acme/api", status="failed", + profile="quick", stage="files", files_total=2, files_processed=1, + selected_paths=["README.md", "src/app.py"], + completed_summaries={"README.md": "resumo salvo"}, + ) + self.analyzer.runs[state.run_id] = state + state.stats = self.analyzer._snapshot(state) + self.assertTrue(self.analyzer.status(state.run_id)["can_resume"]) + self.assertTrue(self.analyzer.status(state.run_id)["stats"]["can_resume"]) + + with patch.object(self.analyzer, "_start_worker") as start_worker: + result = self.analyzer.resume(state.run_id) + + self.assertEqual(result["status"], "queued") + self.assertEqual(self.analyzer.runs[state.run_id].error, "") + start_worker.assert_called_once_with(state) + def test_checkpoint_restores_running_analysis_as_resumable_and_keeps_finished_summaries(self): with TemporaryDirectory() as workspace: settings = AnalyzerSettings(workspace_dir=Path(workspace)) @@ -177,7 +197,7 @@ def __init__(self): def check(self): return None - def chat(self, _system, prompt, should_cancel=None, num_predict=None): + def chat(self, _system, prompt, should_cancel=None, num_predict=None, accept_truncated=False): self.calls.append(prompt) if self.pause_on_first_call: self.pause_on_first_call = False diff --git a/ai-service/tests/test_ollama_client.py b/ai-service/tests/test_ollama_client.py index 5a2190e..6983c72 100644 --- a/ai-service/tests/test_ollama_client.py +++ b/ai-service/tests/test_ollama_client.py @@ -22,6 +22,7 @@ def test_chat_caps_generated_tokens(self, client_factory): payload = client.stream.call_args.kwargs["json"] self.assertEqual(payload["options"]["num_predict"], 700) + self.assertEqual(payload["options"]["num_ctx"], 8192) self.assertTrue(payload["stream"]) @patch("analyzer.ollama_client.httpx.Client") @@ -102,6 +103,22 @@ def stream(*_args, **kwargs): self.assertEqual(ollama.chat("sistema", "prompt"), "completa") self.assertEqual(limits, [512, 1024, 2048, 4096, 8192]) + @patch("analyzer.ollama_client.httpx.Client") + def test_quick_file_summary_keeps_partial_output_instead_of_retrying_slowly(self, client_factory): + client = client_factory.return_value + response = Mock() + response.iter_lines.return_value = [ + '{"message":{"content":"Resumo útil"},"done":true,"done_reason":"length"}' + ] + client.stream.return_value.__enter__.return_value = response + ollama = OllamaClient("http://ollama:11434", "qwen", 300, "5m") + + result = ollama.chat("sistema", "prompt", num_predict=512, accept_truncated=True) + + self.assertIn("Resumo útil", result) + self.assertIn("trate este trecho como parcial", result) + self.assertEqual(client.stream.call_count, 1) + def test_chat_uses_safe_default_token_limit(self): with patch("analyzer.ollama_client.httpx.Client"): ollama = OllamaClient("http://ollama:11434", "qwen", 300, "5m") diff --git a/docker-compose.yml b/docker-compose.yml index 9c2019a..b6bfbf0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,6 +116,7 @@ services: - OLLAMA_BASE_URL=http://ollama:11434 - OLLAMA_LLM_MODEL=${OLLAMA_LLM_MODEL:-qwen2.5:1.5b} - OLLAMA_NUM_PREDICT=${OLLAMA_NUM_PREDICT:-1024} + - OLLAMA_NUM_CTX=${OLLAMA_NUM_CTX:-8192} - OLLAMA_FILE_NUM_PREDICT=${OLLAMA_FILE_NUM_PREDICT:-512} - OLLAMA_SYNTHESIS_NUM_PREDICT=${OLLAMA_SYNTHESIS_NUM_PREDICT:-3072} - OLLAMA_EMBEDDING_MODEL=${OLLAMA_EMBEDDING_MODEL:-bge-m3} diff --git a/docs/Architecture/README.md b/docs/Architecture/README.md index 4edc1cc..0d316f8 100644 --- a/docs/Architecture/README.md +++ b/docs/Architecture/README.md @@ -72,9 +72,9 @@ O backend guarda conversas e mensagens por usuário, valida a posse da conversa O backend valida o projeto e a URL do GitHub, pede ao FastAPI para iniciar uma execução e persiste o identificador recebido. Consultas seguintes sincronizam estágio, progresso e relatório. O acesso ao projeto e o estado de arquivamento são revalidados nas rotas. -O escopo é escolhido na tela: **Rápida** (padrão) prioriza até 8 arquivos, **Equilibrada** até 80 e **Completa** analisa todos os arquivos elegíveis, respeitando o limite global configurado. A varredura ainda inventaria o repositório para relatar cobertura; somente os arquivos selecionados seguem para inferência local. README, manifestos de dependências e código de produção recebem prioridade sobre documentação extensa, exemplos e arquivos de lock. +O escopo é escolhido na tela: **Rápida** (padrão) prioriza até 8 arquivos, **Equilibrada** até 80 e **Completa** analisa todos os arquivos elegíveis, respeitando o limite global configurado. A varredura ainda inventaria o repositório para relatar cobertura; somente os arquivos selecionados seguem para inferência local. README, manifestos de dependências e código de produção recebem prioridade sobre documentação extensa, exemplos e arquivos de lock. O perfil rápido usa contexto local de até 8.192 tokens e, quando um resumo individual atinge seu limite de saída, preserva o trecho gerado, identifica a limitação no relatório e continua a análise dos demais arquivos. -Os resumos de arquivos concluídos e os blocos já processados de arquivos grandes são salvos em `WORKSPACE_DIR/runs//checkpoint.json`. Pausar aguarda a chamada atual ao modelo e salva o checkpoint; retomar continua sem repetir blocos ou arquivos já concluídos. Esses checkpoints e os clones ficam no volume `repo_analysis_data` do serviço Python e sobrevivem à recriação do container. Cancelar encerra a execução em um ponto seguro e mantém o progresso salvo para consulta, mas não permite retomada. +Os resumos de arquivos concluídos e os blocos já processados de arquivos grandes são salvos em `WORKSPACE_DIR/runs//checkpoint.json`. Pausar aguarda a chamada atual ao modelo e salva o checkpoint; retomar continua sem repetir blocos ou arquivos já concluídos. Uma falha depois de algum progresso também permite continuar pelo checkpoint. Esses checkpoints e os clones ficam no volume `repo_analysis_data` do serviço Python e sobrevivem à recriação do container. Cancelar encerra a execução em um ponto seguro e mantém o progresso salvo para consulta, mas não permite retomada. ## Dados e evolução do schema diff --git a/frontend/src/views/projects/RepoAnalyzerView.test.tsx b/frontend/src/views/projects/RepoAnalyzerView.test.tsx index 3ba2f2e..38814af 100644 --- a/frontend/src/views/projects/RepoAnalyzerView.test.tsx +++ b/frontend/src/views/projects/RepoAnalyzerView.test.tsx @@ -193,6 +193,21 @@ it("pausa uma análise e oferece retomada pelo checkpoint", async () => { expect(request.mock.calls[1][0]).toBe("/api/v1/projects/p-1/repo-analyses/a-1/pause"); }); +it("permite continuar uma análise que falhou depois de salvar progresso", async () => { + const failed = analysis({ status: "falha", etapa: "error", erro: "Limite temporário de saída", metadados: { files_total: 8, files_processed: 5, can_resume: true } }); + const resumed = analysis({ status: "iniciado", etapa: "queued", mensagem: "Retomando do último ponto salvo…" }); + const request = vi.fn() + .mockImplementationOnce(() => json([failed])) + .mockImplementationOnce(() => json(resumed)) + .mockImplementationOnce(() => json([resumed])); + vi.stubGlobal("fetch", request); + render(); + + fireEvent.click(await screen.findByRole("button", { name: "Continuar com progresso salvo" })); + expect(request.mock.calls[1][0]).toBe("/api/v1/projects/p-1/repo-analyses/a-1/resume"); + expect(await screen.findByText("Retomando do último ponto salvo…")).toBeInTheDocument(); +}); + it("pede confirmação antes de cancelar e registra o estado cancelado", async () => { const cancelled = analysis({ status: "cancelada", etapa: "files", etapa_label: "Analisando arquivos", progresso: 30, mensagem: "Análise cancelada pelo usuário." }); const request = vi.fn() diff --git a/frontend/src/views/projects/RepoAnalyzerView.tsx b/frontend/src/views/projects/RepoAnalyzerView.tsx index e94e0c2..99d66d8 100644 --- a/frontend/src/views/projects/RepoAnalyzerView.tsx +++ b/frontend/src/views/projects/RepoAnalyzerView.tsx @@ -286,6 +286,7 @@ function AnalysisDetails({ analysis, now, canRetry, canControl, controlBusy, con }) { const view = STATUS_VIEW[analysis.status] ?? STATUS_VIEW.iniciado; const active = isActive(analysis); + const canResume = analysis.status === "pausada" || analysis.metadados?.can_resume === true; const progress = analysis.progresso ?? 0; const stages = stageStates(analysis); const stats = readStats(analysis.metadados); @@ -334,7 +335,7 @@ function AnalysisDetails({ analysis, now, canRetry, canControl, controlBusy, con ))} - {((active) || analysis.status === "pausada" || analysis.status === "cancelada") && ( + {((active) || canResume || analysis.status === "cancelada") && (
    {analysis.etapa_label || "Na fila"} @@ -350,21 +351,22 @@ function AnalysisDetails({ analysis, now, canRetry, canControl, controlBusy, con {stats.etaSeconds ? ` · restante estimado ${formatDuration(stats.etaSeconds)}` : ""}

    )} - {canControl && (active || analysis.status === "pausada") && ( + {canControl && (active || canResume) && (
    {(analysis.status === "em_execucao" || analysis.status === "iniciado") && ( )} - {analysis.status === "pausada" && ( - + {canResume && ( + )} - {analysis.status !== "pausando" && analysis.status !== "cancelando" && ( + {(active || analysis.status === "pausada") && analysis.status !== "pausando" && analysis.status !== "cancelando" && ( )}
    )} {analysis.status === "pausando" &&

    A pausa será aplicada após a chamada atual ao modelo.

    } {analysis.status === "cancelando" &&

    O cancelamento será aplicado após a chamada atual ao modelo.

    } + {analysis.status === "falha" && canResume &&

    O checkpoint preservou os arquivos já concluídos; você pode continuar sem analisá-los novamente.

    }
    )} {controlError && {controlError}} From a2a2e47f9a7d941a038b084668d4441e74b1ef79 Mon Sep 17 00:00:00 2001 From: LoadCG Date: Tue, 29 Sep 2026 15:41:02 -0300 Subject: [PATCH 5/9] fix: bound quick synthesis and resume progress estimates --- ai-service/analyzer/pipeline.py | 5 ++++- ai-service/tests/test_analyzer.py | 3 +++ docs/Architecture/README.md | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ai-service/analyzer/pipeline.py b/ai-service/analyzer/pipeline.py index cd4f659..937c034 100644 --- a/ai-service/analyzer/pipeline.py +++ b/ai-service/analyzer/pipeline.py @@ -481,7 +481,9 @@ def _run(self, run_id: str): state.language_counts = inventory["language_counts"] state.files_total = total state.files_processed = sum(1 for item in files if item.path in state.completed_summaries) - llm_calls_estimated = self._estimate_llm_calls([item for item in files if item.path not in state.completed_summaries]) + llm_calls_estimated = state.llm_calls_done + self._estimate_llm_calls( + [item for item in files if item.path not in state.completed_summaries] + ) self._push( run_id, @@ -537,6 +539,7 @@ def _run(self, run_id: str): ), should_cancel=lambda: self._check_control(run_id), num_predict=self.settings.ollama_synthesis_num_predict, + accept_truncated=state.profile == "quick", ) self._note_llm_call(run_id) with self.lock: diff --git a/ai-service/tests/test_analyzer.py b/ai-service/tests/test_analyzer.py index 61a54d9..bd55144 100644 --- a/ai-service/tests/test_analyzer.py +++ b/ai-service/tests/test_analyzer.py @@ -192,6 +192,7 @@ def clone(_url, destination): class FakeClient: def __init__(self): self.calls = [] + self.truncation_policies = [] self.pause_on_first_call = True def check(self): @@ -199,6 +200,7 @@ def check(self): def chat(self, _system, prompt, should_cancel=None, num_predict=None, accept_truncated=False): self.calls.append(prompt) + self.truncation_policies.append(accept_truncated) if self.pause_on_first_call: self.pause_on_first_call = False analyzer.pause(state.run_id) @@ -224,6 +226,7 @@ def close(self): self.assertFalse(state.worker_thread.is_alive()) self.assertEqual(analyzer.status(state.run_id)["status"], "completed") self.assertEqual(len(client.calls), 3) # Refaz a chamada interrompida e depois sintetiza o projeto. + self.assertEqual(client.truncation_policies, [True, True, True]) if __name__ == "__main__": diff --git a/docs/Architecture/README.md b/docs/Architecture/README.md index 0d316f8..b629e20 100644 --- a/docs/Architecture/README.md +++ b/docs/Architecture/README.md @@ -72,7 +72,7 @@ O backend guarda conversas e mensagens por usuário, valida a posse da conversa O backend valida o projeto e a URL do GitHub, pede ao FastAPI para iniciar uma execução e persiste o identificador recebido. Consultas seguintes sincronizam estágio, progresso e relatório. O acesso ao projeto e o estado de arquivamento são revalidados nas rotas. -O escopo é escolhido na tela: **Rápida** (padrão) prioriza até 8 arquivos, **Equilibrada** até 80 e **Completa** analisa todos os arquivos elegíveis, respeitando o limite global configurado. A varredura ainda inventaria o repositório para relatar cobertura; somente os arquivos selecionados seguem para inferência local. README, manifestos de dependências e código de produção recebem prioridade sobre documentação extensa, exemplos e arquivos de lock. O perfil rápido usa contexto local de até 8.192 tokens e, quando um resumo individual atinge seu limite de saída, preserva o trecho gerado, identifica a limitação no relatório e continua a análise dos demais arquivos. +O escopo é escolhido na tela: **Rápida** (padrão) prioriza até 8 arquivos, **Equilibrada** até 80 e **Completa** analisa todos os arquivos elegíveis, respeitando o limite global configurado. A varredura ainda inventaria o repositório para relatar cobertura; somente os arquivos selecionados seguem para inferência local. README, manifestos de dependências e código de produção recebem prioridade sobre documentação extensa, exemplos e arquivos de lock. O perfil rápido usa contexto local de até 8.192 tokens e, quando um resumo individual ou a síntese final atinge seu limite de saída, preserva o trecho gerado, identifica a limitação no relatório e continua a análise. Os resumos de arquivos concluídos e os blocos já processados de arquivos grandes são salvos em `WORKSPACE_DIR/runs//checkpoint.json`. Pausar aguarda a chamada atual ao modelo e salva o checkpoint; retomar continua sem repetir blocos ou arquivos já concluídos. Uma falha depois de algum progresso também permite continuar pelo checkpoint. Esses checkpoints e os clones ficam no volume `repo_analysis_data` do serviço Python e sobrevivem à recriação do container. Cancelar encerra a execução em um ponto seguro e mantém o progresso salvo para consulta, mas não permite retomada. From 07523f70feaac8c7f664b9ceee303ea54706a54d Mon Sep 17 00:00:00 2001 From: LoadCG Date: Tue, 29 Sep 2026 15:51:38 -0300 Subject: [PATCH 6/9] fix: ground quick reports in repository evidence --- ai-service/analyzer/pipeline.py | 10 +++++++++- ai-service/analyzer/prompts.py | 4 ++-- ai-service/analyzer/scanner.py | 2 ++ ai-service/tests/test_analyzer.py | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/ai-service/analyzer/pipeline.py b/ai-service/analyzer/pipeline.py index 937c034..475636d 100644 --- a/ai-service/analyzer/pipeline.py +++ b/ai-service/analyzer/pipeline.py @@ -639,7 +639,15 @@ def _process_file(self, run_id: str, repo_dir: Path, info) -> str: succeeded = False try: path = repo_dir / info.path - text = path.read_text(encoding="utf-8", errors="replace") + raw = path.read_bytes() + if raw.startswith((b"\x00\x00\xfe\xff", b"\xff\xfe\x00\x00")): + text = raw.decode("utf-32", errors="replace") + elif raw.startswith((b"\xff\xfe", b"\xfe\xff")): + text = raw.decode("utf-16", errors="replace") + elif raw.startswith(b"\xef\xbb\xbf"): + text = raw.decode("utf-8-sig", errors="replace") + else: + text = raw.decode("utf-8", errors="replace") chunks = self._chunks(text, self.settings.max_chunk_chars) symbols_json = json.dumps(info.symbols, ensure_ascii=False) state = self.runs[run_id] diff --git a/ai-service/analyzer/prompts.py b/ai-service/analyzer/prompts.py index 2d8b9c8..fd67edf 100644 --- a/ai-service/analyzer/prompts.py +++ b/ai-service/analyzer/prompts.py @@ -142,13 +142,13 @@ def project_synthesis_prompt(inventory, file_summaries, compact=False): Escreva uma síntese concisa em Markdown, com estas seções e nesta ordem: # Visão Geral # Objetivo Inferido -# Stack Tecnológica + # Stack Tecnológica # Arquitetura e Estrutura # Funcionalidades Observadas # Qualidade e Possíveis Problemas # Limitações da Análise -Use no máximo 2 frases por seção e até 700 palavras no total. Não repita listas de arquivos nem invente fatos. Separe fatos observados de recomendações. Quando faltar evidência, diga isso claramente.""" +Use até 2 frases por seção e até 350 palavras no total. Para cada tecnologia, cite o arquivo que comprova seu uso ou declaração; imports padrão da linguagem/framework não são dependências separadas. Liste dependências externas somente quando aparecerem no inventário/manifests fornecidos. Para cada possível problema, cite arquivo e trecho/função que o demonstra e descreva a condição de falha; se não houver evidência suficiente, não inclua o item. Não transforme hipóteses em funcionalidades, não repita itens e não invente bibliotecas. Separe fatos observados de inferências e recomendações. Quando faltar evidência, escreva “não determinado pelas evidências disponíveis”.""" return f"""Você é o analista principal do repositório. diff --git a/ai-service/analyzer/scanner.py b/ai-service/analyzer/scanner.py index 07c3306..d7a70ba 100644 --- a/ai-service/analyzer/scanner.py +++ b/ai-service/analyzer/scanner.py @@ -110,6 +110,8 @@ def is_probably_binary(path: Path) -> bool: try: with open(path, "rb") as handle: data = handle.read(4096) + if data.startswith((b"\xff\xfe", b"\xfe\xff", b"\x00\x00\xfe\xff", b"\xff\xfe\x00\x00")): + return False return b"\x00" in data except OSError: return True diff --git a/ai-service/tests/test_analyzer.py b/ai-service/tests/test_analyzer.py index bd55144..2743c9b 100644 --- a/ai-service/tests/test_analyzer.py +++ b/ai-service/tests/test_analyzer.py @@ -6,6 +6,7 @@ from analyzer.config import AnalyzerSettings from analyzer.scanner import detect_language, python_symbols, is_probably_binary, select_analysis_files from analyzer.models import FileInfo, RunState +from analyzer.prompts import project_synthesis_prompt class TestAnalyzer(unittest.TestCase): @@ -44,6 +45,12 @@ def test_detect_language(self): self.assertEqual(detect_language(Path("Dockerfile")), "Dockerfile") self.assertEqual(detect_language(Path("unknown.xyz")), "Unknown") + def test_utf16_text_manifest_is_not_misclassified_as_binary(self): + with TemporaryDirectory() as directory: + manifest = Path(directory) / "requirements.txt" + manifest.write_text("Flask==3.1.0\n", encoding="utf-16") + self.assertFalse(is_probably_binary(manifest)) + def test_python_symbols_extraction(self): code = """ import os @@ -87,6 +94,13 @@ def test_stages_consistency(self): for key in STAGE_KEYS: self.assertIn(key, STAGE_LABELS) + def test_quick_project_synthesis_requires_evidence_for_stack_and_findings(self): + prompt = project_synthesis_prompt("manifest: requirements.txt", "src/app.py: imports flask", compact=True) + self.assertIn("até 350 palavras", prompt) + self.assertIn("cite o arquivo", prompt) + self.assertIn("não são dependências separadas", prompt) + self.assertIn("não invente bibliotecas", prompt) + def test_analysis_profiles_prioritize_core_files_and_bound_llm_scope(self): files = [ FileInfo("src/service.py", 100, ".py", "Python", "source/config"), From 1dfbc1418f0703be1f5a7ceff4485bebf14f0678 Mon Sep 17 00:00:00 2001 From: Moretto <72614882+Giomoret@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:20:50 -0300 Subject: [PATCH 7/9] Corrige concorrencia no chat, documentos e analises e protege configuracao do banco Isola envios por conversa e recupera respostas ao retornar ao historico. Agenda atualizacao dos documentos apos reprocessamento durante uma consulta em andamento. Adiciona revisao de estado para rejeitar respostas antigas do RepoAnalyzer. Prioriza DATABASE_URL e verifica o banco efetivamente conectado antes da validacao S1. Alinha o perfil padrao de novas contas a dev nas camadas repository e SQL. Inclui migracoes 017 e 018 e testes de regressao. Validacao anterior: build frontend, typecheck backend, 213 testes frontend, 281 backend e 5 verificacoes adicionais; 21 testes dependentes de banco pulados na suite geral. Migracoes verificadas em banco descartavel. --- backend/scripts/validate-s1.mts | 10 +++-- backend/src/database/db.ts | 12 +----- backend/src/database/pool-config.ts | 25 +++++++++++ .../src/database/validation-target.test.ts | 15 +++++++ backend/src/database/validation-target.ts | 5 +++ backend/src/modules/auth/auth.repository.ts | 4 +- .../repo-analyses.dispatch.db.test.ts | 11 +++++ .../repo-analyses/repo-analyses.repository.ts | 6 ++- .../repo-analyses/repo-analyses.service.ts | 15 +++---- .../repo-analyses/repo-analyses.types.ts | 1 + database/migrations/017_safe_user_role.sql | 2 + database/migrations/018_analysis_revision.sql | 9 ++++ frontend/src/views/chat/ChatView.test.tsx | 35 +++++++++++++++- frontend/src/views/chat/ChatView.tsx | 41 ++++++++++++++----- .../views/documents/DocumentsView.test.tsx | 20 +++++++++ .../src/views/documents/DocumentsView.tsx | 8 +++- 16 files changed, 183 insertions(+), 36 deletions(-) create mode 100644 backend/src/database/pool-config.ts create mode 100644 backend/src/database/validation-target.test.ts create mode 100644 backend/src/database/validation-target.ts create mode 100644 database/migrations/017_safe_user_role.sql create mode 100644 database/migrations/018_analysis_revision.sql diff --git a/backend/scripts/validate-s1.mts b/backend/scripts/validate-s1.mts index 9476671..ad31e80 100644 --- a/backend/scripts/validate-s1.mts +++ b/backend/scripts/validate-s1.mts @@ -5,12 +5,16 @@ import { once } from "node:events"; import { app } from "../src/index.js"; import { pool } from "../src/database/db.js"; import { env } from "../src/config/env.js"; +import { assertValidationDatabase } from "../src/database/validation-target.js"; import { hashPassword } from "../src/modules/auth/pssword.service.js"; -import { apiRequest, readUser } from "../../frontend/src/auth/api.ts"; -import { createProject, getProject, listProjects } from "../../frontend/src/projects/api.ts"; +import { apiRequest, readUser } from "../../frontend/src/api/api_auth.ts"; +import { createProject, getProject, listProjects } from "../../frontend/src/api/api_projects.ts"; assert.equal(env.NODE_ENV, "test", "Execute com NODE_ENV=test"); -assert.match(env.POSTGRES_DB, /_s1_validation$/, "Use um banco descartável terminado em _s1_validation"); +// Check the connected server, including DATABASE_URL and PostgreSQL defaults. +const target = await pool.query('SELECT current_database() AS database'); +try { assertValidationDatabase(target.rows[0]?.database); } +catch (error) { await pool.end(); throw error; } const server = app.listen(0, "127.0.0.1"); await once(server, "listening"); const address = server.address(); diff --git a/backend/src/database/db.ts b/backend/src/database/db.ts index 93cf005..dd5f9fc 100644 --- a/backend/src/database/db.ts +++ b/backend/src/database/db.ts @@ -1,16 +1,8 @@ import { Pool, type PoolClient } from "pg"; import { env } from "../config/env.js"; +import { databasePoolConfig } from './pool-config.js'; -export const pool = new Pool({ - host: env.POSTGRES_HOST, - port: env.POSTGRES_PORT, - user: env.POSTGRES_USER, - password: env.POSTGRES_PASSWORD, - database: env.POSTGRES_DB, - max: 20, - idleTimeoutMillis: 30000, - connectionTimeoutMillis: 2000, -}); +export const pool = new Pool(databasePoolConfig(env)); export async function checkDatabaseConnection(db: Pick = pool): Promise { let client: PoolClient | undefined; diff --git a/backend/src/database/pool-config.ts b/backend/src/database/pool-config.ts new file mode 100644 index 0000000..06842e5 --- /dev/null +++ b/backend/src/database/pool-config.ts @@ -0,0 +1,25 @@ +import type { PoolConfig } from 'pg'; + +interface DatabaseEnvironment { + DATABASE_URL?: string; + POSTGRES_HOST: string; + POSTGRES_PORT: number; + POSTGRES_USER: string; + POSTGRES_PASSWORD: string; + POSTGRES_DB: string; +} + +export function databasePoolConfig(env: DatabaseEnvironment): PoolConfig { + return { + ...(env.DATABASE_URL ? { connectionString: env.DATABASE_URL } : { + host: env.POSTGRES_HOST, + port: env.POSTGRES_PORT, + user: env.POSTGRES_USER, + password: env.POSTGRES_PASSWORD, + database: env.POSTGRES_DB, + }), + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 2000, + }; +} diff --git a/backend/src/database/validation-target.test.ts b/backend/src/database/validation-target.test.ts new file mode 100644 index 0000000..015849b --- /dev/null +++ b/backend/src/database/validation-target.test.ts @@ -0,0 +1,15 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { assertValidationDatabase } from './validation-target'; +import { databasePoolConfig } from './pool-config'; + +test('validação recusa o banco efetivo mesmo com POSTGRES_DB de teste', () => { + const config = databasePoolConfig({ DATABASE_URL: 'postgresql://localhost/production', + POSTGRES_DB: 'sinapse_s1_validation', POSTGRES_HOST: 'localhost', POSTGRES_PORT: 5432, + POSTGRES_USER: 'test', POSTGRES_PASSWORD: 'test' }); + assert.equal(config.connectionString, 'postgresql://localhost/production'); + assert.equal(config.database, undefined); + assert.throws(() => assertValidationDatabase('production')); + assert.throws(() => assertValidationDatabase(undefined)); + assert.doesNotThrow(() => assertValidationDatabase('sinapse_s1_validation')); +}); diff --git a/backend/src/database/validation-target.ts b/backend/src/database/validation-target.ts new file mode 100644 index 0000000..986d6ec --- /dev/null +++ b/backend/src/database/validation-target.ts @@ -0,0 +1,5 @@ +export function assertValidationDatabase(database: unknown): void { + if (typeof database !== 'string' || !/^[a-zA-Z0-9_-]+_s1_validation$/.test(database)) { + throw new Error('Use um banco descartável terminado em _s1_validation'); + } +} diff --git a/backend/src/modules/auth/auth.repository.ts b/backend/src/modules/auth/auth.repository.ts index 975b185..7f9f379 100644 --- a/backend/src/modules/auth/auth.repository.ts +++ b/backend/src/modules/auth/auth.repository.ts @@ -42,7 +42,7 @@ export class AuthRepository { created_at, updated_at `, - [data.nome.trim(), data.email.trim().toLowerCase(), data.senha_hash, data.role ?? "po"], + [data.nome.trim(), data.email.trim().toLowerCase(), data.senha_hash, data.role ?? "dev"], ); return result.rows[0]; @@ -199,4 +199,4 @@ export class AuthRepository { } } -export const authRepository = new AuthRepository(); \ No newline at end of file +export const authRepository = new AuthRepository(); diff --git a/backend/src/modules/repo-analyses/repo-analyses.dispatch.db.test.ts b/backend/src/modules/repo-analyses/repo-analyses.dispatch.db.test.ts index edd432c..dda0be2 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.dispatch.db.test.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.dispatch.db.test.ts @@ -29,6 +29,17 @@ test('solicitação idempotente, lease exclusiva e bloqueio com arquivamento', { const resumed = await repository.queueResume(a.id, project); assert.equal(resumed?.resume_requested,true); assert.equal(resumed?.dispatch_pending,true); + const stale = { status: 'falha', etapa: 'error', etapaLabel: 'Resposta antiga', progresso: 0 }; + await repository.updateStatus(a.run_id, stale, a.revision ?? 0); + assert.equal((await repository.findById(a.id,project))?.status,'iniciado'); + // Even after dispatch finishes, an earlier HTTP response cannot overwrite the resumed run. + await db.query('UPDATE analise_repositorio SET dispatch_pending=FALSE WHERE id=$1',[a.id]); + await repository.updateStatus(a.run_id, stale, a.revision ?? 0); + assert.equal((await repository.findById(a.id,project))?.status,'iniciado'); + const fresh = (await repository.findById(a.id,project))!; + await repository.updateStatus(a.run_id, { ...stale, status: 'em_execucao' }, fresh.revision!); + assert.equal((await repository.findById(a.id,project))?.status,'em_execucao'); + await db.query('UPDATE analise_repositorio SET dispatch_pending=TRUE WHERE id=$1',[a.id]); await assert.rejects(new ProjectArchiveRepository(db).archive(project), /cancele/); const cancelling = await repository.cancelPending(a.id,project); assert.equal(cancelling?.status,'cancelando'); diff --git a/backend/src/modules/repo-analyses/repo-analyses.repository.ts b/backend/src/modules/repo-analyses/repo-analyses.repository.ts index 56cd142..76a0926 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.repository.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.repository.ts @@ -123,7 +123,8 @@ export class RepoAnalysesRepository { erro?: string; relatorioMarkdown?: string; metadados?: any; - } + }, + expectedRevision: number, ): Promise { const query = ` UPDATE analise_repositorio @@ -137,7 +138,7 @@ export class RepoAnalysesRepository { metadados = COALESCE($8::jsonb, metadados), updated_at = NOW(), concluido_em = CASE WHEN $1::varchar IN ('concluido', 'falha', 'cancelada') THEN COALESCE(concluido_em, NOW()) ELSE concluido_em END - WHERE run_id = $9::varchar; + WHERE run_id = $9::varchar AND revision = $10 AND NOT dispatch_pending; `; await this.db.query(query, [ statusData.status, @@ -149,6 +150,7 @@ export class RepoAnalysesRepository { statusData.relatorioMarkdown || null, statusData.metadados ? JSON.stringify(statusData.metadados) : null, runId, + expectedRevision, ]); } } diff --git a/backend/src/modules/repo-analyses/repo-analyses.service.ts b/backend/src/modules/repo-analyses/repo-analyses.service.ts index c426f62..ae0943f 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.service.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.service.ts @@ -82,7 +82,7 @@ export class RepoAnalysesService { code !== undefined && [400, 409, 422].includes(code)); } } - for (const analysis of await this.repository.findSyncable()) await this.syncAnalysisStatus(analysis.run_id); + for (const analysis of await this.repository.findSyncable()) await this.syncAnalysisStatus(analysis); } private needsSync(analysis: RepoAnalysisRecord): boolean { @@ -96,7 +96,7 @@ export class RepoAnalysesService { // Opcional: Atualizar status das análises em andamento consultando o ai-service await Promise.all(analyses .filter((analysis) => this.needsSync(analysis)) - .map((analysis) => this.syncAnalysisStatus(analysis.run_id))); + .map((analysis) => this.syncAnalysisStatus(analysis))); return await this.repository.findByProjectId(projetoId); } @@ -104,7 +104,7 @@ export class RepoAnalysesService { async getById(projetoId: string, id: string): Promise { const analysis = await this.repository.findById(id, projetoId); if (analysis && this.needsSync(analysis)) { - await this.syncAnalysisStatus(analysis.run_id); + await this.syncAnalysisStatus(analysis); return await this.repository.findById(id, projetoId); } return analysis; @@ -121,7 +121,7 @@ export class RepoAnalysesService { if (!analysis.run_id) throw new AppError('Esta análise não possui execução retomável.', 409, 'ANALYSIS_NOT_RESUMABLE'); try { await axios.post(`${this.analyzerBaseUrl}/api/runs/${encodeURIComponent(analysis.run_id)}/${action}`, {}, { headers: serviceHeaders(), timeout: 15_000 }); - await this.syncAnalysisStatus(analysis.run_id); + await this.syncAnalysisStatus(analysis); return await this.repository.findById(id, projetoId); } catch (error: unknown) { const status = axios.isAxiosError(error) ? error.response?.status : undefined; @@ -132,7 +132,8 @@ export class RepoAnalysesService { } } - private async syncAnalysisStatus(runId: string): Promise { + private async syncAnalysisStatus(analysis: RepoAnalysisRecord): Promise { + const runId = analysis.run_id; try { const response = await axios.get(`${this.analyzerBaseUrl}/api/runs/${runId}`, { headers: serviceHeaders(), timeout: 15_000 }); const data = response.data; @@ -160,7 +161,7 @@ export class RepoAnalysesService { erro: data.error, relatorioMarkdown, metadados: stats, - }); + }, analysis.revision ?? 0); } catch (error) { if (axios.isAxiosError(error) && error.response?.status === 404) { await this.repository.updateStatus(runId, { @@ -170,7 +171,7 @@ export class RepoAnalysesService { progresso: 0, mensagem: 'O motor de análise não encontrou esta execução. Inicie uma nova análise para continuar.', erro: 'Execução não encontrada no serviço de análise.', - }); + }, analysis.revision ?? 0); return; } console.warn('[RepoAnalyzer] Falha ao sincronizar o status de uma análise; nova tentativa na próxima consulta.'); diff --git a/backend/src/modules/repo-analyses/repo-analyses.types.ts b/backend/src/modules/repo-analyses/repo-analyses.types.ts index f51002d..910a98b 100644 --- a/backend/src/modules/repo-analyses/repo-analyses.types.ts +++ b/backend/src/modules/repo-analyses/repo-analyses.types.ts @@ -2,6 +2,7 @@ export type RepoAnalysisStatus = 'iniciado' | 'em_execucao' | 'pausando' | 'paus export type RepoAnalysisStep = 'ollama' | 'clone' | 'scan' | 'files' | 'synthesis' | 'done' | 'error' | 'queued'; export interface RepoAnalysisRecord { + revision?: number; id: string; projeto_id: string; usuario_id: string; diff --git a/database/migrations/017_safe_user_role.sql b/database/migrations/017_safe_user_role.sql new file mode 100644 index 0000000..950a61c --- /dev/null +++ b/database/migrations/017_safe_user_role.sql @@ -0,0 +1,2 @@ +-- Preserve existing accounts; only omitted roles in new accounts default to dev. +ALTER TABLE usuario ALTER COLUMN role SET DEFAULT 'dev'; diff --git a/database/migrations/018_analysis_revision.sql b/database/migrations/018_analysis_revision.sql new file mode 100644 index 0000000..093fbc3 --- /dev/null +++ b/database/migrations/018_analysis_revision.sql @@ -0,0 +1,9 @@ +ALTER TABLE analise_repositorio ADD COLUMN revision INTEGER NOT NULL DEFAULT 0; +CREATE FUNCTION increment_analysis_revision() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + NEW.revision := OLD.revision + 1; + RETURN NEW; +END; +$$; +CREATE TRIGGER analysis_revision BEFORE UPDATE ON analise_repositorio +FOR EACH ROW EXECUTE FUNCTION increment_analysis_revision(); diff --git a/frontend/src/views/chat/ChatView.test.tsx b/frontend/src/views/chat/ChatView.test.tsx index b52b46d..d143259 100644 --- a/frontend/src/views/chat/ChatView.test.tsx +++ b/frontend/src/views/chat/ChatView.test.tsx @@ -1,6 +1,6 @@ // @vitest-environment jsdom import { afterEach, expect, it, vi } from "vitest"; -import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { ChatView } from "./ChatView"; const PROJECT = { id: "p-1", nome: "Sinapse", cliente: "C", descricao: "", status: "ativo" }; @@ -241,3 +241,36 @@ it("copia a resposta do assistente", async () => { await waitFor(() => expect(writeText).toHaveBeenCalledWith("Resposta **importante**")); expect(await screen.findByRole("button", { name: "Copiado" })).toBeInTheDocument(); }); + +it("permite enviar em B enquanto A responde e recupera A ao retornar", async () => { + let finish!: (response: Response) => void; + const routes: Routes = { + conversations: [conversation(), conversation({ id: 'c-2', titulo: 'Conversa B' })], + messages: { 'c-1': [message()], 'c-2': [] }, + query: body => body.conversa_id === 'c-1' + ? new Promise(resolve => { finish = resolve; }) + : json({ conversa_id: 'c-2', resposta: 'Resposta B', fontes: [], origem: 'assistente' }), + }; + const request = api(routes); + vi.stubGlobal('fetch', request); + render(); + await screen.findByText('importante'); + fireEvent.change(composer(), { target: { value: 'Pergunta A' } }); + fireEvent.click(screen.getByRole('button', { name: 'Enviar' })); + await waitFor(() => expect(finish).toBeDefined()); + fireEvent.click(screen.getByRole('button', { name: /Conversa B/ })); + expect(composer()).toBeEnabled(); + fireEvent.change(composer(), { target: { value: 'Pergunta B' } }); + fireEvent.click(screen.getByRole('button', { name: 'Enviar' })); + await screen.findByText('Resposta B'); + fireEvent.click(screen.getByRole('button', { name: /Decisões do login/ })); + await screen.findByText('importante'); + expect(composer()).toBeDisabled(); + routes.messages!['c-1'] = [message({ conteudo: 'Resposta A concluída' })]; + await act(async () => { + finish(await json({ conversa_id: 'c-1', resposta: 'Resposta A concluída', fontes: [], origem: 'assistente' })); + }); + expect(await screen.findByText('Resposta A concluída')).toBeInTheDocument(); + expect(screen.queryByText('Resposta B')).not.toBeInTheDocument(); + expect(composer()).toBeEnabled(); +}); diff --git a/frontend/src/views/chat/ChatView.tsx b/frontend/src/views/chat/ChatView.tsx index 08b7448..bb78db6 100644 --- a/frontend/src/views/chat/ChatView.tsx +++ b/frontend/src/views/chat/ChatView.tsx @@ -36,15 +36,20 @@ export function ChatView() { const [messages, setMessages] = useState([]); const [messagesState, setMessagesState] = useState("idle"); const [draft, setDraft] = useState(""); - const [sending, setSending] = useState(false); + const [pendingKeys, setPendingKeys] = useState([]); const [sendError, setSendError] = useState(""); const [copiedId, setCopiedId] = useState(null); const bottom = useRef(null); const composer = useRef(null); const mounted = useRef(true); const messagesRequest = useRef(0); - const sendLock = useRef(false); + const sendLocks = useRef(new Set()); const skipReload = useRef(null); + const conversationRevision = useRef(0); + const selectedConversation = useRef(activeId); + selectedConversation.current = activeId; + const sendKey = activeId ?? `draft-${conversationRevision.current}`; + const sending = pendingKeys.includes(sendKey); useEffect(() => { mounted.current = true; @@ -113,6 +118,8 @@ export function ChatView() { const scopeLocked = Boolean(activeId); const startNewConversation = () => { + conversationRevision.current += 1; + messagesRequest.current += 1; setActiveId(null); setMessages([]); setSendError(""); @@ -121,6 +128,11 @@ export function ChatView() { }; const openConversation = (conversation: ChatConversation) => { + if (conversation.id === activeId) return; + conversationRevision.current += 1; + messagesRequest.current += 1; + setMessages([]); + setDraft(""); setSendError(""); setActiveId(conversation.id); setScopeId(conversation.projeto_id ?? ""); @@ -128,9 +140,10 @@ export function ChatView() { const send = useCallback(async (text: string, retryId?: string) => { const question = text.trim(); - if (!question || sendLock.current || !scopeId) return; - sendLock.current = true; - setSending(true); + if (!question || sendLocks.current.has(sendKey) || !scopeId) return; + sendLocks.current.add(sendKey); + const revision = conversationRevision.current; + setPendingKeys(current => [...current, sendKey]); setSendError(""); const pendingId = retryId ?? `local-${Date.now()}`; setMessages((current) => retryId @@ -140,6 +153,11 @@ export function ChatView() { try { const answer = await askQuestion({ pergunta: question, conversaId: activeId, projetoId: activeId ? null : scopeId }); if (!mounted.current) return; + void loadConversations(false); + if (revision !== conversationRevision.current) { + if (selectedConversation.current === answer.conversa_id) void loadMessages(answer.conversa_id); + return; + } setMessages((current) => [ ...current, { id: `assistant-${Date.now()}`, remetente: "assistant", conteudo: answer.resposta, fontes: answer.fontes, created_at: new Date().toISOString(), origem: answer.origem }, @@ -148,9 +166,12 @@ export function ChatView() { skipReload.current = answer.conversa_id; setActiveId(answer.conversa_id); } - void loadConversations(false); } catch (error) { if (!mounted.current) return; + if (revision !== conversationRevision.current) { + if (activeId && selectedConversation.current === activeId) void loadMessages(activeId); + return; + } setMessages((current) => current.map((item) => (item.id === pendingId ? { ...item, failed: true } : item))); if (error instanceof ApiError && error.details && typeof error.details === "object" && "details" in error.details) { const detail = error.details.details as { conversa_id?: string }; @@ -158,10 +179,10 @@ export function ChatView() { } setSendError(describeChatError(error)); } finally { - sendLock.current = false; - if (mounted.current) setSending(false); + sendLocks.current.delete(sendKey); + if (mounted.current) setPendingKeys(current => current.filter(key => key !== sendKey)); } - }, [activeId, scopeId, loadConversations, loadMessages]); + }, [activeId, scopeId, sendKey, loadConversations, loadMessages]); const copy = async (message: ThreadMessage) => { try { @@ -198,7 +219,7 @@ export function ChatView() { className="ds-input" value={scopeId} disabled={scopeLocked} - onChange={(event) => setScopeId(event.target.value)} + onChange={(event) => { conversationRevision.current += 1; setMessages([]); setSendError(""); setScopeId(event.target.value); }} aria-describedby="chat-scope-help" > diff --git a/frontend/src/views/documents/DocumentsView.test.tsx b/frontend/src/views/documents/DocumentsView.test.tsx index 584f3a0..03edec7 100644 --- a/frontend/src/views/documents/DocumentsView.test.tsx +++ b/frontend/src/views/documents/DocumentsView.test.tsx @@ -372,3 +372,23 @@ for (const operation of ['refresh', 'more', 'upload', 'remove', 'reprocess'] as expect(screen.getByText('Projeto B.txt')).toBeInTheDocument(); }); } + +it('reprocessamento agenda novo refresh se a lista ainda estiver sendo carregada', async () => { + let finish!: (response: Response) => void; + let reads = 0; + vi.stubGlobal('fetch', vi.fn((url: RequestInfo | URL) => { + if (String(url).includes('/reprocess')) return json({}); + reads++; + if (reads === 2) return new Promise(resolve => { finish = resolve; }); + return listing([doc({ status_processamento: reads > 2 ? 'processado' : 'falha' })]); + })); + view(); + await screen.findByText('Escopo.pdf'); + fireEvent.click(screen.getByRole('button', { name: 'Atualizar' })); + await waitFor(() => expect(finish).toBeDefined()); + fireEvent.click(screen.getByRole('button', { name: 'Reprocessar' })); + await screen.findByText('Documento enviado para reprocessamento.'); + await act(async () => { finish(await listing([doc({status_processamento:'falha'})])); }); + expect(await screen.findByText('Disponível no acervo')).toBeInTheDocument(); + expect(reads).toBe(3); +}); diff --git a/frontend/src/views/documents/DocumentsView.tsx b/frontend/src/views/documents/DocumentsView.tsx index e3e85e7..f0cfb36 100644 --- a/frontend/src/views/documents/DocumentsView.tsx +++ b/frontend/src/views/documents/DocumentsView.tsx @@ -75,6 +75,7 @@ function ProjectDocuments({ const scope = useRef(new AbortController()); const pages = useRef(1); const listLock = useRef(false); + const refreshPending = useRef(false); const mutationRevision = useRef(0); useEffect(() => { const controller = new AbortController(); @@ -84,7 +85,9 @@ function ProjectDocuments({ const requestSignal = (timeout = 15_000) => AbortSignal.any([scope.current.signal, AbortSignal.timeout(timeout)]); const refresh = useCallback(async (quiet = false) => { - if (!projectId || listLock.current || scope.current.signal.aborted) return; + if (!projectId || scope.current.signal.aborted) return; + if (listLock.current) { refreshPending.current = true; return; } + refreshPending.current = false; listLock.current = true; const signal = requestSignal(); const revision = mutationRevision.current; @@ -114,6 +117,7 @@ function ProjectDocuments({ if (scope.current.signal.aborted) return; setLoading(false); setRefreshing(false); + if (refreshPending.current) void refresh(true); } }, [projectId]); @@ -144,6 +148,7 @@ function ProjectDocuments({ try { await reprocessDocument(projectId, id, signal); if (signal.aborted) return; + mutationRevision.current += 1; setNotice("Documento enviado para reprocessamento."); await refresh(true); } catch { if (!scope.current.signal.aborted) setNotice("Não foi possível reprocessar o documento. Tente novamente."); } @@ -184,6 +189,7 @@ function ProjectDocuments({ listLock.current = false; if (scope.current.signal.aborted) return; setLoadingMore(false); + if (refreshPending.current) void refresh(true); } }; From da75dbefca70a0a02ebc5db081ef37951a10ad9e Mon Sep 17 00:00:00 2001 From: Moretto <72614882+Giomoret@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:58:43 -0300 Subject: [PATCH 8/9] feat(documents): concluir pipeline assincrono da S2-01 --- backend/scripts/demo-s201.mts | 92 +++++++++++ .../documents/documents.ingestion.db.test.ts | 4 + .../modules/documents/documents.ingestion.ts | 151 +++++++++++------- .../modules/documents/documents.repository.ts | 2 +- .../modules/documents/documents.retry.test.ts | 51 ++++++ .../src/modules/documents/documents.types.ts | 1 + docs/S2-01-DEMONSTRACAO.md | 19 +++ docs/S2-01-PIPELINE-DOCUMENTOS.md | 13 ++ docs/api/openapi.yaml | 5 + 9 files changed, 283 insertions(+), 55 deletions(-) create mode 100644 backend/scripts/demo-s201.mts create mode 100644 backend/src/modules/documents/documents.retry.test.ts create mode 100644 docs/S2-01-DEMONSTRACAO.md create mode 100644 docs/S2-01-PIPELINE-DOCUMENTOS.md diff --git a/backend/scripts/demo-s201.mts b/backend/scripts/demo-s201.mts new file mode 100644 index 0000000..98852ed --- /dev/null +++ b/backend/scripts/demo-s201.mts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve, sep } from "node:path"; +import { Client } from "pg"; +import { applyUntil } from "../src/database/migration-test-utils.js"; +const name="sinapse-s201-demo-"+randomUUID().slice(0,8), password=randomUUID(), evidence:string[]=[]; +function docker(args:string[]){const r=spawnSync("docker",args,{encoding:"utf8"});if(r.status!==0)throw Error("Operação Docker falhou.");return r.stdout.trim();} +const record=(s:string)=>{console.log(s);evidence.push(s);}; +let started=false, server:import("node:http").Server|undefined, db:import("pg").Pool|undefined; +const folder=await mkdtemp(join(tmpdir(),"sinapse-s201-demo-")); +try { + const ai=JSON.parse(docker(["inspect","sinapse-ai-service"]))[0]; + const token=ai.Config.Env.find((s:string)=>s.startsWith("AI_SERVICE_TOKEN=")); + assert.ok(token); + process.env.AI_SERVICE_TOKEN=token.slice(17); + process.env.DOCUMENT_STORAGE_DIR=folder; + process.env.DOCUMENT_INGEST_WEBHOOK_URL="http://localhost:5678/webhook/sinapse-ingest"; + process.env.NODE_ENV="test"; + docker(["run","-d","--name",name,"-e","POSTGRES_USER=sinapse","-e","POSTGRES_PASSWORD="+password,"-e","POSTGRES_DB=sinapse_s201_demo_test","-p","127.0.0.1::5432","pgvector/pgvector:pg16"]);started=true; + const port=JSON.parse(docker(["inspect",name]))[0].NetworkSettings.Ports["5432/tcp"][0].HostPort; + process.env.DATABASE_URL="postgresql://sinapse:"+password+"@127.0.0.1:"+port+"/sinapse_s201_demo_test"; + let ready=false; + for(let i=0;i<40;i++){if(spawnSync("docker",["exec",name,"pg_isready","-U","sinapse","-d","sinapse_s201_demo_test"],{encoding:"utf8"}).status===0){ready=true;break;}await new Promise(r=>setTimeout(r,500));} + assert.ok(ready); + const client=new Client({connectionString:process.env.DATABASE_URL});await client.connect(); + try{await applyUntil(client,true,()=>false);}finally{await client.end();} + const {pool}=await import("../src/database/db.js");db=pool; + const {hashPassword}=await import("../src/modules/auth/pssword.service.js"); + const {app}=await import("../src/app.js"); + const {DocumentIngestionWorker}=await import("../src/modules/documents/documents.ingestion.js"); + const {env}=await import("../src/config/env.js"); + const loginPassword=randomUUID(); + await pool.query("INSERT INTO usuario(id,nome,email,senha_hash,role) VALUES($1,'PO Demonstração','demo-s201@example.test',$2,'admin')",[randomUUID(),await hashPassword(loginPassword)]); + server=app.listen(0,"127.0.0.1");await new Promise(r=>server!.once("listening",r)); + const base="http://127.0.0.1:"+(server.address() as import("node:net").AddressInfo).port; + let authorization=""; + async function api(path:string,method="GET",body?:string|Buffer,headers:Record={},expected=200){ + const r=await fetch(base+path,{method,headers:{...(authorization?{Authorization:authorization}:{}),...headers},body}); + const data=await r.json();assert.equal(r.status,expected,method+" "+path);return data; + } + const login=await api("/api/v1/auth/login","POST",JSON.stringify({email:"demo-s201@example.test",password:loginPassword}),{"Content-Type":"application/json"}); + authorization="Bearer "+login.token; + const project=await api("/api/v1/projects","POST",JSON.stringify({nome:"Demonstração S2-01",cliente:"Dados sintéticos",status:"ativo"}),{"Content-Type":"application/json"},201); + assert.ok(project.id); + const path="/api/v1/projects/"+project.id+"/documents", worker=new DocumentIngestionWorker(); + const lookup=async(id:string)=>(await api(path)).items.find((d:{id:string})=>d.id===id); + const t=Date.now(); + const doc=await api(path,"POST",Buffer.from("A política S201ALFA estabelece revisão de requisitos antes da entrega. Somente dados sintéticos."),{"Content-Type":"application/octet-stream","X-File-Name":"demonstracao.txt"},201); + assert.equal(doc.status_processamento,"pendente");record("Upload HTTP 201 em "+(Date.now()-t)+" ms; estado pendente."); + const serviceToken=env.AI_SERVICE_TOKEN;env.AI_SERVICE_TOKEN="demonstracao-token-invalido"; + try{await worker.tick(project.id);}finally{env.AI_SERVICE_TOKEN=serviceToken;} + const failed=await lookup(doc.id);assert.equal(failed.status_processamento,"falha");assert.equal(failed.erro_processamento_codigo,"SERVICE_HTTP_ERROR"); + record("Falha controlada no caminho backend → n8n → IA real: token inválido recusado; motivo seguro: "+failed.erro_processamento); + const retryPath=path+"/"+doc.id+"/reprocess"; + await api(retryPath,"POST",undefined,{},202); + const before=(await pool.query("SELECT ingest_attempts,ingest_next_attempt_at,updated_at FROM documento WHERE id=$1",[doc.id])).rows; + await api(retryPath,"POST",undefined,{},202); + assert.deepEqual((await pool.query("SELECT ingest_attempts,ingest_next_attempt_at,updated_at FROM documento WHERE id=$1",[doc.id])).rows,before); + record("Dois pedidos de retry HTTP 202 preservaram fila, contadores e timestamps."); + const processing=worker.tick(project.id);let observed=false; + for(let i=0;i<60;i++){if((await lookup(doc.id)).status_processamento==="processando"){observed=true;break;}await new Promise(r=>setTimeout(r,50));} + assert.ok(observed);record("Estado processando observado pela API durante execução real."); + await api("/api/v1/projects");record("Consulta de projetos HTTP 200 durante processamento."); + await processing; + const done=await lookup(doc.id);assert.equal(done.status_processamento,"processado");assert.equal(done.erro_processamento,null); + const chunks=await pool.query("SELECT projeto_id,vector_dims(embedding) n,metadados_json FROM chunk WHERE entidade_id=$1",[doc.id]); + assert.ok(chunks.rowCount); + for(const row of chunks.rows){assert.equal(row.projeto_id,project.id);assert.equal(row.n,1024);assert.equal(row.metadados_json.project_id,project.id);} + const search=await api("/api/v1/search?projeto_id="+project.id+"&q=S201ALFA"); + assert.ok(search.items.some((s:{entidade_id:string})=>s.entidade_id===doc.id)); + record("Estado processado; "+chunks.rowCount+" trecho(s), embeddings reais de 1024 dimensões, origem preservada e documento encontrado na busca."); + const other=await api("/api/v1/projects","POST",JSON.stringify({nome:"Projeto isolado",cliente:"Dados sintéticos"}),{"Content-Type":"application/json"},201); + assert.equal((await api("/api/v1/search?projeto_id="+other.id+"&q=S201ALFA")).items.length,0); + record("Busca no segundo projeto não retornou dados do primeiro."); + const pdf=await api(path,"POST",Buffer.from("%PDF-1.7\nDocumento demonstrativo truncado."),{"Content-Type":"application/octet-stream","X-File-Name":"corrompido.pdf"},201); + await worker.tick(project.id); + const extraction=await lookup(pdf.id);assert.equal(extraction.status_processamento,"falha");assert.equal(extraction.erro_processamento_codigo,"INVALID_DOCUMENT");assert.equal(extraction.nova_tentativa_pendente,false); + record("PDF truncado falhou na extração da IA real: INVALID_DOCUMENT; motivo: "+extraction.erro_processamento); + await api(path+"/"+pdf.id+"/reprocess","POST",undefined,{},202);assert.equal((await lookup(pdf.id)).status_processamento,"pendente"); + record("Falha permanente de extração permite nova tentativa manual HTTP 202."); + await writeFile(resolve("../docs/S2-01-DEMONSTRACAO.md"),"# Demonstração integrada S2-01\n\nData: "+new Date().toLocaleString("pt-BR",{timeZone:"America/Sao_Paulo"})+" (America/Sao_Paulo).\n\nBackend com código local, PostgreSQL 16/pgvector temporário, n8n e IA/Ollama reais. Migrações aplicadas, dados sintéticos, autenticação e chamadas HTTP reais. Worker acionado por tick explícito para controlar a demonstração; agendamento periódico não medido.\n\n"+evidence.map(s=>"- "+s).join("\n")+"\n\nAsserções aprovadas. Banco e arquivos temporários removidos ao terminar; dados de desenvolvimento preservados. Sem credenciais no relatório.\n\nReproduzir no backend com serviços locais ativos: node --import tsx scripts/demo-s201.mts.\n"); + record("Relatório salvo em docs/S2-01-DEMONSTRACAO.md."); +} finally { + if(server){server.closeAllConnections();await new Promise(r=>server!.close(()=>r()));} + if(db)await db.end(); + assert.ok(resolve(folder).startsWith(resolve(tmpdir())+sep));await rm(folder,{recursive:true,force:true}); + if(started)docker(["rm","-f","-v",name]);console.log("Ambiente temporário removido."); +} + diff --git a/backend/src/modules/documents/documents.ingestion.db.test.ts b/backend/src/modules/documents/documents.ingestion.db.test.ts index fb39011..fb0f545 100644 --- a/backend/src/modules/documents/documents.ingestion.db.test.ts +++ b/backend/src/modules/documents/documents.ingestion.db.test.ts @@ -45,7 +45,11 @@ test("ingestão: falha, retry, lease, vetores e remoção sem ressuscitar docume assert.equal(failed.nova_tentativa_pendente,true); assert.equal(failed.erro_processamento_codigo,"SERVICE_HTTP_ERROR"); assert.equal(failed.tentativas_processamento,1); + assert.match(failed.erro_processamento!, /recusou/); await repository.retry(project,id); + await db.query("UPDATE documento SET ingest_attempts=2 WHERE id=$1", [id]); + await repository.retry(project,id); + assert.equal((await db.query("SELECT ingest_attempts FROM documento WHERE id=$1", [id])).rows[0].ingest_attempts, 2); const job=(await repository.claim(project))!; assert.equal(await repository.claim(project),null); await assert.rejects(repository.finish(job,[{text:"Inválido",embedding:[1]}])); diff --git a/backend/src/modules/documents/documents.ingestion.ts b/backend/src/modules/documents/documents.ingestion.ts index f790c6d..f7389f0 100644 --- a/backend/src/modules/documents/documents.ingestion.ts +++ b/backend/src/modules/documents/documents.ingestion.ts @@ -10,104 +10,147 @@ import { ConflictError, NotFoundError } from "../../shared/errors.js"; import { assertWritable, lockHierarchy } from "../projects/hierarchy-archive.js"; import { LocalDocumentStorage } from "./documents.storage.js"; -export interface IngestionFailure { code: string; retryable: boolean; httpStatus?: number } -class IngestionScopeError extends Error {} +export interface IngestionFailure { + code: string; + retryable: boolean; + httpStatus?: number; +} +export function ingestionFailureReason(code: string): string { + const reasons: Record = { + INVALID_SCOPE: "O serviço retornou um documento ou projeto diferente do solicitado.", + INVALID_RESPONSE: "O serviço retornou dados de indexação inválidos.", + INVALID_DOCUMENT: "Não foi possível extrair o conteúdo do documento.", + TIMEOUT: "O serviço de processamento excedeu o tempo limite.", + SERVICE_UNAVAILABLE: "O serviço de processamento está indisponível.", + SERVICE_HTTP_ERROR: "O serviço de processamento recusou a solicitação.", + FILE_NOT_FOUND: "O arquivo armazenado não foi encontrado.", + }; + return reasons[code] ?? "Não foi possível processar o documento."; +} +class IngestionScopeError extends Error { +} export function classifyIngestionFailure(error: unknown): IngestionFailure { - if (error instanceof IngestionScopeError) return {code: "INVALID_SCOPE", retryable: false}; - if (error instanceof z.ZodError) return {code: "INVALID_RESPONSE", retryable: false}; + if (error instanceof IngestionScopeError) + return { code: "INVALID_SCOPE", retryable: false }; + if (error instanceof z.ZodError) + return { code: "INVALID_RESPONSE", retryable: false }; if (axios.isAxiosError(error)) { - if (["ECONNABORTED","ETIMEDOUT"].includes(error.code ?? "")) return {code: "TIMEOUT", retryable: true}; + if (["ECONNABORTED", "ETIMEDOUT"].includes(error.code ?? "")) + return { code: "TIMEOUT", retryable: true }; const status = error.response?.status; - if (status) return {code: status === 422 || status === 400 ? "INVALID_DOCUMENT" : "SERVICE_HTTP_ERROR", retryable: status >= 500 || status === 429, httpStatus: status}; - return {code: "SERVICE_UNAVAILABLE", retryable: true}; + if (status) + return { code: status === 422 || status === 400 ? "INVALID_DOCUMENT" : "SERVICE_HTTP_ERROR", retryable: status >= 500 || status === 429, httpStatus: status }; + return { code: "SERVICE_UNAVAILABLE", retryable: true }; } - if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") return {code: "FILE_NOT_FOUND", retryable: false}; - return {code: "PROCESSING_ERROR", retryable: true}; + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") + return { code: "FILE_NOT_FOUND", retryable: false }; + return { code: "PROCESSING_ERROR", retryable: true }; +} +interface IngestionJob { + id: string; + projeto_id: string; + caminho: string; + nome: string; + ingest_lease: string; } - -interface IngestionJob { id: string; projeto_id: string; caminho: string; nome: string; ingest_lease: string } const chunksSchema = z.array(z.object({ text: z.string().trim().min(1).max(50000), embedding: z.array(z.number().finite()).length(1024).refine(v => v.some(n => n !== 0)) })).min(1).max(5000); export type IngestedChunk = z.infer[number]; - export class DocumentIngestionRepository { constructor(private readonly db: Pool = pool) { } - async claim(projectId?: string): Promise { - await this.db.query("UPDATE documento SET status_processamento='falha',ingest_lease=NULL,ingest_locked_until=NULL WHERE status_processamento='processando' AND ingest_attempts>=5 AND ingest_locked_until<=CURRENT_TIMESTAMP"); - const result = await this.db.query( - `WITH picked AS ( - SELECT d.id FROM documento d JOIN projeto p ON p.id=d.projeto_id - WHERE d.status_processamento <> 'processado' AND p.status <> 'arquivado' AND ($2::uuid IS NULL OR d.projeto_id=$2) - AND d.ingest_retryable AND d.ingest_attempts < 5 AND d.ingest_next_attempt_at <= CURRENT_TIMESTAMP - AND (d.ingest_locked_until IS NULL OR d.ingest_locked_until <= CURRENT_TIMESTAMP) - AND NOT EXISTS (SELECT 1 FROM documento_operacao_armazenamento op WHERE op.documento_id=d.id AND op.acao='finalizar_upload' AND op.status='pendente') - ORDER BY d.ingest_next_attempt_at,d.id LIMIT 1 FOR UPDATE OF d SKIP LOCKED - ) UPDATE documento d SET status_processamento='processando', ingest_lease=$1, - ingest_locked_until=CURRENT_TIMESTAMP + INTERVAL '3 minutes', ingest_attempts=ingest_attempts+1 - FROM picked WHERE d.id=picked.id RETURNING d.id,d.projeto_id,d.caminho,d.nome,d.ingest_lease`, [randomUUID(), projectId ?? null]); + await this.db.query("UPDATE documento SET status_processamento='falha',ingest_lease=NULL,ingest_locked_until=NULL,ingest_error='O processamento excedeu o limite de tentativas.',ingest_error_code='ATTEMPTS_EXHAUSTED',ingest_retryable=FALSE,updated_at=CURRENT_TIMESTAMP WHERE status_processamento='processando' AND ingest_attempts>=5 AND ingest_locked_until<=CURRENT_TIMESTAMP"); + const result = await this.db.query(`WITH picked AS ( + SELECT d.id FROM documento d JOIN projeto p ON p.id=d.projeto_id + WHERE d.status_processamento <> 'processado' AND p.status <> 'arquivado' AND ($2::uuid IS NULL OR d.projeto_id=$2) + AND d.ingest_retryable AND d.ingest_attempts < 5 AND d.ingest_next_attempt_at <= CURRENT_TIMESTAMP + AND (d.ingest_locked_until IS NULL OR d.ingest_locked_until <= CURRENT_TIMESTAMP) + AND NOT EXISTS (SELECT 1 FROM documento_operacao_armazenamento op WHERE op.documento_id=d.id AND op.acao='finalizar_upload' AND op.status='pendente') + ORDER BY d.ingest_next_attempt_at,d.id LIMIT 1 FOR UPDATE OF d SKIP LOCKED + ) UPDATE documento d SET status_processamento='processando', ingest_lease=$1, + ingest_locked_until=CURRENT_TIMESTAMP + INTERVAL '3 minutes', ingest_attempts=ingest_attempts+1,updated_at=CURRENT_TIMESTAMP + FROM picked WHERE d.id=picked.id RETURNING d.id,d.projeto_id,d.caminho,d.nome,d.ingest_lease`, [randomUUID(), projectId ?? null]); return result.rows[0] ?? null; } - async finish(job: IngestionJob, chunks: IngestedChunk[]): Promise { const validated = chunksSchema.parse(chunks); - await withTransaction(this.db, async client => { + await withTransaction(this.db, async (client) => { await lockHierarchy(client, "projeto", job.projeto_id); await assertWritable(client, "projeto", job.projeto_id); const current = await client.query("SELECT id FROM documento WHERE id=$1 AND projeto_id=$2 AND ingest_lease=$3 FOR UPDATE", [job.id, job.projeto_id, job.ingest_lease]); - // A removed document or superseded lease must never be resurrected. - if (!current.rowCount) return; + // Ignora respostas de documentos removidos ou tentativas substituídas. + if (!current.rowCount) + return; await client.query("DELETE FROM chunk WHERE projeto_id=$1 AND entidade_tipo='documento' AND entidade_id=$2", [job.projeto_id, job.id]); for (const [index, chunk] of validated.entries()) { await client.query(`INSERT INTO chunk(id,projeto_id,entidade_tipo,entidade_id,texto,embedding,metadados_json) - VALUES (md5($1::text || ':' || $2::text)::uuid,$3,'documento',$1::uuid,$4,$5::vector,$6::jsonb)`, - [job.id, index, job.projeto_id, chunk.text, JSON.stringify(chunk.embedding), JSON.stringify({ document_id: job.id, project_id: job.projeto_id, chunk_index: index, nome: job.nome })]); + VALUES (md5($1::text || ':' || $2::text)::uuid,$3,'documento',$1::uuid,$4,$5::vector,$6::jsonb)`, [job.id, index, job.projeto_id, chunk.text, JSON.stringify(chunk.embedding), JSON.stringify({ document_id: job.id, project_id: job.projeto_id, chunk_index: index, nome: job.nome })]); } await client.query("UPDATE documento SET status_processamento='processado',ingest_lease=NULL,ingest_locked_until=NULL,ingest_error=NULL,ingest_error_code=NULL,updated_at=CURRENT_TIMESTAMP WHERE id=$1", [job.id]); }); } - async fail(job: IngestionJob, failure: IngestionFailure = { code: "UNKNOWN", retryable: true }): Promise { const result = await this.db.query(`UPDATE documento SET status_processamento='falha',ingest_lease=NULL,ingest_locked_until=NULL, - ingest_error='Não foi possível processar o documento.', ingest_error_code=$3,ingest_retryable=$4, ingest_next_attempt_at=CURRENT_TIMESTAMP + make_interval(secs => LEAST(3600,15 * power(2,ingest_attempts))::int),updated_at=CURRENT_TIMESTAMP - WHERE id=$1 AND ingest_lease=$2 RETURNING ingest_attempts`, [job.id, job.ingest_lease, failure.code, failure.retryable]); - if (result.rowCount) console.warn("[Documents] Ingestion failure", { documentId: job.id, projectId: job.projeto_id, attempt: result.rows[0].ingest_attempts, code: failure.code, retryable: failure.retryable, httpStatus: failure.httpStatus }); + ingest_error=$5, ingest_error_code=$3,ingest_retryable=$4, ingest_next_attempt_at=CURRENT_TIMESTAMP + make_interval(secs => LEAST(3600,15 * power(2,ingest_attempts))::int),updated_at=CURRENT_TIMESTAMP + WHERE id=$1 AND projeto_id=$6 AND ingest_lease=$2 RETURNING ingest_attempts`, [job.id, job.ingest_lease, failure.code, failure.retryable, ingestionFailureReason(failure.code), job.projeto_id]); + if (result.rowCount) + console.warn("[Documents] Ingestion failure", { documentId: job.id, projectId: job.projeto_id, attempt: result.rows[0].ingest_attempts, code: failure.code, retryable: failure.retryable, httpStatus: failure.httpStatus }); } - async retry(projectId: string, documentId: string): Promise { - await withTransaction(this.db, async client => { + await withTransaction(this.db, async (client) => { await lockHierarchy(client, "projeto", projectId); await assertWritable(client, "projeto", projectId); - const row = (await client.query("SELECT status_processamento,ingest_locked_until FROM documento WHERE id=$1 AND projeto_id=$2 FOR UPDATE", [documentId, projectId])).rows[0]; - if (!row) throw new NotFoundError("Documento não encontrado."); - if (row.ingest_locked_until && new Date(row.ingest_locked_until).getTime() > Date.now()) throw new ConflictError("Documento em processamento."); - await client.query("UPDATE documento SET status_processamento='pendente',ingest_attempts=0,ingest_lease=NULL,ingest_locked_until=NULL,ingest_next_attempt_at=CURRENT_TIMESTAMP,ingest_error=NULL,ingest_error_code=NULL,ingest_retryable=TRUE WHERE id=$1", [documentId]); + const result = await client.query( + "SELECT status_processamento,ingest_locked_until FROM documento WHERE id=$1 AND projeto_id=$2 FOR UPDATE", + [documentId, projectId], + ); + const row = result.rows[0]; + if (!row) + throw new NotFoundError("Documento não encontrado."); + // Já está na fila; não reinicia a tentativa. + if (row.status_processamento === "pendente") + return; + if (row.ingest_locked_until && new Date(row.ingest_locked_until).getTime() > Date.now()) + throw new ConflictError("Documento em processamento."); + await client.query("UPDATE documento SET status_processamento='pendente',ingest_attempts=0,ingest_lease=NULL,ingest_locked_until=NULL,ingest_next_attempt_at=CURRENT_TIMESTAMP,ingest_error=NULL,ingest_error_code=NULL,ingest_retryable=TRUE,updated_at=CURRENT_TIMESTAMP WHERE id=$1 AND projeto_id=$2", [documentId, projectId]); }); } } - export class DocumentIngestionWorker { - constructor( - private readonly repository = new DocumentIngestionRepository(), - private readonly storage = new LocalDocumentStorage(env.DOCUMENT_STORAGE_DIR), - private readonly endpoint = env.DOCUMENT_INGEST_WEBHOOK_URL || `${env.AI_SERVICE_URL}/ingest/file`, - ) { } - + constructor(private readonly repository = new DocumentIngestionRepository(), private readonly storage = new LocalDocumentStorage(env.DOCUMENT_STORAGE_DIR), private readonly endpoint = env.DOCUMENT_INGEST_WEBHOOK_URL || `${env.AI_SERVICE_URL}/ingest/file`) { } async tick(projectId?: string): Promise { const job = await this.repository.claim(projectId); - if (!job) return; + if (!job) + return; try { const content = await this.storage.read(job.caminho); const { data } = await axios.post(this.endpoint, { document_id: job.id, project_id: job.projeto_id, file_name: job.nome, content_base64: content.toString("base64") }, { headers: serviceHeaders(), timeout: 120000, maxBodyLength: 150 * 1024 * 1024, maxContentLength: 64 * 1024 * 1024 }); - if (data.document_id !== job.id || data.project_id !== job.projeto_id) throw new IngestionScopeError(); + if (!data || data.document_id !== job.id || data.project_id !== job.projeto_id) + throw new IngestionScopeError(); await this.repository.finish(job, chunksSchema.parse(data.chunks)); - } catch (error) { await this.repository.fail(job, classifyIngestionFailure(error)); } + } + catch (error) { + await this.repository.fail(job, classifyIngestionFailure(error)); + } } } - export function startDocumentIngestionWorker(worker = new DocumentIngestionWorker()): () => void { let running = false; - const tick = async () => { if (running) return; running = true; try { await worker.tick(); } catch { console.warn("[Documents] Ingestion remains queued for retry."); } finally { running = false; } }; + + async function tick() { + if (running) return; + + running = true; + try { + await worker.tick(); + } catch { + console.warn("[Documents] Ingestion remains queued for retry."); + } finally { + running = false; + } + } + const timer = setInterval(() => void tick(), 15000); - timer.unref(); void tick(); + timer.unref(); + void tick(); return () => clearInterval(timer); } diff --git a/backend/src/modules/documents/documents.repository.ts b/backend/src/modules/documents/documents.repository.ts index 9827873..d0ebd7b 100644 --- a/backend/src/modules/documents/documents.repository.ts +++ b/backend/src/modules/documents/documents.repository.ts @@ -37,7 +37,7 @@ export interface DocumentPage { const SELECT_COLUMNS = ` d.id, d.projeto_id, d.nome, d.extensao, d.mime, d.tamanho_bytes, d.status_processamento, - d.ingest_attempts AS tentativas_processamento, d.ingest_error_code AS erro_processamento_codigo, + d.ingest_error AS erro_processamento, d.ingest_attempts AS tentativas_processamento, d.ingest_error_code AS erro_processamento_codigo, (d.status_processamento <> 'processado' AND d.ingest_retryable AND d.ingest_attempts < 5 AND EXISTS (SELECT 1 FROM projeto processing_project WHERE processing_project.id=d.projeto_id AND processing_project.status <> 'arquivado')) AS nova_tentativa_pendente, d.ingest_next_attempt_at::text AS proxima_tentativa_em, diff --git a/backend/src/modules/documents/documents.retry.test.ts b/backend/src/modules/documents/documents.retry.test.ts new file mode 100644 index 0000000..9752948 --- /dev/null +++ b/backend/src/modules/documents/documents.retry.test.ts @@ -0,0 +1,51 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import type { Pool } from "pg"; +import { DocumentIngestionRepository, ingestionFailureReason } from "./documents.ingestion.js"; + +function createRepository(status = "falha", lockedUntil: Date | null = null) { + const state = { status, updates: 0, rollback: false }; + const client = { + async query(sql: string, values?: unknown[]) { + if (sql === "ROLLBACK") + state.rollback = true; + if (sql.includes("AS archived")) + return { rows: [{ archived: false }] }; + if (sql.startsWith("SELECT status_processamento")) { + assert.deepEqual(values, ["document", "project"]); + return { rows: [{ status_processamento: state.status, ingest_locked_until: lockedUntil }] }; + } + if (sql.startsWith("UPDATE documento")) { + assert.deepEqual(values, ["document", "project"]); + assert.ok(sql.includes("projeto_id=$2")); + state.status = "pendente"; + state.updates++; + } + return { rows: [] }; + }, release() { }, + }; + return { state, repository: new DocumentIngestionRepository({ async connect() { return client; } } as unknown as Pool) }; +} +test("retry repetido preserva a tentativa pendente", async () => { + const { state, repository } = createRepository(); + await repository.retry("project", "document"); + await repository.retry("project", "document"); + assert.equal(state.updates, 1); +}); +test("retry ativo retorna conflito sem alterar a tentativa", async () => { + const { state, repository } = createRepository("processando", new Date(Date.now() + 60000)); + await assert.rejects(repository.retry("project", "document"), /Documento em processamento/); + assert.equal(state.updates, 0); + assert.equal(state.rollback, true); +}); +test("retry recupera lease expirado", async () => { + const { state, repository } = createRepository("processando", new Date(0)); + await repository.retry("project", "document"); + assert.equal(state.status, "pendente"); + assert.equal(state.updates, 1); +}); +test("motivos específicos não expõem códigos desconhecidos", () => { + assert.match(ingestionFailureReason("INVALID_DOCUMENT"), /extrair/); + assert.match(ingestionFailureReason("TIMEOUT"), /tempo limite/); + assert.doesNotMatch(ingestionFailureReason("segredo-token"), /segredo/); +}); diff --git a/backend/src/modules/documents/documents.types.ts b/backend/src/modules/documents/documents.types.ts index d393701..73d901a 100644 --- a/backend/src/modules/documents/documents.types.ts +++ b/backend/src/modules/documents/documents.types.ts @@ -16,6 +16,7 @@ export interface DocumentRecord { nova_tentativa_pendente?: boolean; tentativas_processamento?: number; proxima_tentativa_em?: string | null; + erro_processamento?: string | null; erro_processamento_codigo?: string | null; autor_id: string | null; autor_nome: string | null; diff --git a/docs/S2-01-DEMONSTRACAO.md b/docs/S2-01-DEMONSTRACAO.md new file mode 100644 index 0000000..34ee17e --- /dev/null +++ b/docs/S2-01-DEMONSTRACAO.md @@ -0,0 +1,19 @@ +# Demonstração integrada S2-01 + +Data: 07/10/2026, 23:17:44 (America/Sao_Paulo). + +Backend com código local, PostgreSQL 16/pgvector temporário, n8n e IA/Ollama reais. Migrações aplicadas, dados sintéticos, autenticação e chamadas HTTP reais. Worker acionado por tick explícito para controlar a demonstração; agendamento periódico não medido. + +- Upload HTTP 201 em 58 ms; estado pendente. +- Falha controlada no caminho backend → n8n → IA real: token inválido recusado; motivo seguro: O serviço de processamento recusou a solicitação. +- Dois pedidos de retry HTTP 202 preservaram fila, contadores e timestamps. +- Estado processando observado pela API durante execução real. +- Consulta de projetos HTTP 200 durante processamento. +- Estado processado; 1 trecho(s), embeddings reais de 1024 dimensões, origem preservada e documento encontrado na busca. +- Busca no segundo projeto não retornou dados do primeiro. +- PDF truncado falhou na extração da IA real: INVALID_DOCUMENT; motivo: Não foi possível extrair o conteúdo do documento. +- Falha permanente de extração permite nova tentativa manual HTTP 202. + +Asserções aprovadas. Banco e arquivos temporários removidos ao terminar; dados de desenvolvimento preservados. Sem credenciais no relatório. + +Reproduzir no backend com serviços locais ativos: node --import tsx scripts/demo-s201.mts. diff --git a/docs/S2-01-PIPELINE-DOCUMENTOS.md b/docs/S2-01-PIPELINE-DOCUMENTOS.md new file mode 100644 index 0000000..ae10018 --- /dev/null +++ b/docs/S2-01-PIPELINE-DOCUMENTOS.md @@ -0,0 +1,13 @@ +# S2-01 — Pipeline assíncrono de documentos + +Upload retorna com estado pendente. O worker consulta a fila persistida a cada 15 segundos, aguardando finalização do armazenamento e excluindo projetos arquivados. Estados: pendente, processando, processado e falha. + +Tentativas usam lease de três minutos e timeout HTTP de dois minutos. Falhas transitórias usam backoff exponencial e até cinco tentativas por ciclo. Leases expirados são recuperados após reinício; expiração da quinta tentativa informa ATTEMPTS_EXHAUSTED. Falhas permanentes aguardam nova tentativa manual. + +POST /api/v1/projects/{projectId}/documents/{documentId}/reprocess exige admin ou PO com acesso ao projeto e retorna 202. Pedidos repetidos enquanto pendente preservam a fila e os contadores. Lease ativo retorna 409. Após falha ou lease expirado, limpa diagnósticos e reinicia o ciclo. Documentos processados também podem ser reindexados, preservando o contrato existente. + +GET /api/v1/projects/{projectId}/documents expõe estado, erro_processamento (motivo seguro), erro_processamento_codigo, tentativas_processamento e nova_tentativa_pendente. Respostas de leases substituídos e documentos removidos não restauram dados. O Node valida escopo e vetores e substitui os trechos atomicamente com IDs determinísticos e projeto de origem. + +DOCUMENT_INGEST_WEBHOOK_URL seleciona o workflow n8n versionado; sem essa configuração, o backend chama a IA diretamente. Ative o workflow e encaminhe X-Service-Token. O workflow não salva payloads de execução. Logs registram somente IDs, tentativa, código, retry e status HTTP, sem conteúdo, credenciais ou exceções internas. + +Validação: npm test e npm run typecheck no backend. documents.retry.test.ts verifica idempotência, conflito, lease expirado e motivos seguros. documents.ingestion.db.test.ts verifica falha, retry, leases, vetores e remoção com PostgreSQL migrado e ARCHIVE_TEST_DATABASE_URL apontando para um banco de testes permitido. Sem essa variável, os testes de banco são ignorados. Revisão do PR por outra pessoa permanece uma etapa humana. diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index d3a8943..275a249 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -1569,6 +1569,7 @@ paths: - {name: documentId, in: path, required: true, schema: {type: string, format: uuid}} post: summary: Reenfileira a ingestão de um documento (admin ou PO) + description: Pedidos repetidos enquanto pendente preservam a fila e os contadores. Tentativas ativas retornam 409. A indexação substitui atomicamente os trechos do documento. operationId: reprocessDocument security: [{cookieAuth: []}] responses: @@ -3325,6 +3326,10 @@ components: proxima_tentativa_em: type: string nullable: true + erro_processamento: + type: string + nullable: true + description: Motivo seguro da falha, sem conteúdo do documento ou mensagens internas. erro_processamento_codigo: type: string nullable: true From 1f587c2107b9c170533a5727ff2d0b731ab088b7 Mon Sep 17 00:00:00 2001 From: Moretto <72614882+Giomoret@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:59:02 -0300 Subject: [PATCH 9/9] style: remover linha vazia ao final do roteiro S2-01 --- backend/scripts/demo-s201.mts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/scripts/demo-s201.mts b/backend/scripts/demo-s201.mts index 98852ed..26790c6 100644 --- a/backend/scripts/demo-s201.mts +++ b/backend/scripts/demo-s201.mts @@ -89,4 +89,3 @@ try { assert.ok(resolve(folder).startsWith(resolve(tmpdir())+sep));await rm(folder,{recursive:true,force:true}); if(started)docker(["rm","-f","-v",name]);console.log("Ambiente temporário removido."); } -