From 01d15c486e14d6de2201768a827d481496cee4b8 Mon Sep 17 00:00:00 2001 From: Sagent Date: Sat, 17 Oct 2026 21:24:18 +0000 Subject: [PATCH] ci: harden the qdrant integration job The integration test job has been failing intermittently on main and on open pull requests with `httpx.ReadTimeout: timed out` on the first write to Qdrant, while `GET /collections` and `/healthz` kept answering. Cause: the qdrant-client default request timeout is 5 seconds, and the job only waits for `/healthz`, which is served before the storage engine accepts writes. On a shared runner doing several builds at once, the first collection creation or indexing exceeds 5s and the whole fixture errors out. - pin qdrant/qdrant to v1.19 (the digest `latest` and the cluster already run) and let Renovate track the pin in the workflows as well - wait for a real write round-trip (create + delete a probe collection) instead of `/healthz` alone, and dump the container logs if it never becomes writable - make the Qdrant client timeout configurable via QDRANT_TIMEOUT and set it to 60s in CI, where there is no latency SLA --- .gitea/workflows/docker-build.yaml | 34 ++++++++++++++++++++++++---- .gitea/workflows/main.yaml | 36 +++++++++++++++++++++++++----- docker-compose.yml | 2 +- renovate.json | 9 +++++--- src/server.py | 8 +++++-- 5 files changed, 73 insertions(+), 16 deletions(-) diff --git a/.gitea/workflows/docker-build.yaml b/.gitea/workflows/docker-build.yaml index 8972a8e..66753ab 100644 --- a/.gitea/workflows/docker-build.yaml +++ b/.gitea/workflows/docker-build.yaml @@ -32,15 +32,32 @@ jobs: docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true docker run -d --name qdrant-${{ github.run_id }} \ --network "container:$(hostname)" \ - docker.io/qdrant/qdrant:latest + docker.io/qdrant/qdrant:v1.19 - name: Wait for Qdrant run: | - for i in $(seq 1 30); do - curl -s http://localhost:6333/healthz && echo "QDRANT ready" && break - echo "Waiting for Qdrant... ($i/30)" + # /healthz answers before the storage engine accepts writes, and a cold + # container on a loaded host can take several seconds to create its + # first collection. Gate on a real write round-trip, not on /healthz. + ready="" + for i in $(seq 1 60); do + if curl -sf "http://localhost:6333/healthz" > /dev/null 2>&1 \ + && curl -sf -X PUT "http://localhost:6333/collections/ci_readiness_probe?timeout=60" \ + -H 'Content-Type: application/json' \ + -d '{"vectors":{"size":4,"distance":"Cosine"}}' > /dev/null 2>&1; then + curl -sf -X DELETE "http://localhost:6333/collections/ci_readiness_probe" > /dev/null 2>&1 || true + ready="yes" + echo "QDRANT ready: write path OK (attempt $i)" + break + fi + echo "Waiting for Qdrant... ($i/60)" sleep 1 done + if [ -z "$ready" ]; then + echo "Qdrant write path never became ready" + docker logs --tail 100 qdrant-${{ github.run_id }} || true + exit 1 + fi - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: @@ -53,6 +70,15 @@ jobs: env: QDRANT_URL: http://localhost:6333 COLLECTION_NAME: test_mcp_maildir + # The qdrant client defaults to a 5s request timeout, which a cold + # collection on a shared runner can exceed; this job has no SLA. + QDRANT_TIMEOUT: "60" + + - name: Qdrant diagnostics + if: failure() + run: | + docker ps -a || true + docker logs --tail 200 qdrant-${{ github.run_id }} || true - name: Cleanup Qdrant if: always() diff --git a/.gitea/workflows/main.yaml b/.gitea/workflows/main.yaml index b12b0df..22c6d7f 100644 --- a/.gitea/workflows/main.yaml +++ b/.gitea/workflows/main.yaml @@ -45,17 +45,32 @@ jobs: run: | docker run -d --name qdrant-${{ github.run_id }} \ --network "container:$(hostname)" \ - docker.io/qdrant/qdrant:latest + docker.io/qdrant/qdrant:v1.19 - name: Wait for Qdrant run: | - for i in $(seq 1 30); do - curl -s http://localhost:6333/healthz \ - && echo "QDRANT ready" && break - echo "Waiting for Qdrant... ($i/30)" + # /healthz answers before the storage engine accepts writes, and a cold + # container on a loaded host can take several seconds to create its + # first collection. Gate on a real write round-trip, not on /healthz. + ready="" + for i in $(seq 1 60); do + if curl -sf "http://localhost:6333/healthz" > /dev/null 2>&1 \ + && curl -sf -X PUT "http://localhost:6333/collections/ci_readiness_probe?timeout=60" \ + -H 'Content-Type: application/json' \ + -d '{"vectors":{"size":4,"distance":"Cosine"}}' > /dev/null 2>&1; then + curl -sf -X DELETE "http://localhost:6333/collections/ci_readiness_probe" > /dev/null 2>&1 || true + ready="yes" + echo "QDRANT ready: write path OK (attempt $i)" + break + fi + echo "Waiting for Qdrant... ($i/60)" sleep 1 done - curl -s http://localhost:6333/healthz > /dev/null + if [ -z "$ready" ]; then + echo "Qdrant write path never became ready" + docker logs --tail 100 qdrant-${{ github.run_id }} || true + exit 1 + fi - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: @@ -67,6 +82,15 @@ jobs: env: QDRANT_URL: http://localhost:6333 COLLECTION_NAME: test_mcp_maildir + # The qdrant client defaults to a 5s request timeout, which a cold + # collection on a shared runner can exceed; this job has no SLA. + QDRANT_TIMEOUT: "60" + + - name: Qdrant diagnostics + if: failure() + run: | + docker ps -a || true + docker logs --tail 200 qdrant-${{ github.run_id }} || true - name: Cleanup Qdrant if: always() diff --git a/docker-compose.yml b/docker-compose.yml index e29f7e2..cd145ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: qdrant: - image: docker.io/qdrant/qdrant:latest + image: docker.io/qdrant/qdrant:v1.19 container_name: mcp_maildir_qdrant ports: - "127.0.0.1:6333:6333" # REST API diff --git a/renovate.json b/renovate.json index 5e8e69d..e8a2d46 100644 --- a/renovate.json +++ b/renovate.json @@ -3,9 +3,12 @@ "customManagers": [ { "customType": "regex", - "description": "Track qdrant/qdrant Docker image from docker-compose.yml (native manager missed it)", - "managerFilePatterns": ["/^docker-compose\\.ya?ml$/"], - "matchStrings": ["image:\\s*(?:docker\\.io/)?qdrant/qdrant:(?[^\\s]+)"], + "description": "Track the pinned qdrant/qdrant image in docker-compose.yml and the CI workflows (native manager missed it)", + "managerFilePatterns": [ + "/^docker-compose\\.ya?ml$/", + "/^\\.gitea/workflows/.*\\.ya?ml$/" + ], + "matchStrings": ["(?:docker\\.io/)?qdrant/qdrant:(?[^\\s\"']+)"], "depNameTemplate": "qdrant/qdrant", "datasourceTemplate": "docker" } diff --git a/src/server.py b/src/server.py index 79d30d3..8c83d70 100644 --- a/src/server.py +++ b/src/server.py @@ -36,6 +36,10 @@ COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "") EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5") DEFAULT_LIMIT = int(os.environ.get("SEARCH_LIMIT", "50")) +# Per-request timeout (seconds) for Qdrant. The client default is 5s, which is +# tight for a cold collection or a busy host: unset means keep that default. +QDRANT_TIMEOUT = int(os.environ["QDRANT_TIMEOUT"]) if os.environ.get("QDRANT_TIMEOUT") else None + if not QDRANT_URL: raise ValueError("QDRANT_URL environment variable is required.") if not COLLECTION_NAME: @@ -60,8 +64,8 @@ def get_qdrant_client() -> QdrantClient: """Returns a singleton Qdrant client.""" global _qdrant_client if _qdrant_client is None: - logger.info(f"Connecting to Qdrant at {QDRANT_URL}") - _qdrant_client = QdrantClient(url=QDRANT_URL) + logger.info(f"Connecting to Qdrant at {QDRANT_URL} (timeout={QDRANT_TIMEOUT})") + _qdrant_client = QdrantClient(url=QDRANT_URL, timeout=QDRANT_TIMEOUT) return _qdrant_client -- 2.54.0