Merge pull request 'ci: add Renovate workflow' (#44) from add-renovate-workflow into main

Reviewed-on: #44

.gitea/workflows/cron.yaml

@@ -18,60 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
-        with:
+      - run: docker build -f pkg/Dockerfile -t ci-image:${{ github.sha }} .
-          python-version: "3.13"
-      - name: Install dependencies
-        run: pip install -r requirements.txt pytest
-      - name: Run unit tests
-        run: pytest tests/ -v
-        env:
-          # indexer/server read env at import; unit tests need no real backend
-          QDRANT_URL: http://localhost:6333
-          COLLECTION_NAME: test
-
-  integration-test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-
-      - name: Cleanup old Qdrant container
-        run: |
-          docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true
-
-      - name: Start Qdrant
-        run: |
-          docker run -d --name qdrant-${{ github.run_id }} \
-            --network "container:$(hostname)" \
-            docker.io/qdrant/qdrant:latest
-
-      - name: Wait for Qdrant
-        run: |
-          for i in $(seq 1 30); do
-            curl -s http://localhost:6333/healthz \
-              && echo "QDRANT ready" && break
-            echo "Waiting for Qdrant... ($i/30)"
-            sleep 1
-          done
-          curl -s http://localhost:6333/healthz > /dev/null
-
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
-        with:
-          python-version: "3.13"
-      - name: Install dependencies
-        run: pip install -r requirements.txt pytest
-      - name: Run integration tests
-        run: pytest tests/ -v -m integration
-        env:
-          QDRANT_URL: http://localhost:6333
-          COLLECTION_NAME: test_mcp_maildir
-
-      - name: Cleanup Qdrant
-        if: always()
-        run: docker rm -f qdrant-${{ github.run_id }} || true
 
   build-push:
-    needs: [test, integration-test]
+    needs: [test]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

.gitea/workflows/docker-build.yaml

@@ -29,8 +29,8 @@ jobs:
 
       - name: Start Qdrant
         run: |
-          docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true
+          docker rm -f qdrant 2>/dev/null || true
-          docker run -d --name qdrant-${{ github.run_id }} \
+          docker run -d --name qdrant \
             --network "container:$(hostname)" \
             docker.io/qdrant/qdrant:latest
 
@@ -49,14 +49,14 @@ jobs:
         run: |
           pip install -r requirements.txt pytest
       - name: Run integration tests
-        run: pytest tests/ -v -m integration
+        run: pytest tests/test_integration.py -v
         env:
           QDRANT_URL: http://localhost:6333
           COLLECTION_NAME: test_mcp_maildir
 
       - name: Cleanup Qdrant
         if: always()
-        run: docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true
+        run: docker rm -f qdrant 2>/dev/null || true
 
   build:
     runs-on: ubuntu-latest

.gitea/workflows/main.yaml

@@ -18,60 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
-        with:
+      - run: docker build -f pkg/Dockerfile -t ci-image:${{ github.sha }} .
-          python-version: "3.13"
-      - name: Install dependencies
-        run: pip install -r requirements.txt pytest
-      - name: Run unit tests
-        run: pytest tests/ -v
-        env:
-          # indexer/server read env at import; unit tests need no real backend
-          QDRANT_URL: http://localhost:6333
-          COLLECTION_NAME: test
-
-  integration-test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-
-      - name: Cleanup old Qdrant container
-        run: |
-          docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true
-
-      - name: Start Qdrant
-        run: |
-          docker run -d --name qdrant-${{ github.run_id }} \
-            --network "container:$(hostname)" \
-            docker.io/qdrant/qdrant:latest
-
-      - name: Wait for Qdrant
-        run: |
-          for i in $(seq 1 30); do
-            curl -s http://localhost:6333/healthz \
-              && echo "QDRANT ready" && break
-            echo "Waiting for Qdrant... ($i/30)"
-            sleep 1
-          done
-          curl -s http://localhost:6333/healthz > /dev/null
-
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
-        with:
-          python-version: "3.13"
-      - name: Install dependencies
-        run: pip install -r requirements.txt pytest
-      - name: Run integration tests
-        run: pytest tests/ -v -m integration
-        env:
-          QDRANT_URL: http://localhost:6333
-          COLLECTION_NAME: test_mcp_maildir
-
-      - name: Cleanup Qdrant
-        if: always()
-        run: docker rm -f qdrant-${{ github.run_id }} || true
 
   build:
-    needs: [test, integration-test]
+    needs: [test]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

.gitea/workflows/pr.yaml

@@ -14,22 +14,6 @@ jobs:
         with:
           dockerfile: pkg/Dockerfile
 
-  pytest:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
-        with:
-          python-version: "3.13"
-      - name: Install dependencies
-        run: pip install -r requirements.txt pytest
-      - name: Run unit tests
-        run: pytest tests/ -v
-        env:
-          # indexer/server read env at import; unit tests need no real backend
-          QDRANT_URL: http://localhost:6333
-          COLLECTION_NAME: test
-
   build-test:
     runs-on: ubuntu-latest
     steps:

.gitea/workflows/renovate.yaml

@@ -0,0 +1,12 @@
+name: Renovate
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  renovate:
+    uses: perso/gitea-pipelines/.gitea/workflows/renovate.yaml@main
+    secrets: inherit

.gitea/workflows/tag.yaml

@@ -18,60 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
+      - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
-        with:
+      - run: docker build -f pkg/Dockerfile -t ci-image:${{ github.sha }} .
-          python-version: "3.13"
-      - name: Install dependencies
-        run: pip install -r requirements.txt pytest
-      - name: Run unit tests
-        run: pytest tests/ -v
-        env:
-          # indexer/server read env at import; unit tests need no real backend
-          QDRANT_URL: http://localhost:6333
-          COLLECTION_NAME: test
-
-  integration-test:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
-
-      - name: Cleanup old Qdrant container
-        run: |
-          docker rm -f qdrant-${{ github.run_id }} 2>/dev/null || true
-
-      - name: Start Qdrant
-        run: |
-          docker run -d --name qdrant-${{ github.run_id }} \
-            --network "container:$(hostname)" \
-            docker.io/qdrant/qdrant:latest
-
-      - name: Wait for Qdrant
-        run: |
-          for i in $(seq 1 30); do
-            curl -s http://localhost:6333/healthz \
-              && echo "QDRANT ready" && break
-            echo "Waiting for Qdrant... ($i/30)"
-            sleep 1
-          done
-          curl -s http://localhost:6333/healthz > /dev/null
-
-      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7
-        with:
-          python-version: "3.13"
-      - name: Install dependencies
-        run: pip install -r requirements.txt pytest
-      - name: Run integration tests
-        run: pytest tests/ -v -m integration
-        env:
-          QDRANT_URL: http://localhost:6333
-          COLLECTION_NAME: test_mcp_maildir
-
-      - name: Cleanup Qdrant
-        if: always()
-        run: docker rm -f qdrant-${{ github.run_id }} || true
 
   build-push:
-    needs: [test, integration-test]
+    needs: [test]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

pytest.ini

@@ -1,8 +1,3 @@
 [pytest]
 testpaths = tests
 pythonpath = src
-; Integration tests (tests/test_integration.py) require a live Qdrant on
-; localhost:6333 — they only run in the nightly CI job (or manually with -m integration).
-markers =
-    integration: requires a live Qdrant instance (deselected by default)
-addopts = -m "not integration" -p no:cacheprovider

src/indexer.py

@@ -22,27 +22,26 @@ from bs4 import BeautifulSoup
 # Load .env config
 load_dotenv()
 
-# Configuration (validated lazily in main() so the module can be imported
+# Configuration
-# and unit-tested without environment variables)
 MAILDIR_PATH = os.environ.get("MAILDIR_PATH", "")
 MAILDIR_FOLDERS = os.environ.get("MAILDIR_FOLDERS", "")
 QDRANT_URL = os.environ.get("QDRANT_URL", "")
 COLLECTION_NAME = os.environ.get("COLLECTION_NAME", "")
 
+if not MAILDIR_PATH:
+    raise ValueError("MAILDIR_PATH environment variable is required.")
+if not QDRANT_URL:
+    raise ValueError("QDRANT_URL environment variable is required.")
+if not COLLECTION_NAME:
+    raise ValueError("COLLECTION_NAME environment variable is required.")
+
+EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5")
+BATCH_SIZE = int(os.environ.get("BATCH_SIZE", "100"))
+EMBEDDING_BATCH_SIZE = int(os.environ.get("EMBEDDING_BATCH_SIZE", "64"))
 METADATA_COLLECTION = "mcp_indexer_metadata"
 INCREMENTAL_DAYS = int(os.environ.get("INCREMENTAL_DAYS", "7"))
 FORCE_REINDEX = os.environ.get("FORCE_REINDEX", "").lower() in ("1", "true", "yes")
 
-
-def _validate_config():
-    """Raises if required environment variables are missing. Called from main()."""
-    if not MAILDIR_PATH:
-        raise ValueError("MAILDIR_PATH environment variable is required.")
-    if not QDRANT_URL:
-        raise ValueError("QDRANT_URL environment variable is required.")
-    if not COLLECTION_NAME:
-        raise ValueError("COLLECTION_NAME environment variable is required.")
-
 def decode_mime_words(s: str) -> str:
     """Decodes MIME encoded strings (e.g. subjects, filenames)."""
     if not s:
@@ -252,8 +251,6 @@ def main():
     Main ingestion function.
     Reads Maildir, extracts text, generates local embeddings, and pushes to Qdrant.
     """
-    _validate_config()
-
     print(f"Indexing emails from {MAILDIR_PATH} into {QDRANT_URL}...")
 
     if not os.path.exists(MAILDIR_PATH):

tests/test_indexer.py

@@ -1,229 +0,0 @@
-"""Unit tests for mcp-maildir indexer parsing functions.
-
-Pure-function tests: no Qdrant, no embedding model, no network.
-A throwaway Maildir is built in tmp_path for get_recent_keys().
-"""
-
-import mailbox
-import os
-import time
-import uuid
-from datetime import datetime, timezone, timedelta
-from email.message import EmailMessage
-from email.utils import format_datetime
-
-import pytest
-
-from indexer import (
-    decode_mime_words,
-    extract_text_from_html,
-    normalize_email_address,
-    parse_email_message,
-    get_recent_keys,
-    METADATA_COLLECTION,
-)
-
-
-# ---------------------------------------------------------------------------
-# decode_mime_words
-# ---------------------------------------------------------------------------
-
-class TestDecodeMimeWords:
-    def test_empty_and_none(self):
-        assert decode_mime_words("") == ""
-        assert decode_mime_words(None) == ""
-
-    def test_plain_ascii(self):
-        assert decode_mime_words("Hello World") == "Hello World"
-
-    def test_encoded_word_utf8(self):
-        # =?utf-8?q?...?= encoded word
-        assert decode_mime_words("=?utf-8?q?Caf=C3=A9_au_lait?=") == "Café au lait"
-
-    def test_encoded_word_iso8859(self):
-        # decode_mime_words does not strip whitespace around the words
-        assert decode_mime_words("=?iso-8859-1?q?caf=E9?=") == "café"
-
-    def test_mixed_encoded_and_plain(self):
-        assert decode_mime_words("Re: =?utf-8?q?planning_2026?=") == "Re: planning 2026"
-
-
-# ---------------------------------------------------------------------------
-# extract_text_from_html
-# ---------------------------------------------------------------------------
-
-class TestExtractTextFromHtml:
-    def test_simple_paragraphs(self):
-        html = "<html><body><p>Hello</p><p>World</p></body></html>"
-        assert extract_text_from_html(html) == "Hello World"
-
-    def test_strips_tags_and_scripts(self):
-        html = "<div>Sign<span>ature</span><script>evil()</script>End</div>"
-        text = extract_text_from_html(html)
-        assert "evil()" not in text
-        # get_text uses separator=" ", so tags become whitespace
-        assert "Sign" in text and "ature" in text and "End" in text
-
-    def test_fallback_on_invalid_content(self):
-        # Garbage that would make the parser blow up should not raise
-        assert extract_text_from_html(None) is None
-
-
-# ---------------------------------------------------------------------------
-# parse_email_message — real mailbox.Message objects (no mocks)
-# ---------------------------------------------------------------------------
-
-def build_email(
-    subject="Hello",
-    body="This is the body.",
-    html=None,
-    attachments=None,
-    charset="utf-8",
-):
-    """Builds a real EmailMessage, optionally multipart with attachments."""
-    msg = EmailMessage()
-    msg["Subject"] = subject
-    msg["From"] = "Alice <alice@example.com>"
-    msg["To"] = "Bob <bob@example.com>"
-    msg["Message-ID"] = "<build-email@example.com>"
-    msg["Date"] = format_datetime(datetime(2026, 1, 15, 10, 0, 0, tzinfo=timezone.utc))
-
-    if html:
-        msg.set_content(body)
-        msg.add_alternative(html, subtype="html")
-    else:
-        msg.set_content(body, charset=charset)
-
-    for filename, content in (attachments or {}).items():
-        payload = content if isinstance(content, bytes) else content.encode()
-        msg.add_attachment(payload, maintype="application",
-                           subtype="octet-stream", filename=filename)
-    return msg
-
-
-def build_raw_email(raw_bytes: bytes):
-    """Parses raw email bytes (for exotic headers without EmailMessage helpers)."""
-    import email as email_mod
-    return email_mod.message_from_bytes(raw_bytes)
-
-
-class TestParseEmailMessage:
-    def test_plain_text_body(self):
-        msg = build_email(body="Line one.\nLine two.")
-        body, attachments = parse_email_message(msg)
-        assert "Line one." in body and "Line two." in body
-        assert attachments == []
-
-    def test_attachments_detected_and_excluded_from_body(self):
-        msg = build_email(
-            body="See attached.",
-            attachments={"report.pdf": b"%PDF-fake", "notes.txt": b"notes"},
-        )
-        body, attachments = parse_email_message(msg)
-        assert attachments == ["report.pdf", "notes.txt"]
-        assert "%PDF" not in body
-        assert "See attached." in body
-
-    def test_html_alternative_extracts_text(self):
-        msg = build_email(
-            body="Fallback plain text.",
-            html="<html><body><b>Bold</b> intro</body></html>",
-        )
-        body, _ = parse_email_message(msg)
-        assert "Bold" in body and "intro" in body
-
-    def test_unknown_charset_falls_back_not_crash(self):
-        # A payload claiming an unknown charset must not raise.
-        # Built from raw bytes: EmailMessage.set_content(charset=...) would
-        # reject the unknown encoding at build time, real mail does not.
-        raw = (
-            b"Subject: broken charset\n"
-            b"From: alice@example.com\n"
-            b"To: bob@example.com\n"
-            b"Content-Type: text/plain; charset=x-unknown-charset\n"
-            b"Content-Transfer-Encoding: 8bit\n\n"
-            b"body with broken charset\n"
-        )
-        msg = build_raw_email(raw)
-        body, attachments = parse_email_message(msg)
-        # Whatever the fallback, we must get a string body and no exception
-        assert isinstance(body, str)
-        assert attachments == []
-
-    def test_encoded_attachment_filename_decoded(self):
-        msg = build_email(
-            body="body",
-            attachments={},
-        )
-        # Add attachment with non-ascii filename via encoded word
-        msg.add_attachment(b"data", maintype="application", subtype="pdf",
-                           filename="café-rapport.pdf")
-        body, attachments = parse_email_message(msg)
-        assert attachments == ["café-rapport.pdf"]
-
-
-# ---------------------------------------------------------------------------
-# normalize_email_address (indexer copy — same contract as server.py)
-# ---------------------------------------------------------------------------
-
-class TestIndexerNormalizeEmailAddress:
-    def test_display_name(self):
-        assert normalize_email_address("John Doe <john@example.com>") == "john@example.com"
-
-    def test_case_and_whitespace(self):
-        assert normalize_email_address("  USER@Example.COM ") == "user@example.com"
-
-    def test_empty(self):
-        assert normalize_email_address("") == ""
-        assert normalize_email_address(None) == ""
-
-
-# ---------------------------------------------------------------------------
-# get_recent_keys — real Maildir on disk (tmp_path), mtime-driven
-# ---------------------------------------------------------------------------
-
-def make_maildir(root):
-    """Opens a Maildir at root — pre-creates cur/new/tmp because mailbox.Maildir
-    (Python 3.11) does not create them when adding a message."""
-    for sub in ("cur", "new", "tmp"):
-        os.makedirs(os.path.join(str(root), sub), exist_ok=True)
-    return mailbox.Maildir(str(root), create=False)
-
-
-def write_maildir_message(maildir: mailbox.Maildir, msg: EmailMessage):
-    """Adds a message and returns its key."""
-    return maildir.add(msg)
-
-
-class TestGetRecentKeys:
-    def test_recent_file_selected(self, tmp_path):
-        md = make_maildir(tmp_path)
-        msg = build_email()
-        md.add(msg)
-        md.flush()
-        # mtime is now → within the window
-        keys = get_recent_keys(md, days=7)
-        assert len(keys) == 1
-
-    def test_old_file_excluded(self, tmp_path):
-        md = make_maildir(tmp_path)
-        md.add(build_email())
-        md.flush()
-        # Age every file beyond the cutoff
-        old = time.time() - 30 * 86400
-        for path in tmp_path.rglob("*.??*"):
-            if path.is_file():
-                os.utime(path, (old, old))
-        keys = get_recent_keys(md, days=7)
-        assert len(keys) == 0
-
-    def test_key_is_suffixless_filename(self, tmp_path):
-        """Maildir keys strip the ':2,S' info suffix — get_recent_keys must match."""
-        md = make_maildir(tmp_path)
-        key = md.add(build_email())
-        md.flush()
-        keys = get_recent_keys(md, days=7)
-        assert keys  # non-empty set of keys for fresh files
-        assert key.split(":")[0] in keys
-        for k in keys:
-            assert ":" not in k

tests/test_integration.py

@@ -11,10 +11,6 @@ os.environ["EMBEDDING_MODEL_NAME"] = "BAAI/bge-small-en-v1.5"
 from server import get_qdrant_client, get_embedding_model, search_emails, read_email
 from qdrant_client.http import models
 
-# Requires a live Qdrant on localhost:6333 (see pytest.ini: deselected by
-# default; run with `pytest -m integration`, nightly CI does exactly that).
-pytestmark = pytest.mark.integration
-
 
 TEST_EMAILS = [
     {

tests/test_server.py

@@ -8,7 +8,6 @@ from server import (
     normalize_email_address,
     payload_matches_participant,
     format_search_result,
-    build_filter,
 )
 
 
@@ -109,55 +108,3 @@ class TestFormatSearchResult:
         assert result["receiver"] is None
         assert result["subject"] is None
         assert result["attachments"] == []
-
-
-class TestBuildFilter:
-    def test_no_filters_returns_none(self):
-        assert build_filter(None, None, None) is None
-
-    def test_empty_strings_return_none(self):
-        assert build_filter("", "", "") is None
-
-    def test_participant_filter_is_should_clause_on_sender_and_receiver(self):
-        f = build_filter("alice@example.com", None, None)
-        assert f is not None
-        # The date conditions list must be absent and only the participant filter present
-        assert len(f.must) == 1
-        participant = f.must[0]
-        keys = [c.key for c in participant.should]
-        assert keys == ["sender", "receiver"]
-        assert all(c.match.value == "alice@example.com" for c in participant.should)
-
-    def test_participant_normalized_before_match(self):
-        f = build_filter("Alice <Alice@Example.COM>", None, None)
-        should = f.must[0].should
-        assert all(c.match.value == "alice@example.com" for c in should)
-
-    def test_date_range_only(self):
-        f = build_filter(None, "2026-01-01", "2026-06-30")
-        assert len(f.must) == 1
-        cond = f.must[0]
-        assert cond.key == "date"
-        assert cond.range.gte is not None
-        assert cond.range.lte is not None
-
-    def test_start_date_only(self):
-        f = build_filter(None, "2026-03-15", None)
-        cond = f.must[0]
-        assert cond.range.gte is not None
-        assert cond.range.lte is None
-
-    def test_end_date_only(self):
-        f = build_filter(None, None, "2026-03-15")
-        cond = f.must[0]
-        assert cond.range.gte is None
-        assert cond.range.lte is not None
-
-    def test_combined_participant_and_date(self):
-        f = build_filter("bob@example.org", "2026-01-01", "2026-12-31")
-        assert len(f.must) == 2
-        # One is the participant should-filter, the other the date range
-        kinds = sorted(
-            "participant" if getattr(c, "should", None) else "date" for c in f.must
-        )
-        assert kinds == ["date", "participant"]