fix: perms

src/indexer.py

@@ -6,7 +6,7 @@ import os
 import email
 import mailbox
 from datetime import datetime
-from email.utils import parsedate_to_datetime
+from email.utils import parsedate_to_datetime, parseaddr
 from email.header import decode_header
 from typing import List, Dict, Any, Tuple
 import uuid
@@ -14,7 +14,7 @@ import uuid
 from dotenv import load_dotenv
 from qdrant_client import QdrantClient
 from qdrant_client.http import models
-from sentence_transformers import SentenceTransformer
+from fastembed import TextEmbedding
 from bs4 import BeautifulSoup
 
 # Load .env config
@@ -32,7 +32,7 @@ if not QDRANT_URL:
 if not COLLECTION_NAME:
     raise ValueError("COLLECTION_NAME environment variable is required.")
 
-EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2"
+EMBEDDING_MODEL_NAME = os.environ.get("EMBEDDING_MODEL_NAME", "BAAI/bge-small-en-v1.5")
 BATCH_SIZE = 50
 
 def decode_mime_words(s: str) -> str:
@@ -59,6 +59,14 @@ def extract_text_from_html(html_content: str) -> str:
     except Exception:
         return html_content
 
+def normalize_email_address(value: str) -> str:
+    """Extracts and normalizes the bare email address from a header value."""
+    if not value:
+        return ""
+    _, addr = parseaddr(value)
+    return (addr or value).strip().lower()
+
+
 def parse_email_message(msg: mailbox.Message) -> Tuple[str, List[str]]:
     """Extracts plain text body and a list of attachment filenames."""
     body_parts = []
@@ -99,9 +107,8 @@ def parse_email_message(msg: mailbox.Message) -> Tuple[str, List[str]]:
 
     return "\n".join(body_parts).strip(), attachments
 
-def init_qdrant_collection(client: QdrantClient, model: SentenceTransformer):
+def init_qdrant_collection(client: QdrantClient, vector_size: int):
     """Ensures Qdrant collection exists and payload indexes are created."""
-    vector_size = model.get_sentence_embedding_dimension()
     
     # Check if collection exists
     collections = client.get_collections().collections
@@ -151,12 +158,13 @@ def main():
 
     # Initialize model
     print(f"Loading embedding model: {EMBEDDING_MODEL_NAME}...")
-    model = SentenceTransformer(EMBEDDING_MODEL_NAME)
+    model = TextEmbedding(model_name=EMBEDDING_MODEL_NAME)
+    vector_size = len(next(iter(model.embed(["dimension_probe"])) ))
     
     # Initialize Qdrant
     print("Connecting to Qdrant...")
     qdrant_client = QdrantClient(url=QDRANT_URL)
-    init_qdrant_collection(qdrant_client, model)
+    init_qdrant_collection(qdrant_client, vector_size)
 
     points = []
 
@@ -173,8 +181,10 @@ def main():
                 try:
                     # Parse headers
                     subject = decode_mime_words(msg.get("Subject", "No Subject"))
-                    sender = decode_mime_words(msg.get("From", "Unknown"))
-                    receiver = decode_mime_words(msg.get("To", "Unknown"))
+                    sender_raw = decode_mime_words(msg.get("From", "Unknown"))
+                    receiver_raw = decode_mime_words(msg.get("To", "Unknown"))
+                    sender = normalize_email_address(sender_raw)
+                    receiver = normalize_email_address(receiver_raw)
                     message_id = msg.get("Message-ID", str(uuid.uuid4()))
                     
                     # Parse date
@@ -207,14 +217,18 @@ def main():
                     )
                     
                     # Embed the text
-                    vector = model.encode(vector_text).tolist()
+                    # Fastembed returns an iterable of numpy arrays
+                    embeddings = list(model.embed([vector_text]))
+                    vector = embeddings[0].tolist()
                     
                     # Prepare payload (metadata)
                     payload = {
                         "message_id": message_id,
                         "date": iso_date,
                         "sender": sender,
+                        "sender_raw": sender_raw,
                         "receiver": receiver,
+                        "receiver_raw": receiver_raw,
                         "subject": subject,
                         "body_text": body_text,
                         "attachments": attachments