چگونه پایپ‌لاین‌های هوش مصنوعی را با تاخیر زیر ۵۰ میلی‌ثانیه اجرا کنیم؟

# Python 3.12+ / Fast Semantic Cache Guard
import numpy as np
from typing import Optional

class SemanticEdgeCache:
    def __init__(self, vector_store, threshold: float = 0.94):
        self.vector_store = vector_store
        self.threshold = threshold

    async def get_or_compute(self, prompt: str, embed_fn, llm_fn) -> dict:
        # 1. Compute embedding in < 8ms
        embedding = await embed_fn(prompt)
        
        # 2. Search similarity vector
        cached_result = await self.vector_store.find_nearest(embedding)
        if cached_result and cached_result.score >= self.threshold:
            return {
                "source": "cache_hit",
                "latency_ms": 12,
                "data": cached_result.payload
            }

        # 3. Cache miss: Fallback to LLM
        response = await llm_fn(prompt)
        await self.vector_store.upsert(embedding, response)
        return {"source": "llm_fresh", "latency_ms": 480, "data": response}