Auto-classifying customer support tickets with AI and routing them to the right team

# routing_prompt.txt — production prompt
Classify the following support ticket.
Return valid JSON only, no extra text.

Categories:
- billing (payment, invoice, refund)
- technical (integration, error, API)
- sales (pre-sale, pricing)
- bug (software defect)
- cancellation (account close, subscription stop)
- other

Ticket:
{{ticket_text}}

JSON schema:
{
  "category": "billing|technical|sales|bug|cancellation|other",
  "priority": "low|medium|high",
  "team": "billing_support|tech_support|sales|core_eng|account_ops|triage",
  "confidence": 0.0,
  "escalation_required": true,
  "rationale": "short reason, max 80 chars"
}

# router.py — hybrid routing: rule + cheap model + strong fallback
import json
from openai import OpenAI

client = OpenAI(api_key=PAREL_API_KEY, base_url="https://api.parel.cloud/v1")

CONFIDENCE_THRESHOLD = 0.75
PROMPT = open("routing_prompt.txt").read()

def classify(model, ticket):
    response = client.chat.completions.create(
        model=model,
        messages=[
            {"role": "system", "content": "Return valid JSON only."},
            {"role": "user", "content": PROMPT.replace("{{ticket_text}}", ticket)},
        ],
        temperature=0,
        response_format={"type": "json_object"},
    )
    return json.loads(response.choices[0].message.content)

def route(ticket: str) -> dict:
    # Layer 1: rule-based hotwords (fastest, free)
    lower = ticket.lower()
    if any(w in lower for w in ["refund", "chargeback", "money back"]):
        return {"category": "billing", "team": "billing_support", "via": "rule"}

    # Layer 2: cheap model (gpt-4o-mini) — handles ~85% of cases
    decision = classify("gpt-4o-mini", ticket)
    if decision["confidence"] >= CONFIDENCE_THRESHOLD:
        decision["via"] = "gpt-4o-mini"
        return decision

    # Layer 3: strong model fallback — only hard cases
    decision = classify("claude-opus-4-7", ticket)
    decision["via"] = "claude-opus-4-7"
    return decision

# escalation.py — force critical tickets to a human, not the model
HIGH_PRIORITY_KEYWORDS = ["urgent", "asap", "production down", "data loss", "gdpr", "kvkk"]

def needs_human_override(ticket: str, decision: dict) -> bool:
    # 1. Model itself flagged it
    if decision.get("escalation_required"):
        return True

    # 2. High priority and low confidence
    if decision.get("priority") == "high" and decision.get("confidence", 0) < 0.85:
        return True

    # 3. Critical keyword present, skip the model entirely
    lower = ticket.lower()
    if any(kw in lower for kw in HIGH_PRIORITY_KEYWORDS):
        return True

    return False

# weekly_drift.py — re-run the eval set in production weekly
import csv, statistics
from router import classify

def weekly_drift_check(eval_csv: str, baseline: dict) -> dict:
    correct = 0
    confidences = []
    for row in csv.DictReader(open(eval_csv)):
        out = classify("gpt-4o-mini", row["ticket_text"])
        correct += out["category"] == row["expected_category"]
        confidences.append(out["confidence"])

    accuracy = correct / 50
    avg_conf = statistics.mean(confidences)

    drift = {
        "accuracy_drop": baseline["accuracy"] - accuracy,
        "confidence_drop": baseline["avg_confidence"] - avg_conf,
    }

    if drift["accuracy_drop"] > 0.03:  # 3% drop -> alert
        send_slack_alert(f"Routing drift: accuracy {accuracy:.1%}")

    return drift