FAI End-to-End Workshop

{
  "play": "69-it-helpdesk-bot",
  "version": "1.0.0",
  "description": "RAG-powered IT helpdesk knowledge bot",
  "context": {
    "knowledge": ["R2", "O4"],
    "waf": [
      "security",
      "reliability",
      "cost-optimization"
    ],
    "compatible-plays": ["01-enterprise-rag"]
  },
  "primitives": {
    "agents": [
      "fai-rag-architect",
      "fai-azure-ai-search-expert",
      "fai-code-reviewer"
    ],
    "instructions": [
      "typescript-waf",
      "python-waf"
    ],
    "skills": [
      "fai-play-initializer",
      "fai-rag-evaluator"
    ]
  },
  "infrastructure": {
    "services": [
      "azure-openai",
      "azure-ai-search",
      "azure-container-apps",
      "azure-key-vault"
    ],
    "region": "eastus2",
    "iac": "bicep"
  },
  "toolkit": {
    "tunekit": {
      "openai": "config/openai.json",
      "guardrails": "config/guardrails.json"
    }
  },
  "guardrails": {
    "groundedness": 0.8,
    "relevance": 0.7,
    "safety": 0.9
  }
}

# Create the play directory
mkdir -p solution-plays/69-it-helpdesk-bot

# Scaffold domain-specific agents
node scripts/scaffold-primitive.js agent
# Name: fai-it-helpdesk-builder
# Description: "Builds IT helpdesk RAG pipeline components"
# WAF: security, reliability
# Plays: 69-it-helpdesk-bot

node scripts/scaffold-primitive.js agent
# Name: fai-it-helpdesk-reviewer
# Description: "Reviews IT helpdesk bot for quality and security"
# WAF: security, responsible-ai
# Plays: 69-it-helpdesk-bot

# .github/instructions/it-helpdesk-rag.instructions.md
---
description: "IT helpdesk RAG patterns — chunking, search config, grounding"
applyTo: "solution-plays/69-it-helpdesk-bot/**/*.{ts,py,json}"
waf: [security, reliability]
---

## IT Helpdesk RAG Guidelines

- Chunk IT documentation by section headers (H2/H3 boundaries)
- Use hybrid search: keyword for error codes, vector for concepts
- System prompt must include: "Answer ONLY from the provided context"
- PII filter: strip employee IDs, emails, phone numbers from responses
- Max response length: 500 tokens (concise IT answers)

# Verify everything is wired correctly
node scripts/validate-primitives.js --verbose

# Expected: 0 errors, 0 warnings

{
  "content_safety": {
    "enabled": true,
    "categories": {
      "hate":       { "threshold": "medium", "action": "block" },
      "violence":   { "threshold": "medium", "action": "block" },
      "self_harm":  { "threshold": "low",    "action": "block" },
      "sexual":     { "threshold": "medium", "action": "block" }
    }
  },
  "pii_detection": {
    "enabled": true,
    "categories": ["email", "phone", "ssn", "employee_id"],
    "action": "redact"
  },
  "prompt_injection": {
    "enabled": true,
    "detection_model": "azure-content-safety",
    "action": "block_and_log"
  },
  "evaluation": {
    "metrics": {
      "groundedness": { "threshold": 4.0 },
      "relevance":    { "threshold": 3.5 },
      "coherence":    { "threshold": 4.0 },
      "safety":       { "threshold": 1.0 }
    }
  }
}

{
  "model": "gpt-4o",
  "api_version": "2024-12-01-preview",
  "temperature": 0.1,
  "max_tokens": 512,
  "top_p": 0.95,
  "frequency_penalty": 0,
  "presence_penalty": 0,
  "system_prompt": "You are an IT helpdesk assistant. Answer questions using ONLY the provided context from the IT knowledge base. If the context does not contain the answer, respond: 'I don't have information about that. Please contact the IT helpdesk at ext. 4357.' Never speculate or provide information not in the context.",
  "routing": {
    "simple_queries": "gpt-4o-mini",
    "complex_queries": "gpt-4o",
    "complexity_threshold": 0.6
  }
}

# Set variables
$RG = "rg-it-helpdesk-bot"
$LOCATION = "eastus2"

# Create resource group
az group create --name $RG --location $LOCATION

# Deploy Bicep template (creates OpenAI, Search, Container Apps, Key Vault)
az deployment group create \
  --resource-group $RG \
  --template-file infra/main.bicep \
  --parameters location=$LOCATION \
               projectName="it-helpdesk"

# Verify deployment
az deployment group show \
  --resource-group $RG \
  --name "main" \
  --query "properties.provisioningState"

# Upload IT knowledge base documents to the search index
python scripts/index-documents.py \
  --source "./data/it-knowledge-base/" \
  --index-name "it-helpdesk-index" \
  --chunk-strategy "section-headers" \
  --embedding-model "text-embedding-3-large"

# Expected output:
# Indexed 342 documents → 1,284 chunks
# Embedding dimensions: 3072
# Index size: 48 MB

# Build and deploy the container
az containerapp up \
  --name "it-helpdesk-api" \
  --resource-group $RG \
  --source . \
  --ingress external \
  --target-port 8080

# Verify health endpoint
curl https://it-helpdesk-api.<region>.azurecontainerapps.io/health
# Expected: { "status": "healthy" }

# Run full evaluation suite
node engine/index.js \
  solution-plays/69-it-helpdesk-bot/fai-manifest.json \
  --eval \
  --test-set "evaluation/it-helpdesk-queries.jsonl" \
  --output "evaluation/results.json"

# Expected output:
# ┌─────────────────┬───────┬───────────┬────────┐
# │ Metric          │ Score │ Threshold │ Status │
# ├─────────────────┼───────┼───────────┼────────┤
# │ Groundedness    │ 4.3   │ 4.0       │ ✓ PASS │
# │ Relevance       │ 4.1   │ 3.5       │ ✓ PASS │
# │ Coherence       │ 4.5   │ 4.0       │ ✓ PASS │
# │ Safety          │ 1.0   │ 1.0       │ ✓ PASS │
# └─────────────────┴───────┴───────────┴────────┘
# Overall: PASS — all metrics above threshold

# Create the plugin structure
mkdir -p plugins/it-helpdesk-bot

# Create plugin.json
cat > plugins/it-helpdesk-bot/plugin.json << 'EOF'
{
  "name": "it-helpdesk-bot",
  "description": "RAG-powered IT helpdesk with hybrid search and model routing",
  "version": "1.0.0",
  "author": {
    "name": "Your Name",
    "url": "https://github.com/yourname"
  },
  "license": "MIT",
  "play": "69-it-helpdesk-bot",
  "primitives": {
    "agents": 2,
    "instructions": 1,
    "skills": 2
  },
  "tags": ["rag", "helpdesk", "it-support", "azure"]
}
EOF

# Regenerate marketplace index
node scripts/generate-marketplace.js

# Run the full build pipeline
node scripts/validate-primitives.js
node scripts/generate-website-data.js
node scripts/update-readme.js

# Verify plugin appears
npx frootai list | grep "it-helpdesk"
# Expected: it-helpdesk-bot  RAG-powered IT helpdesk...

# Commit and tag
git add .
git commit -m "feat: add play 69 — IT helpdesk knowledge bot"
git tag v2.1.0

# Push (triggers CI/CD pipeline)
git push origin main --tags

# The FAI Factory automatically:
# 1. Validates all primitives
# 2. Generates marketplace listing
# 3. Publishes npm package
# 4. Updates website data