Analytics & Observability

Track conversation metrics, user behavior, and flow performance with FlowKit's analytics system.

Overview

The AnalyticsCollector class provides:

• Event tracking
• Conversation metrics
• Flow performance analysis
• Funnel visualization
• Export capabilities

Basic Usage

import { AnalyticsCollector } from "@andresaya/flowkit";

const analytics = new AnalyticsCollector();

// Track events
analytics.track({
    type: "conversation:start",
    conversationId: "user-123",
    timestamp: Date.now(),
});

analytics.track({
    type: "step:enter",
    conversationId: "user-123",
    timestamp: Date.now(),
    data: { stepId: "greeting" },
});

analytics.track({
    type: "extract:success",
    conversationId: "user-123",
    timestamp: Date.now(),
    data: { extractType: "name", value: "John" },
});

Integration with FlowEngine

import { FlowEngine, AnalyticsCollector, EngineEvent } from "@andresaya/flowkit";

const analytics = new AnalyticsCollector();

// Create engine with event handler
const engine = new FlowEngine(flow, { 
    llm: adapter, 
    storage,
    onEvent: (event: EngineEvent) => {
        // Track all events to analytics
        if (event.type === "step:enter") {
            analytics.track({
                type: "step:enter",
                conversationId: event.conversationId,
                timestamp: Date.now(),
                data: { stepId: event.step },
            });
        }
        
        if (event.type === "extract:success") {
            analytics.track({
                type: "extract:success",
                conversationId: event.conversationId,
                timestamp: Date.now(),
                data: { 
                    slot: event.slot,
                    value: event.value,
                },
            });
        }
        
        if (event.type === "flow:end") {
            analytics.track({
                type: "conversation:end",
                conversationId: event.conversationId,
                timestamp: Date.now(),
                data: { reason: "completed" },
            });
        }
    }
});

Event Types

type AnalyticsEventType =
    | "conversation:start"    // Conversation started
    | "conversation:end"      // Conversation ended
    | "step:enter"           // Entered a step
    | "step:exit"            // Left a step
    | "extract:success"      // Successfully extracted data
    | "extract:fail"         // Failed to extract data
    | "handoff:request"      // Human handoff requested
    | "timeout"              // Conversation timed out
    | "error"                // Error occurred
    | "custom";              // Custom event

Getting Metrics

Conversation Metrics

const conversationMetrics = analytics.getConversationMetrics("user-123");

console.log(conversationMetrics);
// {
//   conversationId: "user-123",
//   startTime: 1703123456789,
//   endTime: 1703123556789,
//   duration: 100000,           // ms
//   messageCount: 5,
//   stepsVisited: ["greeting", "get_name", "confirm", "done"],
//   extractionSuccessRate: 0.8,
//   completed: true,
//   handoffRequested: false,
// }

Flow Metrics

const flowMetrics = analytics.getFlowMetrics();

console.log(flowMetrics);
// {
//   totalConversations: 150,
//   completedConversations: 120,
//   completionRate: 0.8,
//   averageDuration: 45000,     // ms
//   handoffRate: 0.05,
//   stepMetrics: {
//     "greeting": { visits: 150, avgTime: 5000 },
//     "get_name": { visits: 145, avgTime: 8000 },
//     "confirm": { visits: 130, avgTime: 3000 },
//   },
//   extractionMetrics: {
//     "name": { attempts: 145, successes: 140, rate: 0.97 },
//     "email": { attempts: 130, successes: 120, rate: 0.92 },
//   },
// }

Funnel Analysis

const funnel = analytics.getFunnel(["greeting", "get_name", "get_email", "confirm", "done"]);

console.log(funnel);
// {
//   steps: [
//     { step: "greeting", count: 1000, dropoff: 0 },
//     { step: "get_name", count: 950, dropoff: 0.05 },
//     { step: "get_email", count: 800, dropoff: 0.16 },
//     { step: "confirm", count: 750, dropoff: 0.06 },
//     { step: "done", count: 700, dropoff: 0.07 },
//   ],
//   overallConversion: 0.70,
// }

Exporting Data

Export All Events

const allEvents = analytics.export();
// Returns: AnalyticsEvent[]

// Save to file
import { writeFileSync } from "fs";
writeFileSync("analytics.json", JSON.stringify(allEvents, null, 2));

Export to Custom Format

// Export for external analytics (Google Analytics, Mixpanel, etc.)
const events = analytics.export();

for (const event of events) {
    await mixpanel.track(event.type, {
        distinct_id: event.conversationId,
        timestamp: new Date(event.timestamp),
        ...event.data,
    });
}

Custom Events

Track custom business events:

// Track product interest
analytics.track({
    type: "custom",
    conversationId: "user-123",
    timestamp: Date.now(),
    data: {
        name: "product_viewed",
        productId: "PRD-001",
        productName: "Premium Plan",
    },
});

// Track purchase intent
analytics.track({
    type: "custom",
    conversationId: "user-123",
    timestamp: Date.now(),
    data: {
        name: "purchase_intent",
        value: 500,
        currency: "USD",
    },
});

Real-time Dashboard

Build a simple real-time dashboard:

import { AnalyticsCollector, FlowEngine } from "@andresaya/flowkit";

const analytics = new AnalyticsCollector();

// Update dashboard every 5 seconds
setInterval(() => {
    const metrics = analytics.getFlowMetrics();
    
    console.clear();
    console.log("=== FlowKit Dashboard ===");
    console.log(`Active Conversations: ${metrics.totalConversations - metrics.completedConversations}`);
    console.log(`Completion Rate: ${(metrics.completionRate * 100).toFixed(1)}%`);
    console.log(`Avg Duration: ${(metrics.averageDuration / 1000).toFixed(1)}s`);
    console.log(`Handoff Rate: ${(metrics.handoffRate * 100).toFixed(1)}%`);
    console.log("");
    console.log("Step Performance:");
    for (const [step, data] of Object.entries(metrics.stepMetrics)) {
        console.log(`  ${step}: ${data.visits} visits, ${(data.avgTime / 1000).toFixed(1)}s avg`);
    }
}, 5000);

Retention & Cleanup

Manage analytics data size:

// Clear old events (older than 7 days)
const oneWeekAgo = Date.now() - (7 * 24 * 60 * 60 * 1000);
analytics.clearBefore(oneWeekAgo);

// Or export and clear
const events = analytics.export();
await saveToDatabase(events);
analytics.clear();

Integration Examples

With Prometheus/Grafana

import { Registry, Counter, Histogram } from "prom-client";

const registry = new Registry();

const conversationsTotal = new Counter({
    name: "flowkit_conversations_total",
    help: "Total conversations",
    labelNames: ["status"],
    registers: [registry],
});

const stepDuration = new Histogram({
    name: "flowkit_step_duration_seconds",
    help: "Time spent in each step",
    labelNames: ["step"],
    buckets: [1, 5, 10, 30, 60],
    registers: [registry],
});

const stepTimers = new Map<string, number>();

const engine = new FlowEngine(flow, {
    llm: adapter,
    storage,
    onEvent: (event) => {
        if (event.type === "flow:end") {
            conversationsTotal.inc({ status: "completed" });
        }
        
        if (event.type === "step:enter") {
            stepTimers.set(`${event.conversationId}:${event.step}`, Date.now());
        }
        
        if (event.type === "step:exit") {
            const start = stepTimers.get(`${event.conversationId}:${event.step}`);
            if (start) {
                const duration = (Date.now() - start) / 1000;
                stepDuration.observe({ step: event.step }, duration);
                stepTimers.delete(`${event.conversationId}:${event.step}`);
            }
        }
    }
});

With Database Storage

import { Pool } from "pg";

const pool = new Pool({ connectionString: process.env.DATABASE_URL });

async function saveAnalytics(event: AnalyticsEvent) {
    await pool.query(`
        INSERT INTO analytics_events 
        (conversation_id, event_type, timestamp, data)
        VALUES ($1, $2, $3, $4)
    `, [
        event.conversationId,
        event.type,
        new Date(event.timestamp),
        JSON.stringify(event.data),
    ]);
}

Tips

1. Track meaningful events - Focus on business-relevant metrics
2. Use consistent naming - Establish event naming conventions
3. Don't over-track - Too many events can be noise
4. Export regularly - Don't let in-memory data grow too large
5. Set up alerts - Monitor for anomalies (high handoff rate, low completion)