feat: llm analytics

schema.md

@@ -0,0 +1,105 @@
+# LLM Observability Event Schema
+
+This document defines the standard event schema for LLM observability in the Amplitude Python SDK.
+
+## Standard Events
+
+### 1. `llm_run_started`
+Emitted when an LLM interaction/session begins.
+
+**Properties:**
+- `session_id` (string): Unique identifier for the LLM session
+- `user_id` (string, optional): User identifier
+- `model_provider` (string): Provider name (openai, anthropic, google, etc.)
+- `model_name` (string): Model identifier (gpt-4, claude-3, gemini-pro, etc.)
+- `timestamp` (number): Unix timestamp when the run started
+
+### 2. `llm_message` 
+Emitted for each LLM response/completion.
+
+**Properties:**
+- `session_id` (string): Session identifier
+- `message_id` (string): Unique identifier for this message
+- `user_id` (string, optional): User identifier
+- `model_provider` (string): Provider name
+- `model_name` (string): Model identifier
+- `input_tokens` (number, optional): Number of input tokens
+- `output_tokens` (number, optional): Number of output tokens
+- `total_tokens` (number, optional): Total tokens used
+- `latency_ms` (number): Response latency in milliseconds
+- `cost_usd` (number, optional): Estimated cost in USD
+- `input_messages` (array, optional): Input messages/prompts (privacy configurable)
+- `output_content` (string, optional): Generated response (privacy configurable)
+- `finish_reason` (string, optional): Why the completion finished
+- `temperature` (number, optional): Temperature parameter used
+- `max_tokens` (number, optional): Max tokens parameter used
+- `timestamp` (number): Unix timestamp
+
+### 3. `user_message`
+Emitted for user inputs/messages.
+
+**Properties:**
+- `session_id` (string): Session identifier
+- `message_id` (string): Unique identifier for this message
+- `user_id` (string, optional): User identifier
+- `content` (string, optional): User message content (privacy configurable)
+- `message_type` (string): Type of message (text, image, etc.)
+- `timestamp` (number): Unix timestamp
+
+### 4. `tool_called`
+Emitted when LLM calls a function/tool.
+
+**Properties:**
+- `session_id` (string): Session identifier
+- `tool_call_id` (string): Unique identifier for this tool call
+- `user_id` (string, optional): User identifier
+- `tool_name` (string): Name of the tool/function called
+- `tool_parameters` (object, optional): Parameters passed to the tool
+- `tool_result` (object, optional): Result returned by the tool
+- `execution_time_ms` (number, optional): Tool execution time
+- `success` (boolean): Whether the tool call succeeded
+- `error_message` (string, optional): Error message if tool call failed
+- `timestamp` (number): Unix timestamp
+
+### 5. `llm_run_finished`
+Emitted when an LLM interaction/session completes.
+
+**Properties:**
+- `session_id` (string): Session identifier
+- `user_id` (string, optional): User identifier
+- `total_messages` (number): Total number of messages in the session
+- `total_tokens` (number, optional): Total tokens used across all messages
+- `total_cost_usd` (number, optional): Total estimated cost
+- `duration_ms` (number): Total session duration
+- `success` (boolean): Whether the session completed successfully
+- `error_message` (string, optional): Error message if session failed
+- `timestamp` (number): Unix timestamp
+
+## Common Properties
+
+All events share these common Amplitude event properties:
+- `event_type` (string): The event name (llm_run_started, etc.)
+- `user_id` (string, optional): User identifier
+- `device_id` (string, optional): Device identifier
+- `session_id` (string): LLM session identifier
+- `timestamp` (number): Unix timestamp
+- `event_properties` (object): Event-specific properties defined above
+- `user_properties` (object, optional): User properties
+- `groups` (object, optional): Group identifiers
+
+## Privacy Configuration
+
+The following properties can be excluded based on privacy settings:
+- `input_messages` - Exclude user prompts/inputs
+- `output_content` - Exclude LLM responses
+- `content` - Exclude user message content
+- `tool_parameters` - Exclude tool call parameters
+- `tool_result` - Exclude tool call results
+
+## Error Events
+
+When errors occur, standard events should include:
+- `success: false`
+- `error_message` (string): Description of the error
+- `error_code` (string, optional): Error code if available
+- `error_type` (string, optional): Type/category of error

src/amplitude/__init__.py

@@ -7,3 +7,7 @@
 from amplitude.config import Config
 from amplitude.constants import PluginType
 from amplitude.plugin import EventPlugin, DestinationPlugin
+
+# AI observability is available as a submodule
+# Import with: from amplitude.ai import OpenAI, Anthropic, etc.
+# or: from amplitude import ai

src/amplitude/ai/__init__.py

@@ -0,0 +1,111 @@
+"""Amplitude AI observability module for LLM tracking.
+
+This module provides automatic instrumentation and manual event emission for LLM observability.
+Supports OpenAI, Anthropic, Google/Gemini, LangChain, and Pydantic AI with zero-latency impact.
+"""
+
+from .events import (
+    LLMRunStartedEvent,
+    LLMMessageEvent,
+    UserMessageEvent,
+    ToolCalledEvent,
+    LLMRunFinishedEvent,
+    emit_llm_run_started,
+    emit_llm_message,
+    emit_user_message,
+    emit_tool_called,
+    emit_llm_run_finished
+)
+
+from .config import AIConfig
+from .manual import (
+    LLMSessionTracker,
+    llm_session,
+    MessageTimer,
+    calculate_cost,
+    create_session_id,
+    quick_track_message
+)
+from .plugin import (
+    AIObservabilityPlugin,
+    AIEventFilterPlugin,
+    AIMetricsPlugin
+)
+
+# Provider wrappers (imported on-demand)
+__all__ = [
+    # Event classes
+    "LLMRunStartedEvent",
+    "LLMMessageEvent", 
+    "UserMessageEvent",
+    "ToolCalledEvent",
+    "LLMRunFinishedEvent",
+
+    # Basic emission functions
+    "emit_llm_run_started",
+    "emit_llm_message",
+    "emit_user_message", 
+    "emit_tool_called",
+    "emit_llm_run_finished",
+
+    # Configuration
+    "AIConfig",
+
+    # Manual tracking APIs
+    "LLMSessionTracker",
+    "llm_session",
+    "MessageTimer",
+    "calculate_cost",
+    "create_session_id",
+    "quick_track_message",
+
+    # Plugins
+    "AIObservabilityPlugin",
+    "AIEventFilterPlugin",
+    "AIMetricsPlugin"
+]
+
+
+def get_openai():
+    """Import OpenAI wrapper on-demand to avoid unnecessary dependencies."""
+    try:
+        from .openai import OpenAI, AsyncOpenAI
+        return OpenAI, AsyncOpenAI
+    except ImportError as e:
+        raise ImportError("OpenAI not installed. Install with: pip install openai") from e
+
+
+def get_anthropic():
+    """Import Anthropic wrapper on-demand to avoid unnecessary dependencies."""
+    try:
+        from .anthropic import Anthropic, AsyncAnthropic
+        return Anthropic, AsyncAnthropic
+    except ImportError as e:
+        raise ImportError("Anthropic not installed. Install with: pip install anthropic") from e
+
+
+def get_google():
+    """Import Google wrapper on-demand to avoid unnecessary dependencies."""
+    try:
+        from .google import GoogleGenerativeAI
+        return GoogleGenerativeAI
+    except ImportError as e:
+        raise ImportError("Google AI not installed. Install with: pip install google-generativeai") from e
+
+
+def get_langchain():
+    """Import LangChain callback handler on-demand to avoid unnecessary dependencies."""
+    try:
+        from .langchain import AmplitudeLangChainCallback
+        return AmplitudeLangChainCallback
+    except ImportError as e:
+        raise ImportError("LangChain not installed. Install with: pip install langchain-core") from e
+
+
+def get_pydantic():
+    """Import Pydantic AI wrapper on-demand to avoid unnecessary dependencies."""
+    try:
+        from .pydantic import PydanticAI
+        return PydanticAI
+    except ImportError as e:
+        raise ImportError("Pydantic AI not installed. Install with: pip install pydantic-ai") from e

src/amplitude/ai/anthropic.py

@@ -0,0 +1,101 @@
+"""Anthropic wrapper for Amplitude LLM observability."""
+
+from typing import Optional
+
+try:
+    from anthropic import Anthropic as OriginalAnthropic
+    from anthropic import AsyncAnthropic as OriginalAsyncAnthropic
+except ImportError:
+    OriginalAnthropic = None
+    OriginalAsyncAnthropic = None
+
+from amplitude.client import Amplitude
+from .config import AIConfig
+from .instrumentation import InstrumentationMixin, import_guard
+from .parsers import AnthropicResponseParser
+
+
+@import_guard("anthropic", "pip install anthropic")
+class Anthropic(InstrumentationMixin):
+    """Anthropic wrapper with Amplitude observability.
+
+    Drop-in replacement for anthropic.Anthropic that automatically tracks LLM usage.
+    """
+
+    def __init__(self, 
+                 api_key: Optional[str] = None,
+                 amplitude_client: Optional[Amplitude] = None,
+                 ai_config: Optional[AIConfig] = None,
+                 auto_start_session: bool = True,
+                 **kwargs):
+        """Initialize Anthropic wrapper."""
+        # Initialize mixin
+        super().__init__(
+            amplitude_client=amplitude_client,
+            ai_config=ai_config,
+            auto_start_session=auto_start_session
+        )
+
+        # Initialize Anthropic client
+        self._client = OriginalAnthropic(api_key=api_key, **kwargs)
+        self._response_parser = AnthropicResponseParser()
+
+        # Replace methods with instrumented versions
+        self._client.messages.create = self._create_instrumented_method(
+            self._client.messages.create, is_async=False
+        )
+
+    def get_provider_name(self) -> str:
+        """Get the provider name."""
+        return "anthropic"
+
+    def get_response_parser(self):
+        """Get the response parser."""
+        return self._response_parser
+
+    # Delegate all other attributes to the underlying Anthropic client
+    def __getattr__(self, name):
+        return getattr(self._client, name)
+
+
+@import_guard("anthropic", "pip install anthropic")
+class AsyncAnthropic(InstrumentationMixin):
+    """Async Anthropic wrapper with Amplitude observability.
+
+    Drop-in replacement for anthropic.AsyncAnthropic that automatically tracks LLM usage.
+    """
+
+    def __init__(self, 
+                 api_key: Optional[str] = None,
+                 amplitude_client: Optional[Amplitude] = None,
+                 ai_config: Optional[AIConfig] = None,
+                 auto_start_session: bool = True,
+                 **kwargs):
+        """Initialize AsyncAnthropic wrapper."""
+        # Initialize mixin
+        super().__init__(
+            amplitude_client=amplitude_client,
+            ai_config=ai_config,
+            auto_start_session=auto_start_session
+        )
+
+        # Initialize AsyncAnthropic client
+        self._client = OriginalAsyncAnthropic(api_key=api_key, **kwargs)
+        self._response_parser = AnthropicResponseParser()
+
+        # Replace methods with instrumented versions
+        self._client.messages.create = self._create_instrumented_method(
+            self._client.messages.create, is_async=True
+        )
+
+    def get_provider_name(self) -> str:
+        """Get the provider name."""
+        return "anthropic"
+
+    def get_response_parser(self):
+        """Get the response parser."""
+        return self._response_parser
+
+    # Delegate all other attributes to the underlying AsyncAnthropic client
+    def __getattr__(self, name):
+        return getattr(self._client, name)