AI_COUNT_TOKENS

AI_COUNT_TOKENS(
  model_or_function,
  text
)

SELECT AI_COUNT_TOKENS(
    'claude-4-sonnet',
    'This is a sample text for token counting'
) AS

WITH token_check AS (
    SELECT 
        product_id,
        description,
        AI_COUNT_TOKENS('llama3.1-70b', description) AS token_count
    FROM products
)
SELECT 
    product_id,
    CASE 
        WHEN token_count <= 8000 THEN AI_COMPLETE('llama3.1-70b', description)
        ELSE 'Text too long for model'
    END AS result
FROM

SELECT 
    COUNT(*) AS record_count,
    SUM(AI_COUNT_TOKENS('mistral-large2', review_text)) AS total_input_tokens,
    SUM(AI_COUNT_TOKENS('mistral-large2', review_text)) * 0.0001 AS estimated_cost_usd
FROM customer_reviews
WHERE process_date >= CURRENT_DATE - 30

SELECT 
    CASE 
        WHEN AI_COUNT_TOKENS('claude-4-sonnet', content) < 1000 THEN '< 1K tokens'
        WHEN AI_COUNT_TOKENS('claude-4-sonnet', content) < 10000 THEN '1K-10K tokens'
        WHEN AI_COUNT_TOKENS('claude-4-sonnet', content) < 50000 THEN '10K-50K tokens'
        ELSE '> 50K tokens'
    END AS token_range,
    COUNT(*) AS document_count
FROM documents
GROUP BY

SELECT 
    doc_id,
    title,
    AI_COUNT_TOKENS('AI_SENTIMENT', content) AS token_count,
    CASE 
        WHEN AI_COUNT_TOKENS('AI_SENTIMENT', content) > 2048 THEN 'Exceeds limit (2048)'
        ELSE 'OK'
    END AS validation
FROM documents
WHERE validation != 'OK'

-- Create a validation step
WITH validated_inputs AS (
    SELECT 
        input_id,
        input_text,
        AI_COUNT_TOKENS('llama3.1-70b', input_text) AS tokens,
        CASE 
            WHEN AI_COUNT_TOKENS('llama3.1-70b', input_text) <= 128000 THEN true
            ELSE false
        END AS within_limit
    FROM input_queue
)
SELECT 
    input_id,
    AI_COMPLETE('llama3.1-70b', input_text) AS result
FROM validated_inputs
WHERE within_limit = true;

-- Handle oversized inputs
SELECT input_id, 'Input exceeds token limit' AS error
FROM validated_inputs  
WHERE within_limit = false

-- Cost estimation view
CREATE VIEW ai_processing_estimates AS
SELECT 
    'Sentiment Analysis' AS task,
    COUNT(*) AS record_count,
    AVG(AI_COUNT_TOKENS('AI_SENTIMENT', feedback)) AS avg_tokens,
    SUM(AI_COUNT_TOKENS('AI_SENTIMENT', feedback)) AS total_tokens,
    SUM(AI_COUNT_TOKENS('AI_SENTIMENT', feedback)) * 0.0001 AS est_cost_usd
FROM customer_feedback
UNION ALL
SELECT 
    'Summarization' AS task,
    COUNT(*),
    AVG(AI_COUNT_TOKENS('claude-4-sonnet', content)),
    SUM(AI_COUNT_TOKENS('claude-4-sonnet', content)),
    SUM(AI_COUNT_TOKENS('claude-4-sonnet', content)) * 0.0003
FROM

-- Compare token counts for different prompt versions
WITH prompt_options AS (
    SELECT 
        'v1' AS version,
        'Please analyze this text and provide detailed insights: ' AS prompt
    UNION ALL
    SELECT 'v2', 'Analyze: '
    UNION ALL  
    SELECT 'v3', 'Analysis: '
)
SELECT 
    version,
    prompt,
    AI_COUNT_TOKENS('claude-4-sonnet', prompt) AS prompt_tokens,
    AI_COUNT_TOKENS('claude-4-sonnet', prompt || sample_text) AS total_tokens
FROM prompt_options, (SELECT 'Sample data text' AS sample_text)

-- Track token consumption over time
CREATE TABLE token_usage_log AS
SELECT 
    CURRENT_DATE() AS usage_date,
    'AI_COMPLETE' AS function_name,
    COUNT(*) AS calls,
    SUM(AI_COUNT_TOKENS('claude-4-sonnet', input_text)) AS total_tokens
FROM ai_processing_queue
GROUP BY usage_date,

-- Estimate monthly AI costs
WITH monthly_volume AS (
    SELECT 
        DATE_TRUNC('month', created_date) AS month,
        COUNT(*) AS record_count,
        AVG(AI_COUNT_TOKENS('claude-4-sonnet', content)) AS avg_tokens_per_record
    FROM content_queue
    GROUP BY month
)
SELECT 
    month,
    record_count,
    avg_tokens_per_record,
    record_count * avg_tokens_per_record AS total_tokens,
    (record_count * avg_tokens_per_record / 1000000.0) * 0.30 AS estimated_cost_usd
FROM monthly_volume
ORDER BY month DESC

-- Split documents that exceed token limits
WITH doc_analysis AS (
    SELECT 
        doc_id,
        content,
        AI_COUNT_TOKENS('llama3.1-70b', content) AS token_count,
        CEIL(AI_COUNT_TOKENS('llama3.1-70b', content) / 120000.0) AS chunks_needed
    FROM large_documents
)
SELECT 
    doc_id,
    chunk_num,
    SUBSTRING(
        content,
        (chunk_num - 1) * chunk_size + 1,
        chunk_size
    ) AS chunk_content
FROM doc_analysis,
     TABLE(GENERATOR(ROWCOUNT => chunks_needed)),
     (SELECT 120000 AS chunk_size)

-- Choose model based on input size
SELECT 
    request_id,
    input_text,
    CASE 
        WHEN AI_COUNT_TOKENS('llama3.1-8b', input_text) <= 8000 THEN 'llama3.1-8b'
        WHEN AI_COUNT_TOKENS('llama3.1-70b', input_text) <= 128000 THEN 'llama3.1-70b'
        ELSE 'claude-4-sonnet'
    END AS recommended_model,
    AI_COUNT_TOKENS('llama3.1-70b', input_text) AS token_count
FROM

-- Remove unnecessary whitespace
SELECT 
    AI_COUNT_TOKENS('claude-4-sonnet', original_text) AS original_tokens,
    AI_COUNT_TOKENS('claude-4-sonnet', REGEXP_REPLACE(original_text, '\s+', ' ')) AS optimized_tokens,
    AI_COUNT_TOKENS('claude-4-sonnet', original_text) - 
        AI_COUNT_TOKENS('claude-4-sonnet', REGEXP_REPLACE(original_text, '\s+', ' ')) AS tokens_saved
FROM

-- Find optimal batch size
SELECT 
    batch_size,
    COUNT(*) AS batches,
    SUM(batch_tokens) AS total_tokens,
    AVG(batch_tokens) AS avg_batch_tokens
FROM (
    SELECT 
        FLOOR(ROW_NUMBER() OVER (ORDER BY id) / 100) AS batch_id,
        100 AS batch_size,
        SUM(AI_COUNT_TOKENS('mistral-large2', text)) AS batch_tokens
    FROM content
    GROUP BY batch_id
)