globals.py

"""Global variables and configuration for the Inference Provider Testing Dashboard."""

import threading
from typing import Dict, Any, Optional

# Type definition for job result entries
JobResult = Dict[str, Any]  # {model, provider, last_run, status, current_score, previous_score, job_id, start_time, duration, completed_at, runs: [{job_id, score, status, start_time, duration, completed_at}]}

# Global variables to track jobs
job_results: Dict[str, JobResult] = {}  # {model_provider_key: JobResult}
results_lock: threading.Lock = threading.Lock()

# Configuration
NUM_MODELS_RUN: int = 100
NUM_RUNS_PER_JOB: int = 4  # Number of times to run each job for variance reduction
RESULTS_DATASET_NAME: str = "IPTesting/inference-provider-test-results"
LOCAL_CONFIG_FILE: str = "/home/user/app/model_providers.txt"
TASKS: str = "extended|ifeval|0,lighteval|gpqa:diamond|0"
NAMESPACE: str = "huggingface"


def get_model_provider_key(model: str, provider: str) -> str:
    """Create a unique key for model-provider combination."""
    return f"{model}||{provider}"