.flake8
.gitignore
.pre-commit-config.yaml
.readthedocs.yaml
CHANGELOG.md
CITATION.bib
CODE_OF_CONDUCT.md
CONTRIBUTING.md
LEADERBOARD_EXPORT.md
LICENSE
MANIFEST.in
README.md
SECURITY.md
conftest.py
constraints.txt
install-dev.sh
install-heim-extras.sh
install-shelm-extras.sh
mkdocs.yml
pre-commit-frontend.sh
pre-commit.sh
pyproject.toml
requirements.txt
uv.lock
.github/dependabot.yml
.github/workflows/build-frontend.yml
.github/workflows/deploy-docs.yml
.github/workflows/frontend.yml
.github/workflows/manage-python-cache.yml
.github/workflows/publish.yml
.github/workflows/test.yml
.github/workflows/test_scenarios.yml
.github/workflows/update-requirements.yml
.github/workflows/upgrade-requirements.yml
docs/CNAME
docs/Gemfile
docs/README.md
docs/_config.yml
docs/adding_new_models.md
docs/adding_new_scenarios.md
docs/adding_new_tokenizers.md
docs/benchmark.md
docs/code.md
docs/credentials.md
docs/developer_adding_new_models.md
docs/developer_setup.md
docs/docstrings.css
docs/downloading_raw_results.md
docs/editing_documentation.md
docs/enterprise_benchmark.md
docs/get_helm_rank.md
docs/heim.md
docs/helm-eb-climate-2024.png
docs/helm-eb-cybersecurity-2024.png
docs/helm-eb-finance-2024.png
docs/helm-eb-legal-2024.png
docs/huggingface_models.md
docs/importing_custom_modules.md
docs/index.html
docs/installation.md
docs/medhelm.md
docs/metrics.md
docs/mkdocs_hooks.py
docs/mkdocs_macros.py
docs/models.md
docs/old_index.md
docs/perturbations.md
docs/proxy_server.md
docs/quick_start.md
docs/reeval.md
docs/reproducing_leaderboards.md
docs/requirements.txt
docs/run_entries.md
docs/run_entries_configuration_files.md
docs/scenarios.md
docs/schemas.md
docs/tutorial.md
docs/vhelm.md
docs/_includes/docs_sidebar.html
docs/_includes/github_stars.html
docs/_includes/header.html
docs/_includes/theme_head.html
docs/_layouts/default.html
docs/_layouts/docs.html
docs/assets/css/style.css
docs/assets/images/crfm-logo.png
docs/assets/images/heim-logo.png
docs/assets/images/helm-logo.png
docs/assets/images/medhelm-overview.png
docs/assets/images/medhelm_logo.jpg
docs/assets/images/pacific-ai-logo.svg
docs/assets/js/theme-toggle.js
docs/scripts/jekyll_prepare_mkdocstring_pages.py
docs/scripts/serve_jekyll_local.sh
helm-frontend/.eslintrc.cjs
helm-frontend/.gitignore
helm-frontend/README.md
helm-frontend/index.html
helm-frontend/package.json
helm-frontend/postcss.config.js
helm-frontend/tailwind.config.js
helm-frontend/tsconfig.json
helm-frontend/tsconfig.node.json
helm-frontend/vite.config.ts
helm-frontend/yarn.lock
helm-frontend/public/config.js
helm-frontend/public/helm.svg
helm-frontend/src/App.css
helm-frontend/src/App.tsx
helm-frontend/src/main.tsx
helm-frontend/src/vite-env.d.ts
helm-frontend/src/assets/crfm-logo.png
helm-frontend/src/assets/helmhero.png
helm-frontend/src/assets/language-model-helm.png
helm-frontend/src/assets/medhelm_logo.jpg
helm-frontend/src/assets/scenarios-by-metrics.png
helm-frontend/src/assets/taxonomy-scenarios.png
helm-frontend/src/assets/air-bench/air-overview.png
helm-frontend/src/assets/audio/audio-table.png
helm-frontend/src/assets/image2struct/overview.png
helm-frontend/src/assets/image2struct/process-flow.png
helm-frontend/src/assets/instruct/instruct-flowchart.svg
helm-frontend/src/assets/instruct/instruct-graph.svg
helm-frontend/src/assets/medhelm/medhelm-overview.png
helm-frontend/src/assets/medhelm/medhelm-v1-overview.png
helm-frontend/src/assets/roboreward/overview.png
helm-frontend/src/assets/seahelm/aisingapore.png
helm-frontend/src/assets/vhelm/vhelm-aspects.png
helm-frontend/src/assets/vhelm/vhelm-framework.png
helm-frontend/src/assets/vhelm/vhelm-model.png
helm-frontend/src/components/Alert.tsx
helm-frontend/src/components/AnnotationsDisplay.tsx
helm-frontend/src/components/Card.tsx
helm-frontend/src/components/CardGrid.tsx
helm-frontend/src/components/ExtraDataDisplay.tsx
helm-frontend/src/components/GroupLeaderboard.tsx
helm-frontend/src/components/HeadTitle.tsx
helm-frontend/src/components/Hero.tsx
helm-frontend/src/components/InstanceData.tsx
helm-frontend/src/components/Instances.tsx
helm-frontend/src/components/LeaderboardTable.tsx
helm-frontend/src/components/Link.tsx
helm-frontend/src/components/Loading.tsx
helm-frontend/src/components/MediaObjectDisplay.tsx
helm-frontend/src/components/MessagesDisplay.tsx
helm-frontend/src/components/MetricsList.tsx
helm-frontend/src/components/MiniLeaderboard.tsx
helm-frontend/src/components/MinimalNavBar.tsx
helm-frontend/src/components/ModelsList.tsx
helm-frontend/src/components/MultimediaObjectDisplay.tsx
helm-frontend/src/components/NavDropdown.tsx
helm-frontend/src/components/Pagination.tsx
helm-frontend/src/components/Predictions.tsx
helm-frontend/src/components/Preview.tsx
helm-frontend/src/components/ProjectCard.tsx
helm-frontend/src/components/References.tsx
helm-frontend/src/components/ReleaseDropdown.tsx
helm-frontend/src/components/RowValue.tsx
helm-frontend/src/components/RunMetrics.tsx
helm-frontend/src/components/ScenariosList.tsx
helm-frontend/src/components/SimpleHero.tsx
helm-frontend/src/components/StatNameDisplay.tsx
helm-frontend/src/components/Tab.tsx
helm-frontend/src/components/TransposedLeaderboardTable.tsx
helm-frontend/src/components/UserAgreement.tsx
helm-frontend/src/components/VHELMLanding.tsx
helm-frontend/src/components/AccessBadge/AccessBadge.test.tsx
helm-frontend/src/components/AccessBadge/AccessBadge.tsx
helm-frontend/src/components/AccessBadge/index.tsx
helm-frontend/src/components/Indicator/Indicator.test.tsx
helm-frontend/src/components/Indicator/Indicator.tsx
helm-frontend/src/components/Indicator/index.tsx
helm-frontend/src/components/Landing/AIRBenchLanding.tsx
helm-frontend/src/components/Landing/ArabicLanding.tsx
helm-frontend/src/components/Landing/AudioLanding.tsx
helm-frontend/src/components/Landing/CLEVALanding.tsx
helm-frontend/src/components/Landing/CapabilitiesLanding.tsx
helm-frontend/src/components/Landing/EWoKLanding.tsx
helm-frontend/src/components/Landing/FinanceLanding.tsx
helm-frontend/src/components/Landing/HEIMLanding.tsx
helm-frontend/src/components/Landing/HomeLanding.tsx
helm-frontend/src/components/Landing/Image2StructLanding.tsx
helm-frontend/src/components/Landing/InstructLanding.tsx
helm-frontend/src/components/Landing/LiteLanding.tsx
helm-frontend/src/components/Landing/LongContextLanding.tsx
helm-frontend/src/components/Landing/MMLULanding.tsx
helm-frontend/src/components/Landing/MMLUWinograndeAfrLanding.tsx
helm-frontend/src/components/Landing/MedHELMLanding.tsx
helm-frontend/src/components/Landing/MedHELMV1Landing.tsx
helm-frontend/src/components/Landing/RoboRewardLanding.tsx
helm-frontend/src/components/Landing/SEAHELMLanding.tsx
helm-frontend/src/components/Landing/SLPHelmLanding.tsx
helm-frontend/src/components/Landing/SQLLanding.tsx
helm-frontend/src/components/Landing/SafetyLanding.tsx
helm-frontend/src/components/Landing/SpeechLanding.tsx
helm-frontend/src/components/Landing/ThaiExamLanding.tsx
helm-frontend/src/components/Landing/ToRRLanding.tsx
helm-frontend/src/components/Landing/ViLLMLanding.tsx
helm-frontend/src/components/MarkdownValue/MarkdownValue.test.tsx
helm-frontend/src/components/MarkdownValue/MarkdownValue.tsx
helm-frontend/src/components/MarkdownValue/index.tsx
helm-frontend/src/components/NavBar/NavBar.test.tsx
helm-frontend/src/components/NavBar/NavBar.tsx
helm-frontend/src/components/NavBar/index.tsx
helm-frontend/src/components/PageTitle/PageTitle.test.tsx
helm-frontend/src/components/PageTitle/PageTitle.tsx
helm-frontend/src/components/PageTitle/index.tsx
helm-frontend/src/components/Request/Request.test.tsx
helm-frontend/src/components/Request/Request.tsx
helm-frontend/src/components/Request/index.tsx
helm-frontend/src/components/Tabs/Tabs.test.tsx
helm-frontend/src/components/Tabs/Tabs.tsx
helm-frontend/src/components/Tabs/index.tsx
helm-frontend/src/layouts/Main.tsx
helm-frontend/src/routes/Groups.tsx
helm-frontend/src/routes/Home.tsx
helm-frontend/src/routes/Leaderboard.tsx
helm-frontend/src/routes/Models.tsx
helm-frontend/src/routes/Run.tsx
helm-frontend/src/routes/Runs.tsx
helm-frontend/src/routes/Scenarios.tsx
helm-frontend/src/services/getDisplayPredictionsByName.ts
helm-frontend/src/services/getDisplayRequestsByName.ts
helm-frontend/src/services/getGroupTablesByName.ts
helm-frontend/src/services/getGroupsMetadata.ts
helm-frontend/src/services/getGroupsTables.ts
helm-frontend/src/services/getInstances.ts
helm-frontend/src/services/getReleaseSummary.ts
helm-frontend/src/services/getRunSpecByName.ts
helm-frontend/src/services/getRunSpecs.ts
helm-frontend/src/services/getRunsToRunSuites.ts
helm-frontend/src/services/getScenarioByName.ts
helm-frontend/src/services/getScenarioStateByName.ts
helm-frontend/src/services/getSchema.ts
helm-frontend/src/services/getStatsByName.ts
helm-frontend/src/services/getSuiteForRun.ts
helm-frontend/src/types/AdapterField.ts
helm-frontend/src/types/AdapterFieldMap.ts
helm-frontend/src/types/CompletionAnnotation.ts
helm-frontend/src/types/DisplayPrediction.ts
helm-frontend/src/types/DisplayRequest.ts
helm-frontend/src/types/EncryptionDataMap.ts
helm-frontend/src/types/GroupMetadata.ts
helm-frontend/src/types/GroupsMetadata.ts
helm-frontend/src/types/GroupsTable.ts
helm-frontend/src/types/HeaderValue.ts
helm-frontend/src/types/Instance.ts
helm-frontend/src/types/LinkValue.ts
helm-frontend/src/types/MediaObject.ts
helm-frontend/src/types/Message.ts
helm-frontend/src/types/MetricField.ts
helm-frontend/src/types/MetricFieldMap.ts
helm-frontend/src/types/MetricGroup.ts
helm-frontend/src/types/Model.ts
helm-frontend/src/types/MultimediaObject.ts
helm-frontend/src/types/Perturbation.ts
helm-frontend/src/types/ProjectMetadata.ts
helm-frontend/src/types/Reference.ts
helm-frontend/src/types/ReleaseSummary.ts
helm-frontend/src/types/RowValue.ts
helm-frontend/src/types/RunGroup.ts
helm-frontend/src/types/RunSpec.ts
helm-frontend/src/types/Scenario.ts
helm-frontend/src/types/Schema.ts
helm-frontend/src/types/Stat.ts
helm-frontend/src/types/Taxonomy.ts
helm-frontend/src/types/global.d.ts
helm-frontend/src/utils/getBenchmarkEndpoint.ts
helm-frontend/src/utils/getBenchmarkRelease.ts
helm-frontend/src/utils/getBenchmarkSuite.ts
helm-frontend/src/utils/getReleaseUrl.ts
helm-frontend/src/utils/getStatCorrectness.ts
helm-frontend/src/utils/getVersionBaseUrl.ts
helm-frontend/src/utils/helmPortalConfig.ts
helm-frontend/src/utils/isScenarioEncrypted.ts
scripts/compute_request_limits.py
scripts/create_custom_mcqa_dataset.py
scripts/decrypt_scenario_states.py
scripts/encrypt_scenario_states.py
scripts/estimate_cost.py
scripts/estimate_cost_audio.py
scripts/heim_human_eval.py
scripts/helm-run-all.sh
scripts/redact_scenario_states.py
scripts/verify_reproducibility.py
scripts/air_bench/export_air_bench_csv.py
scripts/cache/__init__.py
scripts/cache/copy_cache.py
scripts/cache/fix_anthropic_cache.py
scripts/cache/fix_together_cache.py
scripts/cache/remove_together_api_entries.py
scripts/efficiency/generate_instances.py
scripts/efficiency/generate_run_specs.py
scripts/examples/auto_client_usage.py
scripts/examples/request_json_schema.py
scripts/examples/server_service_usage.py
scripts/fact_completion/README.MD
scripts/fact_completion/create_benchmark.py
scripts/fact_completion/fetch_triples_and_aliases.py
scripts/fact_completion/filter_triples.py
scripts/fact_completion/utils.py
scripts/medhelm/redact_annotations.py
scripts/offline_eval/__init__.py
scripts/offline_eval/export_requests.py
scripts/offline_eval/import_results.py
scripts/offline_eval/deepfloyd/__init__.py
scripts/offline_eval/deepfloyd/deepfloyd.py
scripts/parade/audio_pairs_inst.json
scripts/parade/dataset_creation.py
scripts/scale/create_and_setup_instruction_following_project.py
scripts/scale/create_and_setup_project.py
scripts/scale/finalize_batch.py
scripts/scale/instruction_following_calibration_instances.jsonl
scripts/scale/scale_utils.py
src/helm/__init__.py
src/helm/py.typed
src/helm/benchmark/__init__.py
src/helm/benchmark/annotation_executor.py
src/helm/benchmark/config_registry.py
src/helm/benchmark/data_preprocessor.py
src/helm/benchmark/executor.py
src/helm/benchmark/huggingface_registration.py
src/helm/benchmark/model_deployment_registry.py
src/helm/benchmark/model_metadata_registry.py
src/helm/benchmark/multi_gpu_runner.py
src/helm/benchmark/reeval_run.py
src/helm/benchmark/reeval_runner.py
src/helm/benchmark/run.py
src/helm/benchmark/run_expander.py
src/helm/benchmark/run_spec.py
src/helm/benchmark/run_spec_factory.py
src/helm/benchmark/runner.py
src/helm/benchmark/runner_config_registry.py
src/helm/benchmark/server.py
src/helm/benchmark/slurm_jobs.py
src/helm/benchmark/slurm_runner.py
src/helm/benchmark/test_data_preprocessor.py
src/helm/benchmark/test_plugins.py
src/helm/benchmark/test_run_expander.py
src/helm/benchmark/tokenizer_config_registry.py
src/helm/benchmark/adaptation/__init__.py
src/helm/benchmark/adaptation/adapter_spec.py
src/helm/benchmark/adaptation/common_adapter_specs.py
src/helm/benchmark/adaptation/prompt.py
src/helm/benchmark/adaptation/request_state.py
src/helm/benchmark/adaptation/scenario_state.py
src/helm/benchmark/adaptation/adapters/__init__.py
src/helm/benchmark/adaptation/adapters/adapter.py
src/helm/benchmark/adaptation/adapters/adapter_factory.py
src/helm/benchmark/adaptation/adapters/binary_ranking_adapter.py
src/helm/benchmark/adaptation/adapters/chat_adapter.py
src/helm/benchmark/adaptation/adapters/ehr_instruction_adapter.py
src/helm/benchmark/adaptation/adapters/generation_adapter.py
src/helm/benchmark/adaptation/adapters/in_context_learning_adapter.py
src/helm/benchmark/adaptation/adapters/language_modeling_adapter.py
src/helm/benchmark/adaptation/adapters/multiple_choice_calibrated_adapter.py
src/helm/benchmark/adaptation/adapters/multiple_choice_joint_adapter.py
src/helm/benchmark/adaptation/adapters/multiple_choice_joint_chain_of_thought_adapter.py
src/helm/benchmark/adaptation/adapters/multiple_choice_separate_adapter.py
src/helm/benchmark/adaptation/adapters/test_adapter.py
src/helm/benchmark/adaptation/adapters/test_generation_adapter.py
src/helm/benchmark/adaptation/adapters/test_language_modeling_adapter.py
src/helm/benchmark/adaptation/adapters/test_multiple_choice_joint_adapter.py
src/helm/benchmark/adaptation/adapters/multimodal/__init__.py
src/helm/benchmark/adaptation/adapters/multimodal/generation_multimodal_adapter.py
src/helm/benchmark/adaptation/adapters/multimodal/in_context_learning_multimodal_adapter.py
src/helm/benchmark/adaptation/adapters/multimodal/multimodal_prompt.py
src/helm/benchmark/adaptation/adapters/multimodal/multiple_choice_joint_multimodal_adapter.py
src/helm/benchmark/adaptation/adapters/multimodal/test_in_context_learning_multimodal_adapter.py
src/helm/benchmark/adaptation/adapters/multimodal/test_multimodal_prompt.py
src/helm/benchmark/annotation/__init__.py
src/helm/benchmark/annotation/aci_bench_annotator.py
src/helm/benchmark/annotation/air_bench_annotator.py
src/helm/benchmark/annotation/alrage_annotator.py
src/helm/benchmark/annotation/annotator.py
src/helm/benchmark/annotation/annotator_factory.py
src/helm/benchmark/annotation/anthropic_red_team_annotator.py
src/helm/benchmark/annotation/arabic_content_generation_annotator.py
src/helm/benchmark/annotation/arabic_content_generation_relative_annotator.py
src/helm/benchmark/annotation/arabic_content_generation_similarity_annotator.py
src/helm/benchmark/annotation/autobencher_capabilities_annotator.py
src/helm/benchmark/annotation/autobencher_safety_annotator.py
src/helm/benchmark/annotation/bigcodebench_annotator.py
src/helm/benchmark/annotation/bird_sql_annotator.py
src/helm/benchmark/annotation/chw_care_plan_annotator.py
src/helm/benchmark/annotation/czech_bank_qa_annotator.py
src/helm/benchmark/annotation/dischargeme_annotator.py
src/helm/benchmark/annotation/ehr_sql_annotator.py
src/helm/benchmark/annotation/financebench_annotator.py
src/helm/benchmark/annotation/harm_bench_annotator.py
src/helm/benchmark/annotation/health_bench_annotator.py
src/helm/benchmark/annotation/helpdesk_call_summarization_annotator.py
src/helm/benchmark/annotation/live_qa_annotator.py
src/helm/benchmark/annotation/med_dialog_annotator.py
src/helm/benchmark/annotation/medalign_annotator.py
src/helm/benchmark/annotation/medi_qa_annotator.py
src/helm/benchmark/annotation/medication_qa_annotator.py
src/helm/benchmark/annotation/mental_health_annotator.py
src/helm/benchmark/annotation/mimic_bhc_annotator.py
src/helm/benchmark/annotation/mimic_rrs_annotator.py
src/helm/benchmark/annotation/model_as_judge.py
src/helm/benchmark/annotation/mtsamples_procedures_annotator.py
src/helm/benchmark/annotation/mtsamples_replicate_annotator.py
src/helm/benchmark/annotation/omni_math_annotator.py
src/helm/benchmark/annotation/simple_safety_tests_annotator.py
src/helm/benchmark/annotation/spider_annotator.py
src/helm/benchmark/annotation/starr_patient_instructions_annotator.py
src/helm/benchmark/annotation/test_annotator_factory.py
src/helm/benchmark/annotation/test_dummy_annotator.py
src/helm/benchmark/annotation/wildbench_annotator.py
src/helm/benchmark/annotation/xstest_annotator.py
src/helm/benchmark/annotation/image2struct/__init__.py
src/helm/benchmark/annotation/image2struct/image_compiler_annotator.py
src/helm/benchmark/annotation/image2struct/latex_compiler_annotator.py
src/helm/benchmark/annotation/image2struct/lilypond_compiler_annotator.py
src/helm/benchmark/annotation/image2struct/webpage_compiler_annotator.py
src/helm/benchmark/annotation/omni_math/gpt_evaluation_template.txt
src/helm/benchmark/annotation/omni_math/gpt_evaluation_zero_shot_template.txt
src/helm/benchmark/annotation/wildbench/eval_template.pairwise.v2.md
src/helm/benchmark/annotation/wildbench/eval_template.score.v2.md
src/helm/benchmark/augmentations/__init__.py
src/helm/benchmark/augmentations/cleva_perturbation.py
src/helm/benchmark/augmentations/contraction_expansion_perturbation.py
src/helm/benchmark/augmentations/contrast_sets_perturbation.py
src/helm/benchmark/augmentations/correct_to_misspelling.json
src/helm/benchmark/augmentations/data_augmenter.py
src/helm/benchmark/augmentations/dialect_perturbation.py
src/helm/benchmark/augmentations/extra_space_perturbation.py
src/helm/benchmark/augmentations/filler_words_perturbation.py
src/helm/benchmark/augmentations/gender_perturbation.py
src/helm/benchmark/augmentations/lowercase_perturbation.py
src/helm/benchmark/augmentations/mild_mix_perturbation.py
src/helm/benchmark/augmentations/misspelling_perturbation.py
src/helm/benchmark/augmentations/person_name_perturbation.py
src/helm/benchmark/augmentations/perturbation.py
src/helm/benchmark/augmentations/perturbation_description.py
src/helm/benchmark/augmentations/space_perturbation.py
src/helm/benchmark/augmentations/suffix_perturbation.py
src/helm/benchmark/augmentations/synonym_perturbation.py
src/helm/benchmark/augmentations/test_perturbation.py
src/helm/benchmark/augmentations/translate_perturbation.py
src/helm/benchmark/augmentations/typos_perturbation.py
src/helm/benchmark/efficiency_data/inference_denoised_runtimes.json
src/helm/benchmark/efficiency_data/inference_idealized_runtimes.json
src/helm/benchmark/efficiency_data/training_efficiency.json
src/helm/benchmark/metrics/__init__.py
src/helm/benchmark/metrics/air_bench_metrics.py
src/helm/benchmark/metrics/alrage_metric.py
src/helm/benchmark/metrics/annotation_metrics.py
src/helm/benchmark/metrics/basic_metrics.py
src/helm/benchmark/metrics/bbq_metrics.py
src/helm/benchmark/metrics/bias_metrics.py
src/helm/benchmark/metrics/bias_word_lists.py
src/helm/benchmark/metrics/bigcodebench_metrics.py
src/helm/benchmark/metrics/bird_sql_metrics.py
src/helm/benchmark/metrics/classification_metrics.py
src/helm/benchmark/metrics/cleva_accuracy_metrics.py
src/helm/benchmark/metrics/cleva_harms_metrics.py
src/helm/benchmark/metrics/cleva_metrics_helper.py
src/helm/benchmark/metrics/code_metrics.py
src/helm/benchmark/metrics/code_metrics_helper.py
src/helm/benchmark/metrics/codeinsights_code_efficiency_metrics.py
src/helm/benchmark/metrics/codeinsights_code_evaluation_metrics.py
src/helm/benchmark/metrics/codeinsights_correct_code_metrics.py
src/helm/benchmark/metrics/codeinsights_edge_case_metrics.py
src/helm/benchmark/metrics/codeinsights_metric_specs.py
src/helm/benchmark/metrics/comet_metric.py
src/helm/benchmark/metrics/common_metric_specs.py
src/helm/benchmark/metrics/conv_fin_qa_calc_metrics.py
src/helm/benchmark/metrics/copyright_metrics.py
src/helm/benchmark/metrics/czech_bank_qa_metrics.py
src/helm/benchmark/metrics/decodingtrust_fairness_metrics.py
src/helm/benchmark/metrics/decodingtrust_ood_knowledge_metrics.py
src/helm/benchmark/metrics/decodingtrust_privacy_metrics.py
src/helm/benchmark/metrics/decodingtrust_stereotype_bias_metrics.py
src/helm/benchmark/metrics/disinformation_metrics.py
src/helm/benchmark/metrics/dry_run_metrics.py
src/helm/benchmark/metrics/efficiency_metrics.py
src/helm/benchmark/metrics/ehr_sql_metrics.py
src/helm/benchmark/metrics/evaluate_instances_metric.py
src/helm/benchmark/metrics/evaluate_reference_metrics.py
src/helm/benchmark/metrics/fin_qa_metrics.py
src/helm/benchmark/metrics/fin_qa_metrics_helper.py
src/helm/benchmark/metrics/gpqa_chain_of_thought_metric.py
src/helm/benchmark/metrics/gpt4_audio_critique_metrics.py
src/helm/benchmark/metrics/gpt4_audio_refusal_metrics.py
src/helm/benchmark/metrics/gpt4v_originality_critique_metrics.py
src/helm/benchmark/metrics/ifeval_metrics.py
src/helm/benchmark/metrics/instruction_following_critique_metrics.py
src/helm/benchmark/metrics/kpi_edgar_metrics.py
src/helm/benchmark/metrics/language_modeling_metrics.py
src/helm/benchmark/metrics/live_qa_metrics.py
src/helm/benchmark/metrics/llm_jury_metrics.py
src/helm/benchmark/metrics/lmkt_metric_specs.py
src/helm/benchmark/metrics/lmkt_metrics.py
src/helm/benchmark/metrics/machine_translation_metrics.py
src/helm/benchmark/metrics/medcalc_bench_metrics.py
src/helm/benchmark/metrics/medec_metrics.py
src/helm/benchmark/metrics/melt_bias_metric.py
src/helm/benchmark/metrics/melt_bias_word_lists.py
src/helm/benchmark/metrics/melt_metric_specs.py
src/helm/benchmark/metrics/melt_toxicity_metric.py
src/helm/benchmark/metrics/metric.py
src/helm/benchmark/metrics/metric_name.py
src/helm/benchmark/metrics/metric_service.py
src/helm/benchmark/metrics/mimiciv_billing_code_metrics.py
src/helm/benchmark/metrics/nltk_helper.py
src/helm/benchmark/metrics/omni_math_metrics.py
src/helm/benchmark/metrics/openai_mrcr_metrics.py
src/helm/benchmark/metrics/output_processing_metric.py
src/helm/benchmark/metrics/output_processors.py
src/helm/benchmark/metrics/paraphrase_generation_metrics.py
src/helm/benchmark/metrics/prometheus_vision_critique_metrics.py
src/helm/benchmark/metrics/ranking_metrics.py
src/helm/benchmark/metrics/reference_metric.py
src/helm/benchmark/metrics/reka_vibe_critique_metrics.py
src/helm/benchmark/metrics/ruler_qa_metrics.py
src/helm/benchmark/metrics/safety_metrics.py
src/helm/benchmark/metrics/sct_bench_metrics.py
src/helm/benchmark/metrics/seahelm_metrics.py
src/helm/benchmark/metrics/seahelm_metrics_specs.py
src/helm/benchmark/metrics/spider_metrics.py
src/helm/benchmark/metrics/statistic.py
src/helm/benchmark/metrics/summarization_critique_metrics.py
src/helm/benchmark/metrics/summarization_metrics.py
src/helm/benchmark/metrics/test_bias_metrics.py
src/helm/benchmark/metrics/test_classification_metrics.py
src/helm/benchmark/metrics/test_disinformation_metrics.py
src/helm/benchmark/metrics/test_evaluate_reference_metrics.py
src/helm/benchmark/metrics/test_metric.py
src/helm/benchmark/metrics/test_statistic.py
src/helm/benchmark/metrics/toxicity_metrics.py
src/helm/benchmark/metrics/toxicity_utils.py
src/helm/benchmark/metrics/ultra_suite_asr_classification_metrics.py
src/helm/benchmark/metrics/unitxt_metrics.py
src/helm/benchmark/metrics/wildbench_metrics.py
src/helm/benchmark/metrics/data_stats/__init__.py
src/helm/benchmark/metrics/data_stats/data_stats_metric.py
src/helm/benchmark/metrics/data_stats/fragments.py
src/helm/benchmark/metrics/ifeval/__init__.py
src/helm/benchmark/metrics/ifeval/instructions.py
src/helm/benchmark/metrics/ifeval/instructions_registry.py
src/helm/benchmark/metrics/ifeval/instructions_registry.pyi
src/helm/benchmark/metrics/ifeval/instructions_util.py
src/helm/benchmark/metrics/image_generation/__init__.py
src/helm/benchmark/metrics/image_generation/aesthetics_metrics.py
src/helm/benchmark/metrics/image_generation/aesthetics_scorer.py
src/helm/benchmark/metrics/image_generation/clip_score_metrics.py
src/helm/benchmark/metrics/image_generation/denoised_runtime_metric.py
src/helm/benchmark/metrics/image_generation/detection_metrics.py
src/helm/benchmark/metrics/image_generation/efficiency_metrics.py
src/helm/benchmark/metrics/image_generation/fidelity_metrics.py
src/helm/benchmark/metrics/image_generation/fractal_dimension_metric.py
src/helm/benchmark/metrics/image_generation/gender_metrics.py
src/helm/benchmark/metrics/image_generation/image_critique_metrics.py
src/helm/benchmark/metrics/image_generation/lpips_metrics.py
src/helm/benchmark/metrics/image_generation/multi_scale_ssim_metrics.py
src/helm/benchmark/metrics/image_generation/nsfw_detector.py
src/helm/benchmark/metrics/image_generation/nsfw_metrics.py
src/helm/benchmark/metrics/image_generation/nudity_metrics.py
src/helm/benchmark/metrics/image_generation/photorealism_critique_metrics.py
src/helm/benchmark/metrics/image_generation/psnr_metrics.py
src/helm/benchmark/metrics/image_generation/q16_toxicity_metrics.py
src/helm/benchmark/metrics/image_generation/skin_tone_metrics.py
src/helm/benchmark/metrics/image_generation/uiqi_metrics.py
src/helm/benchmark/metrics/image_generation/watermark_metrics.py
src/helm/benchmark/metrics/image_generation/detectors/__init__.py
src/helm/benchmark/metrics/image_generation/detectors/base_detector.py
src/helm/benchmark/metrics/image_generation/detectors/vitdet.py
src/helm/benchmark/metrics/image_generation/fractal_dimension/__init__.py
src/helm/benchmark/metrics/image_generation/fractal_dimension/fractal_dimension_util.py
src/helm/benchmark/metrics/image_generation/fractal_dimension/test_fractal_dimension_util.py
src/helm/benchmark/metrics/image_generation/fractal_dimension/test_images/cloud.png
src/helm/benchmark/metrics/image_generation/fractal_dimension/test_images/convergence.png
src/helm/benchmark/metrics/image_generation/fractal_dimension/test_images/sea_anemone.png
src/helm/benchmark/metrics/image_generation/fractal_dimension/test_images/snowflakes.png
src/helm/benchmark/metrics/image_generation/q16/__init__.py
src/helm/benchmark/metrics/image_generation/q16/prompts.p
src/helm/benchmark/metrics/image_generation/q16/q16_toxicity_detector.py
src/helm/benchmark/metrics/image_generation/q16/test_q16.py
src/helm/benchmark/metrics/image_generation/q16/test_images/sample_appropriate.jpg
src/helm/benchmark/metrics/image_generation/q16/test_images/sample_inappropriate.png
src/helm/benchmark/metrics/image_generation/watermark/__init__.py
src/helm/benchmark/metrics/image_generation/watermark/test_watermark_detector.py
src/helm/benchmark/metrics/image_generation/watermark/watermark_detector.py
src/helm/benchmark/metrics/image_generation/watermark/test_images/clear_example.png
src/helm/benchmark/metrics/image_generation/watermark/test_images/watermark_example.png
src/helm/benchmark/metrics/summac/__init__.py
src/helm/benchmark/metrics/summac/model_summac.py
src/helm/benchmark/metrics/summac/utils_misc.py
src/helm/benchmark/metrics/tokens/__init__.py
src/helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/auto_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/free_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/openai_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py
src/helm/benchmark/metrics/tokens/token_cost_estimator.py
src/helm/benchmark/metrics/vision_language/__init__.py
src/helm/benchmark/metrics/vision_language/emd_utils.py
src/helm/benchmark/metrics/vision_language/image_metrics.py
src/helm/benchmark/metrics/vision_language/image_utils.py
src/helm/benchmark/presentation/__init__.py
src/helm/benchmark/presentation/contamination.py
src/helm/benchmark/presentation/create_plots.py
src/helm/benchmark/presentation/run_display.py
src/helm/benchmark/presentation/run_entries.conf
src/helm/benchmark/presentation/run_entries_air_bench.conf
src/helm/benchmark/presentation/run_entries_air_bench_reasoning.conf
src/helm/benchmark/presentation/run_entries_arabic.conf
src/helm/benchmark/presentation/run_entries_audio.conf
src/helm/benchmark/presentation/run_entries_audio_debug.conf
src/helm/benchmark/presentation/run_entries_big_bench_lite.conf
src/helm/benchmark/presentation/run_entries_biomedical.conf
src/helm/benchmark/presentation/run_entries_capabilities.conf
src/helm/benchmark/presentation/run_entries_capabilities_reasoning_deepseek_r1.conf
src/helm/benchmark/presentation/run_entries_capabilities_reasoning_v2.conf
src/helm/benchmark/presentation/run_entries_capabilities_v2.conf
src/helm/benchmark/presentation/run_entries_cleva_v1.conf
src/helm/benchmark/presentation/run_entries_codeinsights.conf
src/helm/benchmark/presentation/run_entries_core_scenarios_10.conf
src/helm/benchmark/presentation/run_entries_core_scenarios_100.conf
src/helm/benchmark/presentation/run_entries_core_scenarios_1000.conf
src/helm/benchmark/presentation/run_entries_core_scenarios_20.conf
src/helm/benchmark/presentation/run_entries_core_scenarios_50.conf
src/helm/benchmark/presentation/run_entries_core_scenarios_all.conf
src/helm/benchmark/presentation/run_entries_dec2023.conf
src/helm/benchmark/presentation/run_entries_decodingtrust.conf
src/helm/benchmark/presentation/run_entries_ewok.conf
src/helm/benchmark/presentation/run_entries_extra.conf
src/helm/benchmark/presentation/run_entries_finance.conf
src/helm/benchmark/presentation/run_entries_gpu.conf
src/helm/benchmark/presentation/run_entries_heim.conf
src/helm/benchmark/presentation/run_entries_heim_debug.conf
src/helm/benchmark/presentation/run_entries_heim_human.conf
src/helm/benchmark/presentation/run_entries_heim_human_eval.conf
src/helm/benchmark/presentation/run_entries_image2struct.conf
src/helm/benchmark/presentation/run_entries_interactive_qa.conf
src/helm/benchmark/presentation/run_entries_legal.conf
src/helm/benchmark/presentation/run_entries_lite.conf
src/helm/benchmark/presentation/run_entries_lite_20240424.conf
src/helm/benchmark/presentation/run_entries_lite_20240424_instruct.conf
src/helm/benchmark/presentation/run_entries_lite_20240424_legalbench_instructions.conf
src/helm/benchmark/presentation/run_entries_lite_20240424_nova.conf
src/helm/benchmark/presentation/run_entries_lite_20240424_output_format_instructions.conf
src/helm/benchmark/presentation/run_entries_lite_20240424_palmyra.conf
src/helm/benchmark/presentation/run_entries_lmkt.conf
src/helm/benchmark/presentation/run_entries_long_context.conf
src/helm/benchmark/presentation/run_entries_medhelm_dspy.conf
src/helm/benchmark/presentation/run_entries_medhelm_gated.conf
src/helm/benchmark/presentation/run_entries_medhelm_private_stanford.conf
src/helm/benchmark/presentation/run_entries_medhelm_public.conf
src/helm/benchmark/presentation/run_entries_melt.conf
src/helm/benchmark/presentation/run_entries_mmlu.conf
src/helm/benchmark/presentation/run_entries_mmlu_winogrande_afr.conf
src/helm/benchmark/presentation/run_entries_mmlu_winogrande_afr_instruct.conf
src/helm/benchmark/presentation/run_entries_multimed_qa.conf
src/helm/benchmark/presentation/run_entries_opinions_qa_ai21_default.conf
src/helm/benchmark/presentation/run_entries_opinions_qa_ai21_steer.conf
src/helm/benchmark/presentation/run_entries_opinions_qa_openai_default.conf
src/helm/benchmark/presentation/run_entries_opinions_qa_openai_steer.conf
src/helm/benchmark/presentation/run_entries_safety.conf
src/helm/benchmark/presentation/run_entries_safety_palmyra_x5.conf
src/helm/benchmark/presentation/run_entries_safety_reasoning.conf
src/helm/benchmark/presentation/run_entries_seahelm.conf
src/helm/benchmark/presentation/run_entries_seahelm_zero_shot.conf
src/helm/benchmark/presentation/run_entries_slphelm.conf
src/helm/benchmark/presentation/run_entries_small.conf
src/helm/benchmark/presentation/run_entries_tiny.conf
src/helm/benchmark/presentation/run_entries_torr.conf
src/helm/benchmark/presentation/run_entries_unitxt.conf
src/helm/benchmark/presentation/run_entries_vhelm.conf
src/helm/benchmark/presentation/run_entries_vhelm_debug.conf
src/helm/benchmark/presentation/run_entries_vhelm_lite.conf
src/helm/benchmark/presentation/run_entries_video_understanding.conf
src/helm/benchmark/presentation/run_entry.py
src/helm/benchmark/presentation/schema.py
src/helm/benchmark/presentation/summarize.py
src/helm/benchmark/presentation/table.py
src/helm/benchmark/presentation/taxonomy_info.py
src/helm/benchmark/presentation/test_contamination.py
src/helm/benchmark/presentation/test_create_plots.py
src/helm/benchmark/presentation/test_run_entry.py
src/helm/benchmark/presentation/test_schema.py
src/helm/benchmark/presentation/test_schema_validation.py
src/helm/benchmark/presentation/test_summarize.py
src/helm/benchmark/presentation/torr_robustness_summarizer.py
src/helm/benchmark/run_specs/__init__.py
src/helm/benchmark/run_specs/air_bench_run_specs.py
src/helm/benchmark/run_specs/arabic_enterprise_run_specs.py
src/helm/benchmark/run_specs/arabic_run_specs.py
src/helm/benchmark/run_specs/audio_run_specs.py
src/helm/benchmark/run_specs/bluex_run_specs.py
src/helm/benchmark/run_specs/capabilities_run_specs.py
src/helm/benchmark/run_specs/classic_run_specs.py
src/helm/benchmark/run_specs/cleva_run_specs.py
src/helm/benchmark/run_specs/codeinsights_run_specs.py
src/helm/benchmark/run_specs/decodingtrust_run_specs.py
src/helm/benchmark/run_specs/enem_challenge_specs.py
src/helm/benchmark/run_specs/enterprise_run_specs.py
src/helm/benchmark/run_specs/experimental_run_specs.py
src/helm/benchmark/run_specs/finance_run_specs.py
src/helm/benchmark/run_specs/healthqa_br_run_specs.py
src/helm/benchmark/run_specs/heim_run_specs.py
src/helm/benchmark/run_specs/imdb_ptbr_run_specs.py
src/helm/benchmark/run_specs/instruction_following_run_specs.py
src/helm/benchmark/run_specs/lite_run_specs.py
src/helm/benchmark/run_specs/lmkt_run_specs.py
src/helm/benchmark/run_specs/long_context_run_specs.py
src/helm/benchmark/run_specs/medhelm_run_specs.py
src/helm/benchmark/run_specs/melt_run_specs.py
src/helm/benchmark/run_specs/mmlu_clinical_afr_run_specs.py
src/helm/benchmark/run_specs/multilingual_run_specs.py
src/helm/benchmark/run_specs/oab_exams_specs.py
src/helm/benchmark/run_specs/safety_run_specs.py
src/helm/benchmark/run_specs/seahelm_run_specs.py
src/helm/benchmark/run_specs/simple_run_specs.py
src/helm/benchmark/run_specs/speech_disorder_audio_run_specs.py
src/helm/benchmark/run_specs/sql_run_specs.py
src/helm/benchmark/run_specs/tweetsentbr_run_specs.py
src/helm/benchmark/run_specs/unitxt_run_specs.py
src/helm/benchmark/run_specs/vlm_run_specs.py
src/helm/benchmark/run_specs/winogrande_afr_run_specs.py
src/helm/benchmark/run_specs/medhelm/__init__.py
src/helm/benchmark/run_specs/medhelm/benchmark_config.py
src/helm/benchmark/scenarios/__init__.py
src/helm/benchmark/scenarios/aci_bench_scenario.py
src/helm/benchmark/scenarios/air_bench_scenario.py
src/helm/benchmark/scenarios/alghafa_scenario.py
src/helm/benchmark/scenarios/alrage_scenario.py
src/helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py
src/helm/benchmark/scenarios/anthropic_red_team_scenario.py
src/helm/benchmark/scenarios/arabic_exams_scenario.py
src/helm/benchmark/scenarios/arabic_mmlu_scenario.py
src/helm/benchmark/scenarios/aratrust_scenario.py
src/helm/benchmark/scenarios/autobencher_capabilities_scenario.py
src/helm/benchmark/scenarios/autobencher_safety_scenario.py
src/helm/benchmark/scenarios/babi_qa_scenario.py
src/helm/benchmark/scenarios/banking77_scenario.py
src/helm/benchmark/scenarios/bbq_scenario.py
src/helm/benchmark/scenarios/best_chatgpt_prompts.yaml
src/helm/benchmark/scenarios/big_bench_scenario.py
src/helm/benchmark/scenarios/bigcodebench_scenario.py
src/helm/benchmark/scenarios/bird_sql_scenario.py
src/helm/benchmark/scenarios/bird_sql_scenario_helper.py
src/helm/benchmark/scenarios/blimp_scenario.py
src/helm/benchmark/scenarios/bluex_scenario.py
src/helm/benchmark/scenarios/bold_scenario.py
src/helm/benchmark/scenarios/boolq_scenario.py
src/helm/benchmark/scenarios/casehold_scenario.py
src/helm/benchmark/scenarios/chw_care_plan_scenario.py
src/helm/benchmark/scenarios/ci_mcqa_scenario.py
src/helm/benchmark/scenarios/civil_comments_scenario.py
src/helm/benchmark/scenarios/clear_scenario.py
src/helm/benchmark/scenarios/cleva_scenario.py
src/helm/benchmark/scenarios/code_scenario.py
src/helm/benchmark/scenarios/code_scenario_apps_pinned_file_order.py
src/helm/benchmark/scenarios/code_scenario_helper.py
src/helm/benchmark/scenarios/codeinsights_code_efficiency_scenario.py
src/helm/benchmark/scenarios/codeinsights_correct_code_scenario.py
src/helm/benchmark/scenarios/codeinsights_edge_case_scenario.py
src/helm/benchmark/scenarios/codeinsights_student_coding_scenario.py
src/helm/benchmark/scenarios/codeinsights_student_mistake_scenario.py
src/helm/benchmark/scenarios/commonsense_scenario.py
src/helm/benchmark/scenarios/compositional_instructions.yaml
src/helm/benchmark/scenarios/conv_fin_qa_calc_scenario.py
src/helm/benchmark/scenarios/copyright_scenario.py
src/helm/benchmark/scenarios/covid_dialog_scenario.py
src/helm/benchmark/scenarios/cti_to_mitre_scenario.py
src/helm/benchmark/scenarios/custom_mcqa_scenario.py
src/helm/benchmark/scenarios/czech_bank_qa_scenario.py
src/helm/benchmark/scenarios/decodingtrust_adv_demonstration_scenario.py
src/helm/benchmark/scenarios/decodingtrust_adv_robustness_scenario.py
src/helm/benchmark/scenarios/decodingtrust_fairness_scenario.py
src/helm/benchmark/scenarios/decodingtrust_machine_ethics_scenario.py
src/helm/benchmark/scenarios/decodingtrust_ood_robustness_scenario.py
src/helm/benchmark/scenarios/decodingtrust_privacy_scenario.py
src/helm/benchmark/scenarios/decodingtrust_stereotype_bias_scenario.py
src/helm/benchmark/scenarios/decodingtrust_toxicity_prompts_scenario.py
src/helm/benchmark/scenarios/dialogue_scenarios.py
src/helm/benchmark/scenarios/dischargeme_scenario.py
src/helm/benchmark/scenarios/disinformation_scenario.py
src/helm/benchmark/scenarios/dyck_language_scenario.py
src/helm/benchmark/scenarios/echr_judgment_classification_scenario.py
src/helm/benchmark/scenarios/ehr_sql_scenario.py
src/helm/benchmark/scenarios/ehrshot_scenario.py
src/helm/benchmark/scenarios/enem_challenge_scenario.py
src/helm/benchmark/scenarios/entity_data_imputation_scenario.py
src/helm/benchmark/scenarios/entity_matching_scenario.py
src/helm/benchmark/scenarios/entity_matching_scenario_fixed_random_state.py
src/helm/benchmark/scenarios/ewok_scenario.py
src/helm/benchmark/scenarios/exams_multilingual_scenario.py
src/helm/benchmark/scenarios/fin_qa_scenario.py
src/helm/benchmark/scenarios/financebench_scenario.py
src/helm/benchmark/scenarios/financial_phrasebank_scenario.py
src/helm/benchmark/scenarios/gold_commodity_news_scenario.py
src/helm/benchmark/scenarios/gpqa_scenario.py
src/helm/benchmark/scenarios/grammar.py
src/helm/benchmark/scenarios/grammar_scenario.py
src/helm/benchmark/scenarios/gsm_scenario.py
src/helm/benchmark/scenarios/harm_bench_gcg_transfer_scenario.py
src/helm/benchmark/scenarios/harm_bench_scenario.py
src/helm/benchmark/scenarios/headqa_scenario.py
src/helm/benchmark/scenarios/health_bench_scenario.py
src/helm/benchmark/scenarios/healthqa_br_scenario.py
src/helm/benchmark/scenarios/ice_scenario.py
src/helm/benchmark/scenarios/ice_scenario_pinned_file_order.py
src/helm/benchmark/scenarios/ifeval_scenario.py
src/helm/benchmark/scenarios/imdb_ptbr_scenario.py
src/helm/benchmark/scenarios/imdb_scenario.py
src/helm/benchmark/scenarios/imdb_scenario_pinned_file_order.py
src/helm/benchmark/scenarios/infinite_bench_en_mc_scenario.py
src/helm/benchmark/scenarios/infinite_bench_en_qa_scenario.py
src/helm/benchmark/scenarios/infinite_bench_en_sum_scenario.py
src/helm/benchmark/scenarios/interactive_qa_mmlu_scenario.py
src/helm/benchmark/scenarios/koala_scenario.py
src/helm/benchmark/scenarios/kpi_edgar_scenario.py
src/helm/benchmark/scenarios/legal_contract_summarization_scenario.py
src/helm/benchmark/scenarios/legal_opinion_sentiment_classification_scenario.py
src/helm/benchmark/scenarios/legal_summarization_scenario.py
src/helm/benchmark/scenarios/legal_support_scenario.py
src/helm/benchmark/scenarios/legalbench_scenario.py
src/helm/benchmark/scenarios/lex_glue_scenario.py
src/helm/benchmark/scenarios/lextreme_scenario.py
src/helm/benchmark/scenarios/live_qa_scenario.py
src/helm/benchmark/scenarios/lm_entry_scenario.py
src/helm/benchmark/scenarios/lmkt_scenarios.py
src/helm/benchmark/scenarios/lsat_qa_scenario.py
src/helm/benchmark/scenarios/madinah_qa_scenario.py
src/helm/benchmark/scenarios/math_scenario.py
src/helm/benchmark/scenarios/mbzuai_human_translated_arabic_mmlu.py
src/helm/benchmark/scenarios/me_q_sum_scenario.py
src/helm/benchmark/scenarios/med_dialog_scenario.py
src/helm/benchmark/scenarios/med_mcqa_scenario.py
src/helm/benchmark/scenarios/med_paragraph_simplification_scenario.py
src/helm/benchmark/scenarios/med_qa_scenario.py
src/helm/benchmark/scenarios/medalign_scenario.py
src/helm/benchmark/scenarios/medalign_scenario_helper.py
src/helm/benchmark/scenarios/medbullets_scenario.py
src/helm/benchmark/scenarios/medcalc_bench_scenario.py
src/helm/benchmark/scenarios/medec_scenario.py
src/helm/benchmark/scenarios/medhallu_scenario.py
src/helm/benchmark/scenarios/medhelm_configurable_scenario.py
src/helm/benchmark/scenarios/medi_qa_scenario.py
src/helm/benchmark/scenarios/medication_qa_scenario.py
src/helm/benchmark/scenarios/medxpert_qa_text_scenario.py
src/helm/benchmark/scenarios/melt_ir_scenario.py
src/helm/benchmark/scenarios/melt_knowledge_scenario.py
src/helm/benchmark/scenarios/melt_lm_scenarios.py
src/helm/benchmark/scenarios/melt_scenarios.py
src/helm/benchmark/scenarios/melt_srn_scenario.py
src/helm/benchmark/scenarios/melt_synthetic_reasoning_scenario.py
src/helm/benchmark/scenarios/melt_translation_scenario.py
src/helm/benchmark/scenarios/mental_health_scenario.py
src/helm/benchmark/scenarios/mimic_bhc_scenario.py
src/helm/benchmark/scenarios/mimic_rrs_scenario.py
src/helm/benchmark/scenarios/mimiciv_billing_code_scenario.py
src/helm/benchmark/scenarios/mmlu_clinical_afr_scenario.py
src/helm/benchmark/scenarios/mmlu_pro_scenario.py
src/helm/benchmark/scenarios/mmlu_scenario.py
src/helm/benchmark/scenarios/mmmlu_scenario.py
src/helm/benchmark/scenarios/msmarco_scenario.py
src/helm/benchmark/scenarios/mtsamples_procedures_scenario.py
src/helm/benchmark/scenarios/mtsamples_replicate_scenario.py
src/helm/benchmark/scenarios/n2c2_ct_matching_scenario.py
src/helm/benchmark/scenarios/narrativeqa_scenario.py
src/helm/benchmark/scenarios/natural_qa_scenario.py
src/helm/benchmark/scenarios/newsqa_scenario.py
src/helm/benchmark/scenarios/oab_exams_scenario.py
src/helm/benchmark/scenarios/omni_math_scenario.py
src/helm/benchmark/scenarios/open_assistant_scenario.py
src/helm/benchmark/scenarios/openai_mrcr_scenario.py
src/helm/benchmark/scenarios/opinions_qa_scenario.py
src/helm/benchmark/scenarios/pubmed_qa_scenario.py
src/helm/benchmark/scenarios/quac_scenario.py
src/helm/benchmark/scenarios/race_based_med_scenario.py
src/helm/benchmark/scenarios/raft_scenario.py
src/helm/benchmark/scenarios/real_toxicity_prompts_scenario.py
src/helm/benchmark/scenarios/ruler_qa_scenario_helper.py
src/helm/benchmark/scenarios/ruler_qa_scenarios.py
src/helm/benchmark/scenarios/scenario.py
src/helm/benchmark/scenarios/sct_bench_scenario.py
src/helm/benchmark/scenarios/seahelm_scenario.py
src/helm/benchmark/scenarios/self_instruct_scenario.py
src/helm/benchmark/scenarios/shc_bmt_scenario.py
src/helm/benchmark/scenarios/shc_cdi_scenario.py
src/helm/benchmark/scenarios/shc_conf_scenario.py
src/helm/benchmark/scenarios/shc_ent_scenario.py
src/helm/benchmark/scenarios/shc_gip_scenario.py
src/helm/benchmark/scenarios/shc_privacy_scenario.py
src/helm/benchmark/scenarios/shc_proxy_scenario.py
src/helm/benchmark/scenarios/shc_ptbm_scenario.py
src/helm/benchmark/scenarios/shc_sei_scenario.py
src/helm/benchmark/scenarios/shc_sequoia_scenario.py
src/helm/benchmark/scenarios/simple_safety_tests_scenario.py
src/helm/benchmark/scenarios/simple_scenarios.py
src/helm/benchmark/scenarios/situation_prompts.yaml
src/helm/benchmark/scenarios/spider_scenario.py
src/helm/benchmark/scenarios/starr_patient_instructions_scenario.py
src/helm/benchmark/scenarios/summarization_scenario.py
src/helm/benchmark/scenarios/sumosum_scenario.py
src/helm/benchmark/scenarios/synthetic_efficiency_scenario.py
src/helm/benchmark/scenarios/synthetic_reasoning_natural_scenario.py
src/helm/benchmark/scenarios/synthetic_reasoning_scenario.py
src/helm/benchmark/scenarios/test_aci_bench_scenario.py
src/helm/benchmark/scenarios/test_air_bench_scenario.py
src/helm/benchmark/scenarios/test_alghafa_scenario.py
src/helm/benchmark/scenarios/test_alrage_scenario.py
src/helm/benchmark/scenarios/test_arabic_exams_scenario.py
src/helm/benchmark/scenarios/test_aratrust_scenario.py
src/helm/benchmark/scenarios/test_bigcodebench_scenario.py
src/helm/benchmark/scenarios/test_bluex_scenario.py
src/helm/benchmark/scenarios/test_chw_care_plan_scenario.py
src/helm/benchmark/scenarios/test_clear_scenario.py
src/helm/benchmark/scenarios/test_commonsense_scenario.py
src/helm/benchmark/scenarios/test_czech_bank_qa_scenario.py
src/helm/benchmark/scenarios/test_dischargeme_scenario.py
src/helm/benchmark/scenarios/test_ehr_sql_scenario.py
src/helm/benchmark/scenarios/test_ehrshot_scenario.py
src/helm/benchmark/scenarios/test_enem_challenge_scenario.py
src/helm/benchmark/scenarios/test_ewok_scenario.py
src/helm/benchmark/scenarios/test_exams_multilingual_scenario.py
src/helm/benchmark/scenarios/test_financebench_scenario.py
src/helm/benchmark/scenarios/test_gold_commodity_news_scenario.py
src/helm/benchmark/scenarios/test_gpqa_scenario.py
src/helm/benchmark/scenarios/test_grammar.py
src/helm/benchmark/scenarios/test_gsm_scenario.py
src/helm/benchmark/scenarios/test_headqa_scenario.py
src/helm/benchmark/scenarios/test_health_bench_scenario.py
src/helm/benchmark/scenarios/test_healtha_br_scenario.py
src/helm/benchmark/scenarios/test_ifeval_scenario.py
src/helm/benchmark/scenarios/test_imdb_ptbr_scenario.py
src/helm/benchmark/scenarios/test_infinite_bench_en_qa_scenario.py
src/helm/benchmark/scenarios/test_infinite_bench_en_sum_scenario.py
src/helm/benchmark/scenarios/test_legalbench_scenario.py
src/helm/benchmark/scenarios/test_math_scenario.py
src/helm/benchmark/scenarios/test_med_dialog_scenario.py
src/helm/benchmark/scenarios/test_med_mcqa_scenario.py
src/helm/benchmark/scenarios/test_med_qa_scenario.py
src/helm/benchmark/scenarios/test_medalign_scenario.py
src/helm/benchmark/scenarios/test_medalign_scenario_helper.py
src/helm/benchmark/scenarios/test_medbullets_scenario.py
src/helm/benchmark/scenarios/test_medcalc_bench_scenario.py
src/helm/benchmark/scenarios/test_medec_scenario.py
src/helm/benchmark/scenarios/test_medhallu_scenario.py
src/helm/benchmark/scenarios/test_medi_qa_scenario.py
src/helm/benchmark/scenarios/test_medication_qa_scenario.py
src/helm/benchmark/scenarios/test_mental_health_scenario.py
src/helm/benchmark/scenarios/test_mimic_bhc_scenario.py
src/helm/benchmark/scenarios/test_mimic_rrs_scenario.py
src/helm/benchmark/scenarios/test_mimiciv_billing_code_scenario.py
src/helm/benchmark/scenarios/test_mmlu_clinical_afr_scenario.py
src/helm/benchmark/scenarios/test_mmlu_pro_scenario.py
src/helm/benchmark/scenarios/test_mmlu_scenario.py
src/helm/benchmark/scenarios/test_mtsamples_procedures_scenario.py
src/helm/benchmark/scenarios/test_mtsamples_replicate_scenario.py
src/helm/benchmark/scenarios/test_n2c2_ct_matching_scenario.py
src/helm/benchmark/scenarios/test_narrativeqa_scenario.py
src/helm/benchmark/scenarios/test_oab_exams_scenario.py
src/helm/benchmark/scenarios/test_omni_math_scenario.py
src/helm/benchmark/scenarios/test_pubmed_qa_scenario.py
src/helm/benchmark/scenarios/test_race_based_med_scenario.py
src/helm/benchmark/scenarios/test_scenario.py
src/helm/benchmark/scenarios/test_shc_bmt_scenario.py
src/helm/benchmark/scenarios/test_shc_cdi_scenario.py
src/helm/benchmark/scenarios/test_shc_conf_scenario.py
src/helm/benchmark/scenarios/test_shc_ent_scenario.py
src/helm/benchmark/scenarios/test_shc_gip_scenario.py
src/helm/benchmark/scenarios/test_shc_privacy_scenario.py
src/helm/benchmark/scenarios/test_shc_proxy_scenario.py
src/helm/benchmark/scenarios/test_shc_ptbm_scenario.py
src/helm/benchmark/scenarios/test_shc_sequoia_scenario.py
src/helm/benchmark/scenarios/test_simple_scenarios.py
src/helm/benchmark/scenarios/test_starr_patient_instructions_scenario.py
src/helm/benchmark/scenarios/test_tweetsentbr_scenario.py
src/helm/benchmark/scenarios/test_wildbench_scenario.py
src/helm/benchmark/scenarios/test_winogrande_afr_scenario.py
src/helm/benchmark/scenarios/thai_exam_scenario.py
src/helm/benchmark/scenarios/the_pile_scenario.py
src/helm/benchmark/scenarios/truthful_qa_scenario.py
src/helm/benchmark/scenarios/tweetsentbr_scenario.py
src/helm/benchmark/scenarios/twitter_aae_scenario.py
src/helm/benchmark/scenarios/unitxt_scenario.py
src/helm/benchmark/scenarios/verifiability_judgment_scenario.py
src/helm/benchmark/scenarios/vicuna_scenario.py
src/helm/benchmark/scenarios/wikifact_scenario.py
src/helm/benchmark/scenarios/wikitext_103_scenario.py
src/helm/benchmark/scenarios/wildbench_scenario.py
src/helm/benchmark/scenarios/winogrande_afr_scenario.py
src/helm/benchmark/scenarios/wmt_14_scenario.py
src/helm/benchmark/scenarios/xstest_scenario.py
src/helm/benchmark/scenarios/audio_language/__init__.py
src/helm/benchmark/scenarios/audio_language/air_bench_chat_scenario.py
src/helm/benchmark/scenarios/audio_language/air_bench_foundation_scenario.py
src/helm/benchmark/scenarios/audio_language/ami_scenario.py
src/helm/benchmark/scenarios/audio_language/audio_mnist_scenario.py
src/helm/benchmark/scenarios/audio_language/audio_pairs_scenario.py
src/helm/benchmark/scenarios/audio_language/audiocaps_scenario.py
src/helm/benchmark/scenarios/audio_language/casual_conversations2_scenario.py
src/helm/benchmark/scenarios/audio_language/common_voice_15_scenario.py
src/helm/benchmark/scenarios/audio_language/corebench_scenario.py
src/helm/benchmark/scenarios/audio_language/covost2_scenario.py
src/helm/benchmark/scenarios/audio_language/fleurs_fairness_scenario.py
src/helm/benchmark/scenarios/audio_language/fleurs_scenario.py
src/helm/benchmark/scenarios/audio_language/iemocap_audio_scenario.py
src/helm/benchmark/scenarios/audio_language/librispeech_fairness_scenario.py
src/helm/benchmark/scenarios/audio_language/librispeech_scenario.py
src/helm/benchmark/scenarios/audio_language/meld_audio_scenario.py
src/helm/benchmark/scenarios/audio_language/multilingual_librispeech_scenario.py
src/helm/benchmark/scenarios/audio_language/mustard_scenario.py
src/helm/benchmark/scenarios/audio_language/mutox_scenario.py
src/helm/benchmark/scenarios/audio_language/parade_scenario.py
src/helm/benchmark/scenarios/audio_language/speech_robust_bench_scenario.py
src/helm/benchmark/scenarios/audio_language/ultra_suite_asr_classification_scenario.py
src/helm/benchmark/scenarios/audio_language/ultra_suite_asr_transcription_scenario.py
src/helm/benchmark/scenarios/audio_language/ultra_suite_classification_scenario.py
src/helm/benchmark/scenarios/audio_language/ultra_suite_disorder_breakdown_scenario.py
src/helm/benchmark/scenarios/audio_language/ultra_suite_disorder_symptoms_scenario.py
src/helm/benchmark/scenarios/audio_language/vocal_sound_scenario.py
src/helm/benchmark/scenarios/audio_language/voice_jailbreak_attacks_scenario.py
src/helm/benchmark/scenarios/audio_language/voxceleb2_scenario.py
src/helm/benchmark/scenarios/image_generation/__init__.py
src/helm/benchmark/scenarios/image_generation/common_syntactic_processes_scenario.py
src/helm/benchmark/scenarios/image_generation/cub200_scenario.py
src/helm/benchmark/scenarios/image_generation/daily_dalle_scenario.py
src/helm/benchmark/scenarios/image_generation/demographic_stereotypes_scenario.py
src/helm/benchmark/scenarios/image_generation/detection_scenario.py
src/helm/benchmark/scenarios/image_generation/draw_bench_scenario.py
src/helm/benchmark/scenarios/image_generation/i2p_scenario.py
src/helm/benchmark/scenarios/image_generation/landing_page_scenario.py
src/helm/benchmark/scenarios/image_generation/logos_scenario.py
src/helm/benchmark/scenarios/image_generation/magazine_cover_scenario.py
src/helm/benchmark/scenarios/image_generation/mental_disorders_scenario.py
src/helm/benchmark/scenarios/image_generation/mscoco_scenario.py
src/helm/benchmark/scenarios/image_generation/paint_skills_scenario.py
src/helm/benchmark/scenarios/image_generation/parti_prompts_scenario.py
src/helm/benchmark/scenarios/image_generation/radiology_scenario.py
src/helm/benchmark/scenarios/image_generation/relational_understanding_scenario.py
src/helm/benchmark/scenarios/image_generation/time_most_significant_historical_figures_scenario.py
src/helm/benchmark/scenarios/image_generation/winoground_scenario.py
src/helm/benchmark/scenarios/medhelm/__init__.py
src/helm/benchmark/scenarios/medhelm/judges.yaml
src/helm/benchmark/scenarios/vision_language/__init__.py
src/helm/benchmark/scenarios/vision_language/a_okvqa_scenario.py
src/helm/benchmark/scenarios/vision_language/bingo_scenario.py
src/helm/benchmark/scenarios/vision_language/blink_scenario.py
src/helm/benchmark/scenarios/vision_language/crossmodal_3600_scenario.py
src/helm/benchmark/scenarios/vision_language/exams_v_scenario.py
src/helm/benchmark/scenarios/vision_language/fair_face_scenario.py
src/helm/benchmark/scenarios/vision_language/flickr30k_scenario.py
src/helm/benchmark/scenarios/vision_language/gqa_scenario.py
src/helm/benchmark/scenarios/vision_language/hateful_memes_scenario.py
src/helm/benchmark/scenarios/vision_language/heim_human_eval_scenario.py
src/helm/benchmark/scenarios/vision_language/math_vista_scenario.py
src/helm/benchmark/scenarios/vision_language/mementos_scenario.py
src/helm/benchmark/scenarios/vision_language/mm_safety_bench_scenario.py
src/helm/benchmark/scenarios/vision_language/mm_star_scenario.py
src/helm/benchmark/scenarios/vision_language/mme_scenario.py
src/helm/benchmark/scenarios/vision_language/mmmu_scenario.py
src/helm/benchmark/scenarios/vision_language/mscoco_captioning_scenario.py
src/helm/benchmark/scenarios/vision_language/mscoco_categorization_scenario.py
src/helm/benchmark/scenarios/vision_language/msr_vtt_scenario.py
src/helm/benchmark/scenarios/vision_language/multipanelvqa_scenario.py
src/helm/benchmark/scenarios/vision_language/originality_scenario.py
src/helm/benchmark/scenarios/vision_language/pairs_scenario.py
src/helm/benchmark/scenarios/vision_language/pope_scenario.py
src/helm/benchmark/scenarios/vision_language/real_world_qa_scenario.py
src/helm/benchmark/scenarios/vision_language/seed_bench_scenario.py
src/helm/benchmark/scenarios/vision_language/unicorn_scenario.py
src/helm/benchmark/scenarios/vision_language/vibe_eval_scenario.py
src/helm/benchmark/scenarios/vision_language/viz_wiz_scenario.py
src/helm/benchmark/scenarios/vision_language/vqa_rad_scenario.py
src/helm/benchmark/scenarios/vision_language/vqa_scenario.py
src/helm/benchmark/scenarios/vision_language/image2struct/__init__.py
src/helm/benchmark/scenarios/vision_language/image2struct/chart2csv_scenario.py
src/helm/benchmark/scenarios/vision_language/image2struct/image2struct_scenario.py
src/helm/benchmark/scenarios/vision_language/image2struct/latex_scenario.py
src/helm/benchmark/scenarios/vision_language/image2struct/musicsheet_scenario.py
src/helm/benchmark/scenarios/vision_language/image2struct/utils_latex.py
src/helm/benchmark/scenarios/vision_language/image2struct/webpage_scenario.py
src/helm/benchmark/scenarios/vision_language/image2struct/webpage/Gemfile.default
src/helm/benchmark/scenarios/vision_language/image2struct/webpage/__init__.py
src/helm/benchmark/scenarios/vision_language/image2struct/webpage/_config.default.yml
src/helm/benchmark/scenarios/vision_language/image2struct/webpage/driver.py
src/helm/benchmark/scenarios/vision_language/image2struct/webpage/jekyll_server.py
src/helm/benchmark/scenarios/vision_language/image2struct/webpage/utils.py
src/helm/benchmark/static/contamination.yaml
src/helm/benchmark/static/schema_air_bench.yaml
src/helm/benchmark/static/schema_arabic.yaml
src/helm/benchmark/static/schema_audio.yaml
src/helm/benchmark/static/schema_autobencher.yaml
src/helm/benchmark/static/schema_capabilities.yaml
src/helm/benchmark/static/schema_classic.yaml
src/helm/benchmark/static/schema_cleva.yaml
src/helm/benchmark/static/schema_czech_bank.yaml
src/helm/benchmark/static/schema_decodingtrust.yaml
src/helm/benchmark/static/schema_enem_challenge.yaml
src/helm/benchmark/static/schema_enterprise.yaml
src/helm/benchmark/static/schema_ewok.yaml
src/helm/benchmark/static/schema_finance.yaml
src/helm/benchmark/static/schema_heim.yaml
src/helm/benchmark/static/schema_image2struct.yaml
src/helm/benchmark/static/schema_instruction_following.yaml
src/helm/benchmark/static/schema_legal.yaml
src/helm/benchmark/static/schema_lite.yaml
src/helm/benchmark/static/schema_long_context.yaml
src/helm/benchmark/static/schema_medhelm.yaml
src/helm/benchmark/static/schema_melt.yaml
src/helm/benchmark/static/schema_mmlu.yaml
src/helm/benchmark/static/schema_mmlu_winogrande_afr.yaml
src/helm/benchmark/static/schema_safety.yaml
src/helm/benchmark/static/schema_seahelm.yaml
src/helm/benchmark/static/schema_slp.yaml
src/helm/benchmark/static/schema_slphelm.yaml
src/helm/benchmark/static/schema_social_audio.yaml
src/helm/benchmark/static/schema_sql.yaml
src/helm/benchmark/static/schema_thai.yaml
src/helm/benchmark/static/schema_torr.yaml
src/helm/benchmark/static/schema_tweetsentbr.yaml
src/helm/benchmark/static/schema_unitxt.yaml
src/helm/benchmark/static/schema_vhelm.yaml
src/helm/benchmark/static/schema_vhelm_lite.yaml
src/helm/benchmark/static/schema_video.yaml
src/helm/benchmark/window_services/__init__.py
src/helm/benchmark/window_services/default_window_service.py
src/helm/benchmark/window_services/encoder_decoder_window_service.py
src/helm/benchmark/window_services/ice_window_service.py
src/helm/benchmark/window_services/local_window_service.py
src/helm/benchmark/window_services/mock_ai21_tokenizer_request_results.pkl
src/helm/benchmark/window_services/no_decoding_window_service.py
src/helm/benchmark/window_services/no_tokenizer_window_service.py
src/helm/benchmark/window_services/test_anthropic_window_service.py
src/helm/benchmark/window_services/test_bloom_window_service.py
src/helm/benchmark/window_services/test_flan_t5_window_service.py
src/helm/benchmark/window_services/test_gpt2_window_service.py
src/helm/benchmark/window_services/test_gpt4_window_service.py
src/helm/benchmark/window_services/test_gptj_window_service.py
src/helm/benchmark/window_services/test_gptneox_window_service.py
src/helm/benchmark/window_services/test_openai_window_service.py
src/helm/benchmark/window_services/test_opt_window_service.py
src/helm/benchmark/window_services/test_palmyra_window_service.py
src/helm/benchmark/window_services/test_t0pp_window_service.py
src/helm/benchmark/window_services/test_t511b_window_service.py
src/helm/benchmark/window_services/test_ul2_window_service.py
src/helm/benchmark/window_services/test_utils.py
src/helm/benchmark/window_services/test_yalm_window_service.py
src/helm/benchmark/window_services/tokenizer_service.py
src/helm/benchmark/window_services/window_service.py
src/helm/benchmark/window_services/window_service_factory.py
src/helm/benchmark/window_services/yalm_window_service.py
src/helm/benchmark/window_services/image_generation/__init__.py
src/helm/benchmark/window_services/image_generation/clip_window_service.py
src/helm/benchmark/window_services/image_generation/lexica_search_window_service.py
src/helm/benchmark/window_services/image_generation/openai_dalle_window_service.py
src/helm/benchmark/window_services/image_generation/test_clip_window_service.py
src/helm/benchmark/window_services/image_generation/test_openai_dalle_window_service.py
src/helm/clients/__init__.py
src/helm/clients/ai21_client.py
src/helm/clients/ai21_utils.py
src/helm/clients/aleph_alpha_client.py
src/helm/clients/anthropic_client.py
src/helm/clients/auto_client.py
src/helm/clients/azure_openai_client.py
src/helm/clients/bedrock_client.py
src/helm/clients/bedrock_utils.py
src/helm/clients/client.py
src/helm/clients/clip_score_client.py
src/helm/clients/cohere_client.py
src/helm/clients/cohere_utils.py
src/helm/clients/dspy_client.py
src/helm/clients/gcs_client.py
src/helm/clients/google_client.py
src/helm/clients/google_genai_client.py
src/helm/clients/google_translate_client.py
src/helm/clients/grok_client.py
src/helm/clients/http_model_client.py
src/helm/clients/huggingface_client.py
src/helm/clients/huggingface_inference_providers_client.py
src/helm/clients/huggingface_pipeline_client.py
src/helm/clients/ibm_client.py
src/helm/clients/lit_gpt_client.py
src/helm/clients/lit_gpt_generate.py
src/helm/clients/litellm_client.py
src/helm/clients/megatron_client.py
src/helm/clients/mistral_client.py
src/helm/clients/moderation_api_client.py
src/helm/clients/nvidia_nim_client.py
src/helm/clients/open_lm_client.py
src/helm/clients/openai_client.py
src/helm/clients/openai_responses_client.py
src/helm/clients/openrouter_client.py
src/helm/clients/palmyra_client.py
src/helm/clients/perspective_api_client.py
src/helm/clients/reka_client.py
src/helm/clients/simple_client.py
src/helm/clients/stanfordhealthcare_azure_openai_client.py
src/helm/clients/stanfordhealthcare_claude_client.py
src/helm/clients/stanfordhealthcare_google_client.py
src/helm/clients/stanfordhealthcare_http_model_client.py
src/helm/clients/stanfordhealthcare_openai_client.py
src/helm/clients/stanfordhealthcare_shc_openai_client.py
src/helm/clients/test_auto_client.py
src/helm/clients/test_client.py
src/helm/clients/test_huggingface_client.py
src/helm/clients/test_openrouter_client.py
src/helm/clients/test_simple_client.py
src/helm/clients/test_together_client.py
src/helm/clients/together_client.py
src/helm/clients/toxicity_classifier_client.py
src/helm/clients/upstage_client.py
src/helm/clients/vertexai_client.py
src/helm/clients/vllm_client.py
src/helm/clients/vllm_granite_thinking_client.py
src/helm/clients/writer_client.py
src/helm/clients/yi_client.py
src/helm/clients/audio_language/__init__.py
src/helm/clients/audio_language/diva_llama_client.py
src/helm/clients/audio_language/llama_omni_client.py
src/helm/clients/audio_language/qwen2_5_omni_client.py
src/helm/clients/audio_language/qwen2_audiolm_client.py
src/helm/clients/audio_language/qwen_audiolm_client.py
src/helm/clients/audio_language/test.py
src/helm/clients/audio_language/llama_omni/arguments.py
src/helm/clients/audio_language/llama_omni/constants.py
src/helm/clients/audio_language/llama_omni/conversation.py
src/helm/clients/audio_language/llama_omni/preprocess.py
src/helm/clients/audio_language/llama_omni/utils.py
src/helm/clients/audio_language/llama_omni/model/__init__.py
src/helm/clients/audio_language/llama_omni/model/builder.py
src/helm/clients/audio_language/llama_omni/model/omni_speech_arch.py
src/helm/clients/audio_language/llama_omni/model/language_model/omni_speech2s_llama.py
src/helm/clients/audio_language/llama_omni/model/language_model/omni_speech_llama.py
src/helm/clients/audio_language/llama_omni/model/speech_encoder/builder.py
src/helm/clients/audio_language/llama_omni/model/speech_encoder/speech_encoder.py
src/helm/clients/audio_language/llama_omni/model/speech_generator/builder.py
src/helm/clients/audio_language/llama_omni/model/speech_generator/generation.py
src/helm/clients/audio_language/llama_omni/model/speech_generator/speech_generator.py
src/helm/clients/audio_language/llama_omni/model/speech_projector/builder.py
src/helm/clients/audio_language/llama_omni/model/speech_projector/speech_projector.py
src/helm/clients/audio_language/qwen_omni/configuration_qwen2_5_omni.py
src/helm/clients/audio_language/qwen_omni/modeling_qwen2_5_omni.py
src/helm/clients/audio_language/qwen_omni/processing_qwen2_5_omni.py
src/helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/__init__.py
src/helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/v2_5/__init__.py
src/helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/v2_5/audio_process.py
src/helm/clients/audio_language/qwen_omni/qwen2_5_omni_utils/v2_5/vision_process.py
src/helm/clients/clip_scorers/__init__.py
src/helm/clients/clip_scorers/base_clip_scorer.py
src/helm/clients/clip_scorers/clip_scorer.py
src/helm/clients/clip_scorers/multilingual_clip_scorer.py
src/helm/clients/image_generation/__init__.py
src/helm/clients/image_generation/adobe_vision_client.py
src/helm/clients/image_generation/aleph_alpha_image_generation_client.py
src/helm/clients/image_generation/cogview2_client.py
src/helm/clients/image_generation/dalle2_client.py
src/helm/clients/image_generation/dalle3_client.py
src/helm/clients/image_generation/dalle_mini_client.py
src/helm/clients/image_generation/deep_floyd_client.py
src/helm/clients/image_generation/huggingface_diffusers_client.py
src/helm/clients/image_generation/image_generation_client_utils.py
src/helm/clients/image_generation/lexica_client.py
src/helm/clients/image_generation/mindalle_client.py
src/helm/clients/image_generation/nudity_check_client.py
src/helm/clients/image_generation/together_image_generation_client.py
src/helm/clients/image_generation/cogview2/__init__.py
src/helm/clients/image_generation/cogview2/cluster_label.npy
src/helm/clients/image_generation/cogview2/coglm_strategy.py
src/helm/clients/image_generation/cogview2/coglm_utils.py
src/helm/clients/image_generation/cogview2/sr_pipeline/__init__.py
src/helm/clients/image_generation/cogview2/sr_pipeline/direct_sr.py
src/helm/clients/image_generation/cogview2/sr_pipeline/dsr_model.py
src/helm/clients/image_generation/cogview2/sr_pipeline/dsr_sampling.py
src/helm/clients/image_generation/cogview2/sr_pipeline/iterative_sr.py
src/helm/clients/image_generation/cogview2/sr_pipeline/itersr_model.py
src/helm/clients/image_generation/cogview2/sr_pipeline/itersr_sampling.py
src/helm/clients/image_generation/cogview2/sr_pipeline/sr_group.py
src/helm/clients/image_generation/dalle_mini/__init__.py
src/helm/clients/image_generation/dalle_mini/data.py
src/helm/clients/image_generation/dalle_mini/model/__init__.py
src/helm/clients/image_generation/dalle_mini/model/configuration.py
src/helm/clients/image_generation/dalle_mini/model/modeling.py
src/helm/clients/image_generation/dalle_mini/model/partitions.py
src/helm/clients/image_generation/dalle_mini/model/processor.py
src/helm/clients/image_generation/dalle_mini/model/text.py
src/helm/clients/image_generation/dalle_mini/model/tokenizer.py
src/helm/clients/image_generation/dalle_mini/model/utils.py
src/helm/clients/image_generation/dalle_mini/vqgan_jax/__init__.py
src/helm/clients/image_generation/dalle_mini/vqgan_jax/configuration_vqgan.py
src/helm/clients/image_generation/dalle_mini/vqgan_jax/convert_pt_model_to_jax.py
src/helm/clients/image_generation/dalle_mini/vqgan_jax/modeling_flax_vqgan.py
src/helm/clients/image_generation/mindalle/__init__.py
src/helm/clients/image_generation/mindalle/models/__init__.py
src/helm/clients/image_generation/mindalle/models/tokenizer.py
src/helm/clients/image_generation/mindalle/models/stage1/__init__.py
src/helm/clients/image_generation/mindalle/models/stage1/layers.py
src/helm/clients/image_generation/mindalle/models/stage1/vqgan.py
src/helm/clients/image_generation/mindalle/models/stage2/__init__.py
src/helm/clients/image_generation/mindalle/models/stage2/layers.py
src/helm/clients/image_generation/mindalle/models/stage2/transformer.py
src/helm/clients/image_generation/mindalle/utils/__init__.py
src/helm/clients/image_generation/mindalle/utils/config.py
src/helm/clients/image_generation/mindalle/utils/sampling.py
src/helm/clients/image_generation/mindalle/utils/utils.py
src/helm/clients/vision_language/__init__.py
src/helm/clients/vision_language/huggingface_vision2seq_client.py
src/helm/clients/vision_language/huggingface_vlm_client.py
src/helm/clients/vision_language/idefics_client.py
src/helm/clients/vision_language/open_flamingo_client.py
src/helm/clients/vision_language/paligemma_client.py
src/helm/clients/vision_language/palmyra_vision_client.py
src/helm/clients/vision_language/qwen2_vlm_client.py
src/helm/clients/vision_language/qwen_vlm_client.py
src/helm/clients/vision_language/open_flamingo/__init__.py
src/helm/clients/vision_language/open_flamingo/src/__init__.py
src/helm/clients/vision_language/open_flamingo/src/factory.py
src/helm/clients/vision_language/open_flamingo/src/flamingo.py
src/helm/clients/vision_language/open_flamingo/src/flamingo_lm.py
src/helm/clients/vision_language/open_flamingo/src/helpers.py
src/helm/clients/vision_language/open_flamingo/src/utils.py
src/helm/common/__init__.py
src/helm/common/audio_utils.py
src/helm/common/authentication.py
src/helm/common/cache.py
src/helm/common/cache_backend_config.py
src/helm/common/clip_score_request.py
src/helm/common/codec.py
src/helm/common/concurrency.py
src/helm/common/context.py
src/helm/common/credentials_utils.py
src/helm/common/critique_request.py
src/helm/common/file_upload_request.py
src/helm/common/general.py
src/helm/common/gpu_utils.py
src/helm/common/hierarchical_logger.py
src/helm/common/image_generation_parameters.py
src/helm/common/images_utils.py
src/helm/common/key_value_store.py
src/helm/common/local_context.py
src/helm/common/media_object.py
src/helm/common/moderations_api_request.py
src/helm/common/mongo_key_value_store.py
src/helm/common/multimodal_request_utils.py
src/helm/common/nudity_check_request.py
src/helm/common/object_spec.py
src/helm/common/optional_dependencies.py
src/helm/common/perspective_api_request.py
src/helm/common/reeval_parameters.py
src/helm/common/remote_context.py
src/helm/common/request.py
src/helm/common/response_format.py
src/helm/common/test_cache.py
src/helm/common/test_codec.py
src/helm/common/test_general.py
src/helm/common/test_logging.py
src/helm/common/test_media_object.py
src/helm/common/tokenization_request.py
src/helm/common/file_caches/__init__.py
src/helm/common/file_caches/file_cache.py
src/helm/common/file_caches/local_file_cache.py
src/helm/common/file_caches/test_local_file_cache.py
src/helm/config/__init__.py
src/helm/config/model_deployments.yaml
src/helm/config/model_metadata.yaml
src/helm/config/tokenizer_configs.yaml
src/helm/proxy/__init__.py
src/helm/proxy/accounts.py
src/helm/proxy/cli.py
src/helm/proxy/example_queries.py
src/helm/proxy/query.py
src/helm/proxy/retry.py
src/helm/proxy/server.py
src/helm/proxy/test_accounts.py
src/helm/proxy/test_retry.py
src/helm/proxy/critique/__init__.py
src/helm/proxy/critique/critique_client.py
src/helm/proxy/critique/mechanical_turk_critique_client.py
src/helm/proxy/critique/mechanical_turk_critique_exporter.py
src/helm/proxy/critique/mechanical_turk_critique_importer.py
src/helm/proxy/critique/mechanical_turk_utils.py
src/helm/proxy/critique/model_critique_client.py
src/helm/proxy/critique/scale_critique_client.py
src/helm/proxy/critique/surge_ai_critique_client.py
src/helm/proxy/services/__init__.py
src/helm/proxy/services/remote_service.py
src/helm/proxy/services/server_service.py
src/helm/proxy/services/service.py
src/helm/proxy/services/test_remote_service.py
src/helm/proxy/services/test_service.py
src/helm/proxy/static/general.js
src/helm/proxy/static/help.html
src/helm/proxy/static/index.css
src/helm/proxy/static/index.html
src/helm/proxy/static/index.js
src/helm/proxy/static/info-icon.png
src/helm/proxy/token_counters/__init__.py
src/helm/proxy/token_counters/auto_token_counter.py
src/helm/proxy/token_counters/test_auto_token_counter.py
src/helm/proxy/token_counters/token_counter.py
src/helm/tokenizers/__init__.py
src/helm/tokenizers/ai21_tokenizer.py
src/helm/tokenizers/aleph_alpha_tokenizer.py
src/helm/tokenizers/auto_tokenizer.py
src/helm/tokenizers/caching_tokenizer.py
src/helm/tokenizers/cohere_tokenizer.py
src/helm/tokenizers/grok_tokenizer.py
src/helm/tokenizers/http_model_tokenizer.py
src/helm/tokenizers/huggingface_tokenizer.py
src/helm/tokenizers/lit_gpt_tokenizer.py
src/helm/tokenizers/simple_tokenizer.py
src/helm/tokenizers/test_ai21_tokenizer.py
src/helm/tokenizers/test_anthropic_tokenizer.py
src/helm/tokenizers/test_cohere_tokenizer.py
src/helm/tokenizers/test_grok_tokenizer.py
src/helm/tokenizers/test_huggingface_tokenizer.py
src/helm/tokenizers/test_simple_tokenizer.py
src/helm/tokenizers/test_yalm_tokenizer.py
src/helm/tokenizers/tiktoken_tokenizer.py
src/helm/tokenizers/tokenizer.py
src/helm/tokenizers/vertexai_tokenizer.py
src/helm/tokenizers/yalm_tokenizer.py
src/helm/tokenizers/yalm_tokenizer_data/__init__.py
src/helm/tokenizers/yalm_tokenizer_data/test_yalm_tokenizer.py
src/helm/tokenizers/yalm_tokenizer_data/voc_100b.sp
src/helm/tokenizers/yalm_tokenizer_data/yalm_tokenizer.py
src/medhelm.egg-info/PKG-INFO
src/medhelm.egg-info/SOURCES.txt
src/medhelm.egg-info/dependency_links.txt
src/medhelm.egg-info/entry_points.txt
src/medhelm.egg-info/not-zip-safe
src/medhelm.egg-info/requires.txt
src/medhelm.egg-info/top_level.txt