lighteval:mmmu_pro:standard-4 lighteval:mmmu_pro:standard-10 lighteval:mmmu_pro:vision lighteval:bigbench:abstract_narrative_understanding lighteval:agieval:aqua-rat lighteval:agieval:gaokao-biology lighteval:agieval:gaokao-chemistry lighteval:agieval:gaokao-chinese lighteval:agieval:gaokao-english lighteval:agieval:gaokao-geography lighteval:agieval:gaokao-history lighteval:agieval:gaokao-mathqa lighteval:agieval:gaokao-physics lighteval:agieval:logiqa-en lighteval:agieval:logiqa-zh lighteval:agieval:lsat-ar lighteval:agieval:lsat-lr lighteval:agieval:lsat-rc lighteval:agieval:sat-en lighteval:agieval:sat-en-without-passage lighteval:agieval:sat-math lighteval:aime24 lighteval:aime24_avg lighteval:aime24_gpassk lighteval:aime25 lighteval:aime25_gpassk lighteval:bigbench:anachronisms lighteval:bigbench:analogical_similarity lighteval:bigbench:analytic_entailment lighteval:anli:r1 lighteval:anli:r2 lighteval:anli:r3 lighteval:arc_agi_2 lighteval:arc:easy lighteval:arithmetic:1dc lighteval:arithmetic:2da lighteval:arithmetic:2dm lighteval:arithmetic:2ds lighteval:arithmetic:3da lighteval:arithmetic:3ds lighteval:arithmetic:4da lighteval:arithmetic:4ds lighteval:arithmetic:5da lighteval:arithmetic:5ds lighteval:bigbench:arithmetic_bb lighteval:bigbench:ascii_word_recognition lighteval:asdiv lighteval:bigbench:authorship_verification lighteval:bigbench:auto_categorization lighteval:bigbench_lite:auto_debugging lighteval:bigbench:causal_judgment lighteval:bigbench:date_understanding lighteval:bigbench:disambiguation_qa lighteval:bigbench:geometric_shapes lighteval:bigbench:logical_deduction_five_objects lighteval:bigbench:logical_deduction_seven_objects lighteval:bigbench:logical_deduction_three_objects lighteval:bigbench:movie_recommendation lighteval:bigbench:navigate lighteval:bigbench:reasoning_about_colored_objects lighteval:bigbench:ruin_names lighteval:bigbench:salient_translation_error_detection lighteval:bigbench:snarks lighteval:bigbench:sports_understanding lighteval:bigbench:temporal_sequences lighteval:bigbench:tracking_shuffled_objects_five_objects lighteval:bigbench:tracking_shuffled_objects_seven_objects lighteval:bigbench:tracking_shuffled_objects_three_objects lighteval:bigbench_lite:bbq_lite_json lighteval:blimp:adjunct_island lighteval:blimp:anaphor_gender_agreement lighteval:blimp:anaphor_number_agreement lighteval:blimp:animate_subject_passive lighteval:blimp:animate_subject_trans lighteval:blimp:causative lighteval:blimp:complex_NP_island lighteval:blimp:coordinate_structure_constraint_complex_left_branch lighteval:blimp:coordinate_structure_constraint_object_extraction lighteval:blimp:determiner_noun_agreement_1 lighteval:blimp:determiner_noun_agreement_2 lighteval:blimp:determiner_noun_agreement_irregular_1 lighteval:blimp:determiner_noun_agreement_irregular_2 lighteval:blimp:determiner_noun_agreement_with_adj_2 lighteval:blimp:determiner_noun_agreement_with_adj_irregular_1 lighteval:blimp:determiner_noun_agreement_with_adj_irregular_2 lighteval:blimp:determiner_noun_agreement_with_adjective_1 lighteval:blimp:distractor_agreement_relational_noun lighteval:blimp:distractor_agreement_relative_clause lighteval:blimp:drop_argument lighteval:blimp:ellipsis_n_bar_1 lighteval:blimp:ellipsis_n_bar_2 lighteval:blimp:existential_there_object_raising lighteval:blimp:existential_there_quantifiers_1 lighteval:blimp:existential_there_quantifiers_2 lighteval:blimp:existential_there_subject_raising lighteval:blimp:expletive_it_object_raising lighteval:blimp:inchoative lighteval:blimp:intransitive lighteval:blimp:irregular_past_participle_adjectives lighteval:blimp:irregular_past_participle_verbs lighteval:blimp:irregular_plural_subject_verb_agreement_1 lighteval:blimp:irregular_plural_subject_verb_agreement_2 lighteval:blimp:left_branch_island_echo_question lighteval:blimp:left_branch_island_simple_question lighteval:blimp:matrix_question_npi_licensor_present lighteval:blimp:npi_present_1 lighteval:blimp:npi_present_2 lighteval:blimp:only_npi_licensor_present lighteval:blimp:only_npi_scope lighteval:blimp:passive_1 lighteval:blimp:passive_2 lighteval:blimp:principle_A_c_command lighteval:blimp:principle_A_case_1 lighteval:blimp:principle_A_case_2 lighteval:blimp:principle_A_domain_1 lighteval:blimp:principle_A_domain_2 lighteval:blimp:principle_A_domain_3 lighteval:blimp:principle_A_reconstruction lighteval:blimp:regular_plural_subject_verb_agreement_1 lighteval:blimp:regular_plural_subject_verb_agreement_2 lighteval:blimp:sentential_negation_npi_licensor_present lighteval:blimp:sentential_negation_npi_scope lighteval:blimp:sentential_subject_island lighteval:blimp:superlative_quantifiers_1 lighteval:blimp:superlative_quantifiers_2 lighteval:blimp:tough_vs_raising_1 lighteval:blimp:tough_vs_raising_2 lighteval:blimp:transitive lighteval:blimp:wh_island lighteval:blimp:wh_questions_object_gap lighteval:blimp:wh_questions_subject_gap lighteval:blimp:wh_questions_subject_gap_long_distance lighteval:blimp:wh_vs_that_no_gap lighteval:blimp:wh_vs_that_no_gap_long_distance lighteval:blimp:wh_vs_that_with_gap lighteval:blimp:wh_vs_that_with_gap_long_distance lighteval:bigbench:bridging_anaphora_resolution_barqa lighteval:bigbench:cause_and_effect lighteval:bigbench:checkmate_in_one lighteval:bigbench:chess_state_tracking lighteval:bigbench:chinese_remainder_theorem lighteval:bigbench:cifar10_classification lighteval:bigbench_lite:code_line_description lighteval:bigbench:codenames lighteval:bigbench:color lighteval:bigbench:common_morpheme lighteval:bigbench_lite:conceptual_combinations lighteval:bigbench_lite:conlang_translation lighteval:bigbench:contextual_parametric_knowledge_conflicts lighteval:coqa lighteval:coqa_bb lighteval:bigbench:crash_blossom lighteval:bigbench:crass_ai lighteval:bigbench:cryobiology_spanish lighteval:bigbench:cryptonite lighteval:bigbench:cs_algorithms lighteval:bigbench:dark_humor_detection lighteval:bigbench:discourse_marker_prediction lighteval:bigbench:disfl_qa lighteval:drop lighteval:bigbench:dyck_languages lighteval:bigbench:elementary_math_qa lighteval:bigbench_lite:emoji_movie lighteval:bigbench:emojis_emotion_prediction lighteval:bigbench:empirical_judgments lighteval:bigbench:english_proverbs lighteval:bigbench:english_russian_proverbs lighteval:bigbench:entailed_polarity lighteval:bigbench:entailed_polarity_hindi lighteval:bigbench:epistemic_reasoning lighteval:ethics:commonsense lighteval:ethics:deontology lighteval:ethics:justice lighteval:ethics:utilitarianism lighteval:ethics:virtue lighteval:bigbench:evaluating_information_essentiality lighteval:bigbench:fact_checker lighteval:bigbench:fantasy_reasoning lighteval:bigbench:few_shot_nlg lighteval:bigbench:figure_of_speech_detection lighteval:bigbench_lite:formal_fallacies_syllogisms_negation lighteval:bigbench:gem lighteval:bigbench:gender_inclusive_sentences_german lighteval:bigbench:general_knowledge lighteval:glue:cola lighteval:glue:mnli lighteval:glue:mnli_mismatched lighteval:glue:mrpc lighteval:glue:qnli lighteval:glue:qqp lighteval:glue:rte lighteval:glue:sst2 lighteval:glue:stsb lighteval:glue:wnli lighteval:bigbench:goal_step_wikihow lighteval:gpqa:mc lighteval:gpqa:diamond lighteval:gpqa:extended lighteval:gpqa:main lighteval:bigbench:gre_reading_comprehension lighteval:gsm_plus lighteval:gsm8k lighteval:headqa:en lighteval:headqa:es lighteval:bigbench:hhh_alignment lighteval:bigbench:hindi_question_answering lighteval:bigbench_lite:hindu_knowledge lighteval:bigbench:hinglish_toxicity lighteval:bigbench:human_organs_senses lighteval:bigbench:hyperbaton lighteval:bigbench:identify_math_theorems lighteval:bigbench:identify_odd_metaphor lighteval:bigbench:implicatures lighteval:bigbench:implicit_relations lighteval:bigbench:intent_recognition lighteval:bigbench:international_phonetic_alphabet_nli lighteval:bigbench:international_phonetic_alphabet_transliterate lighteval:bigbench:intersect_geometry lighteval:bigbench:irony_identification lighteval:iwslt17:ar-en lighteval:iwslt17:de-en lighteval:iwslt17:en-ar lighteval:iwslt17:en-de lighteval:iwslt17:en-fr lighteval:iwslt17:en-ja lighteval:iwslt17:en-ko lighteval:iwslt17:en-zh lighteval:iwslt17:fr-en lighteval:iwslt17:ja-en lighteval:iwslt17:ko-en lighteval:iwslt17:zh-en lighteval:jeopardy lighteval:bigbench:kanji_ascii lighteval:bigbench:kannada lighteval:bigbench:key_value_maps lighteval:bigbench_lite:known_unknowns lighteval:lambada:standard lighteval:lambada:standard_cloze lighteval:lambada:openai lighteval:lambada:openai:de lighteval:lambada:openai:en lighteval:lambada:openai:es lighteval:lambada:openai:fr lighteval:lambada:openai:it lighteval:lambada:openai_cloze lighteval:bigbench:language_games lighteval:bigbench_lite:language_identification lighteval:bigbench:linguistic_mappings lighteval:bigbench_lite:linguistics_puzzles lighteval:bigbench_lite:logic_grid_puzzle lighteval:bigbench:logical_args lighteval:bigbench_lite:logical_deduction lighteval:bigbench:logical_fallacy_detection lighteval:bigbench:logical_sequence lighteval:logiqa lighteval:math_500 lighteval:math_500_gpassk lighteval:math:algebra lighteval:math:counting_and_probability lighteval:math:geometry lighteval:math:intermediate_algebra lighteval:math:number_theory lighteval:math:prealgebra lighteval:math:precalculus lighteval:math_cot:algebra lighteval:math_cot:counting_and_probability lighteval:math_cot:geometry lighteval:math_cot:intermediate_algebra lighteval:math_cot:number_theory lighteval:math_cot:prealgebra lighteval:math_cot:precalculus lighteval:bigbench:mathematical_induction lighteval:mathqa lighteval:bigbench:matrixshapes lighteval:bigbench:metaphor_boolean lighteval:bigbench:metaphor_understanding lighteval:mgsm:en lighteval:mgsm:es lighteval:mgsm:fr lighteval:mgsm:de lighteval:mgsm:ru lighteval:mgsm:zh lighteval:mgsm:ja lighteval:mgsm:th lighteval:mgsm:sw lighteval:mgsm:bn lighteval:mgsm:te lighteval:bigbench:minute_mysteries_qa lighteval:bigbench:misconceptions lighteval:bigbench_lite:misconceptions_russian lighteval:bigbench:mnist_ascii lighteval:bigbench:modified_arithmetic lighteval:bigbench:moral_permissibility lighteval:bigbench:movie_dialog_same_or_different lighteval:mtnt2019:en-fr lighteval:mtnt2019:en-ja lighteval:mtnt2019:fr-en lighteval:mtnt2019:ja-en lighteval:bigbench:mult_data_wrangling lighteval:bigbench:multiemo lighteval:musr:murder_mysteries lighteval:musr:object_placements lighteval:musr:team_allocation lighteval:mutual lighteval:mutual_plus lighteval:bigbench:natural_instructions lighteval:natural_questions lighteval:bigbench:nonsense_words_grammar lighteval:bigbench_lite:novel_concepts lighteval:bigbench:object_counting lighteval:bigbench:odd_one_out lighteval:openbookqa lighteval:bigbench_lite:operators lighteval:bigbench:paragraph_segmentation lighteval:bigbench:parsinlu_qa lighteval:bigbench_lite:parsinlu_reading_comprehension lighteval:bigbench:penguins_in_a_table lighteval:bigbench:periodic_elements lighteval:bigbench:persian_idioms lighteval:bigbench:phrase_relatedness lighteval:bigbench:physical_intuition lighteval:bigbench:physics lighteval:bigbench:physics_questions lighteval:piqa lighteval:bigbench_lite:play_dialog_same_or_different lighteval:bigbench:polish_sequence_labeling lighteval:bigbench:presuppositions_as_nli lighteval:prost lighteval:pubmedqa lighteval:qa4mre:2011 lighteval:qa4mre:2012 lighteval:qa4mre:2013 lighteval:bigbench:qa_wikidata lighteval:qasper lighteval:qasper_ll lighteval:bigbench:question_selection lighteval:race:high lighteval:bigbench:real_or_fake_text lighteval:bigbench_lite:repeat_copy_logic lighteval:bigbench:rephrase lighteval:bigbench:rhyming lighteval:bigbench:riddle_sense lighteval:bigbench:scientific_press_release lighteval:sciq lighteval:bigbench:semantic_parsing_in_context_sparc lighteval:bigbench:semantic_parsing_spider lighteval:bigbench:sentence_ambiguity lighteval:bigbench:similarities_abstraction lighteval:bigbench:simp_turing_concept lighteval:simpleqa lighteval:bigbench:simple_arithmetic_json lighteval:bigbench:simple_arithmetic_json_multiple_choice lighteval:bigbench:simple_arithmetic_json_subtasks lighteval:bigbench:simple_arithmetic_multiple_targets_json lighteval:bigbench:simple_ethical_questions lighteval:bigbench:simple_text_editing lighteval:bigbench:social_iqa lighteval:bigbench:social_support lighteval:squad_v2 lighteval:storycloze:2016 lighteval:storycloze:2018 lighteval:bigbench_lite:strange_stories lighteval:bigbench_lite:strategyqa lighteval:bigbench:sufficient_information lighteval:bigbench:suicide_risk lighteval:super_glue:boolq lighteval:super_glue:cb lighteval:super_glue:copa lighteval:super_glue:rte lighteval:super_glue:multirc lighteval:super_glue:wic lighteval:super_glue:wsc lighteval:bigbench:swahili_english_proverbs lighteval:swag lighteval:bigbench:swedish_to_german_proverbs lighteval:bigbench_lite:symbol_interpretation lighteval:bigbench:tellmewhy lighteval:bigbench:tense lighteval:bigbench:timedial lighteval:toxigen lighteval:bigbench:topical_chat lighteval:bigbench:tracking_shuffled_objects lighteval:triviaqa lighteval:truthfulqa:gen lighteval:bigbench:understanding_fables lighteval:bigbench:undo_permutation lighteval:bigbench:unit_conversion lighteval:bigbench:unit_interpretation lighteval:bigbench:unnatural_in_context_learning lighteval:unscramble:anagrams1 lighteval:unscramble:anagrams2 lighteval:unscramble:cycle_letters lighteval:unscramble:random_insertion lighteval:unscramble:reversed_words lighteval:bigbench_lite:vitaminc_fact_verification lighteval:webqs lighteval:bigbench:what_is_the_tao lighteval:bigbench:which_wiki_edit lighteval:wikitext:2 lighteval:bigbench:wino_x_german lighteval:bigbench_lite:winowhy lighteval:wmt08:cs-en lighteval:wmt08:de-en lighteval:wmt08:en-cs lighteval:wmt08:en-de lighteval:wmt08:en-es lighteval:wmt08:en-fr lighteval:wmt08:en-hu lighteval:wmt08:es-en lighteval:wmt08:fr-en lighteval:wmt08:hu-en lighteval:wmt09:cs-en lighteval:wmt09:de-en lighteval:wmt09:en-cs lighteval:wmt09:en-de lighteval:wmt09:en-es lighteval:wmt09:en-fr lighteval:wmt09:en-hu lighteval:wmt09:en-it lighteval:wmt09:es-en lighteval:wmt09:fr-en lighteval:wmt09:hu-en lighteval:wmt09:it-en lighteval:wmt10:cs-en lighteval:wmt10:de-en lighteval:wmt10:en-cs lighteval:wmt10:en-de lighteval:wmt10:en-es lighteval:wmt10:en-fr lighteval:wmt10:es-en lighteval:wmt10:fr-en lighteval:wmt11:cs-en lighteval:wmt11:de-en lighteval:wmt11:en-cs lighteval:wmt11:en-de lighteval:wmt11:en-es lighteval:wmt11:en-fr lighteval:wmt11:es-en lighteval:wmt11:fr-en lighteval:wmt12:cs-en lighteval:wmt12:de-en lighteval:wmt12:en-cs lighteval:wmt12:en-de lighteval:wmt12:en-es lighteval:wmt12:en-fr lighteval:wmt12:es-en lighteval:wmt12:fr-en lighteval:wmt13:cs-en lighteval:wmt13:de-en lighteval:wmt13:en-cs lighteval:wmt13:en-de lighteval:wmt13:en-es lighteval:wmt13:en-fr lighteval:wmt13:en-ru lighteval:wmt13:es-en lighteval:wmt13:fr-en lighteval:wmt13:ru-en lighteval:wmt14:cs-en lighteval:wmt14:de-en lighteval:wmt14:en-cs lighteval:wmt14:en-de lighteval:wmt14:en-fr lighteval:wmt14:en-hi lighteval:wmt14:en-ru lighteval:wmt14:fr-en lighteval:wmt14:hi-en lighteval:wmt14:ru-en lighteval:wmt15:cs-en lighteval:wmt15:de-en lighteval:wmt15:en-cs lighteval:wmt15:en-de lighteval:wmt15:en-fi lighteval:wmt15:en-fr lighteval:wmt15:en-ru lighteval:wmt15:fi-en lighteval:wmt15:fr-en lighteval:wmt15:ru-en lighteval:wmt16:cs-en lighteval:wmt16:de-en lighteval:wmt16:en-cs lighteval:wmt16:en-de lighteval:wmt16:en-fi lighteval:wmt16:en-ro lighteval:wmt16:en-ru lighteval:wmt16:en-tr lighteval:wmt16:fi-en lighteval:wmt16:ro-en lighteval:wmt16:ru-en lighteval:wmt16:tr-en lighteval:wmt17:cs-en lighteval:wmt17:de-en lighteval:wmt17:en-cs lighteval:wmt17:en-de lighteval:wmt17:en-fi lighteval:wmt17:en-lv lighteval:wmt17:en-ru lighteval:wmt17:en-tr lighteval:wmt17:en-zh lighteval:wmt17:fi-en lighteval:wmt17:lv-en lighteval:wmt17:ru-en lighteval:wmt17:tr-en lighteval:wmt17:zh-en lighteval:wmt18:cs-en lighteval:wmt18:de-en lighteval:wmt18:en-cs lighteval:wmt18:en-de lighteval:wmt18:en-et lighteval:wmt18:en-fi lighteval:wmt18:en-ru lighteval:wmt18:en-tr lighteval:wmt18:en-zh lighteval:wmt18:et-en lighteval:wmt18:fi-en lighteval:wmt18:ru-en lighteval:wmt18:tr-en lighteval:wmt18:zh-en lighteval:wmt19:cs-de lighteval:wmt19:de-cs lighteval:wmt19:de-en lighteval:wmt19:de-fr lighteval:wmt19:en-cs lighteval:wmt19:en-de lighteval:wmt19:en-fi lighteval:wmt19:en-gu lighteval:wmt19:en-kk lighteval:wmt19:en-lt lighteval:wmt19:en-ru lighteval:wmt19:en-zh lighteval:wmt19:fi-en lighteval:wmt19:fr-de lighteval:wmt19:gu-en lighteval:wmt19:kk-en lighteval:wmt19:lt-en lighteval:wmt19:ru-en lighteval:wmt19:zh-en lighteval:wmt20:cs-en lighteval:wmt20:de-en lighteval:wmt20:de-fr lighteval:wmt20:en-cs lighteval:wmt20:en-de lighteval:wmt20:en-iu lighteval:wmt20:en-ja lighteval:wmt20:en-km lighteval:wmt20:en-pl lighteval:wmt20:en-ps lighteval:wmt20:en-ru lighteval:wmt20:en-ta lighteval:wmt20:en-zh lighteval:wmt20:fr-de lighteval:wmt20:iu-en lighteval:wmt20:ja-en lighteval:wmt20:km-en lighteval:wmt20:pl-en lighteval:wmt20:ps-en lighteval:wmt20:ru-en lighteval:wmt20:ta-en lighteval:wmt20:zh-en lighteval:bigbench:word_sorting lighteval:bigbench:word_unscrambling lighteval:wsc273 lighteval:xcopa:en lighteval:xcopa:et lighteval:xcopa:ht lighteval:xcopa:it lighteval:xcopa:id lighteval:xcopa:qu lighteval:xcopa:sw lighteval:xcopa:zh lighteval:xcopa:ta lighteval:xcopa:th lighteval:xcopa:tr lighteval:xcopa:vi lighteval:xstory_cloze:en lighteval:xstory_cloze:ru lighteval:xstory_cloze:zh lighteval:xstory_cloze:es lighteval:xstory_cloze:ar lighteval:xstory_cloze:hi lighteval:xstory_cloze:id lighteval:xstory_cloze:te lighteval:xstory_cloze:sw lighteval:xstory_cloze:eu lighteval:xstory_cloze:my lighteval:xwinograd:en lighteval:xwinograd:fr lighteval:xwinograd:jp lighteval:xwinograd:pt lighteval:xwinograd:ru lighteval:xwinograd:zh lighteval:mmlu_redux_2: lighteval:hle