{"benchmarkName":"MAEB(v1)","taskTypes":["AudioClassification","AudioReranking","AudioClustering","AudioPairClassification","Any2AnyRetrieval","AudioMultilabelClassification","AudioZeroshotClassification"],"tasks":["VoxCelebSA","GTZANGenre","VehicleSoundClustering","VoxPopuliGenderClustering","MridinghamTonic","VoxPopuliAccentPairClassification","NMSQAPairClassification","IEMOCAPGender","VoxPopuliLanguageID","MACST2ARetrieval","BirdCLEF","SIBFLEURS","JamAltArtistA2ARetrieval","ClothoT2ARetrieval","CREMADPairClassification","FSD2019Kaggle","CommonVoiceMini21T2ARetrieval","SpeechCommandsZeroshotv0.02","CREMA_DClustering","MInDS14","UrbanSound8KT2ARetrieval","GTZANAudioReranking","BeijingOpera","CommonLanguageAgeDetection","FleursT2ARetrieval","CREMA_D","GigaSpeechT2ARetrieval","RavdessZeroshot","SpokenSQuADT2ARetrieval","JamAltLyricA2TRetrieval"],"tasksMeta":[{"name":"ClothoT2ARetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic","Written"],"modalities":["text","audio"],"description":"An audio captioning datasetst containing audio clips from the Freesound platform and their corresponding captions.","reference":"https://github.com/audio-captioning/clotho-dataset","citation":"\n@inproceedings{drossos2019clothoaudiocaptioningdataset,\n  author = {Drossos, Konstantinos and Lipping, Samuel and Virtanen, Tuomas},\n  booktitle = {ICASSP 2020 - 2020 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp40776.2020.9052990},\n  month = {May},\n  pages = {736–740},\n  publisher = {IEEE},\n  title = {Clotho: an Audio Captioning Dataset},\n  url = {https://doi.org/10.1109/icassp40776.2020.9052990},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/Clotho","license":"mit","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_5","numModels":0},{"name":"CommonVoiceMini21T2ARetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["Abkhazian","Afrikaans","Albanian","Amharic","Arabic","Armenian","Assamese","Asturian","Azerbaijani","Basa (Cameroon)","Bashkir","Basque","Belarusian","Bengali","Breton","Bulgarian","Catalan","Central Kurdish","Chinese","Chuvash","Czech","Danish","Dhivehi","Dutch","Dyula","Eastern Mari","English","Erzya","Esperanto","Estonian","Finnish","French","Galician","Ganda","Georgian","German","Guarani","Hakha Chin","Hausa","Hebrew","Hindi","Hungarian","Indonesian","Interlingua (International Auxiliary Language Association)","Iranian Persian","Irish","Italian","Japanese","Kabyle","Kalenjin","Kazakh","Kinyarwanda","Kirghiz","Korean","Latgalian","Latvian","Ligurian","Lithuanian","Luo (Kenya and Tanzania)","Macedonian","Malayalam","Maltese","Marathi","Min Nan Chinese","Modern Greek (1453-)","Moksha","Mongolian","Nepali (macrolanguage)","Northern Kurdish","Northern Uzbek","Norwegian Nynorsk","Occitan (post 1500)","Oriya (macrolanguage)","Ossetian","Panjabi","Polish","Portuguese","Pushto","Romanian","Romansh","Russian","Santali","Saraiki","Sardinian","Serbian","Slovak","Slovenian","Spanish","Standard Moroccan Tamazight","Swahili (individual language)","Swedish","Taita","Tamil","Tatar","Telugu","Thai","Tigre","Toki Pona","Tswana","Turkish","Turkmen","Uighur","Ukrainian","Upper Sorbian","Urdu","Vietnamese","Welsh","Western Frisian","Western Mari","Yakut","Yiddish","Yoruba","Yue Chinese","Zaza"],"domains":["Spoken"],"modalities":["text","audio"],"description":"Speech recordings with corresponding text transcriptions from CommonVoice dataset.","reference":"https://commonvoice.mozilla.org/","citation":"\n@inproceedings{ardila-etal-2020-common,\n  archiveprefix = {arXiv},\n  eprint = {1912.06670},\n  title = {Common Voice: A Massively-Multilingual Speech Corpus},\n  author = {Ardila, Rosana  and\n      Branson, Megan  and\n      Davis, Kelly  and\n      Kohler, Michael  and\n      Meyer, Josh  and\n      Henretty, Michael  and\n      Morais, Reuben  and\n      Saunders, Lindsay  and\n      Tyers, Francis  and\n      Weber, Gregor},\n  editor = {Calzolari, Nicoletta  and\n      B{\\'e}chet, Fr{\\'e}d{\\'e}ric  and\n      Blache, Philippe  and\n      Choukri, Khalid  and\n      Cieri, Christopher  and\n      Declerck, Thierry  and\n      Goggi, Sara  and\n      Isahara, Hitoshi  and\n      Maegaard, Bente  and\n      Mariani, Joseph  and\n      Mazo, H{\\'e}l{\\`e}ne  and\n      Moreno, Asuncion  and\n      Odijk, Jan  and\n      Piperidis, Stelios},\n  booktitle = {Proceedings of the Twelfth Language Resources and Evaluation Conference},\n  month = may,\n  year = {2020},\n  address = {Marseille, France},\n  publisher = {European Language Resources Association},\n  url = {https://aclanthology.org/2020.lrec-1.520/},\n  pages = {4218--4222},\n  language = {eng},\n  ISBN = {979-10-95546-34-4},\n  abstract = {The Common Voice corpus is a massively-multilingual collection of transcribed speech intended for speech technology research and development. Common Voice is designed for Automatic Speech Recognition purposes but can be useful in other domains (e.g. language identification). To achieve scale and sustainability, the Common Voice project employs crowdsourcing for both data collection and data validation. The most recent release includes 29 languages, and as of November 2019 there are a total of 38 languages collecting data. Over 50,000 individuals have participated so far, resulting in 2,500 hours of collected audio. To our knowledge this is the largest audio corpus in the public domain for speech recognition, both in terms of number of hours and number of languages. As an example use case for Common Voice, we present speech recognition experiments using Mozilla{'}s DeepSpeech Speech-to-Text toolkit. By applying transfer learning from a source English model, we find an average Character Error Rate improvement of 5.99 {\\ensuremath{\\pm}} 5.48 for twelve target languages (German, French, Italian, Turkish, Catalan, Slovenian, Welsh, Irish, Breton, Tatar, Chuvash, and Kabyle). For most of these languages, these are the first ever published results on end-to-end Automatic Speech Recognition.},\n}\n","isPublic":true,"sourceDataset":"mteb/common_voice_21_0_mini","license":"cc0-1.0","dateFrom":"2020-01-01","dateTo":"2024-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_5","numModels":0},{"name":"FleursT2ARetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["Afrikaans","Amharic","Arabic","Armenian","Assamese","Asturian","Azerbaijani","Belarusian","Bengali","Bosnian","Bulgarian","Burmese","Catalan","Cebuano","Central Kurdish","Croatian","Czech","Danish","Dutch","English","Estonian","Filipino","Finnish","French","Fulah","Galician","Ganda","Georgian","German","Gujarati","Hausa","Hebrew","Hindi","Hungarian","Icelandic","Igbo","Indonesian","Irish","Italian","Japanese","Javanese","Kabuverdianu","Kamba (Kenya)","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Lingala","Lithuanian","Luo (Kenya and Tanzania)","Luxembourgish","Macedonian","Malay (macrolanguage)","Malayalam","Maltese","Mandarin Chinese","Maori","Marathi","Modern Greek (1453-)","Mongolian","Nepali (individual language)","Northern Uzbek","Norwegian Bokmål","Nyanja","Occitan (post 1500)","Oriya (macrolanguage)","Oromo","Panjabi","Pedi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Serbian","Shona","Sindhi","Slovak","Slovenian","Somali","Spanish","Standard Latvian","Swahili (individual language)","Swedish","Tajik","Tamil","Telugu","Thai","Turkish","Ukrainian","Umbundu","Urdu","Vietnamese","Welsh","Wolof","Xhosa","Yoruba","Yue Chinese","Zulu"],"domains":["Spoken"],"modalities":["text","audio"],"description":"Speech recordings with corresponding text transcriptions from the FLEURS dataset.","reference":"https://github.com/google-research-datasets/fleurs","citation":"\n@inproceedings{conneau2023fleurs,\n  author = {Conneau, Alexis and Ma, Min and Khanuja, Simran and Zhang, Yu and Axelrod, Vera and Dalmia, Siddharth and Riesa, Jason and Rivera, Clara and Bapna, Ankur},\n  booktitle = {2022 IEEE Spoken Language Technology Workshop (SLT)},\n  organization = {IEEE},\n  pages = {798--805},\n  title = {Fleurs: Few-shot learning evaluation of universal representations of speech},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/fleurs","license":"apache-2.0","dateFrom":"2021-01-01","dateTo":"2024-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_5","numModels":0},{"name":"GigaSpeechT2ARetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Spoken"],"modalities":["text","audio"],"description":"Given an English transcription, retrieve its corresponding audio segment. Audio comes from the 10 000‑hour training subset of GigaSpeech, sourced from ≈40 000 hours of transcribed audiobooks, podcasts, and YouTube.","reference":"https://github.com/SpeechColab/GigaSpeech","citation":"\n@inproceedings{GigaSpeech2021,\n  author = {Chen, Guoguo and Chai, Shuzhou and Wang, Guanbo and Du, Jiayu and Zhang, Wei-Qiang and Weng, Chao and Su, Dan and Povey, Daniel and Trmal, Jan and Zhang, Junbo and Jin, Mingjie and Khudanpur, Sanjeev and Watanabe, Shinji and Zhao, Shuaijiang and Zou, Wei and Li, Xiangang and Yao, Xuchen and Wang, Yongqing and Wang, Yujun and You, Zhao and Yan, Zhiyong},\n  booktitle = {Proc. Interspeech 2021},\n  title = {GigaSpeech: An Evolving, Multi-domain ASR Corpus with 10,000 Hours of Transcribed Audio},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/gigaspeech_t2a","license":"https://github.com/SpeechColab/GigaSpeech/blob/main/LICENSE","dateFrom":"2021-01-01","dateTo":"2021-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_5","numModels":0},{"name":"JamAltArtistA2ARetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English","French","German","Spanish"],"domains":["Music"],"modalities":["audio"],"description":"Given audio clip of a song (query), retrieve all songs from the same artist in the Jam-Alt-Lines dataset","reference":"https://huggingface.co/datasets/jamendolyrics/jam-alt-lines","citation":"\n@inproceedings{cifka-2024-jam-alt,\n  author = {Ond{\\v{r}}ej C{\\'{\\i}}fka and\nHendrik Schreiber and\nLuke Miner and\nFabian{-}Robert St{\\\"{o}}ter},\n  booktitle = {Proceedings of the 25th International Society for\nMusic Information Retrieval Conference},\n  doi = {10.5281/ZENODO.14877443},\n  pages = {737--744},\n  publisher = {ISMIR},\n  title = {Lyrics Transcription for Humans: {A} Readability-Aware Benchmark},\n  url = {https://doi.org/10.5281/zenodo.14877443},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/jam-alt-lines","license":"cc-by-nc-sa-4.0","dateFrom":"2023-01-01","dateTo":"2024-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"JamAltLyricA2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English","French","German","Spanish"],"domains":["Music"],"modalities":["text","audio"],"description":"From audio clips of songs (query), retrieve corresponding textual lyric from the Jam-Alt-Lines dataset","reference":"https://huggingface.co/datasets/jamendolyrics/jam-alt-lines","citation":"\n@inproceedings{cifka-2024-jam-alt,\n  author = {Ond{\\v{r}}ej C{\\'{\\i}}fka and\nHendrik Schreiber and\nLuke Miner and\nFabian{-}Robert St{\\\"{o}}ter},\n  booktitle = {Proceedings of the 25th International Society for\nMusic Information Retrieval Conference},\n  doi = {10.5281/ZENODO.14877443},\n  pages = {737--744},\n  publisher = {ISMIR},\n  title = {Lyrics Transcription for Humans: {A} Readability-Aware Benchmark},\n  url = {https://doi.org/10.5281/zenodo.14877443},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/jam-alt-lines","license":"cc-by-nc-sa-4.0","dateFrom":"2023-01-01","dateTo":"2024-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"MACST2ARetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["AudioScene"],"modalities":["text","audio"],"description":"Audio captions and tags for urban acoustic scenes in TAU Urban Acoustic Scenes 2019 development dataset.","reference":"https://zenodo.org/records/5114771","citation":"\n@misc{martinmorato2021groundtruthreliabilitymultiannotator,\n  archiveprefix = {arXiv},\n  author = {Irene Martin-Morato and Annamaria Mesaros},\n  eprint = {2104.04214},\n  primaryclass = {eess.AS},\n  title = {What is the ground truth? Reliability of multi-annotator data for audio tagging},\n  url = {https://arxiv.org/abs/2104.04214},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/MACS_t2a","license":"https://zenodo.org/records/5114771","dateFrom":"2021-01-01","dateTo":"2021-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_5","numModels":0},{"name":"SpokenSQuADT2ARetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Academic","Encyclopaedic","Non-fiction"],"modalities":["text","audio"],"description":"Text-to-audio retrieval task based on SpokenSQuAD dataset. Given a text question, retrieve relevant audio segments that contain the answer. Questions are derived from SQuAD reading comprehension dataset with corresponding spoken passages.","reference":"https://github.com/Chia-Hsuan-Lee/Spoken-SQuAD","citation":"\n@article{lee2018spoken,\n  author = {Lee, Chia-Hsuan and Wu, Szu-Lin and Liu, Chi-Liang and Lee, Hung-yi},\n  journal = {Proc. Interspeech 2018},\n  pages = {3459--3463},\n  title = {Spoken SQuAD: A Study of Mitigating the Impact of Speech Recognition Errors on Listening Comprehension},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/spoken-squad-t2a","license":"cc-by-sa-4.0","dateFrom":"2018-03-01","dateTo":"2018-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_5","numModels":0},{"name":"UrbanSound8KT2ARetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["No linguistic content"],"domains":["AudioScene"],"modalities":["text","audio"],"description":"UrbanSound8K: Text-to-audio retrieval of urban sound events.","reference":"https://huggingface.co/datasets/CLAPv2/Urbansound8K","citation":"\n@inproceedings{Salamon:UrbanSound:ACMMM:14,\n  author = {Salamon, Justin and Jacoby, Christopher and Bello, Juan Pablo},\n  booktitle = {Proceedings of the 22nd ACM international conference on Multimedia},\n  organization = {ACM},\n  pages = {1041--1044},\n  title = {A Dataset and Taxonomy for Urban Sound Research},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/Urbansound8K_t2a","license":"cc-by-nc-sa-3.0","dateFrom":"2014-11-01","dateTo":"2014-11-03","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_5","numModels":0},{"name":"BeijingOpera","type":"AudioClassification","simplifiedType":"classification","languages":["No linguistic content"],"domains":["Music"],"modalities":["audio"],"description":"Audio classification of percussion instruments into one of 4 classes: `Bangu`, `Naobo`, `Daluo`, and `Xiaoluo`","reference":"https://huggingface.co/datasets/silky1708/BeijingOpera","citation":"\n@inproceedings{6853981,\n  author = {Tian, Mi and Srinivasamurthy, Ajay and Sandler, Mark and Serra, Xavier},\n  booktitle = {2014 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/ICASSP.2014.6853981},\n  keywords = {Decision support systems;Conferences;Acoustics;Speech;Speech processing;Time-frequency analysis;Beijing Opera;Onset Detection;Drum Transcription;Non-negative matrix factorization},\n  number = {},\n  pages = {2159-2163},\n  title = {A study of instrument-wise onset detection in Beijing Opera percussion ensembles},\n  volume = {},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/beijing-opera","license":"cc-by-4.0","dateFrom":"2014-01-01","dateTo":"2014-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"BirdCLEF","type":"AudioClassification","simplifiedType":"classification","languages":["No linguistic content"],"domains":["Spoken","Speech","Bioacoustics"],"modalities":["audio"],"description":"BirdCLEF+ 2025 dataset for species identification from audio, focused on birds, amphibians, mammals and insects from the Middle Magdalena Valley of Colombia. Downsampled to 50 classes with 20 samples each.","reference":"https://huggingface.co/datasets/christopher/birdclef-2025","citation":"\n@inproceedings{canas2025overview,\n  author = {Ca{\\~n}as, Juan Sebasti{\\'a}n and Kahl, Stefan and Denton, Tom and Toro-G{\\'o}mez, Maria Paula and Rodriguez-Buritica, Susana and Benavides-Lopez, Jose Luis and Ulloa, Juan Sebasti{\\'a}n and Caycedo-Rosales, Paula and Klinck, Holger and Go{\\\"e}au, Herv{\\'e} and others},\n  booktitle = {Conference and Labs of the Evaluation Forum (CLEF 2025)},\n  number = {4038},\n  organization = {CEUR-WS},\n  pages = {2909--2919},\n  title = {Overview of BirdCLEF+ 2025: Multi-taxonomic sound identification in the Middle Magdalena, Colombia},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/birdclef25-mini","license":"cc-by-nc-4.0","dateFrom":"2025-01-01","dateTo":"2025-12-31","annotationsCreators":"expert-annotated","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"CREMA_D","type":"AudioClassification","simplifiedType":"classification","languages":["English"],"domains":["Speech"],"modalities":["audio"],"description":"Emotion classification of audio into one of 6 classes: Anger, Disgust, Fear, Happy, Neutral, Sad.","reference":"https://huggingface.co/datasets/silky1708/CREMA-D","citation":"\n@article{Cao2014-ih,\n  author = {Cao, Houwei and Cooper, David G and Keutmann, Michael K and Gur,\nRuben C and Nenkova, Ani and Verma, Ragini},\n  copyright = {https://ieeexplore.ieee.org/Xplorehelp/downloads/license-information/IEEE.html},\n  journal = {IEEE Transactions on Affective Computing},\n  keywords = {Emotional corpora; facial expression; multi-modal recognition;\nvoice expression},\n  language = {en},\n  month = oct,\n  number = {4},\n  pages = {377--390},\n  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},\n  title = {{CREMA-D}: Crowd-sourced emotional multimodal actors dataset},\n  volume = {5},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/crema-d","license":"http://opendatacommons.org/licenses/odbl/1.0/","dateFrom":"2014-01-01","dateTo":"2014-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"CommonLanguageAgeDetection","type":"AudioClassification","simplifiedType":"classification","languages":["English"],"domains":["Spoken","Scene","Speech"],"modalities":["audio"],"description":"Age Classification. This is a stratified subsampled version of the original CommonLanguage dataset.","reference":"https://huggingface.co/datasets/speechbrain/common_language","citation":"\n@dataset{ganesh_sinisetty_2021_5036977,\n  author = {Ganesh Sinisetty and\nPavlo Ruban and\nOleksandr Dymov and\nMirco Ravanelli},\n  doi = {10.5281/zenodo.5036977},\n  month = jun,\n  publisher = {Zenodo},\n  title = {CommonLanguage},\n  url = {https://doi.org/10.5281/zenodo.5036977},\n  version = {0.1},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/commonlanguage-age-mini","license":"cc-by-4.0","dateFrom":"2021-01-01","dateTo":"2021-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"GTZANGenre","type":"AudioClassification","simplifiedType":"classification","languages":["No linguistic content"],"domains":["Music"],"modalities":["audio"],"description":"Music Genre Classification (10 classes)","reference":"https://ieeexplore.ieee.org/abstract/document/1021072","citation":"\n@article{1021072,\n  author = {Tzanetakis, G. and Cook, P.},\n  doi = {10.1109/TSA.2002.800560},\n  journal = {IEEE Transactions on Speech and Audio Processing},\n  keywords = {Humans;Music information retrieval;Instruments;Computer science;Multiple signal classification;Signal analysis;Pattern recognition;Feature extraction;Wavelet analysis;Cultural differences},\n  number = {5},\n  pages = {293-302},\n  title = {Musical genre classification of audio signals},\n  volume = {10},\n  year = {2002},\n}\n","isPublic":true,"sourceDataset":"mteb/gtzan-genre","license":"not specified","dateFrom":"2000-01-01","dateTo":"2001-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"IEMOCAPGender","type":"AudioClassification","simplifiedType":"classification","languages":["English"],"domains":["Spoken","Speech"],"modalities":["audio"],"description":"Classification of speech samples by speaker gender (male/female) from the IEMOCAP database of interactive emotional dyadic conversations.","reference":"https://doi.org/10.1007/s10579-008-9076-6","citation":"\n@article{busso2008iemocap,\n  author = {Busso, Carlos and Bulut, Murtaza and Lee, Chi-Chun and Kazemzadeh, Abe and Mower, Emily and Kim, Samuel and Chang, Jeannette N and Lee, Sungbok and Narayanan, Shrikanth S},\n  journal = {Language resources and evaluation},\n  number = {4},\n  pages = {335--359},\n  publisher = {Springer},\n  title = {IEMOCAP: Interactive emotional dyadic motion capture database},\n  volume = {42},\n  year = {2008},\n}\n","isPublic":true,"sourceDataset":"mteb/iemocap","license":"cc-by-nc-sa-3.0","dateFrom":"2008-01-01","dateTo":"2008-12-31","annotationsCreators":"expert-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"MInDS14","type":"AudioClassification","simplifiedType":"classification","languages":["Chinese","Czech","Dutch","English","French","German","Italian","Korean","Polish","Portuguese","Russian","Spanish"],"domains":["Speech","Spoken"],"modalities":["audio"],"description":"MInDS-14 is an evaluation resource for intent detection with spoken data in 14 diverse languages.","reference":"https://aclanthology.org/2021.emnlp-main.591/","citation":"\n@inproceedings{DBLP:journals/corr/abs-2104-08524,\n  address = {Online and Punta Cana, Dominican Republic},\n  author = {Gerz, Daniela and Su, Pei-Hao and Kusztos, Razvan and Mondal, Avishek and Lis, Micha{\\l} and Singhal, Eshan and Mrk{\\v{s}}i{\\'c}, Nikola and Wen, Tsung-Hsien and Vuli{\\'c}, Ivan},\n  booktitle = {Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},\n  doi = {10.18653/v1/2021.emnlp-main.591},\n  editor = {Moens, Marie-Francine and Huang, Xuanjing and Specia, Lucia and Yih, Scott Wen-tau},\n  month = {nov},\n  pages = {7468--7475},\n  publisher = {Association for Computational Linguistics},\n  title = {Multilingual and Cross-Lingual Intent Detection from Spoken Data},\n  url = {https://aclanthology.org/2021.emnlp-main.591/},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/minds14-multilingual","license":"cc-by-4.0","dateFrom":"2021-04-01","dateTo":"2021-04-30","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"MridinghamTonic","type":"AudioClassification","simplifiedType":"classification","languages":["No linguistic content"],"domains":["Music"],"modalities":["audio"],"description":"Tonic classification of Mridingham (a pitched percussion instrument) into one of 6 classes: B,C,C#,D,D#,E","reference":"https://huggingface.co/datasets/silky1708/Mridingham-Tonic","citation":"\n@inproceedings{6637633,\n  author = {Anantapadmanabhan, Akshay and Bellur, Ashwin and Murthy, Hema A},\n  booktitle = {2013 IEEE International Conference on Acoustics, Speech and Signal Processing},\n  doi = {10.1109/ICASSP.2013.6637633},\n  keywords = {Instruments;Vectors;Hidden Markov models;Harmonic analysis;Modal analysis;Dictionaries;Music;Modal Analysis;Mridangam;automatic transcription;Non-negative Matrix Factorization;Hidden Markov models},\n  number = {},\n  pages = {181-185},\n  title = {Modal analysis and transcription of strokes of the mridangam using non-negative matrix factorization},\n  volume = {},\n  year = {2013},\n}\n","isPublic":true,"sourceDataset":"mteb/mridingham-tonic","license":"cc-by-4.0","dateFrom":"2020-01-01","dateTo":"2020-10-06","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"SIBFLEURS","type":"AudioClassification","simplifiedType":"classification","languages":["Afrikaans","Amharic","Armenian","Assamese","Asturian","Belarusian","Bengali","Bosnian","Bulgarian","Burmese","Catalan","Cebuano","Central Kurdish","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Ganda","Georgian","German","Gujarati","Halh Mongolian","Hausa","Hebrew","Hindi","Hungarian","Icelandic","Igbo","Indonesian","Iranian Persian","Irish","Italian","Japanese","Javanese","Kabuverdianu","Kamba (Kenya)","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Lingala","Lithuanian","Luo (Kenya and Tanzania)","Luxembourgish","Macedonian","Malayalam","Maltese","Maori","Marathi","Modern Greek (1453-)","Nepali (individual language)","Nigerian Fulfulde","North Azerbaijani","Northern Uzbek","Norwegian Bokmål","Nyanja","Occitan (post 1500)","Odia","Panjabi","Pedi","Polish","Portuguese","Romanian","Russian","Serbian","Shona","Sindhi","Slovak","Slovenian","Somali","Southern Pashto","Spanish","Standard Arabic","Standard Latvian","Standard Malay","Swahili (individual language)","Swedish","Tagalog","Tajik","Tamil","Telugu","Thai","Turkish","Ukrainian","Umbundu","Urdu","Vietnamese","Welsh","West Central Oromo","Wolof","Xhosa","Yoruba","Zulu"],"domains":["Encyclopaedic"],"modalities":["audio"],"description":"Topic Classification for multilingual audio dataset. This dataset is a stratified and downsampled subset of the SIBFLEURS dataset, which is a collection of 1000+ hours of audio data in 100+ languages.","reference":"https://huggingface.co/datasets/WueNLP/sib-fleurs","citation":"\n@misc{schmidt2025fleursslumassivelymultilingualbenchmark,\n  archiveprefix = {arXiv},\n  author = {Fabian David Schmidt and Ivan Vulić and Goran Glavaš and David Ifeoluwa Adelani},\n  eprint = {2501.06117},\n  primaryclass = {cs.CL},\n  title = {Fleurs-SLU: A Massively Multilingual Benchmark for Spoken Language Understanding},\n  url = {https://arxiv.org/abs/2501.06117},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/sib-fleurs-multilingual-mini","license":"cc-by-4.0","dateFrom":"2024-12-09","dateTo":"2024-12-13","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"VoxCelebSA","type":"AudioClassification","simplifiedType":"classification","languages":["English"],"domains":["Spoken"],"modalities":["audio"],"description":"VoxCeleb dataset augmented for Sentiment Analysis task","reference":"https://huggingface.co/datasets/DynamicSuperb/Sentiment_Analysis_SLUE-VoxCeleb","citation":"\n@inproceedings{shon2022sluenewbenchmarktasks,\n  author = {Shon, Suwon and Pasad, Ankita and Wu, Felix and Brusco, Pablo and Artzi, Yoav and Livescu, Karen and Han, Kyu J.},\n  booktitle = {ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp43922.2022.9746137},\n  month = {May},\n  pages = {7927–7931},\n  publisher = {IEEE},\n  title = {SLUE: New Benchmark Tasks For Spoken Language Understanding Evaluation on Natural Speech},\n  url = {https://doi.org/10.1109/icassp43922.2022.9746137},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/voxceleb-sentiment","license":"cc-by-4.0","dateFrom":"2024-06-27","dateTo":"2024-06-28","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"VoxPopuliLanguageID","type":"AudioClassification","simplifiedType":"classification","languages":["English","French","German","Polish","Spanish"],"domains":["Spoken","Speech"],"modalities":["audio"],"description":"Subsampled Dataset for classification of speech samples into one of 5 European languages (English, German, French, Spanish, Polish) from European Parliament recordings.","reference":"https://huggingface.co/datasets/facebook/voxpopuli","citation":"\n@inproceedings{wang-etal-2021-voxpopuli,\n  address = {Online},\n  author = {Wang, Changhan  and\nRiviere, Morgane  and\nLee, Ann  and\nWu, Anne  and\nTalnikar, Chaitanya  and\nHaziza, Daniel  and\nWilliamson, Mary  and\nPino, Juan  and\nDupoux, Emmanuel},\n  booktitle = {Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},\n  doi = {10.18653/v1/2021.acl-long.80},\n  month = aug,\n  pages = {993--1003},\n  publisher = {Association for Computational Linguistics},\n  title = {{V}ox{P}opuli: A Large-Scale Multilingual Speech Corpus for Representation Learning, Semi-Supervised Learning and Interpretation},\n  url = {https://aclanthology.org/2021.acl-long.80},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/voxpopuli-mini","license":"cc0-1.0","dateFrom":"2009-01-01","dateTo":"2020-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"CREMA_DClustering","type":"AudioClustering","simplifiedType":"clustering","languages":["English"],"domains":["Speech"],"modalities":["audio"],"description":"Emotion clustering task with audio data for 6 emotions: Anger, Disgust, Fear, Happy, Neutral, Sad.","reference":"https://ieeexplore.ieee.org/document/6849440","citation":"\n@article{Cao2014-ih,\n  author = {Cao, Houwei and Cooper, David G and Keutmann, Michael K and Gur,\nRuben C and Nenkova, Ani and Verma, Ragini},\n  copyright = {https://ieeexplore.ieee.org/Xplorehelp/downloads/license-information/IEEE.html},\n  journal = {IEEE Transactions on Affective Computing},\n  keywords = {Emotional corpora; facial expression; multi-modal recognition;\nvoice expression},\n  language = {en},\n  month = oct,\n  number = {4},\n  pages = {377--390},\n  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},\n  title = {{CREMA-D}: Crowd-sourced emotional multimodal actors dataset},\n  volume = {5},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/crema-d","license":"http://opendatacommons.org/licenses/odbl/1.0/","dateFrom":"2014-01-01","dateTo":"2014-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"v_measure","numModels":0},{"name":"VehicleSoundClustering","type":"AudioClustering","simplifiedType":"clustering","languages":["No linguistic content"],"domains":["Scene"],"modalities":["audio"],"description":"Clustering vehicle sounds recorded from smartphones (0 (car class), 1 (truck, bus and van class), 2 (motorcycle class))","reference":"https://huggingface.co/datasets/DynamicSuperb/Vehicle_sounds_classification_dataset","citation":"\n@inproceedings{bazilinskyy2018auditory,\n  author = {Bazilinskyy, Pavlo and van der Aa, Arne and Schoustra, Michael and Spruit, John and Staats, Laurens and van der Vlist, Klaas Jan and de Winter, Joost},\n  booktitle = {12th International Symposium on Tools and Methods of Competitive Engineering (TMCE 2018)},\n  pages = {417--422},\n  title = {An auditory dataset of passing vehicles recorded with a smartphone},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/Vehicle_sounds_classification_dataset","license":"not specified","dateFrom":"2024-06-20","dateTo":"2024-06-20","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"v_measure","numModels":0},{"name":"VoxPopuliGenderClustering","type":"AudioClustering","simplifiedType":"clustering","languages":["English","French","German","Polish","Spanish"],"domains":["Spoken","Speech"],"modalities":["audio"],"description":"Subsampled Dataset for clustering speech samples by speaker gender (male/female) from European Parliament recordings.","reference":"https://huggingface.co/datasets/facebook/voxpopuli","citation":"\n@inproceedings{wang-etal-2021-voxpopuli,\n  address = {Online},\n  author = {Wang, Changhan  and\nRiviere, Morgane  and\nLee, Ann  and\nWu, Anne  and\nTalnikar, Chaitanya  and\nHaziza, Daniel  and\nWilliamson, Mary  and\nPino, Juan  and\nDupoux, Emmanuel},\n  booktitle = {Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},\n  doi = {10.18653/v1/2021.acl-long.80},\n  month = aug,\n  pages = {993--1003},\n  publisher = {Association for Computational Linguistics},\n  title = {{V}ox{P}opuli: A Large-Scale Multilingual Speech Corpus for Representation Learning, Semi-Supervised Learning and Interpretation},\n  url = {https://aclanthology.org/2021.acl-long.80},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/mini-voxpopuli","license":"cc0-1.0","dateFrom":"2009-01-01","dateTo":"2020-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"v_measure","numModels":0},{"name":"CREMADPairClassification","type":"AudioPairClassification","simplifiedType":"pair-classification","languages":["English"],"domains":["Spoken"],"modalities":["audio"],"description":"Classifying pairs as having same or different emotions in actor's voice recordings of text spoken in 6 different emotions","reference":"https://pmc.ncbi.nlm.nih.gov/articles/PMC4313618/","citation":"\n@article{Cao2014-ih,\n  author = {Cao, Houwei and Cooper, David G and Keutmann, Michael K and Gur,\nRuben C and Nenkova, Ani and Verma, Ragini},\n  copyright = {https://ieeexplore.ieee.org/Xplorehelp/downloads/license-information/IEEE.html},\n  journal = {IEEE Transactions on Affective Computing},\n  keywords = {Emotional corpora; facial expression; multi-modal recognition;\nvoice expression},\n  language = {en},\n  month = oct,\n  number = {4},\n  pages = {377--390},\n  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},\n  title = {{CREMA-D}: Crowd-sourced emotional multimodal actors dataset},\n  volume = {5},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/CREMADPairClassification","license":"http://opendatacommons.org/licenses/odbl/1.0/","dateFrom":"2014-01-01","dateTo":"2014-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"max_ap","numModels":0},{"name":"NMSQAPairClassification","type":"AudioPairClassification","simplifiedType":"pair-classification","languages":["English"],"domains":["Spoken"],"modalities":["audio"],"description":"A textless Q&A dataset. Given a pair of audio question and audio answer, is the answer relevant to the question?","reference":"https://www.researchgate.net/publication/311458869_FMA_A_Dataset_For_Music_Analysis","citation":"\n@inproceedings{lin2022dualdiscretespokenunit,\n  author = {Lin, Guan-Ting and Chuang, Yung-Sung and Chung, Ho-Lam and Yang, Shu-wen and Chen, Hsuan-Jui and Dong, Shuyan Annie and Li, Shang-Wen and Mohamed, Abdelrahman and Lee, Hung-yi and Lee, Lin-shan},\n  booktitle = {Interspeech 2022},\n  collection = {interspeech_2022},\n  doi = {10.21437/interspeech.2022-612},\n  month = {Sept},\n  pages = {5165–5169},\n  publisher = {ISCA},\n  series = {interspeech_2022},\n  title = {DUAL: Discrete Spoken Unit Adaptive Learning for Textless Spoken Question Answering},\n  url = {https://doi.org/10.21437/interspeech.2022-612},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/NMSQAPairClassification","license":"cc-by-sa-4.0","dateFrom":"2016-01-01","dateTo":"2016-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"max_ap","numModels":0},{"name":"VoxPopuliAccentPairClassification","type":"AudioPairClassification","simplifiedType":"pair-classification","languages":["English"],"domains":["Spoken"],"modalities":["audio"],"description":"Classifying same or different regional accent of English (Empty Audio Samples filtered out)","reference":"https://aclanthology.org/2021.acl-long.80/","citation":"\n@inproceedings{wang-etal-2021-voxpopuli,\n  address = {Online},\n  author = {Wang, Changhan  and\nRiviere, Morgane  and\nLee, Ann  and\nWu, Anne  and\nTalnikar, Chaitanya  and\nHaziza, Daniel  and\nWilliamson, Mary  and\nPino, Juan  and\nDupoux, Emmanuel},\n  booktitle = {Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)},\n  doi = {10.18653/v1/2021.acl-long.80},\n  editor = {Zong, Chengqing  and\nXia, Fei  and\nLi, Wenjie  and\nNavigli, Roberto},\n  month = aug,\n  pages = {993--1003},\n  publisher = {Association for Computational Linguistics},\n  title = {{V}ox{P}opuli: A Large-Scale Multilingual Speech Corpus for Representation Learning, Semi-Supervised Learning and Interpretation},\n  url = {https://aclanthology.org/2021.acl-long.80/},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/VoxPopuliAccentPairClassificationFiltered","license":"cc0-1.0","dateFrom":"2021-01-01","dateTo":"2021-08-01","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"max_ap","numModels":0},{"name":"GTZANAudioReranking","type":"AudioReranking","simplifiedType":"retrieval","languages":["No linguistic content"],"domains":["Music"],"modalities":["audio"],"description":"GTZAN music genre dataset adapted for audio reranking. Given a query audio from one of 10 music genres, rank 3 relevant audio samples higher than 10 irrelevant ones from different genres. Contains 100 queries across 10 music genres for comprehensive evaluation.","reference":"https://www.kaggle.com/datasets/andradaolteanu/gtzan-dataset-music-genre-classification","citation":"\n@article{1021072,\n  author = {Tzanetakis, G. and Cook, P.},\n  doi = {10.1109/TSA.2002.800560},\n  journal = {IEEE Transactions on Speech and Audio Processing},\n  keywords = {Humans;Music information retrieval;Instruments;Computer science;Multiple signal classification;Signal analysis;Pattern recognition;Feature extraction;Wavelet analysis;Cultural differences},\n  number = {5},\n  pages = {293-302},\n  title = {Musical genre classification of audio signals},\n  volume = {10},\n  year = {2002},\n}\n","isPublic":true,"sourceDataset":"mteb/GTZANAudioReranking","license":"not specified","dateFrom":"2001-01-01","dateTo":"2001-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"map_at_1000","numModels":0},{"name":"RavdessZeroshot","type":"AudioZeroshotClassification","simplifiedType":"classification","languages":["English"],"domains":["Spoken"],"modalities":["audio","text"],"description":"Emotion classification Dataset. RAVDESS contains 24 professional actors (12 female, 12 male), vocalizing two lexically-matched statements in a neutral North American accent. Speech emotions includes neutral,calm, happy, sad, angry, fearful, surprise, and disgust expressions. These 8 emtoions also serve as labels for the dataset.","reference":"https://huggingface.co/datasets/narad/ravdess","citation":"\n@article{10.1371/journal.pone.0196391,\n  author = {Livingstone, Steven R. AND Russo, Frank A.},\n  doi = {10.1371/journal.pone.0196391},\n  journal = {PLOS ONE},\n  month = {05},\n  number = {5},\n  pages = {1-35},\n  publisher = {Public Library of Science},\n  title = {The Ryerson Audio-Visual Database of Emotional Speech and Song (RAVDESS): A dynamic, multimodal set of facial and vocal expressions in North American English},\n  url = {https://doi.org/10.1371/journal.pone.0196391},\n  volume = {13},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/RavdessZeroshot","license":"cc-by-nc-sa-3.0","dateFrom":"2018-03-01","dateTo":"2018-03-16","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"SpeechCommandsZeroshotv0.02","type":"AudioZeroshotClassification","simplifiedType":"classification","languages":["English"],"domains":["Spoken"],"modalities":["audio","text"],"description":"Sound Classification/Keyword Spotting Dataset. This is a set of one-second audio clips containing a single spoken English word or background noise. These words are from a small set of commands such as 'yes', 'no', and 'stop' spoken by various speakers. With a total of 10 labels/commands for keyword spotting and a total of 30 labels for other auxiliary tasks","reference":"https://huggingface.co/datasets/google/speech_commands","citation":"\n@article{DBLP:journals/corr/abs-1804-03209,\n  author = {Pete Warden},\n  bibsource = {dblp computer science bibliography, https://dblp.org},\n  biburl = {https://dblp.org/rec/journals/corr/abs-1804-03209.bib},\n  eprint = {1804.03209},\n  eprinttype = {arXiv},\n  journal = {CoRR},\n  timestamp = {Mon, 13 Aug 2018 16:48:32 +0200},\n  title = {Speech Commands: {A} Dataset for Limited-Vocabulary Speech Recognition},\n  url = {http://arxiv.org/abs/1804.03209},\n  volume = {abs/1804.03209},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/SpeechCommandsZeroshotv0.02","license":"cc-by-4.0","dateFrom":"2018-07-07","dateTo":"2018-07-13","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"FSD2019Kaggle","type":"AudioMultilabelClassification","simplifiedType":"classification","languages":["English"],"domains":["Web"],"modalities":["audio"],"description":"Multilabel Audio Classification.","reference":"https://huggingface.co/datasets/confit/fsdkaggle2019-parquet","citation":"\n@dataset{eduardo_fonseca_2020_3612637,\n  author = {Eduardo Fonseca and\nManoj Plakal and\nFrederic Font and\nDaniel P. W. Ellis and\nXavier Serra},\n  doi = {10.5281/zenodo.3612637},\n  month = jan,\n  publisher = {Zenodo},\n  title = {FSDKaggle2019},\n  url = {https://doi.org/10.5281/zenodo.3612637},\n  version = {1.0},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/fsdkaggle2019-parquet","license":"cc-by-4.0","dateFrom":"2020-01-01","dateTo":"2020-01-30","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"found","mainScore":"lrap","numModels":0}],"rows":[{"rank":1,"model":{"name":"BrunoHays/embeddinggemma-2-nullspace-audio","url":"https://huggingface.co/BrunoHays/embeddinggemma-2-nullspace-audio","zeroShotPct":100,"activeParamsB":0.442,"totalParamsB":0.577,"embeddingDim":768,"maxTokens":8192.0,"releaseDate":"2026-10-24","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["text","audio"],"languages":["Bengali","Chinese","Chinese (Traditional)","English","Finnish","French","German","Hindi","Indonesian","Iranian Persian","Japanese","Korean","Russian","Spanish","Standard Arabic","Swahili (individual language)","Telugu","Thai","Yoruba"],"citation":"\n@misc{vera2025embeddinggemmapowerfullightweighttext,\n      title={EmbeddingGemma: Powerful and Lightweight Text Representations},\n      author={Henrique Schechter Vera and Sahil Dua and Biao Zhang and Daniel Salz and Ryan Mullins and Sindhu Raghuram Panyam and Sara Smoot and Iftekhar Naim and Joe Zou and Feiyang Chen and Daniel Cer and Alice Lisak and Min Choi and Lucas Gonzalez and Omar Sanseviero and Glenn Cameron and Ian Ballantyne and Kat Black and Kaifeng Chen and Weiyi Wang and Zhe Li and Gus Martins and Jinhyuk Lee and Mark Sherwood and Juyeong Ji and Renjie Wu and Jingxiao Zheng and Jyotinder Singh and Abheesht Sharma and Divyashree Sreepathihalli and Aashi Jain and Adham Elarabawy and AJ Co and Andreas Doumanoglou and Babak Samari and Ben Hora and Brian Potetz and Dahun Kim and Enrique Alfonseca and Fedor Moiseev and Feng Han and Frank Palma Gomez and Gustavo Hernández Ábrego and Hesen Zhang and Hui Hui and Jay Han and Karan Gill and Ke Chen and Koert Chen and Madhuri Shanbhogue and Michael Boratko and Paul Suganthan and Sai Meher Karthik Duddu and Sandeep Mariserla and Setareh Ariafar and Shanfeng Zhang and Shijie Zhang and Simon Baumgartner and Sonam Goenka and Steve Qiu and Tanmaya Dabral and Trevor Walker and Vikram Rao and Waleed Khawaja and Wenlei Zhou and Xiaoqi Ren and Ye Xia and Yichang Chen and Yi-Ting Chen and Zhe Dong and Zhongli Ding and Francesco Visin and Gaël Liu and Jiageng Zhang and Kathleen Kenealy and Michelle Casbon and Ravin Kumar and Thomas Mesnard and Zach Gleicher and Cormac Brick and Olivier Lacombe and Adam Roberts and Qin Yin and Yunhsuan Sung and Raphael Hoffmann and Tris Warkentin and Armand Joulin and Tom Duerig and Mojtaba Seyedhosseini},\n      year={2025},\n      eprint={2509.20354},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2509.20354},\n}","memoryUsageMb":2200.0,"license":"gemma","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"google/embeddinggemma-2","supersededBy":null,"extraRequirementsGroups":["embeddinggemma"],"trainingDatasets":["FEVERHardNegatives","HotpotQAHardNegatives","MIRACLRetrievalHardNegatives","NQHardNegatives"]},"zeroShotPct":100,"activeParamsB":0.442,"totalParamsB":0.577,"embeddingDim":768,"maxTokens":8192,"meanTask":0.5439946571727836,"meanTaskType":0.5596143742103938,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5370283199643494,"AudioReranking":0.75823,"AudioClustering":0.27563766666666667,"AudioPairClassification":0.652648,"Any2AnyRetrieval":0.5777871328417407,"AudioMultilabelClassification":0.562582,"AudioZeroshotClassification":0.5533875},"scoresByTask":{"VoxCelebSA":0.354011,"GTZANGenre":0.715,"VehicleSoundClustering":0.146704,"VoxPopuliGenderClustering":0.630935,"MridinghamTonic":0.395296,"VoxPopuliAccentPairClassification":0.598207,"NMSQAPairClassification":0.786665,"IEMOCAPGender":0.857659,"VoxPopuliLanguageID":0.97,"MACST2ARetrieval":0.0687,"BirdCLEF":0.205,"SIBFLEURS":0.24992251960784315,"JamAltArtistA2ARetrieval":0.9646250000000001,"ClothoT2ARetrieval":0.10671,"CREMADPairClassification":0.573072,"FSD2019Kaggle":0.562582,"CommonVoiceMini21T2ARetrieval":0.9307521367521367,"SpeechCommandsZeroshotv0.02":0.968581,"CREMA_DClustering":0.049274,"MInDS14":0.62797,"UrbanSound8KT2ARetrieval":0.00491,"GTZANAudioReranking":0.75823,"BeijingOpera":0.936348,"CommonLanguageAgeDetection":0.1946,"FleursT2ARetrieval":0.9973270588235295,"CREMA_D":0.401505,"GigaSpeechT2ARetrieval":0.81956,"RavdessZeroshot":0.138194,"SpokenSQuADT2ARetrieval":0.69333,"JamAltLyricA2TRetrieval":0.61417},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.5481591326109391}},"trainedOnTasks":[],"experiments":null},{"rank":2,"model":{"name":"Haon-Chen/e5-omni-7B","url":"https://huggingface.co/Haon-Chen/e5-omni-7B","zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768.0,"releaseDate":"2026-01-06","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@inproceedings{chen2026e5omni,\n  address = {San Diego, California, United States},\n  author = {Chen, Haonan and Gao, Sicheng and Timofte, Radu and Sakai, Tetsuya and Dou, Zhicheng},\n  booktitle = {Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026},\n  doi = {10.18653/v1/2026.findings-acl.970},\n  editor = {Liakata, Maria and Moreira, Viviane P. and Zhang, Jiajun and Jurgens, David},\n  isbn = {979-8-89176-395-1},\n  month = {jul},\n  pages = {19430--19443},\n  publisher = {Association for Computational Linguistics},\n  title = {e5-omni: Explicit Cross-modal Alignment for Omni-modal Embeddings},\n  url = {https://aclanthology.org/2026.findings-acl.970/},\n  year = {2026},\n}\n","memoryUsageMb":17036.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen2.5-Omni-7B","supersededBy":null,"extraRequirementsGroups":["multimodal-sbert"],"trainingDatasets":["AudioCapsT2ARetrieval","MSRVTTT2V","MSRVTTV2T"]},"zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768,"meanTask":0.524540184263449,"meanTaskType":0.571222512991366,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5217233547237076,"AudioReranking":0.867,"AudioClustering":0.017841333333333334,"AudioPairClassification":0.655367,"Any2AnyRetrieval":0.5607434028825206,"AudioMultilabelClassification":0.687832,"AudioZeroshotClassification":0.6880505},"scoresByTask":{"VoxCelebSA":0.358658,"GTZANGenre":0.851,"VehicleSoundClustering":0.024321,"VoxPopuliGenderClustering":0.019478,"MridinghamTonic":0.308166,"VoxPopuliAccentPairClassification":0.521257,"NMSQAPairClassification":0.890584,"IEMOCAPGender":0.812031,"VoxPopuliLanguageID":0.796,"MACST2ARetrieval":0.29262,"BirdCLEF":0.248,"SIBFLEURS":0.3839882352941177,"JamAltArtistA2ARetrieval":0.9204225,"ClothoT2ARetrieval":0.42525,"CREMADPairClassification":0.55426,"FSD2019Kaggle":0.687832,"CommonVoiceMini21T2ARetrieval":0.6629464102564102,"SpeechCommandsZeroshotv0.02":0.970545,"CREMA_DClustering":0.009725,"MInDS14":0.7052596666666666,"UrbanSound8KT2ARetrieval":0.00943,"GTZANAudioReranking":0.867,"BeijingOpera":0.818085,"CommonLanguageAgeDetection":0.1518,"FleursT2ARetrieval":0.5997592156862745,"CREMA_D":0.305969,"GigaSpeechT2ARetrieval":0.828,"RavdessZeroshot":0.405556,"SpokenSQuADT2ARetrieval":0.76667,"JamAltLyricA2TRetrieval":0.5415925},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.5024212843137255}},"trainedOnTasks":[],"experiments":null},{"rank":3,"model":{"name":"LCO-Embedding/LCO-Embedding-Omni-7B","url":"https://huggingface.co/LCO-Embedding/LCO-Embedding-Omni-7B","zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768.0,"releaseDate":"2025-10-15","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio","image","text","video"],"languages":["English"],"citation":"\n@inproceedings{xiao2025scalinglanguagecentricomnimodalrepresentation,\n  archiveprefix = {arXiv},\n  author = {Chenghao Xiao and Hou Pong Chan and Hao Zhang and Weiwen Xu and Mahani Aljunied and Yu Rong},\n  booktitle = {The Thirty-ninth Annual Conference on Neural Information Processing Systems},\n  eprint = {2510.11693},\n  title = {Scaling Language-centric Omnimodal Representation Learning},\n  url = {https://openreview.net/forum?id=eA3Aum0rpA},\n  year = {2025},\n}","memoryUsageMb":17043.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768,"meanTask":0.5353717961580359,"meanTaskType":0.5705994656888137,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5623962731729055,"AudioReranking":0.78711,"AudioClustering":0.016615333333333333,"AudioPairClassification":0.6729626666666667,"Any2AnyRetrieval":0.5522989866487906,"AudioMultilabelClassification":0.757366,"AudioZeroshotClassification":0.645447},"scoresByTask":{"VoxCelebSA":0.434042,"GTZANGenre":0.823,"VehicleSoundClustering":0.036289,"VoxPopuliGenderClustering":0.007334,"MridinghamTonic":0.420671,"VoxPopuliAccentPairClassification":0.507262,"NMSQAPairClassification":0.976016,"IEMOCAPGender":0.707539,"VoxPopuliLanguageID":0.642,"MACST2ARetrieval":0.29771,"BirdCLEF":0.341,"SIBFLEURS":0.47388975490196084,"JamAltArtistA2ARetrieval":0.7501450000000001,"ClothoT2ARetrieval":0.33894,"CREMADPairClassification":0.53561,"FSD2019Kaggle":0.757366,"CommonVoiceMini21T2ARetrieval":0.6000306837606838,"SpeechCommandsZeroshotv0.02":0.974227,"CREMA_DClustering":0.006223,"MInDS14":0.89421925,"UrbanSound8KT2ARetrieval":0.00923,"GTZANAudioReranking":0.78711,"BeijingOpera":0.927926,"CommonLanguageAgeDetection":0.16155,"FleursT2ARetrieval":0.6859551960784313,"CREMA_D":0.360522,"GigaSpeechT2ARetrieval":0.83289,"RavdessZeroshot":0.316667,"SpokenSQuADT2ARetrieval":0.74,"JamAltLyricA2TRetrieval":0.7157899999999999},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.5153867371001032}},"trainedOnTasks":[],"experiments":null},{"rank":4,"model":{"name":"jinaai/jina-embeddings-v5-omni-small","url":"https://huggingface.co/jinaai/jina-embeddings-v5-omni-small","zeroShotPct":-1,"activeParamsB":1.471,"totalParamsB":1.626,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2026-04-01","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["Afrikaans","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Latvian","Lithuanian","Macedonian","Malay (macrolanguage)","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Panjabi","Persian","Polish","Portuguese","Quechua (Latin)","Romanian","Russian","Sinhala","Slovak","Slovenian","Spanish","Swahili (macrolanguage)","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Yoruba"],"citation":"@misc{akram2026jinaembeddingsv5texttasktargetedembeddingdistillation,\n      title={jina-embeddings-v5-text: Task-Targeted Embedding Distillation},\n      author={Mohammad Kalim Akram and Saba Sturua and Nastia Havriushenko and Quentin Herreros and Michael Günther and Maximilian Werk and Han Xiao},\n      year={2026},\n      eprint={2602.15547},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2602.15547},\n}","memoryUsageMb":3102.0,"license":"cc-by-nc-4.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"jinaai/jina-embeddings-v5-text-small","supersededBy":null,"extraRequirementsGroups":["peft"],"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":1.471,"totalParamsB":1.626,"embeddingDim":1024,"maxTokens":32768,"meanTask":0.5040760285118342,"meanTaskType":0.555794564380382,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5199451292335117,"AudioReranking":0.84377,"AudioClustering":0.06132333333333332,"AudioPairClassification":0.653770993179344,"Any2AnyRetrieval":0.4975584949164851,"AudioMultilabelClassification":0.692583,"AudioZeroshotClassification":0.621611},"scoresByTask":{"VoxCelebSA":0.429088,"GTZANGenre":0.856,"VehicleSoundClustering":0.048423,"VoxPopuliGenderClustering":0.001513,"MridinghamTonic":0.322786,"VoxPopuliAccentPairClassification":0.49974745277133903,"NMSQAPairClassification":0.9338365214450436,"IEMOCAPGender":0.715009,"VoxPopuliLanguageID":0.956,"MACST2ARetrieval":0.14504,"BirdCLEF":0.252,"SIBFLEURS":0.3035069215686274,"JamAltArtistA2ARetrieval":0.8389225,"ClothoT2ARetrieval":0.2385,"CREMADPairClassification":0.5277290053216492,"FSD2019Kaggle":0.692583,"CommonVoiceMini21T2ARetrieval":0.8312744444444444,"SpeechCommandsZeroshotv0.02":0.974472,"CREMA_DClustering":0.134034,"MInDS14":0.6670225,"UrbanSound8KT2ARetrieval":0.00825,"GTZANAudioReranking":0.84377,"BeijingOpera":0.839184,"CommonLanguageAgeDetection":0.1763,"FleursT2ARetrieval":0.6892395098039216,"CREMA_D":0.2025,"GigaSpeechT2ARetrieval":0.82148,"RavdessZeroshot":0.26875,"SpokenSQuADT2ARetrieval":0.52667,"JamAltLyricA2TRetrieval":0.37865000000000004},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.5024932579529819}},"trainedOnTasks":[],"experiments":null},{"rank":5,"model":{"name":"Tevatron/OmniEmbed-v0.1","url":"https://huggingface.co/Tevatron/OmniEmbed-v0.1","zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768.0,"releaseDate":"2025-04-12","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@inproceedings{zhuang2025tevatron,\n  author = {Ma, Xueguang and Gao, Luyu and Zhuang, Shengyao and Zhan, Jiaqi Samantha and Callan, Jamie and Lin, Jimmy},\n  booktitle = {Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval},\n  collection = {SIGIR ’25},\n  doi = {10.1145/3726302.3730135},\n  month = {July},\n  pages = {4061–4065},\n  publisher = {ACM},\n  series = {SIGIR ’25},\n  title = {Tevatron 2.0: Unified Document Retrieval Toolkit across Scale, Language, and Modality},\n  url = {https://doi.org/10.1145/3726302.3730135},\n  year = {2025},\n}\n","memoryUsageMb":17036.0,"license":"mit","publicTrainingCode":"https://github.com/texttron/tevatron/tree/qwenomni","publicTrainingData":"https://huggingface.co/Tevatron/OmniEmbed-v0.1#training-data","adaptedFrom":"Qwen/Qwen2.5-Omni-7B","supersededBy":null,"extraRequirementsGroups":["multimodal-sbert","peft"],"trainingDatasets":["AudioCapsT2ARetrieval","FEVER","MSMARCO","MSRVTTT2V","MSRVTTV2T","NQ","SCIDOCS","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768,"meanTask":0.43865448054717615,"meanTaskType":0.4790111725266383,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.49064702361853835,"AudioReranking":0.83135,"AudioClustering":0.032208999999999995,"AudioPairClassification":0.5890313333333334,"Any2AnyRetrieval":0.4194993507345959,"AudioMultilabelClassification":0.688731,"AudioZeroshotClassification":0.3016105},"scoresByTask":{"VoxCelebSA":0.369391,"GTZANGenre":0.786,"VehicleSoundClustering":0.014678,"VoxPopuliGenderClustering":0.018498,"MridinghamTonic":0.283225,"VoxPopuliAccentPairClassification":0.515209,"NMSQAPairClassification":0.650421,"IEMOCAPGender":0.790311,"VoxPopuliLanguageID":0.644,"MACST2ARetrieval":0.25445,"BirdCLEF":0.242,"SIBFLEURS":0.2762613431372549,"JamAltArtistA2ARetrieval":0.89917,"ClothoT2ARetrieval":0.42256,"CREMADPairClassification":0.601464,"FSD2019Kaggle":0.688731,"CommonVoiceMini21T2ARetrieval":0.5870560683760684,"SpeechCommandsZeroshotv0.02":0.328915,"CREMA_DClustering":0.063451,"MInDS14":0.5331809166666667,"UrbanSound8KT2ARetrieval":0.00864,"GTZANAudioReranking":0.83135,"BeijingOpera":0.835018,"CommonLanguageAgeDetection":0.147,"FleursT2ARetrieval":0.29834058823529414,"CREMA_D":0.49073,"GigaSpeechT2ARetrieval":0.62059,"RavdessZeroshot":0.274306,"SpokenSQuADT2ARetrieval":0.62,"JamAltLyricA2TRetrieval":0.06468750000000001},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4732293820949432}},"trainedOnTasks":[],"experiments":null},{"rank":6,"model":{"name":"BidirLM/BidirLM-Omni-2.5B-Embedding","url":"https://huggingface.co/BidirLM/BidirLM-Omni-2.5B-Embedding","zeroShotPct":100,"activeParamsB":2.13,"totalParamsB":2.445,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2026-04-07","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bosnian","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hausa","Hebrew","Hindi","Hungarian","Icelandic","Igbo","Indonesian","Irish","Italian","Japanese","Javanese","Kannada","Kazakh","Kirghiz","Korean","Latvian","Lithuanian","Macedonian","Malagasy (Latin)","Malay (macrolanguage)","Malayalam","Maltese","Marathi","Modern Greek (1453-)","Nepali (macrolanguage)","Norwegian Bokmål","Nyanja","Panjabi","Pedi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Serbian","Shona","Sindhi","Sinhala","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Wolof","Xhosa","Yoruba","Zulu"],"citation":"@misc{boizard2026bidirlmtextomnimodalbidirectional,\n      title={BidirLM: From Text to Omnimodal Bidirectional Encoders by Adapting and Composing Causal LLMs},\n      author={Nicolas Boizard and Théo Deschamps-Berger and Hippolyte Gisserot-Boukhlef and Céline Hudelot and Pierre Colombo},\n      year={2026},\n      eprint={2604.02045},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2604.02045},\n}","memoryUsageMb":4663.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/datasets/BidirLM/BidirLM-Omni-Contrastive","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["bidirlm-omni"],"trainingDatasets":["AFQMC","ATEC","AdvertiseGen","BQ","CAIL2019-SCM","CHEF","CINLID","ChatMed_Consult_Dataset","ChineseSTS","CmedqaRetrieval","Cmnli","CodeFeedback","DRCD","DuRetrieval","ELI5_custom","ESCIReranking","EmotionClassification","Expertqa","FollowIR","GooAQ","InF-IR","JW300","LAION-Audio-300M","LCSTS","MAmmoTH2","MEDI2BGE","MIRACLRetrieval","MSMARCO","MS_COCO","Multi-CPR","MultiLongDocReranking","NFCorpus","NaturalQuestions","OPUS-100","OpenOrca","PAQ","PubMedQA","QBQTC","RefGPT","SQuAD","SearchQA","SimCLUE","SyntheticClassificationData","T2Ranking","TED Talks","THUCNews","TriviaQA","UMETRIP-QA","WebCPM","WikiAnswers","WikiMatrix","cCOVID-News","ccnews","cmrc2018","colpali_train_set","contract-nli","csl","dureader_mrc","law-gpt","lawzhidao","librispeech_asr","lima-chinese","mmarco-chinese","mnli","natcap","nli_zh","nllb","ocnli","rag-dataset-12000","simcse_sup_nli","webgpt_comparisons","webqa","wikipedia-nq","xnli_zh","yahoo-answers"]},"zeroShotPct":100,"activeParamsB":2.13,"totalParamsB":2.445,"embeddingDim":2048,"maxTokens":32768,"meanTask":0.5235733849254232,"meanTaskType":0.5452033570097296,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5976667780748663,"AudioReranking":0.74758,"AudioClustering":0.061831666666666674,"AudioPairClassification":0.6674596666666668,"Any2AnyRetrieval":0.47473088765990723,"AudioMultilabelClassification":0.609474,"AudioZeroshotClassification":0.6576805},"scoresByTask":{"VoxCelebSA":0.494942,"GTZANGenre":0.741,"VehicleSoundClustering":0.051074,"VoxPopuliGenderClustering":0.128471,"MridinghamTonic":0.45349,"VoxPopuliAccentPairClassification":0.510499,"NMSQAPairClassification":0.959874,"IEMOCAPGender":0.853766,"VoxPopuliLanguageID":0.962,"MACST2ARetrieval":0.08397,"BirdCLEF":0.331,"SIBFLEURS":0.47532355882352945,"JamAltArtistA2ARetrieval":0.669175,"ClothoT2ARetrieval":0.1205,"CREMADPairClassification":0.532006,"FSD2019Kaggle":0.609474,"CommonVoiceMini21T2ARetrieval":0.6360670085470085,"SpeechCommandsZeroshotv0.02":0.973,"CREMA_DClustering":0.00595,"MInDS14":0.910736,"UrbanSound8KT2ARetrieval":0.00648,"GTZANAudioReranking":0.74758,"BeijingOpera":0.902482,"CommonLanguageAgeDetection":0.15035,"FleursT2ARetrieval":0.7350609803921568,"CREMA_D":0.299245,"GigaSpeechT2ARetrieval":0.83052,"RavdessZeroshot":0.342361,"SpokenSQuADT2ARetrieval":0.74333,"JamAltLyricA2TRetrieval":0.447475},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.5357349241486068}},"trainedOnTasks":[],"experiments":null},{"rank":7,"model":{"name":"LCO-Embedding/LCO-Embedding-Omni-3B","url":"https://huggingface.co/LCO-Embedding/LCO-Embedding-Omni-3B","zeroShotPct":100,"activeParamsB":4.392,"totalParamsB":4.703,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2025-10-23","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio","image","text","video"],"languages":["English"],"citation":"\n@inproceedings{xiao2025scalinglanguagecentricomnimodalrepresentation,\n  archiveprefix = {arXiv},\n  author = {Chenghao Xiao and Hou Pong Chan and Hao Zhang and Weiwen Xu and Mahani Aljunied and Yu Rong},\n  booktitle = {The Thirty-ninth Annual Conference on Neural Information Processing Systems},\n  eprint = {2510.11693},\n  title = {Scaling Language-centric Omnimodal Representation Learning},\n  url = {https://openreview.net/forum?id=eA3Aum0rpA},\n  year = {2025},\n}","memoryUsageMb":8978.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":4.392,"totalParamsB":4.703,"embeddingDim":2048,"maxTokens":32768,"meanTask":0.5225122528531926,"meanTaskType":0.5550467970607824,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5445804353832443,"AudioReranking":0.75365,"AudioClustering":0.01275,"AudioPairClassification":0.6666250000000001,"Any2AnyRetrieval":0.5454716440422324,"AudioMultilabelClassification":0.740538,"AudioZeroshotClassification":0.6217125},"scoresByTask":{"VoxCelebSA":0.489697,"GTZANGenre":0.81,"VehicleSoundClustering":0.021795,"VoxPopuliGenderClustering":0.01022,"MridinghamTonic":0.395444,"VoxPopuliAccentPairClassification":0.506657,"NMSQAPairClassification":0.969203,"IEMOCAPGender":0.59418,"VoxPopuliLanguageID":0.634,"MACST2ARetrieval":0.22646,"BirdCLEF":0.316,"SIBFLEURS":0.45062387254901964,"JamAltArtistA2ARetrieval":0.6712225,"ClothoT2ARetrieval":0.32695,"CREMADPairClassification":0.524015,"FSD2019Kaggle":0.740538,"CommonVoiceMini21T2ARetrieval":0.6515320512820513,"SpeechCommandsZeroshotv0.02":0.973981,"CREMA_DClustering":0.006235,"MInDS14":0.8892889166666667,"UrbanSound8KT2ARetrieval":0.00884,"GTZANAudioReranking":0.75365,"BeijingOpera":0.932181,"CommonLanguageAgeDetection":0.1687,"FleursT2ARetrieval":0.7134027450980391,"CREMA_D":0.31027,"GigaSpeechT2ARetrieval":0.83126,"RavdessZeroshot":0.269444,"SpokenSQuADT2ARetrieval":0.72,"JamAltLyricA2TRetrieval":0.7595775},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4975464362745098}},"trainedOnTasks":[],"experiments":null},{"rank":8,"model":{"name":"Qwen/Qwen2-Audio-7B","url":"https://huggingface.co/Qwen/Qwen2-Audio-7B","zeroShotPct":-1,"activeParamsB":null,"totalParamsB":7.0,"embeddingDim":4096,"maxTokens":131072.0,"releaseDate":"2024-08-09","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@misc{chu2024qwen2audiotechnicalreport,\n      title={Qwen2-Audio Technical Report},\n      author={Yunfei Chu and Jin Xu and Qian Yang and Haojie Wei and Xipin Wei and Zhifang Guo and Yichong Leng and Yuanjun Lv and Jinzheng He and Junyang Lin and Chang Zhou and Jingren Zhou},\n      year={2024},\n      eprint={2407.10759},\n      archivePrefix={arXiv},\n      primaryClass={eess.AS},\n      url={https://arxiv.org/abs/2407.10759},\n}\n","memoryUsageMb":null,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":null,"totalParamsB":7.0,"embeddingDim":4096,"maxTokens":131072,"meanTask":0.3454015185478465,"meanTaskType":0.3700506069139648,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5834079104278075,"AudioReranking":0.808484,"AudioClustering":0.12652766666666668,"AudioPairClassification":0.5688656666666666,"Any2AnyRetrieval":0.05287950463661248,"AudioMultilabelClassification":0.3264,"AudioZeroshotClassification":0.1237895},"scoresByTask":{"VoxCelebSA":0.295448,"GTZANGenre":0.931,"VehicleSoundClustering":0.05518,"VoxPopuliGenderClustering":0.000726,"MridinghamTonic":0.61172,"VoxPopuliAccentPairClassification":0.52933,"NMSQAPairClassification":0.488547,"IEMOCAPGender":0.929574,"VoxPopuliLanguageID":0.99,"MACST2ARetrieval":0.01527,"BirdCLEF":0.371,"SIBFLEURS":0.15437109803921567,"JamAltArtistA2ARetrieval":0.37577,"ClothoT2ARetrieval":0.00448,"CREMADPairClassification":0.68872,"FSD2019Kaggle":0.3264,"CommonVoiceMini21T2ARetrieval":0.042005982905982904,"SpeechCommandsZeroshotv0.02":0.103829,"CREMA_DClustering":0.323677,"MInDS14":0.2441489166666667,"UrbanSound8KT2ARetrieval":0.00216,"GTZANAudioReranking":0.808484,"BeijingOpera":0.974468,"CommonLanguageAgeDetection":0.1759,"FleursT2ARetrieval":0.015092058823529412,"CREMA_D":0.739857,"GigaSpeechT2ARetrieval":0.00089,"RavdessZeroshot":0.14375,"SpokenSQuADT2ARetrieval":0.01,"JamAltLyricA2TRetrieval":0.0102475},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.5098905797213622}},"trainedOnTasks":[],"experiments":null},{"rank":9,"model":{"name":"jinaai/jina-embeddings-v5-omni-nano","url":"https://huggingface.co/jinaai/jina-embeddings-v5-omni-nano","zeroShotPct":-1,"activeParamsB":0.887,"totalParamsB":0.986,"embeddingDim":768,"maxTokens":8192.0,"releaseDate":"2026-04-01","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["Afrikaans","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Latvian","Lithuanian","Macedonian","Malay (macrolanguage)","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Panjabi","Persian","Polish","Portuguese","Quechua (Latin)","Romanian","Russian","Sinhala","Slovak","Slovenian","Spanish","Swahili (macrolanguage)","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Yoruba"],"citation":"@misc{akram2026jinaembeddingsv5texttasktargetedembeddingdistillation,\n      title={jina-embeddings-v5-text: Task-Targeted Embedding Distillation},\n      author={Mohammad Kalim Akram and Saba Sturua and Nastia Havriushenko and Quentin Herreros and Michael Günther and Maximilian Werk and Han Xiao},\n      year={2026},\n      eprint={2602.15547},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2602.15547},\n}","memoryUsageMb":1881.0,"license":"cc-by-nc-4.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"jinaai/jina-embeddings-v5-text-nano","supersededBy":null,"extraRequirementsGroups":["peft"],"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":0.887,"totalParamsB":0.986,"embeddingDim":768,"maxTokens":8192,"meanTask":0.5013828527882487,"meanTaskType":0.5531055582981785,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5134403471479501,"AudioReranking":0.83193,"AudioClustering":0.059630666666666665,"AudioPairClassification":0.6555646305722808,"Any2AnyRetrieval":0.500294763700352,"AudioMultilabelClassification":0.708284,"AudioZeroshotClassification":0.6025945},"scoresByTask":{"VoxCelebSA":0.424169,"GTZANGenre":0.849,"VehicleSoundClustering":0.016566,"VoxPopuliGenderClustering":0.003876,"MridinghamTonic":0.309885,"VoxPopuliAccentPairClassification":0.4976771363146085,"NMSQAPairClassification":0.9392026751895308,"IEMOCAPGender":0.684329,"VoxPopuliLanguageID":0.924,"MACST2ARetrieval":0.1374,"BirdCLEF":0.257,"SIBFLEURS":0.303069068627451,"JamAltArtistA2ARetrieval":0.8252649999999999,"ClothoT2ARetrieval":0.26052,"CREMADPairClassification":0.5298140802127032,"FSD2019Kaggle":0.708284,"CommonVoiceMini21T2ARetrieval":0.8202284615384615,"SpeechCommandsZeroshotv0.02":0.973245,"CREMA_DClustering":0.15845,"MInDS14":0.69139075,"UrbanSound8KT2ARetrieval":0.00825,"GTZANAudioReranking":0.83193,"BeijingOpera":0.830851,"CommonLanguageAgeDetection":0.16305,"FleursT2ARetrieval":0.630554411764706,"CREMA_D":0.2111,"GigaSpeechT2ARetrieval":0.81556,"RavdessZeroshot":0.231944,"SpokenSQuADT2ARetrieval":0.46,"JamAltLyricA2TRetrieval":0.544875},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4974013005444365}},"trainedOnTasks":[],"experiments":null},{"rank":10,"model":{"name":"laion/larger_clap_general","url":"https://huggingface.co/laion/larger_clap_general","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.194,"embeddingDim":512,"maxTokens":null,"releaseDate":"2023-05-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{wu2024largescalecontrastivelanguageaudiopretraining,\n  author = {Wu, Yusong and Chen, Ke and Zhang, Tianyu and Hui, Yuchen and Berg-Kirkpatrick, Taylor and Dubnov, Shlomo},\n  booktitle = {ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp49357.2023.10095969},\n  month = {June},\n  pages = {1–5},\n  publisher = {IEEE},\n  title = {Large-Scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation},\n  url = {https://doi.org/10.1109/icassp49357.2023.10095969},\n  year = {2023},\n}\n","memoryUsageMb":740.0,"license":"mit","publicTrainingCode":"https://github.com/LAION-AI/CLAP","publicTrainingData":"https://laion.ai/blog/laion-audio-630k/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.194,"embeddingDim":512,"maxTokens":null,"meanTask":0.3498426850343556,"meanTaskType":0.41083900528048567,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4787523591800356,"AudioReranking":0.667839,"AudioClustering":0.06624633333333334,"AudioPairClassification":0.5189336666666667,"Any2AnyRetrieval":0.18910517778336405,"AudioMultilabelClassification":0.806314,"AudioZeroshotClassification":0.1486825},"scoresByTask":{"VoxCelebSA":0.304737,"GTZANGenre":0.845,"VehicleSoundClustering":0.033663,"VoxPopuliGenderClustering":0.033347,"MridinghamTonic":0.494479,"VoxPopuliAccentPairClassification":0.535466,"NMSQAPairClassification":0.474208,"IEMOCAPGender":0.89282,"VoxPopuliLanguageID":0.846,"MACST2ARetrieval":0.33079,"BirdCLEF":0.17,"SIBFLEURS":0.09385411764705882,"JamAltArtistA2ARetrieval":0.956425,"ClothoT2ARetrieval":0.33715,"CREMADPairClassification":0.547127,"FSD2019Kaggle":0.806314,"CommonVoiceMini21T2ARetrieval":0.02516811965811966,"SpeechCommandsZeroshotv0.02":0.124448,"CREMA_DClustering":0.131729,"MInDS14":0.07983583333333333,"UrbanSound8KT2ARetrieval":0.00963,"GTZANAudioReranking":0.667839,"BeijingOpera":0.93617,"CommonLanguageAgeDetection":0.2051,"FleursT2ARetrieval":0.012210980392156862,"CREMA_D":0.39828,"GigaSpeechT2ARetrieval":0.00163,"RavdessZeroshot":0.172917,"SpokenSQuADT2ARetrieval":0.02,"JamAltLyricA2TRetrieval":0.008942499999999999},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.45505683952528375}},"trainedOnTasks":[],"experiments":null},{"rank":11,"model":{"name":"Haon-Chen/e5-omni-3B","url":"https://huggingface.co/Haon-Chen/e5-omni-3B","zeroShotPct":100,"activeParamsB":4.392,"totalParamsB":4.703,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2026-01-06","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@inproceedings{chen2026e5omni,\n  address = {San Diego, California, United States},\n  author = {Chen, Haonan and Gao, Sicheng and Timofte, Radu and Sakai, Tetsuya and Dou, Zhicheng},\n  booktitle = {Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026},\n  doi = {10.18653/v1/2026.findings-acl.970},\n  editor = {Liakata, Maria and Moreira, Viviane P. and Zhang, Jiajun and Jurgens, David},\n  isbn = {979-8-89176-395-1},\n  month = {jul},\n  pages = {19430--19443},\n  publisher = {Association for Computational Linguistics},\n  title = {e5-omni: Explicit Cross-modal Alignment for Omni-modal Embeddings},\n  url = {https://aclanthology.org/2026.findings-acl.970/},\n  year = {2026},\n}\n","memoryUsageMb":8971.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen2.5-Omni-3B","supersededBy":null,"extraRequirementsGroups":["multimodal-sbert"],"trainingDatasets":["AudioCapsT2ARetrieval","MSRVTTT2V","MSRVTTV2T"]},"zeroShotPct":100,"activeParamsB":4.392,"totalParamsB":4.703,"embeddingDim":2048,"maxTokens":32768,"meanTask":0.47996317443857883,"meanTaskType":0.5159785816371355,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.48191904500891264,"AudioReranking":0.84475,"AudioClustering":0.018595666666666667,"AudioPairClassification":0.5947706666666667,"Any2AnyRetrieval":0.5121031931177029,"AudioMultilabelClassification":0.515415,"AudioZeroshotClassification":0.6442965},"scoresByTask":{"VoxCelebSA":0.380974,"GTZANGenre":0.787,"VehicleSoundClustering":0.037803,"VoxPopuliGenderClustering":0.010142,"MridinghamTonic":0.299417,"VoxPopuliAccentPairClassification":0.510206,"NMSQAPairClassification":0.728942,"IEMOCAPGender":0.796888,"VoxPopuliLanguageID":0.594,"MACST2ARetrieval":0.2799,"BirdCLEF":0.241,"SIBFLEURS":0.3291909117647059,"JamAltArtistA2ARetrieval":0.869335,"ClothoT2ARetrieval":0.39875,"CREMADPairClassification":0.545164,"FSD2019Kaggle":0.515415,"CommonVoiceMini21T2ARetrieval":0.6805399145299146,"SpeechCommandsZeroshotv0.02":0.937899,"CREMA_DClustering":0.007842,"MInDS14":0.6403985833333333,"UrbanSound8KT2ARetrieval":0.00864,"GTZANAudioReranking":0.84475,"BeijingOpera":0.775532,"CommonLanguageAgeDetection":0.167,"FleursT2ARetrieval":0.5440838235294118,"CREMA_D":0.289709,"GigaSpeechT2ARetrieval":0.79689,"RavdessZeroshot":0.350694,"SpokenSQuADT2ARetrieval":0.72667,"JamAltLyricA2TRetrieval":0.30412},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.46606807868937045}},"trainedOnTasks":[],"experiments":null},{"rank":12,"model":{"name":"laion/larger_clap_music_and_speech","url":"https://huggingface.co/laion/larger_clap_music_and_speech","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.194,"embeddingDim":512,"maxTokens":null,"releaseDate":"2023-05-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{wu2024largescalecontrastivelanguageaudiopretraining,\n  author = {Wu, Yusong and Chen, Ke and Zhang, Tianyu and Hui, Yuchen and Berg-Kirkpatrick, Taylor and Dubnov, Shlomo},\n  booktitle = {ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp49357.2023.10095969},\n  month = {June},\n  pages = {1–5},\n  publisher = {IEEE},\n  title = {Large-Scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation},\n  url = {https://doi.org/10.1109/icassp49357.2023.10095969},\n  year = {2023},\n}\n","memoryUsageMb":740.0,"license":"mit","publicTrainingCode":"https://github.com/LAION-AI/CLAP","publicTrainingData":"https://laion.ai/blog/laion-audio-630k/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.194,"embeddingDim":512,"maxTokens":null,"meanTask":0.34697450011312214,"meanTaskType":0.40744001639373695,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4740244852941176,"AudioReranking":0.656464,"AudioClustering":0.07653233333333333,"AudioPairClassification":0.520954,"Any2AnyRetrieval":0.18628929612870787,"AudioMultilabelClassification":0.806193,"AudioZeroshotClassification":0.131623},"scoresByTask":{"VoxCelebSA":0.337788,"GTZANGenre":0.835,"VehicleSoundClustering":0.047243,"VoxPopuliGenderClustering":0.038033,"MridinghamTonic":0.489895,"VoxPopuliAccentPairClassification":0.53487,"NMSQAPairClassification":0.461364,"IEMOCAPGender":0.931169,"VoxPopuliLanguageID":0.814,"MACST2ARetrieval":0.30534,"BirdCLEF":0.164,"SIBFLEURS":0.08903358823529413,"JamAltArtistA2ARetrieval":0.9688625,"ClothoT2ARetrieval":0.3239,"CREMADPairClassification":0.566628,"FSD2019Kaggle":0.806193,"CommonVoiceMini21T2ARetrieval":0.024643076923076924,"SpeechCommandsZeroshotv0.02":0.090329,"CREMA_DClustering":0.144321,"MInDS14":0.07046175,"UrbanSound8KT2ARetrieval":0.00943,"GTZANAudioReranking":0.656464,"BeijingOpera":0.915248,"CommonLanguageAgeDetection":0.1659,"FleursT2ARetrieval":0.01262058823529412,"CREMA_D":0.401774,"GigaSpeechT2ARetrieval":0.00207,"RavdessZeroshot":0.172917,"SpokenSQuADT2ARetrieval":0.02,"JamAltLyricA2TRetrieval":0.0097375},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.45431867569659445}},"trainedOnTasks":[],"experiments":null},{"rank":13,"model":{"name":"EximiusLabs/fusion-embedding-2-2b-preview","url":"https://huggingface.co/EximiusLabs/fusion-embedding-2-2b-preview","zeroShotPct":90,"activeParamsB":2.489,"totalParamsB":2.8,"embeddingDim":1024,"maxTokens":512.0,"releaseDate":"2026-07-12","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio","image","text","video"],"languages":["English"],"citation":"\n@article{tonmoy2026fusion,\n  title   = {Fusion Embedding: A Unified Embedding Space for Text, Image, Video, and Audio},\n  author  = {Tonmoy, Abdul Basit and Hoque, Kazi Fardinul and Arham, Md. Shahrier Islam and Luthra, Arman},\n  journal = {arXiv preprint arXiv:2607.18666},\n  year    = {2026}\n}\n","memoryUsageMb":10681.0,"license":"cc-by-nc-4.0","publicTrainingCode":"https://github.com/Eximius-Labs/fusion-embedding","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["fusion-embedding"],"trainingDatasets":["AudioCapsA2TRetrieval","AudioCapsAVA2VRetrieval","AudioCapsAVAT2VRetrieval","AudioCapsAVT2VARetrieval","AudioCapsAVT2VRetrieval","AudioCapsAVV2ARetrieval","AudioCapsAVV2TRetrieval","AudioCapsAVVA2TRetrieval","AudioCapsAVVT2ARetrieval","AudioCapsT2ARetrieval","AudioSet","AudioSetMini","CommonLanguageAgeDetection","CommonVoiceMini21T2ARetrieval","FSD2019Kaggle","FSD50K"]},"zeroShotPct":90,"activeParamsB":2.489,"totalParamsB":2.8,"embeddingDim":1024,"maxTokens":512,"meanTask":0.4021769813683593,"meanTaskType":0.4905208556554194,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.45745301737967914,"AudioReranking":0.7166,"AudioClustering":0.036719666666666664,"AudioPairClassification":0.615257,"Any2AnyRetrieval":0.2750958055415899,"AudioMultilabelClassification":0.780107,"AudioZeroshotClassification":0.5524135},"scoresByTask":{"VoxCelebSA":0.272827,"GTZANGenre":0.644,"VehicleSoundClustering":0.024577,"VoxPopuliGenderClustering":0.079354,"MridinghamTonic":0.53992,"VoxPopuliAccentPairClassification":0.50894,"NMSQAPairClassification":0.793594,"IEMOCAPGender":0.93774,"VoxPopuliLanguageID":0.616,"MACST2ARetrieval":0.1374,"BirdCLEF":0.184,"SIBFLEURS":0.1758389411764706,"JamAltArtistA2ARetrieval":0.73205,"ClothoT2ARetrieval":0.29955,"CREMADPairClassification":0.543237,"FSD2019Kaggle":0.780107,"CommonVoiceMini21T2ARetrieval":0.5972447008547008,"SpeechCommandsZeroshotv0.02":0.893716,"CREMA_DClustering":0.006228,"MInDS14":0.28946225,"UrbanSound8KT2ARetrieval":0.00982,"GTZANAudioReranking":0.7166,"BeijingOpera":0.923582,"CommonLanguageAgeDetection":0.1714,"FleursT2ARetrieval":0.14283754901960785,"CREMA_D":0.277213,"GigaSpeechT2ARetrieval":0.39244,"RavdessZeroshot":0.211111,"SpokenSQuADT2ARetrieval":0.08333,"JamAltLyricA2TRetrieval":0.08119},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.44402964164086695}},"trainedOnTasks":["CommonLanguageAgeDetection","CommonVoiceMini21T2ARetrieval","FSD2019Kaggle"],"experiments":null},{"rank":14,"model":{"name":"laion/clap-htsat-unfused","url":"https://huggingface.co/laion/clap-htsat-unfused","zeroShotPct":100,"activeParamsB":0.115,"totalParamsB":0.153,"embeddingDim":512,"maxTokens":null,"releaseDate":"2023-05-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{wu2024largescalecontrastivelanguageaudiopretraining,\n  author = {Wu, Yusong and Chen, Ke and Zhang, Tianyu and Hui, Yuchen and Berg-Kirkpatrick, Taylor and Dubnov, Shlomo},\n  booktitle = {ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp49357.2023.10095969},\n  month = {June},\n  pages = {1–5},\n  publisher = {IEEE},\n  title = {Large-Scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation},\n  url = {https://doi.org/10.1109/icassp49357.2023.10095969},\n  year = {2023},\n}\n","memoryUsageMb":586.0,"license":"mit","publicTrainingCode":"https://github.com/LAION-AI/CLAP","publicTrainingData":"https://laion.ai/blog/laion-audio-630k/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.115,"totalParamsB":0.153,"embeddingDim":512,"maxTokens":null,"meanTask":0.32960527080610025,"meanTaskType":0.40532349580303506,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4189671942959002,"AudioReranking":0.664536,"AudioClustering":0.125463,"AudioPairClassification":0.5261596666666667,"Any2AnyRetrieval":0.18061210965867827,"AudioMultilabelClassification":0.808447,"AudioZeroshotClassification":0.1130795},"scoresByTask":{"VoxCelebSA":0.337205,"GTZANGenre":0.749,"VehicleSoundClustering":0.026642,"VoxPopuliGenderClustering":0.221965,"MridinghamTonic":0.495194,"VoxPopuliAccentPairClassification":0.539609,"NMSQAPairClassification":0.469791,"IEMOCAPGender":0.92579,"VoxPopuliLanguageID":0.3,"MACST2ARetrieval":0.2799,"BirdCLEF":0.164,"SIBFLEURS":0.08674730392156862,"JamAltArtistA2ARetrieval":0.9296075,"ClothoT2ARetrieval":0.34181,"CREMADPairClassification":0.569079,"FSD2019Kaggle":0.808447,"CommonVoiceMini21T2ARetrieval":0.029378888888888892,"SpeechCommandsZeroshotv0.02":0.09352,"CREMA_DClustering":0.127782,"MInDS14":0.09574083333333333,"UrbanSound8KT2ARetrieval":0.00982,"GTZANAudioReranking":0.664536,"BeijingOpera":0.919592,"CommonLanguageAgeDetection":0.1598,"FleursT2ARetrieval":0.009995098039215687,"CREMA_D":0.37557,"GigaSpeechT2ARetrieval":0.00356,"RavdessZeroshot":0.132639,"SpokenSQuADT2ARetrieval":0.01333,"JamAltLyricA2TRetrieval":0.0081075},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.42935003353973167}},"trainedOnTasks":[],"experiments":null},{"rank":15,"model":{"name":"microsoft/msclap-2023","url":"https://github.com/microsoft/CLAP","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.16,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2023-09-01","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{CLAP2023,\n  author = {Elizalde, Benjamin and Deshmukh, Soham and Wang, Huaming},\n  booktitle = {ICASSP 2024 - 2024 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp48485.2024.10448504},\n  month = {Apr},\n  pages = {336–340},\n  publisher = {IEEE},\n  title = {Natural Language Supervision For General-Purpose Audio Representations},\n  url = {https://doi.org/10.1109/icassp48485.2024.10448504},\n  year = {2024},\n}\n","memoryUsageMb":610.0,"license":"mit","publicTrainingCode":"https://github.com/microsoft/CLAP","publicTrainingData":"https://github.com/microsoft/CLAP","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["msclap"],"trainingDatasets":["AudioCapsA2TRetrieval","ClothoA2TRetrieval","FSD50K"]},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.16,"embeddingDim":1024,"maxTokens":null,"meanTask":0.3396758205086308,"meanTaskType":0.4249404845276365,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.42207254500891267,"AudioReranking":0.75429,"AudioClustering":0.15223666666666666,"AudioPairClassification":0.5359813333333333,"Any2AnyRetrieval":0.18655184668454275,"AudioMultilabelClassification":0.797336,"AudioZeroshotClassification":0.126115},"scoresByTask":{"VoxCelebSA":0.261801,"GTZANGenre":0.781,"VehicleSoundClustering":0.0286,"VoxPopuliGenderClustering":0.320854,"MridinghamTonic":0.520859,"VoxPopuliAccentPairClassification":0.520462,"NMSQAPairClassification":0.510934,"IEMOCAPGender":0.858649,"VoxPopuliLanguageID":0.396,"MACST2ARetrieval":0.27226,"BirdCLEF":0.173,"SIBFLEURS":0.1407347450980392,"JamAltArtistA2ARetrieval":0.9278275,"ClothoT2ARetrieval":0.41934,"CREMADPairClassification":0.576548,"FSD2019Kaggle":0.797336,"CommonVoiceMini21T2ARetrieval":0.02791264957264957,"SpeechCommandsZeroshotv0.02":0.100147,"CREMA_DClustering":0.107256,"MInDS14":0.07557625,"UrbanSound8KT2ARetrieval":0.00943,"GTZANAudioReranking":0.75429,"BeijingOpera":0.911082,"CommonLanguageAgeDetection":0.1535,"FleursT2ARetrieval":0.009991470588235294,"CREMA_D":0.370596,"GigaSpeechT2ARetrieval":0.00104,"RavdessZeroshot":0.152083,"SpokenSQuADT2ARetrieval":0.0,"JamAltLyricA2TRetrieval":0.011165000000000001},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4415562892156862}},"trainedOnTasks":[],"experiments":null},{"rank":16,"model":{"name":"laion/clap-htsat-fused","url":"https://huggingface.co/laion/clap-htsat-fused","zeroShotPct":100,"activeParamsB":0.115,"totalParamsB":0.154,"embeddingDim":512,"maxTokens":null,"releaseDate":"2023-05-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{wu2024largescalecontrastivelanguageaudiopretraining,\n  author = {Wu, Yusong and Chen, Ke and Zhang, Tianyu and Hui, Yuchen and Berg-Kirkpatrick, Taylor and Dubnov, Shlomo},\n  booktitle = {ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp49357.2023.10095969},\n  month = {June},\n  pages = {1–5},\n  publisher = {IEEE},\n  title = {Large-Scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation},\n  url = {https://doi.org/10.1109/icassp49357.2023.10095969},\n  year = {2023},\n}\n","memoryUsageMb":586.0,"license":"mit","publicTrainingCode":"https://github.com/LAION-AI/CLAP","publicTrainingData":"https://laion.ai/blog/laion-audio-630k/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.115,"totalParamsB":0.154,"embeddingDim":512,"maxTokens":null,"meanTask":0.33474351414865094,"meanTaskType":0.41298979518212364,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.41170841399286995,"AudioReranking":0.613031,"AudioClustering":0.22737200000000002,"AudioPairClassification":0.519942,"Any2AnyRetrieval":0.1759516522819954,"AudioMultilabelClassification":0.810872,"AudioZeroshotClassification":0.1320515},"scoresByTask":{"VoxCelebSA":0.29285,"GTZANGenre":0.679,"VehicleSoundClustering":0.047132,"VoxPopuliGenderClustering":0.526757,"MridinghamTonic":0.469975,"VoxPopuliAccentPairClassification":0.531809,"NMSQAPairClassification":0.478507,"IEMOCAPGender":0.936249,"VoxPopuliLanguageID":0.32,"MACST2ARetrieval":0.29262,"BirdCLEF":0.18,"SIBFLEURS":0.0906959705882353,"JamAltArtistA2ARetrieval":0.8765875,"ClothoT2ARetrieval":0.33572,"CREMADPairClassification":0.54951,"FSD2019Kaggle":0.810872,"CommonVoiceMini21T2ARetrieval":0.028831880341880344,"SpeechCommandsZeroshotv0.02":0.117575,"CREMA_DClustering":0.108227,"MInDS14":0.09212958333333333,"UrbanSound8KT2ARetrieval":0.00943,"GTZANAudioReranking":0.613031,"BeijingOpera":0.92766,"CommonLanguageAgeDetection":0.1527,"FleursT2ARetrieval":0.009950490196078432,"CREMA_D":0.387533,"GigaSpeechT2ARetrieval":0.00178,"RavdessZeroshot":0.146528,"SpokenSQuADT2ARetrieval":0.02,"JamAltLyricA2TRetrieval":0.008645},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.43475542389060884}},"trainedOnTasks":[],"experiments":null},{"rank":16,"model":{"name":"google/embeddinggemma-2","url":"https://huggingface.co/google/embeddinggemma-2","zeroShotPct":100,"activeParamsB":0.137,"totalParamsB":0.271,"embeddingDim":768,"maxTokens":8192.0,"releaseDate":"2026-10-06","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["Bengali","Chinese","Chinese (Traditional)","English","Finnish","French","German","Hindi","Indonesian","Iranian Persian","Japanese","Korean","Russian","Spanish","Standard Arabic","Swahili (individual language)","Telugu","Thai","Yoruba"],"citation":"\n@misc{vera2026embeddinggemma2unifiedefficient,\n      title={EmbeddingGemma 2: Unified and Efficient Multimodal Representations},\n      author={Henrique Schechter Vera and Sahil Dua and Jovan Andonov and Sara Smoot and Sebastian Russo and Ivan Llanos and Max Matveev and Jae Hyeon Yoo and Kat Black and Shih-Cheng Huang and Josiah Aklilu and Maarten Grootendorst and Omar Sanseviero and Chandramouli Amarnath and Ian Ballantyne and Zhe Li and Daniel Salz and Ibrahim Alabdulmohsin and Ilya Philippov and Kristen Quan and Glenn Cameron and Edouard Yvinec and Luciano Martins and Doug Reid and Sindhu Raghuram Panyam and Weiyi Wang and Hengxuan Ying and Madhuri Shanbhogue and Tanmaya Dabral and Shanfeng Zhang and Juyeong Ji and Renjie Wu and Joe Zou and Xu Chen and Yunan Zhang and Yu-Hui Chen and Matthew Soulanille and Vinod Mamillapalli and Yi-Chun Kuo and Wai Hon Law and Matt Kreileder and Alice Zheng and Suril Shah and Mogan Shieh and Pooja Consul and Alberto Montes and Andreas Hess and Andres Sevilla and Babak Samari and Brendan McCloskey and Brian Potetz and Daniel Cer and Georgios Kathareios and Gustavo Hernández Ábrego and Hesen Zhang and Ji Ma and Jiaxing Wu and Keegan Mosley and Paul Suganthan and Rui Meng and Shen Gao and Shengcao Cao and Yi-Ting Chen and Yichang Chen and Zhongli Ding and Kendra Byrne and Cormac Brick and Olivier Lacombe and Qin Yin and Tris Warkentin and Armand Joulin and Tom Duerig and Mojtaba Seyedhosseini},\n      year={2026},\n      eprint={2610.37364},\n      archivePrefix={arXiv},\n      primaryClass={cs.LG},\n      url={https://arxiv.org/abs/2610.37364},\n}\n","memoryUsageMb":1034.0,"license":"gemma","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["embeddinggemma"],"trainingDatasets":["FEVERHardNegatives","HotpotQAHardNegatives","MIRACLRetrievalHardNegatives","NQHardNegatives"]},"zeroShotPct":100,"activeParamsB":0.137,"totalParamsB":0.271,"embeddingDim":768,"maxTokens":8192,"meanTask":0.4931071783852857,"meanTaskType":0.4801917517818057,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5233757446524065,"AudioReranking":0.6646,"AudioClustering":0.026834666666666663,"AudioPairClassification":0.6745619999999999,"Any2AnyRetrieval":0.5301243511535669,"AudioMultilabelClassification":0.387518,"AudioZeroshotClassification":0.5543275000000001},"scoresByTask":{"VoxCelebSA":0.449681,"GTZANGenre":0.576,"VehicleSoundClustering":0.074988,"VoxPopuliGenderClustering":0.000452,"MridinghamTonic":0.314034,"VoxPopuliAccentPairClassification":0.516983,"NMSQAPairClassification":0.981411,"IEMOCAPGender":0.664008,"VoxPopuliLanguageID":0.996,"MACST2ARetrieval":0.06616,"BirdCLEF":0.152,"SIBFLEURS":0.5465241078431372,"JamAltArtistA2ARetrieval":0.5537974999999999,"ClothoT2ARetrieval":0.10725,"CREMADPairClassification":0.525292,"FSD2019Kaggle":0.387518,"CommonVoiceMini21T2ARetrieval":0.9325317094017095,"SpeechCommandsZeroshotv0.02":0.969072,"CREMA_DClustering":0.005064,"MInDS14":0.8771310833333333,"UrbanSound8KT2ARetrieval":0.00472,"GTZANAudioReranking":0.6646,"BeijingOpera":0.809397,"CommonLanguageAgeDetection":0.1751,"FleursT2ARetrieval":0.9785474509803922,"CREMA_D":0.197258,"GigaSpeechT2ARetrieval":0.8197,"RavdessZeroshot":0.139583,"SpokenSQuADT2ARetrieval":0.69333,"JamAltLyricA2TRetrieval":0.6150825},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.47788003637770887}},"trainedOnTasks":[],"experiments":null},{"rank":18,"model":{"name":"encord-team/ebind-full","url":"https://huggingface.co/encord-team/ebind-full","zeroShotPct":100,"activeParamsB":1.739,"totalParamsB":1.79,"embeddingDim":1024,"maxTokens":32.0,"releaseDate":"2025-11-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@misc{broadbent2025ebindpracticalapproachspace,\n      title={{EBind}: a practical approach to space binding},\n      author={Jim Broadbent and Felix Cohen and Frederik Hvilshøj and Eric Landau and Eren Sasoglu},\n      year={2025},\n      eprint={2511.14229},\n      archivePrefix={arXiv},\n      primaryClass={cs.LG},\n      url={https://arxiv.org/abs/2511.14229},\n}\n","memoryUsageMb":6828.0,"license":"cc-by-nc-sa-4.0","publicTrainingCode":"https://github.com/encord-team/ebind","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["audio","video"],"trainingDatasets":["AudioCapsA2TRetrieval","AudioCapsAVA2VRetrieval","AudioCapsAVAT2VRetrieval","AudioCapsAVT2VARetrieval","AudioCapsAVT2VRetrieval","AudioCapsAVV2ARetrieval","AudioCapsAVV2TRetrieval","AudioCapsAVVA2TRetrieval","AudioCapsAVVT2ARetrieval","AudioCapsT2ARetrieval","Imagenet1k","Imagenet1kZeroShot","VALOR32KA2VRetrieval","VALOR32KAT2VRetrieval","VALOR32KT2VARetrieval","VALOR32KT2VRetrieval","VALOR32KV2ARetrieval","VALOR32KV2TRetrieval","VALOR32KVA2TRetrieval","VALOR32KVT2ARetrieval","VGGSoundAVA2VRetrieval","VGGSoundAVAT2VRetrieval","VGGSoundAVT2VARetrieval","VGGSoundAVT2VRetrieval","VGGSoundAVV2ARetrieval","VGGSoundAVV2TRetrieval","VGGSoundAVVA2TRetrieval","VGGSoundAVVT2ARetrieval","VGGSoundV","VGGSoundVA","VGGSoundVideoAudioZeroshot","VGGSoundVideoZeroshot"]},"zeroShotPct":100,"activeParamsB":1.739,"totalParamsB":1.79,"embeddingDim":1024,"maxTokens":32,"meanTask":0.3275857660968661,"meanTaskType":0.41382440512718943,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3983372299465241,"AudioReranking":0.81205,"AudioClustering":0.259888,"AudioPairClassification":0.523046,"Any2AnyRetrieval":0.15232860594380204,"AudioMultilabelClassification":0.588188,"AudioZeroshotClassification":0.162933},"scoresByTask":{"VoxCelebSA":0.242095,"GTZANGenre":0.834,"VehicleSoundClustering":0.064883,"VoxPopuliGenderClustering":0.606781,"MridinghamTonic":0.301563,"VoxPopuliAccentPairClassification":0.550747,"NMSQAPairClassification":0.460069,"IEMOCAPGender":0.904669,"VoxPopuliLanguageID":0.436,"MACST2ARetrieval":0.10433,"BirdCLEF":0.086,"SIBFLEURS":0.14727586274509805,"JamAltArtistA2ARetrieval":0.9384925,"ClothoT2ARetrieval":0.25389,"CREMADPairClassification":0.558322,"FSD2019Kaggle":0.588188,"CommonVoiceMini21T2ARetrieval":0.02614264957264957,"SpeechCommandsZeroshotv0.02":0.116838,"CREMA_DClustering":0.108,"MInDS14":0.07130066666666666,"UrbanSound8KT2ARetrieval":0.00923,"GTZANAudioReranking":0.81205,"BeijingOpera":0.842996,"CommonLanguageAgeDetection":0.15435,"FleursT2ARetrieval":0.019844803921568625,"CREMA_D":0.36146,"GigaSpeechT2ARetrieval":0.004,"RavdessZeroshot":0.209028,"SpokenSQuADT2ARetrieval":0.0,"JamAltLyricA2TRetrieval":0.015027500000000001},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4463712647058824}},"trainedOnTasks":[],"experiments":null},{"rank":19,"model":{"name":"OpenMuQ/MuQ-MuLan-large","url":"https://huggingface.co/OpenMuQ/MuQ-MuLan-large","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.63,"embeddingDim":512,"maxTokens":null,"releaseDate":"2025-01-01","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["Chinese","English"],"citation":"\n@article{zhu2025muqselfsupervisedmusicrepresentation,\n  author = {Zhu, Haina and Zhou, Yizhi and Chen, Hangting and Yu, Jianwei and Ma, Ziyang and Gu, Rongzhi and Luo, Yi and Tan, Wei and Chen, Xie},\n  doi = {10.1109/taslpro.2025.3602320},\n  issn = {2998-4173},\n  journal = {IEEE Transactions on Audio, Speech and Language Processing},\n  pages = {3653–3664},\n  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},\n  title = {MuQ: Self-Supervised Music Representation Learning With Mel Residual Vector Quantization},\n  url = {https://doi.org/10.1109/taslpro.2025.3602320},\n  volume = {33},\n  year = {2025},\n}","memoryUsageMb":2530.0,"license":"cc-by-nc-4.0","publicTrainingCode":"https://github.com/tencent-ailab/MuQ","publicTrainingData":"https://github.com/tencent-ailab/MuQ","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["muq","transformers-v4"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.63,"embeddingDim":512,"maxTokens":null,"meanTask":0.2815288970294956,"meanTaskType":0.3397078629999267,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3880045209447415,"AudioReranking":0.85406,"AudioClustering":0.042896333333333335,"AudioPairClassification":0.5189246666666667,"Any2AnyRetrieval":0.11727602005474552,"AudioMultilabelClassification":0.330777,"AudioZeroshotClassification":0.1260165},"scoresByTask":{"VoxCelebSA":0.251078,"GTZANGenre":0.883,"VehicleSoundClustering":0.009234,"VoxPopuliGenderClustering":0.007615,"MridinghamTonic":0.382543,"VoxPopuliAccentPairClassification":0.520424,"NMSQAPairClassification":0.461921,"IEMOCAPGender":0.574462,"VoxPopuliLanguageID":0.414,"MACST2ARetrieval":0.01781,"BirdCLEF":0.074,"SIBFLEURS":0.2320778137254902,"JamAltArtistA2ARetrieval":0.962765,"ClothoT2ARetrieval":0.02041,"CREMADPairClassification":0.574429,"FSD2019Kaggle":0.330777,"CommonVoiceMini21T2ARetrieval":0.026296239316239315,"SpeechCommandsZeroshotv0.02":0.104811,"CREMA_DClustering":0.11184,"MInDS14":0.15687091666666667,"UrbanSound8KT2ARetrieval":0.00275,"GTZANAudioReranking":0.85406,"BeijingOpera":0.763209,"CommonLanguageAgeDetection":0.19335,"FleursT2ARetrieval":0.009907941176470588,"CREMA_D":0.343459,"GigaSpeechT2ARetrieval":0.00015,"RavdessZeroshot":0.147222,"SpokenSQuADT2ARetrieval":0.01,"JamAltLyricA2TRetrieval":0.0053950000000000005},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4089651437048504}},"trainedOnTasks":[],"experiments":null},{"rank":20,"model":{"name":"lyrebird/wav2clip","url":"https://github.com/descriptinc/lyrebird-wav2clip","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.163,"embeddingDim":512,"maxTokens":null,"releaseDate":"2022-03-15","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{wu2022wav2cliplearningrobustaudio,\n  author = {Wu, Ho-Hsiang and Seetharaman, Prem and Kumar, Kundan and Bello, Juan Pablo},\n  booktitle = {ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp43922.2022.9747669},\n  month = {May},\n  pages = {4563–4567},\n  publisher = {IEEE},\n  title = {Wav2CLIP: Learning Robust Audio Representations from Clip},\n  url = {https://doi.org/10.1109/icassp43922.2022.9747669},\n  year = {2022},\n}","memoryUsageMb":622.0,"license":"mit","publicTrainingCode":"https://github.com/descriptinc/lyrebird-wav2clip","publicTrainingData":"https://github.com/descriptinc/lyrebird-wav2clip#data","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["wav2clip"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.163,"embeddingDim":512,"maxTokens":null,"meanTask":0.2662072791938998,"meanTaskType":0.31148418286034957,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.37445292780748657,"AudioReranking":0.68942,"AudioClustering":0.05974166666666667,"AudioPairClassification":0.5359790000000001,"Any2AnyRetrieval":0.09527868554829338,"AudioMultilabelClassification":0.317888,"AudioZeroshotClassification":0.107629},"scoresByTask":{"VoxCelebSA":0.25777,"GTZANGenre":0.591,"VehicleSoundClustering":0.039723,"VoxPopuliGenderClustering":0.007115,"MridinghamTonic":0.402753,"VoxPopuliAccentPairClassification":0.5256,"NMSQAPairClassification":0.494725,"IEMOCAPGender":0.658334,"VoxPopuliLanguageID":0.314,"MACST2ARetrieval":0.01781,"BirdCLEF":0.097,"SIBFLEURS":0.21968920588235294,"JamAltArtistA2ARetrieval":0.7850125,"ClothoT2ARetrieval":0.01146,"CREMADPairClassification":0.587612,"FSD2019Kaggle":0.317888,"CommonVoiceMini21T2ARetrieval":0.02448111111111111,"SpeechCommandsZeroshotv0.02":0.097202,"CREMA_DClustering":0.132387,"MInDS14":0.12185499999999999,"UrbanSound8KT2ARetrieval":0.00177,"GTZANAudioReranking":0.68942,"BeijingOpera":0.881028,"CommonLanguageAgeDetection":0.13105,"FleursT2ARetrieval":0.010272058823529412,"CREMA_D":0.444503,"GigaSpeechT2ARetrieval":0.00089,"RavdessZeroshot":0.118056,"SpokenSQuADT2ARetrieval":0.0,"JamAltLyricA2TRetrieval":0.0058125},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3884514055727554}},"trainedOnTasks":[],"experiments":null},{"rank":21,"model":{"name":"microsoft/msclap-2022","url":"https://github.com/microsoft/CLAP","zeroShotPct":96,"activeParamsB":null,"totalParamsB":0.196,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2022-12-01","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{CLAP2022,\n  title={Clap learning audio concepts from natural language supervision},\n  author={Elizalde, Benjamin and Deshmukh, Soham and Al Ismail, Mahmoud and Wang, Huaming},\n  booktitle={ICASSP 2023-2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  pages={1--5},\n  year={2023},\n  organization={IEEE}\n}\n","memoryUsageMb":750.0,"license":"mit","publicTrainingCode":"https://github.com/microsoft/CLAP","publicTrainingData":"https://github.com/microsoft/CLAP","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["msclap"],"trainingDatasets":["AudioCapsA2TRetrieval","AudioSetMini","ClothoA2TRetrieval","ESC50","FSD50K","GTZANGenre","UrbanSound8k"]},"zeroShotPct":96,"activeParamsB":null,"totalParamsB":0.196,"embeddingDim":1024,"maxTokens":null,"meanTask":0.3216497276101894,"meanTaskType":0.3932099681741642,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.36078716666666666,"AudioReranking":0.62909,"AudioClustering":0.1990333333333333,"AudioPairClassification":0.517419,"Any2AnyRetrieval":0.21695777721914977,"AudioMultilabelClassification":0.708599,"AudioZeroshotClassification":0.12058350000000001},"scoresByTask":{"VoxCelebSA":0.255394,"GTZANGenre":0.587,"VehicleSoundClustering":0.079798,"VoxPopuliGenderClustering":0.463659,"MridinghamTonic":0.29139,"VoxPopuliAccentPairClassification":0.520807,"NMSQAPairClassification":0.500859,"IEMOCAPGender":0.893515,"VoxPopuliLanguageID":0.242,"MACST2ARetrieval":0.4173,"BirdCLEF":0.132,"SIBFLEURS":0.14461366666666667,"JamAltArtistA2ARetrieval":0.8718325,"ClothoT2ARetrieval":0.58729,"CREMADPairClassification":0.530591,"FSD2019Kaggle":0.708599,"CommonVoiceMini21T2ARetrieval":0.026561367521367524,"SpeechCommandsZeroshotv0.02":0.105056,"CREMA_DClustering":0.053643,"MInDS14":0.06873316666666666,"UrbanSound8KT2ARetrieval":0.00884,"GTZANAudioReranking":0.62909,"BeijingOpera":0.932004,"CommonLanguageAgeDetection":0.14265,"FleursT2ARetrieval":0.009313627450980392,"CREMA_D":0.279359,"GigaSpeechT2ARetrieval":0.00163,"RavdessZeroshot":0.136111,"SpokenSQuADT2ARetrieval":0.02333,"JamAltLyricA2TRetrieval":0.0065225000000000005},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.40099675438596494}},"trainedOnTasks":["GTZANGenre"],"experiments":null},{"rank":22,"model":{"name":"LanguageBind/LanguageBind_Omni","url":"https://github.com/PKU-YuanGroup/LanguageBind","zeroShotPct":100,"activeParamsB":1.162,"totalParamsB":1.2,"embeddingDim":768,"maxTokens":77.0,"releaseDate":"2023-10-03","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text","audio","video"],"languages":["English"],"citation":"\n@inproceedings{zhu2024languagebind,\n  archiveprefix = {arXiv},\n  author = {Bin Zhu and Bin Lin and Munan Ning and Yang Yan and Jiaxi Cui and WANG HongFa and Yatian Pang and Wenhao Jiang and Junwu Zhang and Zongwei Li and Cai Wan Zhang and Zhifeng Li and Wei Liu and Li Yuan},\n  booktitle = {The Twelfth International Conference on Learning Representations},\n  eprint = {2310.01852},\n  title = {LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment},\n  url = {https://openreview.net/forum?id=QmZKc7UZCy},\n  year = {2024},\n}\n","memoryUsageMb":4578.0,"license":"mit","publicTrainingCode":"https://github.com/PKU-YuanGroup/LanguageBind","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["languagebind"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":1.162,"totalParamsB":1.2,"embeddingDim":768,"maxTokens":77,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioReranking":0.7921,"AudioClustering":0.028951666666666667,"AudioPairClassification":0.5263503333333333,"AudioMultilabelClassification":0.779693,"AudioZeroshotClassification":0.168301},"scoresByTask":{"VoxCelebSA":0.253442,"GTZANGenre":0.791,"VehicleSoundClustering":0.025643,"VoxPopuliGenderClustering":0.000416,"VoxPopuliAccentPairClassification":0.535252,"NMSQAPairClassification":0.50053,"VoxPopuliLanguageID":0.302,"MACST2ARetrieval":0.11196,"BirdCLEF":0.235,"SIBFLEURS":0.14720610784313726,"ClothoT2ARetrieval":0.25765,"CREMADPairClassification":0.543269,"FSD2019Kaggle":0.779693,"CommonVoiceMini21T2ARetrieval":0.028831025641025642,"SpeechCommandsZeroshotv0.02":0.167158,"CREMA_DClustering":0.060796,"UrbanSound8KT2ARetrieval":0.00727,"GTZANAudioReranking":0.7921,"BeijingOpera":0.957535,"CommonLanguageAgeDetection":0.1441,"CREMA_D":0.334452,"GigaSpeechT2ARetrieval":0.00193,"RavdessZeroshot":0.169444,"SpokenSQuADT2ARetrieval":0.0},"scoresByCustomGroup":{"Benchmark Variant":{}},"trainedOnTasks":[],"experiments":null},{"rank":23,"model":{"name":"microsoft/speecht5_multimodal","url":"https://huggingface.co/microsoft/speecht5_asr","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.298,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-05-16","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{ao2022speecht5unifiedmodalencoderdecoderpretraining,\n  address = {Dublin, Ireland},\n  author = {Ao, Junyi and Wang, Rui and Zhou, Long and Wang, Chengyi and Ren, Shuo and Wu, Yu and Liu, Shujie and Ko, Tom and Li, Qing and Zhang, Yu and Wei, Zhihua and Qian, Yao and Li, Jinyu and Wei, Furu},\n  booktitle = {Proceedings of the 60th Annual Meeting of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)},\n  doi = {10.18653/v1/2022.acl-long.393},\n  editor = {Muresan, Smaranda and Nakov, Preslav and Villavicencio, Aline},\n  month = {may},\n  pages = {5723--5738},\n  publisher = {Association for Computational Linguistics},\n  title = {{S}peech{T}5: Unified-Modal Encoder-Decoder Pre-Training for Spoken Language Processing},\n  url = {https://aclanthology.org/2022.acl-long.393/},\n  year = {2022},\n}","memoryUsageMb":1136.0,"license":"mit","publicTrainingCode":"https://github.com/microsoft/SpeechT5","publicTrainingData":"http://www.festvox.org/cmu_arctic/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["speecht5"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.298,"embeddingDim":768,"maxTokens":null,"meanTask":0.2639633326546003,"meanTaskType":0.3003821881299528,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3846843137254902,"AudioReranking":0.56502,"AudioClustering":0.010570999999999999,"AudioPairClassification":0.5793136666666666,"Any2AnyRetrieval":0.078833836517513,"AudioMultilabelClassification":0.325311,"AudioZeroshotClassification":0.1589415},"scoresByTask":{"VoxCelebSA":0.283851,"GTZANGenre":0.525,"VehicleSoundClustering":0.004677,"VoxPopuliGenderClustering":0.000088,"MridinghamTonic":0.307438,"VoxPopuliAccentPairClassification":0.531399,"NMSQAPairClassification":0.662751,"IEMOCAPGender":0.521368,"VoxPopuliLanguageID":0.788,"MACST2ARetrieval":0.01527,"BirdCLEF":0.158,"SIBFLEURS":0.16217545098039218,"JamAltArtistA2ARetrieval":0.5955675,"ClothoT2ARetrieval":0.00483,"CREMADPairClassification":0.543791,"FSD2019Kaggle":0.325311,"CommonVoiceMini21T2ARetrieval":0.031071538461538463,"SpeechCommandsZeroshotv0.02":0.188022,"CREMA_DClustering":0.026948,"MInDS14":0.194238,"UrbanSound8KT2ARetrieval":0.00177,"GTZANAudioReranking":0.56502,"BeijingOpera":0.771277,"CommonLanguageAgeDetection":0.195,"FleursT2ARetrieval":0.013800490196078432,"CREMA_D":0.32518,"GigaSpeechT2ARetrieval":0.00385,"RavdessZeroshot":0.129861,"SpokenSQuADT2ARetrieval":0.03333,"JamAltLyricA2TRetrieval":0.010015},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.37693520794633645}},"trainedOnTasks":[],"experiments":null},{"rank":24,"model":{"name":"Qwen/Qwen3-Omni-30B-A3B-Instruct","url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct","zeroShotPct":-1,"activeParamsB":35.104,"totalParamsB":35.26,"embeddingDim":2048,"maxTokens":65536.0,"releaseDate":"2025-09-20","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@misc{xu2025qwen3omnitechnicalreport,\n    title={Qwen3-Omni Technical Report},\n    author={Jin Xu and Zhifang Guo and Hangrui Hu and Yunfei Chu and Xiong Wang and Jinzheng He and Yuxuan Wang and Xian Shi and Ting He and Xinfa Zhu and Yuanjun Lv and Yongqi Wang and Dake Guo and He Wang and Linhan Ma and Pei Zhang and Xinyu Zhang and Hongkun Hao and Zishan Guo and Baosong Yang and Bin Zhang and Ziyang Ma and Xipin Wei and Shuai Bai and Keqin Chen and Xuejing Liu and Peng Wang and Mingkun Yang and Dayiheng Liu and Xingzhang Ren and Bo Zheng and Rui Men and Fan Zhou and Bowen Yu and Jianxin Yang and Le Yu and Jingren Zhou and Junyang Lin},\n    year={2025},\n    eprint={2509.17765},\n    archivePrefix={arXiv},\n    primaryClass={cs.CL},\n    url={https://arxiv.org/abs/2509.17765},\n}","memoryUsageMb":67253.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":35.104,"totalParamsB":35.26,"embeddingDim":2048,"maxTokens":65536,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioReranking":0.55453,"AudioClustering":0.007377333333333334,"AudioPairClassification":0.5388563333333333,"AudioMultilabelClassification":0.129997,"AudioZeroshotClassification":0.1944245},"scoresByTask":{"VoxCelebSA":0.334885,"GTZANGenre":0.424,"VehicleSoundClustering":0.012456,"VoxPopuliGenderClustering":0.000504,"MridinghamTonic":0.324927,"VoxPopuliAccentPairClassification":0.509927,"NMSQAPairClassification":0.581434,"IEMOCAPGender":0.534318,"VoxPopuliLanguageID":0.94,"MACST2ARetrieval":0.02036,"SIBFLEURS":0.21777107843137256,"JamAltArtistA2ARetrieval":0.6320875,"ClothoT2ARetrieval":0.00627,"CREMADPairClassification":0.525208,"FSD2019Kaggle":0.129997,"CommonVoiceMini21T2ARetrieval":0.06673564102564103,"SpeechCommandsZeroshotv0.02":0.222877,"CREMA_DClustering":0.009172,"MInDS14":0.35382441666666664,"UrbanSound8KT2ARetrieval":0.00177,"GTZANAudioReranking":0.55453,"BeijingOpera":0.750266,"CommonLanguageAgeDetection":0.18705,"CREMA_D":0.240528,"GigaSpeechT2ARetrieval":0.02444,"RavdessZeroshot":0.165972,"SpokenSQuADT2ARetrieval":0.04,"JamAltLyricA2TRetrieval":0.02271},"scoresByCustomGroup":{"Benchmark Variant":{}},"trainedOnTasks":[],"experiments":null},{"rank":25,"model":{"name":"openai/whisper-medium","url":"https://huggingface.co/openai/whisper-medium","zeroShotPct":100,"activeParamsB":0.769,"totalParamsB":0.769,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2022-09-27","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Assamese","Azerbaijani","Bashkir","Basque","Belarusian","Bengali","Bosnian","Breton","Bulgarian","Burmese","Catalan","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Faroese","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hausa","Hawaiian (Latin)","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Korean","Lao","Latin","Latvian","Lingala","Lithuanian","Luxembourgish","Macedonian","Malagasy (Latin)","Malay (macrolanguage)","Malayalam","Maltese","Maori","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian Bokmål","Norwegian Nynorsk","Occitan (post 1500)","Panjabi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Sanskrit","Serbian","Shona","Sindhi","Sinhala","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tajik","Tamil","Tatar","Telugu","Thai","Tibetan","Turkish","Turkmen","Ukrainian","Urdu","Uzbek","Vietnamese","Welsh","Yiddish","Yoruba"],"citation":"\n@inproceedings{radford2022robustspeechrecognitionlargescale,\n  archiveprefix = {arXiv},\n  author = {Alec Radford and Jong Wook Kim and Tao Xu and Greg Brockman and Christine McLeavey and Ilya Sutskever},\n  booktitle = {Proceedings of the 40th International Conference on Machine Learning},\n  eprint = {2212.04356},\n  pages = {28492--28518},\n  primaryclass = {eess.AS},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Robust Speech Recognition via Large-Scale Weak Supervision},\n  url = {https://proceedings.mlr.press/v202/radford23a.html},\n  volume = {202},\n  year = {2023},\n}","memoryUsageMb":2914.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.769,"totalParamsB":0.769,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5484707299465241,"AudioReranking":0.676449,"AudioClustering":0.050336,"AudioPairClassification":0.539338,"AudioMultilabelClassification":0.567577},"scoresByTask":{"VoxCelebSA":0.339244,"GTZANGenre":0.76,"VehicleSoundClustering":0.010773,"VoxPopuliGenderClustering":0.000064,"MridinghamTonic":0.495055,"VoxPopuliAccentPairClassification":0.553958,"NMSQAPairClassification":0.464547,"IEMOCAPGender":0.760534,"VoxPopuliLanguageID":0.994,"BirdCLEF":0.29,"SIBFLEURS":0.2659110294117647,"JamAltArtistA2ARetrieval":0.721075,"CREMADPairClassification":0.599509,"FSD2019Kaggle":0.567577,"CREMA_DClustering":0.140171,"MInDS14":0.49693000000000004,"GTZANAudioReranking":0.676449,"BeijingOpera":0.915426,"CommonLanguageAgeDetection":0.1763,"CREMA_D":0.539778},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.48419600154798764}},"trainedOnTasks":[],"experiments":null},{"rank":26,"model":{"name":"Qwen/Qwen2.5-Omni-7B","url":"https://huggingface.co/Qwen/Qwen2.5-Omni-7B","zeroShotPct":-1,"activeParamsB":10.187,"totalParamsB":10.732,"embeddingDim":3584,"maxTokens":32768.0,"releaseDate":"2025-03-22","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@misc{xu2025qwen25omnitechnicalreport,\n    title={Qwen2.5-Omni Technical Report},\n    author={Jin Xu and Zhifang Guo and Jinzheng He and Hangrui Hu and Ting He and Shuai Bai and Keqin Chen and Jialin Wang and Yang Fan and Kai Dang and Bin Zhang and Xiong Wang and Yunfei Chu and Junyang Lin},\n    year={2025},\n    eprint={2503.20215},\n    archivePrefix={arXiv},\n    primaryClass={cs.CL},\n    url={https://arxiv.org/abs/2503.20215},\n}","memoryUsageMb":21327.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":10.187,"totalParamsB":10.732,"embeddingDim":3584,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioReranking":0.58926,"AudioClustering":0.006859666666666667,"AudioPairClassification":0.528479,"Any2AnyRetrieval":0.0862617556281772,"AudioMultilabelClassification":0.09847,"AudioZeroshotClassification":0.176632},"scoresByTask":{"VoxCelebSA":0.295735,"GTZANGenre":0.361,"VehicleSoundClustering":0.008655,"VoxPopuliGenderClustering":0.001235,"MridinghamTonic":0.306729,"VoxPopuliAccentPairClassification":0.504013,"NMSQAPairClassification":0.557551,"IEMOCAPGender":0.512104,"VoxPopuliLanguageID":0.828,"MACST2ARetrieval":0.01018,"SIBFLEURS":0.1812175294117647,"JamAltArtistA2ARetrieval":0.6206725,"ClothoT2ARetrieval":0.0077,"CREMADPairClassification":0.523873,"FSD2019Kaggle":0.09847,"CommonVoiceMini21T2ARetrieval":0.03845222222222223,"SpeechCommandsZeroshotv0.02":0.219931,"CREMA_DClustering":0.010689,"MInDS14":0.21248408333333332,"UrbanSound8KT2ARetrieval":0.00138,"GTZANAudioReranking":0.58926,"BeijingOpera":0.766755,"CommonLanguageAgeDetection":0.1527,"FleursT2ARetrieval":0.02272607843137255,"CREMA_D":0.219703,"GigaSpeechT2ARetrieval":0.03452,"RavdessZeroshot":0.133333,"SpokenSQuADT2ARetrieval":0.03,"JamAltLyricA2TRetrieval":0.010725},"scoresByCustomGroup":{"Benchmark Variant":{}},"trainedOnTasks":[],"experiments":null},{"rank":27,"model":{"name":"Qwen/Qwen2.5-Omni-3B","url":"https://huggingface.co/Qwen/Qwen2.5-Omni-3B","zeroShotPct":-1,"activeParamsB":5.226,"totalParamsB":5.537,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2025-04-30","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@misc{xu2025qwen25omnitechnicalreport,\n    title={Qwen2.5-Omni Technical Report},\n    author={Jin Xu and Zhifang Guo and Jinzheng He and Hangrui Hu and Ting He and Shuai Bai and Keqin Chen and Jialin Wang and Yang Fan and Kai Dang and Bin Zhang and Xiong Wang and Yunfei Chu and Junyang Lin},\n    year={2025},\n    eprint={2503.20215},\n    archivePrefix={arXiv},\n    primaryClass={cs.CL},\n    url={https://arxiv.org/abs/2503.20215},\n}","memoryUsageMb":11418.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":5.226,"totalParamsB":5.537,"embeddingDim":2048,"maxTokens":32768,"meanTask":0.2341380898273839,"meanTaskType":0.25728193634651475,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.33428210249554363,"AudioReranking":0.57773,"AudioClustering":0.006349666666666667,"AudioPairClassification":0.511662,"Any2AnyRetrieval":0.0877012852633931,"AudioMultilabelClassification":0.140534,"AudioZeroshotClassification":0.1427145},"scoresByTask":{"VoxCelebSA":0.289359,"GTZANGenre":0.418,"VehicleSoundClustering":0.011603,"VoxPopuliGenderClustering":0.001588,"MridinghamTonic":0.264449,"VoxPopuliAccentPairClassification":0.509825,"NMSQAPairClassification":0.51359,"IEMOCAPGender":0.550256,"VoxPopuliLanguageID":0.632,"MACST2ARetrieval":0.02799,"BirdCLEF":0.049,"SIBFLEURS":0.1973339607843137,"JamAltArtistA2ARetrieval":0.561475,"ClothoT2ARetrieval":0.00591,"CREMADPairClassification":0.511571,"FSD2019Kaggle":0.140534,"CommonVoiceMini21T2ARetrieval":0.06069367521367521,"SpeechCommandsZeroshotv0.02":0.175012,"CREMA_DClustering":0.005858,"MInDS14":0.2732141666666667,"UrbanSound8KT2ARetrieval":0.00118,"GTZANAudioReranking":0.57773,"BeijingOpera":0.64016,"CommonLanguageAgeDetection":0.16325,"FleursT2ARetrieval":0.027930392156862743,"CREMA_D":0.200081,"GigaSpeechT2ARetrieval":0.08356,"RavdessZeroshot":0.110417,"SpokenSQuADT2ARetrieval":0.00333,"JamAltLyricA2TRetrieval":0.0172425},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3352812172342621}},"trainedOnTasks":[],"experiments":null},{"rank":28,"model":{"name":"openai/whisper-base","url":"https://huggingface.co/openai/whisper-base","zeroShotPct":100,"activeParamsB":0.074,"totalParamsB":0.074,"embeddingDim":512,"maxTokens":null,"releaseDate":"2022-09-27","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Assamese","Azerbaijani","Bashkir","Basque","Belarusian","Bengali","Bosnian","Breton","Bulgarian","Burmese","Catalan","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Faroese","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hausa","Hawaiian (Latin)","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Korean","Lao","Latin","Latvian","Lingala","Lithuanian","Luxembourgish","Macedonian","Malagasy (Latin)","Malay (macrolanguage)","Malayalam","Maltese","Maori","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian Bokmål","Norwegian Nynorsk","Occitan (post 1500)","Panjabi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Sanskrit","Serbian","Shona","Sindhi","Sinhala","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tajik","Tamil","Tatar","Telugu","Thai","Tibetan","Turkish","Turkmen","Ukrainian","Urdu","Uzbek","Vietnamese","Welsh","Yiddish","Yoruba"],"citation":"\n@inproceedings{radford2022robustspeechrecognitionlargescale,\n  archiveprefix = {arXiv},\n  author = {Alec Radford and Jong Wook Kim and Tao Xu and Greg Brockman and Christine McLeavey and Ilya Sutskever},\n  booktitle = {Proceedings of the 40th International Conference on Machine Learning},\n  eprint = {2212.04356},\n  pages = {28492--28518},\n  primaryclass = {eess.AS},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Robust Speech Recognition via Large-Scale Weak Supervision},\n  url = {https://proceedings.mlr.press/v202/radford23a.html},\n  volume = {202},\n  year = {2023},\n}","memoryUsageMb":277.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.074,"totalParamsB":0.074,"embeddingDim":512,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.49650603475935834,"AudioReranking":0.649784,"AudioClustering":0.05024400000000001,"AudioPairClassification":0.5211026666666666,"AudioMultilabelClassification":0.462637},"scoresByTask":{"VoxCelebSA":0.312568,"GTZANGenre":0.709,"VehicleSoundClustering":0.024958,"VoxPopuliGenderClustering":0.000373,"MridinghamTonic":0.44675,"VoxPopuliAccentPairClassification":0.515407,"NMSQAPairClassification":0.469942,"IEMOCAPGender":0.721589,"VoxPopuliLanguageID":0.98,"BirdCLEF":0.275,"SIBFLEURS":0.1636518823529412,"JamAltArtistA2ARetrieval":0.6973199999999999,"CREMADPairClassification":0.577959,"FSD2019Kaggle":0.462637,"CREMA_DClustering":0.125401,"MInDS14":0.2968075,"GTZANAudioReranking":0.649784,"BeijingOpera":0.893883,"CommonLanguageAgeDetection":0.1818,"CREMA_D":0.480517},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.44856370433436527}},"trainedOnTasks":[],"experiments":null},{"rank":29,"model":{"name":"Qwen/Qwen3-Omni-30B-A3B-Thinking","url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Thinking","zeroShotPct":-1,"activeParamsB":31.563,"totalParamsB":31.719,"embeddingDim":2048,"maxTokens":65536.0,"releaseDate":"2025-09-15","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@misc{xu2025qwen3omnitechnicalreport,\n    title={Qwen3-Omni Technical Report},\n    author={Jin Xu and Zhifang Guo and Hangrui Hu and Yunfei Chu and Xiong Wang and Jinzheng He and Yuxuan Wang and Xian Shi and Ting He and Xinfa Zhu and Yuanjun Lv and Yongqi Wang and Dake Guo and He Wang and Linhan Ma and Pei Zhang and Xinyu Zhang and Hongkun Hao and Zishan Guo and Baosong Yang and Bin Zhang and Ziyang Ma and Xipin Wei and Shuai Bai and Keqin Chen and Xuejing Liu and Peng Wang and Mingkun Yang and Dayiheng Liu and Xingzhang Ren and Bo Zheng and Rui Men and Fan Zhou and Bowen Yu and Jianxin Yang and Le Yu and Jingren Zhou and Junyang Lin},\n    year={2025},\n    eprint={2509.17765},\n    archivePrefix={arXiv},\n    primaryClass={cs.CL},\n    url={https://arxiv.org/abs/2509.17765},\n}","memoryUsageMb":60500.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":31.563,"totalParamsB":31.719,"embeddingDim":2048,"maxTokens":65536,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioReranking":0.56653,"AudioClustering":0.013467333333333333,"AudioPairClassification":0.507972,"AudioMultilabelClassification":0.151906,"AudioZeroshotClassification":0.15216000000000002},"scoresByTask":{"VoxCelebSA":0.303876,"GTZANGenre":0.445,"VehicleSoundClustering":0.006526,"VoxPopuliGenderClustering":0.004131,"MridinghamTonic":0.279917,"VoxPopuliAccentPairClassification":0.504347,"NMSQAPairClassification":0.487676,"IEMOCAPGender":0.520673,"VoxPopuliLanguageID":0.936,"MACST2ARetrieval":0.01272,"SIBFLEURS":0.19916688235294117,"JamAltArtistA2ARetrieval":0.5046824999999999,"ClothoT2ARetrieval":0.00662,"CREMADPairClassification":0.531893,"FSD2019Kaggle":0.151906,"CommonVoiceMini21T2ARetrieval":0.03792846153846154,"SpeechCommandsZeroshotv0.02":0.10432,"CREMA_DClustering":0.029745,"MInDS14":0.20762725,"UrbanSound8KT2ARetrieval":0.00079,"GTZANAudioReranking":0.56653,"BeijingOpera":0.745922,"CommonLanguageAgeDetection":0.16135,"CREMA_D":0.248455,"GigaSpeechT2ARetrieval":0.00474,"RavdessZeroshot":0.2,"SpokenSQuADT2ARetrieval":0.05667,"JamAltLyricA2TRetrieval":0.015072499999999999},"scoresByCustomGroup":{"Benchmark Variant":{}},"trainedOnTasks":[],"experiments":null},{"rank":30,"model":{"name":"laion/larger_clap_music","url":"https://huggingface.co/laion/larger_clap_music","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.194,"embeddingDim":512,"maxTokens":null,"releaseDate":"2023-05-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{wu2024largescalecontrastivelanguageaudiopretraining,\n  author = {Wu, Yusong and Chen, Ke and Zhang, Tianyu and Hui, Yuchen and Berg-Kirkpatrick, Taylor and Dubnov, Shlomo},\n  booktitle = {ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp49357.2023.10095969},\n  month = {June},\n  pages = {1–5},\n  publisher = {IEEE},\n  title = {Large-Scale Contrastive Language-Audio Pretraining with Feature Fusion and Keyword-to-Caption Augmentation},\n  url = {https://doi.org/10.1109/icassp49357.2023.10095969},\n  year = {2023},\n}\n","memoryUsageMb":740.0,"license":"mit","publicTrainingCode":"https://github.com/LAION-AI/CLAP","publicTrainingData":"https://laion.ai/blog/laion-audio-630k/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.194,"embeddingDim":512,"maxTokens":null,"meanTask":0.20740449728506788,"meanTaskType":0.227985493246405,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.26712767201426024,"AudioReranking":0.485542,"AudioClustering":0.05323733333333334,"AudioPairClassification":0.5028076666666667,"Any2AnyRetrieval":0.09095028071057482,"AudioMultilabelClassification":0.080966,"AudioZeroshotClassification":0.1152675},"scoresByTask":{"VoxCelebSA":0.270503,"GTZANGenre":0.258,"VehicleSoundClustering":0.037272,"VoxPopuliGenderClustering":0.013973,"MridinghamTonic":0.183887,"VoxPopuliAccentPairClassification":0.516037,"NMSQAPairClassification":0.44058,"IEMOCAPGender":0.657529,"VoxPopuliLanguageID":0.282,"MACST2ARetrieval":0.01272,"BirdCLEF":0.037,"SIBFLEURS":0.05552189215686274,"JamAltArtistA2ARetrieval":0.7136075,"ClothoT2ARetrieval":0.00466,"CREMADPairClassification":0.551806,"FSD2019Kaggle":0.080966,"CommonVoiceMini21T2ARetrieval":0.023956153846153846,"SpeechCommandsZeroshotv0.02":0.097202,"CREMA_DClustering":0.108467,"MInDS14":0.08685550000000002,"UrbanSound8KT2ARetrieval":0.00098,"GTZANAudioReranking":0.485542,"BeijingOpera":0.622252,"CommonLanguageAgeDetection":0.1762,"FleursT2ARetrieval":0.009171372549019608,"CREMA_D":0.308656,"GigaSpeechT2ARetrieval":0.00074,"RavdessZeroshot":0.133333,"SpokenSQuADT2ARetrieval":0.04333,"JamAltLyricA2TRetrieval":0.0093875},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.30556257327141384}},"trainedOnTasks":[],"experiments":null},{"rank":31,"model":{"name":"openai/whisper-small","url":"https://huggingface.co/openai/whisper-small","zeroShotPct":100,"activeParamsB":0.244,"totalParamsB":0.244,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-09-27","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Assamese","Azerbaijani","Bashkir","Basque","Belarusian","Bengali","Bosnian","Breton","Bulgarian","Burmese","Catalan","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Faroese","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hausa","Hawaiian (Latin)","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Korean","Lao","Latin","Latvian","Lingala","Lithuanian","Luxembourgish","Macedonian","Malagasy (Latin)","Malay (macrolanguage)","Malayalam","Maltese","Maori","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian Bokmål","Norwegian Nynorsk","Occitan (post 1500)","Panjabi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Sanskrit","Serbian","Shona","Sindhi","Sinhala","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tajik","Tamil","Tatar","Telugu","Thai","Tibetan","Turkish","Turkmen","Ukrainian","Urdu","Uzbek","Vietnamese","Welsh","Yiddish","Yoruba"],"citation":"\n@inproceedings{radford2022robustspeechrecognitionlargescale,\n  archiveprefix = {arXiv},\n  author = {Alec Radford and Jong Wook Kim and Tao Xu and Greg Brockman and Christine McLeavey and Ilya Sutskever},\n  booktitle = {Proceedings of the 40th International Conference on Machine Learning},\n  eprint = {2212.04356},\n  pages = {28492--28518},\n  primaryclass = {eess.AS},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Robust Speech Recognition via Large-Scale Weak Supervision},\n  url = {https://proceedings.mlr.press/v202/radford23a.html},\n  volume = {202},\n  year = {2023},\n}","memoryUsageMb":922.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.244,"totalParamsB":0.244,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.5037780343137256,"AudioReranking":0.641627,"AudioClustering":0.038774333333333334,"AudioPairClassification":0.525541,"AudioMultilabelClassification":0.513812},"scoresByTask":{"VoxCelebSA":0.324456,"GTZANGenre":0.715,"VehicleSoundClustering":0.01292,"VoxPopuliGenderClustering":0.000105,"MridinghamTonic":0.454917,"VoxPopuliAccentPairClassification":0.523648,"NMSQAPairClassification":0.468116,"IEMOCAPGender":0.699273,"VoxPopuliLanguageID":0.992,"BirdCLEF":0.264,"SIBFLEURS":0.19066496078431372,"JamAltArtistA2ARetrieval":0.699585,"CREMADPairClassification":0.584859,"FSD2019Kaggle":0.513812,"CREMA_DClustering":0.103298,"MInDS14":0.3695154166666667,"GTZANAudioReranking":0.641627,"BeijingOpera":0.881294,"CommonLanguageAgeDetection":0.1585,"CREMA_D":0.491938},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4513534935500516}},"trainedOnTasks":[],"experiments":null},{"rank":32,"model":{"name":"openai/whisper-large-v3","url":"https://huggingface.co/openai/whisper-large-v3","zeroShotPct":100,"activeParamsB":1.55,"totalParamsB":1.55,"embeddingDim":1280,"maxTokens":null,"releaseDate":"2022-09-27","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Assamese","Azerbaijani","Bashkir","Basque","Belarusian","Bengali","Bosnian","Breton","Bulgarian","Burmese","Catalan","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Faroese","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hausa","Hawaiian (Latin)","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Korean","Lao","Latin","Latvian","Lingala","Lithuanian","Luxembourgish","Macedonian","Malagasy (Latin)","Malay (macrolanguage)","Malayalam","Maltese","Maori","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian Bokmål","Norwegian Nynorsk","Occitan (post 1500)","Panjabi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Sanskrit","Serbian","Shona","Sindhi","Sinhala","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tajik","Tamil","Tatar","Telugu","Thai","Tibetan","Turkish","Turkmen","Ukrainian","Urdu","Uzbek","Vietnamese","Welsh","Yiddish","Yoruba"],"citation":"\n@inproceedings{radford2022robustspeechrecognitionlargescale,\n  archiveprefix = {arXiv},\n  author = {Alec Radford and Jong Wook Kim and Tao Xu and Greg Brockman and Christine McLeavey and Ilya Sutskever},\n  booktitle = {Proceedings of the 40th International Conference on Machine Learning},\n  eprint = {2212.04356},\n  pages = {28492--28518},\n  primaryclass = {eess.AS},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Robust Speech Recognition via Large-Scale Weak Supervision},\n  url = {https://proceedings.mlr.press/v202/radford23a.html},\n  volume = {202},\n  year = {2023},\n}","memoryUsageMb":5887.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":1.55,"totalParamsB":1.55,"embeddingDim":1280,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4827372295008912,"AudioReranking":0.639337,"AudioClustering":0.033639333333333334,"AudioPairClassification":0.5249166666666666,"AudioMultilabelClassification":0.475849},"scoresByTask":{"VoxCelebSA":0.314591,"GTZANGenre":0.719,"VehicleSoundClustering":0.013834,"VoxPopuliGenderClustering":0.000065,"MridinghamTonic":0.375376,"VoxPopuliAccentPairClassification":0.515548,"NMSQAPairClassification":0.484145,"IEMOCAPGender":0.602549,"VoxPopuliLanguageID":0.982,"BirdCLEF":0.216,"SIBFLEURS":0.22948927450980391,"JamAltArtistA2ARetrieval":0.7287475,"CREMADPairClassification":0.575057,"FSD2019Kaggle":0.475849,"CREMA_DClustering":0.087019,"MInDS14":0.32881125,"GTZANAudioReranking":0.639337,"BeijingOpera":0.860106,"CommonLanguageAgeDetection":0.1928,"CREMA_D":0.489387},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4396769486584107}},"trainedOnTasks":[],"experiments":null},{"rank":33,"model":{"name":"openai/whisper-tiny","url":"https://huggingface.co/openai/whisper-tiny","zeroShotPct":100,"activeParamsB":0.039,"totalParamsB":0.039,"embeddingDim":384,"maxTokens":null,"releaseDate":"2022-09-27","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Assamese","Azerbaijani","Bashkir","Basque","Belarusian","Bengali","Bosnian","Breton","Bulgarian","Burmese","Catalan","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Faroese","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hausa","Hawaiian (Latin)","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Korean","Lao","Latin","Latvian","Lingala","Lithuanian","Luxembourgish","Macedonian","Malagasy (Latin)","Malay (macrolanguage)","Malayalam","Maltese","Maori","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian Bokmål","Norwegian Nynorsk","Occitan (post 1500)","Panjabi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Sanskrit","Serbian","Shona","Sindhi","Sinhala","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tajik","Tamil","Tatar","Telugu","Thai","Tibetan","Turkish","Turkmen","Ukrainian","Urdu","Uzbek","Vietnamese","Welsh","Yiddish","Yoruba"],"citation":"\n@inproceedings{radford2022robustspeechrecognitionlargescale,\n  archiveprefix = {arXiv},\n  author = {Alec Radford and Jong Wook Kim and Tao Xu and Greg Brockman and Christine McLeavey and Ilya Sutskever},\n  booktitle = {Proceedings of the 40th International Conference on Machine Learning},\n  eprint = {2212.04356},\n  pages = {28492--28518},\n  primaryclass = {eess.AS},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Robust Speech Recognition via Large-Scale Weak Supervision},\n  url = {https://proceedings.mlr.press/v202/radford23a.html},\n  volume = {202},\n  year = {2023},\n}","memoryUsageMb":144.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.039,"totalParamsB":0.039,"embeddingDim":384,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.472617825311943,"AudioReranking":0.633982,"AudioClustering":0.07413499999999999,"AudioPairClassification":0.5148166666666666,"AudioMultilabelClassification":0.413679},"scoresByTask":{"VoxCelebSA":0.296603,"GTZANGenre":0.684,"VehicleSoundClustering":0.123883,"VoxPopuliGenderClustering":0.000359,"MridinghamTonic":0.440302,"VoxPopuliAccentPairClassification":0.518697,"NMSQAPairClassification":0.450565,"IEMOCAPGender":0.688812,"VoxPopuliLanguageID":0.966,"BirdCLEF":0.251,"SIBFLEURS":0.15773074509803922,"JamAltArtistA2ARetrieval":0.6915625,"CREMADPairClassification":0.575188,"FSD2019Kaggle":0.413679,"CREMA_DClustering":0.098163,"MInDS14":0.25173233333333334,"GTZANAudioReranking":0.633982,"BeijingOpera":0.83883,"CommonLanguageAgeDetection":0.1645,"CREMA_D":0.459286},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.43637871465428285}},"trainedOnTasks":[],"experiments":null},{"rank":34,"model":{"name":"matthewagi/HeAR-s1.1","url":"https://huggingface.co/matthewagi/HeAR-s1.1","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.022,"embeddingDim":384,"maxTokens":null,"releaseDate":"2026-03-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":true,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":[],"citation":null,"memoryUsageMb":84.0,"license":"https://developers.google.com/health-ai-developer-foundations/terms","publicTrainingCode":"https://github.com/Matthew-agi/hear-distillation-datalake","publicTrainingData":"https://huggingface.co/datasets/laion/LAION-Audio-300M","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["audio","timm"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.022,"embeddingDim":384,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.40876865819964353,"AudioReranking":0.59835,"AudioClustering":0.06012133333333334,"AudioPairClassification":0.5438156666666666},"scoresByTask":{"VoxCelebSA":0.290828,"GTZANGenre":0.521,"VehicleSoundClustering":0.03828,"VoxPopuliGenderClustering":0.00059,"MridinghamTonic":0.450619,"VoxPopuliAccentPairClassification":0.523919,"NMSQAPairClassification":0.534887,"IEMOCAPGender":0.719992,"VoxPopuliLanguageID":0.364,"BirdCLEF":0.22,"SIBFLEURS":0.27898549019607843,"JamAltArtistA2ARetrieval":0.90025,"CREMADPairClassification":0.572641,"CREMA_DClustering":0.141494,"MInDS14":0.12412675000000001,"GTZANAudioReranking":0.59835,"BeijingOpera":0.915426,"CommonLanguageAgeDetection":0.1538,"CREMA_D":0.457678},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.41088769685242515}},"trainedOnTasks":[],"experiments":null},{"rank":35,"model":{"name":"MIT/ast-finetuned-audioset-10-10-0.4593","url":"https://huggingface.co/MIT/ast-finetuned-audioset-10-10-0.4593","zeroShotPct":100,"activeParamsB":0.087,"totalParamsB":0.087,"embeddingDim":768,"maxTokens":null,"releaseDate":"2021-07-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{gong2021astaudiospectrogramtransformer,\n  author = {Gong, Yuan and Chung, Yu-An and Glass, James},\n  booktitle = {Interspeech 2021},\n  collection = {interspeech_2021},\n  doi = {10.21437/interspeech.2021-698},\n  month = {Aug},\n  pages = {571–575},\n  publisher = {ISCA},\n  series = {interspeech_2021},\n  title = {AST: Audio Spectrogram Transformer},\n  url = {https://doi.org/10.21437/interspeech.2021-698},\n  year = {2021},\n}","memoryUsageMb":330.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/YuanGongND/ast","publicTrainingData":"https://research.google.com/audioset/dataset/index.html","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["AudioSetMini"]},"zeroShotPct":100,"activeParamsB":0.087,"totalParamsB":0.087,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.456959444741533,"AudioReranking":0.77646,"AudioClustering":0.06866166666666666,"AudioPairClassification":0.512191,"AudioMultilabelClassification":0.726825},"scoresByTask":{"VoxCelebSA":0.283278,"GTZANGenre":0.807,"VehicleSoundClustering":0.133667,"VoxPopuliGenderClustering":0.000304,"MridinghamTonic":0.541632,"VoxPopuliAccentPairClassification":0.518348,"NMSQAPairClassification":0.475602,"IEMOCAPGender":0.870006,"VoxPopuliLanguageID":0.378,"BirdCLEF":0.452,"SIBFLEURS":0.14204839215686274,"JamAltArtistA2ARetrieval":0.9463475,"CREMADPairClassification":0.542623,"FSD2019Kaggle":0.726825,"CREMA_DClustering":0.072014,"MInDS14":0.0756965,"GTZANAudioReranking":0.77646,"BeijingOpera":0.970301,"CommonLanguageAgeDetection":0.1282,"CREMA_D":0.378392},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4469431259029928}},"trainedOnTasks":[],"experiments":null},{"rank":36,"model":{"name":"google/vggish","url":"https://github.com/tensorflow/models/tree/master/research/audioset/vggish","zeroShotPct":100,"activeParamsB":0.072,"totalParamsB":0.072,"embeddingDim":128,"maxTokens":null,"releaseDate":"2019-06-13","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{hershey2017cnn,\n    author = {Hershey, Shawn and Chaudhuri, Sourish and Ellis, Daniel P. W. and Gemmeke, Jort F. and Jansen, Aren and Moore, R. Channing and Plakal, Manoj and Platt, Devin and Saurous, Rif A. and Seybold, Bryan and Slaney, Malcolm and Weiss, Ron J. and Wilson, Kevin},\n    title = {CNN architectures for large-scale audio classification},\n    year = {2017},\n    publisher = {IEEE Press},\n    url = {https://doi.org/10.1109/ICASSP.2017.7952132},\n    doi = {10.1109/ICASSP.2017.7952132},\n    booktitle = {2017 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n    pages = {131–135},\n    numpages = {5},\n    location = {New Orleans, LA, USA}\n}\n","memoryUsageMb":275.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/tensorflow/models/tree/master/research/audioset/vggish","publicTrainingData":"https://research.google.com/audioset/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["torch-vggish-yamnet"],"trainingDatasets":["AudioSetMini"]},"zeroShotPct":100,"activeParamsB":0.072,"totalParamsB":0.072,"embeddingDim":128,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.39213565240641707,"AudioReranking":0.78668,"AudioClustering":0.07818533333333334,"AudioPairClassification":0.527991,"AudioMultilabelClassification":0.470438},"scoresByTask":{"VoxCelebSA":0.270804,"GTZANGenre":0.793,"VehicleSoundClustering":0.044608,"VoxPopuliGenderClustering":0.082016,"MridinghamTonic":0.327654,"VoxPopuliAccentPairClassification":0.548289,"NMSQAPairClassification":0.477822,"IEMOCAPGender":0.915431,"VoxPopuliLanguageID":0.32,"BirdCLEF":0.105,"SIBFLEURS":0.14067267647058823,"JamAltArtistA2ARetrieval":0.8898125,"CREMADPairClassification":0.557862,"FSD2019Kaggle":0.470438,"CREMA_DClustering":0.107932,"MInDS14":0.06904249999999999,"GTZANAudioReranking":0.78668,"BeijingOpera":0.87695,"CommonLanguageAgeDetection":0.14705,"CREMA_D":0.347888},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.41097440402476787}},"trainedOnTasks":[],"experiments":null},{"rank":37,"model":{"name":"microsoft/wavlm-large","url":"https://huggingface.co/microsoft/wavlm-large","zeroShotPct":96,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2022-07-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{Chen_2022,\n   title={WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing},\n   volume={16},\n   ISSN={1941-0484},\n   url={http://dx.doi.org/10.1109/JSTSP.2022.3188113},\n   DOI={10.1109/jstsp.2022.3188113},\n   number={6},\n   journal={IEEE Journal of Selected Topics in Signal Processing},\n   publisher={Institute of Electrical and Electronics Engineers (IEEE)},\n   author={Chen, Sanyuan and Wang, Chengyi and Chen, Zhengyang and Wu, Yu and Liu, Shujie and Chen, Zhuo and Li, Jinyu and Kanda, Naoyuki and Yoshioka, Takuya and Xiao, Xiong and Wu, Jian and Zhou, Long and Ren, Shuo and Qian, Yanmin and Qian, Yao and Wu, Jian and Zeng, Michael and Yu, Xiangzhan and Wei, Furu},\n   year={2022},\n   month=oct, pages={1505–1518} }\n","memoryUsageMb":1208.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["GigaSpeechA2TRetrieval","Libri-Light","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4035227401960784,"AudioReranking":0.687882,"AudioClustering":0.024456666666666668,"AudioPairClassification":0.5231686666666667,"AudioMultilabelClassification":0.406797},"scoresByTask":{"VoxCelebSA":0.334351,"GTZANGenre":0.677,"VehicleSoundClustering":0.001755,"VoxPopuliGenderClustering":0.000651,"MridinghamTonic":0.314178,"VoxPopuliAccentPairClassification":0.521648,"NMSQAPairClassification":0.489843,"IEMOCAPGender":0.63054,"VoxPopuliLanguageID":0.89,"BirdCLEF":0.198,"SIBFLEURS":0.1254475588235294,"JamAltArtistA2ARetrieval":0.7851675,"CREMADPairClassification":0.558015,"FSD2019Kaggle":0.406797,"CREMA_DClustering":0.070964,"MInDS14":0.13458258333333334,"GTZANAudioReranking":0.687882,"BeijingOpera":0.576773,"CommonLanguageAgeDetection":0.1686,"CREMA_D":0.389278},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3976145074819401}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":38,"model":{"name":"google/yamnet","url":"https://tfhub.dev/google/yamnet/1","zeroShotPct":100,"activeParamsB":0.004,"totalParamsB":0.004,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2020-10-06","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{audioset,\n  title={Audio set: An ontology and human-labeled dataset for audio events},\n  author={Gemmeke, Jort F and Ellis, Daniel PW and Freedman, Dylan and Jansen, Aren and Lawrence, Wade and Moore, R Channing and Plakal, Manoj and Ritter, Marvin},\n  booktitle={2017 IEEE international conference on acoustics, speech and signal processing (ICASSP)},\n  pages={776--780},\n  year={2017},\n  organization={IEEE}\n}\n","memoryUsageMb":14.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/tensorflow/models/tree/master/research/audioset/yamnet","publicTrainingData":"https://research.google.com/audioset/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["torch-vggish-yamnet"],"trainingDatasets":["AudioSetMini"]},"zeroShotPct":100,"activeParamsB":0.004,"totalParamsB":0.004,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.37807732397504457,"AudioReranking":0.81735,"AudioClustering":0.015557333333333333,"AudioPairClassification":0.5461726666666666,"AudioMultilabelClassification":0.573766},"scoresByTask":{"VoxCelebSA":0.274575,"GTZANGenre":0.793,"VehicleSoundClustering":0.00351,"VoxPopuliGenderClustering":0.002928,"MridinghamTonic":0.351585,"VoxPopuliAccentPairClassification":0.524236,"NMSQAPairClassification":0.582803,"IEMOCAPGender":0.769101,"VoxPopuliLanguageID":0.272,"BirdCLEF":0.168,"SIBFLEURS":0.13906064705882354,"JamAltArtistA2ARetrieval":0.79762,"CREMADPairClassification":0.531479,"FSD2019Kaggle":0.573766,"CREMA_DClustering":0.040234,"MInDS14":0.06718291666666666,"GTZANAudioReranking":0.81735,"BeijingOpera":0.893883,"CommonLanguageAgeDetection":0.1718,"CREMA_D":0.258663},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3925795033539732}},"trainedOnTasks":[],"experiments":null},{"rank":39,"model":{"name":"facebook/hubert-base-ls960","url":"https://huggingface.co/facebook/hubert-base-ls960","zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2021-06-14","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{hsu2021hubertselfsupervisedspeechrepresentation,\n  author = {Hsu, Wei-Ning and Bolte, Benjamin and Tsai, Yao-Hung Hubert and Lakhotia, Kushal and Salakhutdinov, Ruslan and Mohamed, Abdelrahman},\n  doi = {10.1109/taslp.2021.3122291},\n  issn = {2329-9304},\n  journal = {IEEE/ACM Transactions on Audio, Speech, and Language Processing},\n  pages = {3451–3460},\n  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},\n  title = {HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units},\n  url = {https://doi.org/10.1109/taslp.2021.3122291},\n  volume = {29},\n  year = {2021},\n}","memoryUsageMb":360.0,"license":"mit","publicTrainingCode":"https://github.com/pytorch/fairseq/tree/master/examples/hubert","publicTrainingData":"https://www.openslr.org/12","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.40228734180035647,"AudioReranking":0.662893,"AudioClustering":0.027081666666666667,"AudioPairClassification":0.5192293333333333,"AudioMultilabelClassification":0.308237},"scoresByTask":{"VoxCelebSA":0.329125,"GTZANGenre":0.69,"VehicleSoundClustering":0.001305,"VoxPopuliGenderClustering":0.00065,"MridinghamTonic":0.315753,"VoxPopuliAccentPairClassification":0.512886,"NMSQAPairClassification":0.484268,"IEMOCAPGender":0.76193,"VoxPopuliLanguageID":0.676,"BirdCLEF":0.183,"SIBFLEURS":0.1311168431372549,"JamAltArtistA2ARetrieval":0.7697275,"CREMADPairClassification":0.560534,"FSD2019Kaggle":0.308237,"CREMA_DClustering":0.07929,"MInDS14":0.12574591666666665,"GTZANAudioReranking":0.662893,"BeijingOpera":0.657624,"CommonLanguageAgeDetection":0.15,"CREMA_D":0.404866},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3945639084107327}},"trainedOnTasks":[],"experiments":null},{"rank":40,"model":{"name":"facebook/wav2vec2-xls-r-2b","url":"https://huggingface.co/facebook/wav2vec2-xls-r-2b","zeroShotPct":96,"activeParamsB":2.0,"totalParamsB":2.0,"embeddingDim":1920,"maxTokens":null,"releaseDate":"2024-09-10","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Abkhazian","Afrikaans","Albanian","Amharic (Latin)","Arabic (Latin)","Armenian (Latin)","Assamese (Latin)","Azerbaijani","Bashkir","Basque","Belarusian (Latin)","Bengali","Bosnian","Breton","Bulgarian (Latin)","Burmese (Latin)","Catalan","Cebuano","Chinese (Latin)","Chuvash","Croatian","Czech","Danish","Dhivehi","Dutch","English","Esperanto","Estonian","Faroese","Finnish","French","Galician","Georgian (Latin)","German","Guarani","Gujarati (Latin)","Haitian","Hausa","Hawaiian (Latin)","Hebrew (Latin)","Hindi","Hungarian","Icelandic","Indonesian","Interlingua (International Auxiliary Language Association)","Irish","Italian","Japanese (Latin)","Javanese","Kannada (Latin)","Kazakh (Latin)","Khmer (Latin)","Kinyarwanda","Kirghiz (Latin)","Modern Greek (1453-) (Latin)"],"citation":"\n@inproceedings{babu2021xlsrselfsupervisedcrosslingualspeech,\n  author = {Babu, Arun and Wang, Changhan and Tjandra, Andros and Lakhotia, Kushal and Xu, Qiantong and Goyal, Naman and Singh, Kritika and von Platen, Patrick and Saraf, Yatharth and Pino, Juan and Baevski, Alexei and Conneau, Alexis and Auli, Michael},\n  booktitle = {Interspeech 2022},\n  collection = {interspeech_2022},\n  doi = {10.21437/interspeech.2022-143},\n  month = {Sept},\n  pages = {2278–2282},\n  publisher = {ISCA},\n  series = {interspeech_2022},\n  title = {XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale},\n  url = {https://doi.org/10.21437/interspeech.2022-143},\n  year = {2022},\n}","memoryUsageMb":9000.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/pytorch/fairseq","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["CommonVoiceMini17A2TRetrieval","VoxLingua107_Top10","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":2.0,"totalParamsB":2.0,"embeddingDim":1920,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.45129493805704096,"AudioReranking":0.628668,"AudioClustering":0.014229666666666668,"AudioPairClassification":0.50802,"AudioMultilabelClassification":0.427766},"scoresByTask":{"VoxCelebSA":0.287336,"GTZANGenre":0.732,"VehicleSoundClustering":0.009359,"VoxPopuliGenderClustering":0.000245,"MridinghamTonic":0.446177,"VoxPopuliAccentPairClassification":0.509671,"NMSQAPairClassification":0.486125,"IEMOCAPGender":0.702665,"VoxPopuliLanguageID":0.748,"BirdCLEF":0.312,"SIBFLEURS":0.1360769019607843,"JamAltArtistA2ARetrieval":0.6224375,"CREMADPairClassification":0.528264,"FSD2019Kaggle":0.427766,"CREMA_DClustering":0.033085,"MInDS14":0.13639041666666665,"GTZANAudioReranking":0.628668,"BeijingOpera":0.830585,"CommonLanguageAgeDetection":0.1736,"CREMA_D":0.459414},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.40958414834881324}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":41,"model":{"name":"facebook/wav2vec2-lv-60-espeak-cv-ft","url":"https://huggingface.co/facebook/wav2vec2-lv-60-espeak-cv-ft","zeroShotPct":100,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2020-10-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{baevski2020wav2vec20frameworkselfsupervised,\n  archiveprefix = {arXiv},\n  author = {Alexei Baevski and Yuhao Zhou and Abdelrahman Mohamed and Michael Auli},\n  booktitle = {NeurIPS},\n  eprint = {2006.11477},\n  title = {wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations},\n  url = {https://proceedings.neurips.cc/paper/2020/hash/92d1e1eb1cd6f9fba3227870bb6d7f07-Abstract.html},\n  year = {2020},\n}","memoryUsageMb":1209.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["CommonVoiceMini17A2TRetrieval"]},"zeroShotPct":100,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4409799108734403,"AudioReranking":0.556419,"AudioClustering":0.015693333333333333,"AudioPairClassification":0.5367523333333333,"AudioMultilabelClassification":0.35031},"scoresByTask":{"VoxCelebSA":0.281249,"GTZANGenre":0.552,"VehicleSoundClustering":0.023741,"VoxPopuliGenderClustering":0.000164,"MridinghamTonic":0.372942,"VoxPopuliAccentPairClassification":0.550836,"NMSQAPairClassification":0.527305,"IEMOCAPGender":0.683138,"VoxPopuliLanguageID":0.97,"BirdCLEF":0.205,"SIBFLEURS":0.14815935294117646,"JamAltArtistA2ARetrieval":0.5036725,"CREMADPairClassification":0.532116,"FSD2019Kaggle":0.35031,"CREMA_DClustering":0.023175,"MInDS14":0.2612626666666667,"GTZANAudioReranking":0.556419,"BeijingOpera":0.873227,"CommonLanguageAgeDetection":0.1543,"CREMA_D":0.349501},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.39832671155830757}},"trainedOnTasks":[],"experiments":null},{"rank":42,"model":{"name":"facebook/mms-1b-l1107","url":"https://huggingface.co/facebook/mms-1b-l1107","zeroShotPct":100,"activeParamsB":1.0,"totalParamsB":1.0,"embeddingDim":1280,"maxTokens":null,"releaseDate":"2023-05-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{pratap2023scalingspeechtechnology1000,\n  archiveprefix = {arXiv},\n  author = {Vineel Pratap and Andros Tjandra and Bowen Shi and Paden Tomasello and Arun Babu and Sayani Kundu and Ali Elkahky and Zhaoheng Ni and Apoorv Vyas and Maryam Fazel-Zarandi and Alexei Baevski and Yossi Adi and Xiaohui Zhang and Wei-Ning Hsu and Alexis Conneau and Michael Auli},\n  eprint = {2305.13516},\n  journal = {J. Mach. Learn. Res.},\n  pages = {97:1-97:52},\n  title = {Scaling Speech Technology to 1, 000+ Languages},\n  url = {https://jmlr.org/papers/v25/23-1318.html},\n  volume = {25},\n  year = {2024},\n}","memoryUsageMb":3680.0,"license":"mit","publicTrainingCode":"https://github.com/facebookresearch/fairseq/tree/main/examples/mms","publicTrainingData":"https://github.com/facebookresearch/fairseq/tree/main/examples/mms#data","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["FleursA2TRetrieval"]},"zeroShotPct":100,"activeParamsB":1.0,"totalParamsB":1.0,"embeddingDim":1280,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.44671943360071303,"AudioReranking":0.587874,"AudioClustering":0.009511,"AudioPairClassification":0.514964,"AudioMultilabelClassification":0.347033},"scoresByTask":{"VoxCelebSA":0.306194,"GTZANGenre":0.583,"VehicleSoundClustering":0.004373,"VoxPopuliGenderClustering":0.001637,"MridinghamTonic":0.307008,"VoxPopuliAccentPairClassification":0.514024,"NMSQAPairClassification":0.500694,"IEMOCAPGender":0.546867,"VoxPopuliLanguageID":0.952,"BirdCLEF":0.21,"SIBFLEURS":0.21601568627450982,"JamAltArtistA2ARetrieval":0.52196,"CREMADPairClassification":0.530174,"FSD2019Kaggle":0.347033,"CREMA_DClustering":0.022523,"MInDS14":0.5270690833333334,"GTZANAudioReranking":0.587874,"BeijingOpera":0.78883,"CommonLanguageAgeDetection":0.1848,"CREMA_D":0.29213},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3998511984004128}},"trainedOnTasks":[],"experiments":null},{"rank":43,"model":{"name":"speechbrain/cnn14-esc50","url":"https://huggingface.co/speechbrain/cnn14-esc50","zeroShotPct":100,"activeParamsB":0.081,"totalParamsB":0.081,"embeddingDim":2048,"maxTokens":null,"releaseDate":"2022-11-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{wang2022CRL,\n    title={Learning Representations for New Sound Classes With Continual Self-Supervised Learning},\n    author={Zhepei Wang, Cem Subakan, Xilin Jiang, Junkai Wu, Efthymios Tzinis, Mirco Ravanelli, Paris Smaragdis},\n    year={2022},\n    booktitle={Accepted to IEEE Signal Processing Letters}\n}\n","memoryUsageMb":308.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/speechbrain/speechbrain","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["speechbrain"],"trainingDatasets":["ESC50"]},"zeroShotPct":100,"activeParamsB":0.081,"totalParamsB":0.081,"embeddingDim":2048,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.31847373395721923,"AudioReranking":0.53777,"AudioClustering":0.07370366666666667,"AudioPairClassification":0.5422253333333333,"AudioMultilabelClassification":0.277021},"scoresByTask":{"VoxCelebSA":0.230747,"GTZANGenre":0.414,"VehicleSoundClustering":0.024725,"VoxPopuliGenderClustering":0.07717,"MridinghamTonic":0.260576,"VoxPopuliAccentPairClassification":0.525752,"NMSQAPairClassification":0.530214,"IEMOCAPGender":0.593693,"VoxPopuliLanguageID":0.298,"BirdCLEF":0.112,"SIBFLEURS":0.17489732352941176,"JamAltArtistA2ARetrieval":0.769995,"CREMADPairClassification":0.57071,"FSD2019Kaggle":0.277021,"CREMA_DClustering":0.119216,"MInDS14":0.09019575,"GTZANAudioReranking":0.53777,"BeijingOpera":0.838564,"CommonLanguageAgeDetection":0.1452,"CREMA_D":0.345338},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3504612143962848}},"trainedOnTasks":[],"experiments":null},{"rank":44,"model":{"name":"facebook/mms-1b-all","url":"https://huggingface.co/facebook/mms-1b-all","zeroShotPct":100,"activeParamsB":1.0,"totalParamsB":1.0,"embeddingDim":1280,"maxTokens":null,"releaseDate":"2023-05-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Arabic","English","French","German","Mandarin Chinese","Russian","Spanish"],"citation":"\n@article{pratap2023scalingspeechtechnology1000,\n  archiveprefix = {arXiv},\n  author = {Vineel Pratap and Andros Tjandra and Bowen Shi and Paden Tomasello and Arun Babu and Sayani Kundu and Ali Elkahky and Zhaoheng Ni and Apoorv Vyas and Maryam Fazel-Zarandi and Alexei Baevski and Yossi Adi and Xiaohui Zhang and Wei-Ning Hsu and Alexis Conneau and Michael Auli},\n  eprint = {2305.13516},\n  journal = {J. Mach. Learn. Res.},\n  pages = {97:1-97:52},\n  title = {Scaling Speech Technology to 1, 000+ Languages},\n  url = {https://jmlr.org/papers/v25/23-1318.html},\n  volume = {25},\n  year = {2024},\n}","memoryUsageMb":3680.0,"license":"mit","publicTrainingCode":"https://github.com/facebookresearch/fairseq/tree/main/examples/mms","publicTrainingData":"https://github.com/facebookresearch/fairseq/tree/main/examples/mms#data","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["FleursA2TRetrieval"]},"zeroShotPct":100,"activeParamsB":1.0,"totalParamsB":1.0,"embeddingDim":1280,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4532385806595365,"AudioReranking":0.594962,"AudioClustering":0.016388999999999997,"AudioPairClassification":0.5279706666666666,"AudioMultilabelClassification":0.310338},"scoresByTask":{"VoxCelebSA":0.283568,"GTZANGenre":0.571,"VehicleSoundClustering":0.003203,"VoxPopuliGenderClustering":0.000566,"MridinghamTonic":0.274186,"VoxPopuliAccentPairClassification":0.512387,"NMSQAPairClassification":0.539002,"IEMOCAPGender":0.56231,"VoxPopuliLanguageID":0.994,"BirdCLEF":0.211,"SIBFLEURS":0.2413314705882353,"JamAltArtistA2ARetrieval":0.51978,"CREMADPairClassification":0.532523,"FSD2019Kaggle":0.310338,"CREMA_DClustering":0.045398,"MInDS14":0.5853929166666666,"GTZANAudioReranking":0.594962,"BeijingOpera":0.788121,"CommonLanguageAgeDetection":0.1584,"CREMA_D":0.316315},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.4070234414344685}},"trainedOnTasks":[],"experiments":null},{"rank":45,"model":{"name":"vitouphy/wav2vec2-xls-r-300m-phoneme","url":"https://huggingface.co/vitouphy/wav2vec2-xls-r-300m-phoneme","zeroShotPct":96,"activeParamsB":0.3,"totalParamsB":0.3,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2022-05-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{babu2021xlsrselfsupervisedcrosslingualspeech,\n  author = {Babu, Arun and Wang, Changhan and Tjandra, Andros and Lakhotia, Kushal and Xu, Qiantong and Goyal, Naman and Singh, Kritika and von Platen, Patrick and Saraf, Yatharth and Pino, Juan and Baevski, Alexei and Conneau, Alexis and Auli, Michael},\n  booktitle = {Interspeech 2022},\n  collection = {interspeech_2022},\n  doi = {10.21437/interspeech.2022-143},\n  month = {Sept},\n  pages = {2278–2282},\n  publisher = {ISCA},\n  series = {interspeech_2022},\n  title = {XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale},\n  url = {https://doi.org/10.21437/interspeech.2022-143},\n  year = {2022},\n}","memoryUsageMb":1200.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["CommonVoiceMini17A2TRetrieval","VoxLingua107_Top10","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":0.3,"totalParamsB":0.3,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.40505959581105166,"AudioReranking":0.58876,"AudioClustering":0.021170333333333333,"AudioPairClassification":0.5040523333333334,"AudioMultilabelClassification":0.361828},"scoresByTask":{"VoxCelebSA":0.271369,"GTZANGenre":0.609,"VehicleSoundClustering":0.012399,"VoxPopuliGenderClustering":0.002458,"MridinghamTonic":0.339687,"VoxPopuliAccentPairClassification":0.510295,"NMSQAPairClassification":0.470849,"IEMOCAPGender":0.511606,"VoxPopuliLanguageID":0.772,"BirdCLEF":0.175,"SIBFLEURS":0.1631054705882353,"JamAltArtistA2ARetrieval":0.617455,"CREMADPairClassification":0.531013,"FSD2019Kaggle":0.361828,"CREMA_DClustering":0.048654,"MInDS14":0.21704408333333333,"GTZANAudioReranking":0.58876,"BeijingOpera":0.860106,"CommonLanguageAgeDetection":0.1996,"CREMA_D":0.337138},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.38092308178534573}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":46,"model":{"name":"facebook/mms-1b-fl102","url":"https://huggingface.co/facebook/mms-1b-fl102","zeroShotPct":100,"activeParamsB":1.0,"totalParamsB":1.0,"embeddingDim":1280,"maxTokens":null,"releaseDate":"2023-05-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{pratap2023scalingspeechtechnology1000,\n  archiveprefix = {arXiv},\n  author = {Vineel Pratap and Andros Tjandra and Bowen Shi and Paden Tomasello and Arun Babu and Sayani Kundu and Ali Elkahky and Zhaoheng Ni and Apoorv Vyas and Maryam Fazel-Zarandi and Alexei Baevski and Yossi Adi and Xiaohui Zhang and Wei-Ning Hsu and Alexis Conneau and Michael Auli},\n  eprint = {2305.13516},\n  journal = {J. Mach. Learn. Res.},\n  pages = {97:1-97:52},\n  title = {Scaling Speech Technology to 1, 000+ Languages},\n  url = {https://jmlr.org/papers/v25/23-1318.html},\n  volume = {25},\n  year = {2024},\n}","memoryUsageMb":3680.0,"license":"mit","publicTrainingCode":"https://github.com/facebookresearch/fairseq/tree/main/examples/mms","publicTrainingData":"https://github.com/facebookresearch/fairseq/tree/main/examples/mms#data","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["FleursA2TRetrieval"]},"zeroShotPct":100,"activeParamsB":1.0,"totalParamsB":1.0,"embeddingDim":1280,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4675719496434938,"AudioReranking":0.57328,"AudioClustering":0.011231333333333334,"AudioPairClassification":0.5200723333333334,"AudioMultilabelClassification":0.336803},"scoresByTask":{"VoxCelebSA":0.319523,"GTZANGenre":0.568,"VehicleSoundClustering":0.00146,"VoxPopuliGenderClustering":0.000119,"MridinghamTonic":0.30887,"VoxPopuliAccentPairClassification":0.514014,"NMSQAPairClassification":0.520583,"IEMOCAPGender":0.524354,"VoxPopuliLanguageID":0.972,"BirdCLEF":0.199,"SIBFLEURS":0.2619661960784314,"JamAltArtistA2ARetrieval":0.5493,"CREMADPairClassification":0.52562,"FSD2019Kaggle":0.336803,"CREMA_DClustering":0.032115,"MInDS14":0.73271725,"GTZANAudioReranking":0.57328,"BeijingOpera":0.779787,"CommonLanguageAgeDetection":0.1703,"CREMA_D":0.306774},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.41367276031991745}},"trainedOnTasks":[],"experiments":null},{"rank":47,"model":{"name":"facebook/seamless-m4t-v2-large","url":"https://huggingface.co/facebook/seamless-m4t-v2-large","zeroShotPct":100,"activeParamsB":2.038,"totalParamsB":2.3,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2023-11-06","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@misc{communication2023seamlessmultilingualexpressivestreaming,\n  doi = {10.48550/arXiv.2312.05187},\n  title = {Seamless: Multilingual Expressive and Streaming Speech Translation},\n  author = {Seamless Communication and Loïc Barrault and Yu-An Chung and Mariano Coria Meglioli and David Dale and Ning Dong and Mark Duppenthaler and Paul-Ambroise Duquenne and Brian Ellis and Hady Elsahar and Justin Haaheim and John Hoffman and Min-Jae Hwang and Hirofumi Inaguma and Christopher Klaiber and Ilia Kulikov and Pengwei Li and Daniel Licht and Jean Maillard and Ruslan Mavlyutov and Alice Rakotoarison and Kaushik Ram Sadagopan and Abinesh Ramakrishnan and Tuan Tran and Guillaume Wenzek and Yilin Yang and Ethan Ye and Ivan Evtimov and Pierre Fernandez and Cynthia Gao and Prangthip Hansanti and Elahe Kalbassi and Amanda Kallet and Artyom Kozhevnikov and Gabriel Mejia Gonzalez and Robin San Roman and Christophe Touret and Corinne Wong and Carleigh Wood and Bokai Yu and Pierre Andrews and Can Balioglu and Peng-Jen Chen and Marta R. Costa-jussà and Maha Elbayad and Hongyu Gong and Francisco Guzmán and Kevin Heffernan and Somya Jain and Justine Kao and Ann Lee and Xutai Ma and Alex Mourachko and Benjamin Peloquin and Juan Pino and Sravya Popuri and Christophe Ropers and Safiyyah Saleem and Holger Schwenk and Anna Sun and Paden Tomasello and Changhan Wang and Jeff Wang and Skyler Wang and Mary Williamson},\n  year = {2023},\n  eprint = {2312.05187},\n  archivePrefix = {arXiv},\n  primaryClass = {cs.CL},\n  url = {https://arxiv.org/abs/2312.05187},\n}\n","memoryUsageMb":8809.0,"license":"mit","publicTrainingCode":"https://github.com/facebookresearch/seamless_communication","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["seamless-m4t"],"trainingDatasets":["FleursA2TRetrieval"]},"zeroShotPct":100,"activeParamsB":2.038,"totalParamsB":2.3,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.4760045646167557,"AudioReranking":0.502091,"AudioClustering":0.0045309999999999994,"AudioPairClassification":0.512971,"AudioMultilabelClassification":0.287953},"scoresByTask":{"VoxCelebSA":0.43551,"GTZANGenre":0.525,"VehicleSoundClustering":0.004163,"VoxPopuliGenderClustering":0.001084,"MridinghamTonic":0.353743,"VoxPopuliAccentPairClassification":0.514944,"NMSQAPairClassification":0.491255,"IEMOCAPGender":0.537208,"VoxPopuliLanguageID":0.89,"BirdCLEF":0.111,"SIBFLEURS":0.41915446078431373,"JamAltArtistA2ARetrieval":0.55826,"CREMADPairClassification":0.532714,"FSD2019Kaggle":0.287953,"CREMA_DClustering":0.008346,"MInDS14":0.85620275,"GTZANAudioReranking":0.502091,"BeijingOpera":0.661525,"CommonLanguageAgeDetection":0.1656,"CREMA_D":0.281107},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.41310037951496387}},"trainedOnTasks":[],"experiments":null},{"rank":48,"model":{"name":"facebook/wav2vec2-base","url":"https://huggingface.co/facebook/wav2vec2-base","zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2020-10-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{baevski2020wav2vec20frameworkselfsupervised,\n  archiveprefix = {arXiv},\n  author = {Alexei Baevski and Yuhao Zhou and Abdelrahman Mohamed and Michael Auli},\n  booktitle = {NeurIPS},\n  eprint = {2006.11477},\n  title = {wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations},\n  url = {https://proceedings.neurips.cc/paper/2020/hash/92d1e1eb1cd6f9fba3227870bb6d7f07-Abstract.html},\n  year = {2020},\n}","memoryUsageMb":362.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3692817682709448,"AudioReranking":0.58514,"AudioClustering":0.05174633333333333,"AudioPairClassification":0.5172416666666667,"AudioMultilabelClassification":0.249958},"scoresByTask":{"VoxCelebSA":0.288811,"GTZANGenre":0.624,"VehicleSoundClustering":0.030933,"VoxPopuliGenderClustering":0.00008,"MridinghamTonic":0.351446,"VoxPopuliAccentPairClassification":0.521515,"NMSQAPairClassification":0.469956,"IEMOCAPGender":0.688815,"VoxPopuliLanguageID":0.488,"BirdCLEF":0.111,"SIBFLEURS":0.13128345098039215,"JamAltArtistA2ARetrieval":0.67925,"CREMADPairClassification":0.560254,"FSD2019Kaggle":0.249958,"CREMA_DClustering":0.124226,"MInDS14":0.094731,"GTZANAudioReranking":0.58514,"BeijingOpera":0.754078,"CommonLanguageAgeDetection":0.1502,"CREMA_D":0.379735},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3701817605779154}},"trainedOnTasks":[],"experiments":null},{"rank":49,"model":{"name":"microsoft/wavlm-base","url":"https://huggingface.co/microsoft/wavlm-base","zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-07-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{Chen_2022,\n   title={WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing},\n   volume={16},\n   ISSN={1941-0484},\n   url={http://dx.doi.org/10.1109/JSTSP.2022.3188113},\n   DOI={10.1109/jstsp.2022.3188113},\n   number={6},\n   journal={IEEE Journal of Selected Topics in Signal Processing},\n   publisher={Institute of Electrical and Electronics Engineers (IEEE)},\n   author={Chen, Sanyuan and Wang, Chengyi and Chen, Zhengyang and Wu, Yu and Liu, Shujie and Chen, Zhuo and Li, Jinyu and Kanda, Naoyuki and Yoshioka, Takuya and Xiao, Xiong and Wu, Jian and Zhou, Long and Ren, Shuo and Qian, Yanmin and Qian, Yao and Wu, Jian and Zeng, Michael and Yu, Xiangzhan and Wei, Furu},\n   year={2022},\n   month=oct, pages={1505–1518} }\n","memoryUsageMb":361.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["Librispeech"]},"zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.37915409937611405,"AudioReranking":0.591956,"AudioClustering":0.02318266666666667,"AudioPairClassification":0.5305223333333333,"AudioMultilabelClassification":0.201766},"scoresByTask":{"VoxCelebSA":0.336089,"GTZANGenre":0.63,"VehicleSoundClustering":0.004094,"VoxPopuliGenderClustering":0.000408,"MridinghamTonic":0.296838,"VoxPopuliAccentPairClassification":0.51314,"NMSQAPairClassification":0.529964,"IEMOCAPGender":0.673176,"VoxPopuliLanguageID":0.654,"BirdCLEF":0.102,"SIBFLEURS":0.1331318431372549,"JamAltArtistA2ARetrieval":0.62584,"CREMADPairClassification":0.548463,"FSD2019Kaggle":0.201766,"CREMA_DClustering":0.065046,"MInDS14":0.13356525,"GTZANAudioReranking":0.591956,"BeijingOpera":0.657092,"CommonLanguageAgeDetection":0.1537,"CREMA_D":0.401103},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3710318996388029}},"trainedOnTasks":[],"experiments":null},{"rank":50,"model":{"name":"microsoft/wavlm-base-sd","url":"https://huggingface.co/microsoft/wavlm-base-sd","zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-07-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{Chen_2022,\n   title={WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing},\n   volume={16},\n   ISSN={1941-0484},\n   url={http://dx.doi.org/10.1109/JSTSP.2022.3188113},\n   DOI={10.1109/jstsp.2022.3188113},\n   number={6},\n   journal={IEEE Journal of Selected Topics in Signal Processing},\n   publisher={Institute of Electrical and Electronics Engineers (IEEE)},\n   author={Chen, Sanyuan and Wang, Chengyi and Chen, Zhengyang and Wu, Yu and Liu, Shujie and Chen, Zhuo and Li, Jinyu and Kanda, Naoyuki and Yoshioka, Takuya and Xiao, Xiong and Wu, Jian and Zhou, Long and Ren, Shuo and Qian, Yanmin and Qian, Yao and Wu, Jian and Zeng, Michael and Yu, Xiangzhan and Wei, Furu},\n   year={2022},\n   month=oct, pages={1505–1518} }\n","memoryUsageMb":361.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["LibriMix","Librispeech"]},"zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.37915409937611405,"AudioReranking":0.591956,"AudioClustering":0.02318266666666667,"AudioPairClassification":0.5305223333333333,"AudioMultilabelClassification":0.201766},"scoresByTask":{"VoxCelebSA":0.336089,"GTZANGenre":0.63,"VehicleSoundClustering":0.004094,"VoxPopuliGenderClustering":0.000408,"MridinghamTonic":0.296838,"VoxPopuliAccentPairClassification":0.51314,"NMSQAPairClassification":0.529964,"IEMOCAPGender":0.673176,"VoxPopuliLanguageID":0.654,"BirdCLEF":0.102,"SIBFLEURS":0.1331318431372549,"JamAltArtistA2ARetrieval":0.62422,"CREMADPairClassification":0.548463,"FSD2019Kaggle":0.201766,"CREMA_DClustering":0.065046,"MInDS14":0.13356525,"GTZANAudioReranking":0.591956,"BeijingOpera":0.657092,"CommonLanguageAgeDetection":0.1537,"CREMA_D":0.401103},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3709466364809082}},"trainedOnTasks":[],"experiments":null},{"rank":50,"model":{"name":"microsoft/wavlm-base-sv","url":"https://huggingface.co/microsoft/wavlm-base-sv","zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-07-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{Chen_2022,\n   title={WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing},\n   volume={16},\n   ISSN={1941-0484},\n   url={http://dx.doi.org/10.1109/JSTSP.2022.3188113},\n   DOI={10.1109/jstsp.2022.3188113},\n   number={6},\n   journal={IEEE Journal of Selected Topics in Signal Processing},\n   publisher={Institute of Electrical and Electronics Engineers (IEEE)},\n   author={Chen, Sanyuan and Wang, Chengyi and Chen, Zhengyang and Wu, Yu and Liu, Shujie and Chen, Zhuo and Li, Jinyu and Kanda, Naoyuki and Yoshioka, Takuya and Xiao, Xiong and Wu, Jian and Zhou, Long and Ren, Shuo and Qian, Yanmin and Qian, Yao and Wu, Jian and Zeng, Michael and Yu, Xiangzhan and Wei, Furu},\n   year={2022},\n   month=oct, pages={1505–1518} }\n","memoryUsageMb":361.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["Librispeech","VoxCeleb1"]},"zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.37915409937611405,"AudioReranking":0.591956,"AudioClustering":0.02318266666666667,"AudioPairClassification":0.5305223333333333,"AudioMultilabelClassification":0.201766},"scoresByTask":{"VoxCelebSA":0.336089,"GTZANGenre":0.63,"VehicleSoundClustering":0.004094,"VoxPopuliGenderClustering":0.000408,"MridinghamTonic":0.296838,"VoxPopuliAccentPairClassification":0.51314,"NMSQAPairClassification":0.529964,"IEMOCAPGender":0.673176,"VoxPopuliLanguageID":0.654,"BirdCLEF":0.102,"SIBFLEURS":0.1331318431372549,"JamAltArtistA2ARetrieval":0.62422,"CREMADPairClassification":0.548463,"FSD2019Kaggle":0.201766,"CREMA_DClustering":0.065046,"MInDS14":0.13356525,"GTZANAudioReranking":0.591956,"BeijingOpera":0.657092,"CommonLanguageAgeDetection":0.1537,"CREMA_D":0.401103},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3709466364809082}},"trainedOnTasks":[],"experiments":null},{"rank":52,"model":{"name":"facebook/wav2vec2-xls-r-1b","url":"https://huggingface.co/facebook/wav2vec2-xls-r-1b","zeroShotPct":96,"activeParamsB":1.0,"totalParamsB":1.0,"embeddingDim":1280,"maxTokens":null,"releaseDate":"2024-09-10","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Abkhazian","Afrikaans","Albanian","Amharic (Latin)","Arabic (Latin)","Armenian (Latin)","Assamese (Latin)","Azerbaijani","Bashkir","Basque","Belarusian (Latin)","Bengali","Bosnian","Breton","Bulgarian (Latin)","Burmese (Latin)","Catalan","Cebuano","Chinese (Latin)","Chuvash","Croatian","Czech","Danish","Dhivehi","Dutch","English","Esperanto","Estonian","Faroese","Finnish","French","Galician","Georgian (Latin)","German","Guarani","Gujarati (Latin)","Haitian","Hausa","Hawaiian (Latin)","Hebrew (Latin)","Hindi","Hungarian","Icelandic","Indonesian","Interlingua (International Auxiliary Language Association)","Irish","Italian","Japanese (Latin)","Javanese","Kannada (Latin)","Kazakh (Latin)","Khmer (Latin)","Kinyarwanda","Kirghiz (Latin)","Modern Greek (1453-) (Latin)"],"citation":"\n@inproceedings{babu2021xlsrselfsupervisedcrosslingualspeech,\n  author = {Babu, Arun and Wang, Changhan and Tjandra, Andros and Lakhotia, Kushal and Xu, Qiantong and Goyal, Naman and Singh, Kritika and von Platen, Patrick and Saraf, Yatharth and Pino, Juan and Baevski, Alexei and Conneau, Alexis and Auli, Michael},\n  booktitle = {Interspeech 2022},\n  collection = {interspeech_2022},\n  doi = {10.21437/interspeech.2022-143},\n  month = {Sept},\n  pages = {2278–2282},\n  publisher = {ISCA},\n  series = {interspeech_2022},\n  title = {XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale},\n  url = {https://doi.org/10.21437/interspeech.2022-143},\n  year = {2022},\n}","memoryUsageMb":4500.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/pytorch/fairseq","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["CommonVoiceMini17A2TRetrieval","VoxLingua107_Top10","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":1.0,"totalParamsB":1.0,"embeddingDim":1280,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3925027459893048,"AudioReranking":0.534862,"AudioClustering":0.010541,"AudioPairClassification":0.5113196666666667,"AudioMultilabelClassification":0.366448},"scoresByTask":{"VoxCelebSA":0.286185,"GTZANGenre":0.661,"VehicleSoundClustering":0.001275,"VoxPopuliGenderClustering":0.003289,"MridinghamTonic":0.39845,"VoxPopuliAccentPairClassification":0.515137,"NMSQAPairClassification":0.493205,"IEMOCAPGender":0.618193,"VoxPopuliLanguageID":0.628,"BirdCLEF":0.168,"SIBFLEURS":0.12637753921568629,"JamAltArtistA2ARetrieval":0.595375,"CREMADPairClassification":0.525617,"FSD2019Kaggle":0.366448,"CREMA_DClustering":0.027059,"MInDS14":0.12456666666666666,"GTZANAudioReranking":0.534862,"BeijingOpera":0.707447,"CommonLanguageAgeDetection":0.1798,"CREMA_D":0.419511},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3691236424148607}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":53,"model":{"name":"microsoft/wavlm-base-plus","url":"https://huggingface.co/microsoft/wavlm-base-plus","zeroShotPct":96,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-07-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{Chen_2022,\n   title={WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing},\n   volume={16},\n   ISSN={1941-0484},\n   url={http://dx.doi.org/10.1109/JSTSP.2022.3188113},\n   DOI={10.1109/jstsp.2022.3188113},\n   number={6},\n   journal={IEEE Journal of Selected Topics in Signal Processing},\n   publisher={Institute of Electrical and Electronics Engineers (IEEE)},\n   author={Chen, Sanyuan and Wang, Chengyi and Chen, Zhengyang and Wu, Yu and Liu, Shujie and Chen, Zhuo and Li, Jinyu and Kanda, Naoyuki and Yoshioka, Takuya and Xiao, Xiong and Wu, Jian and Zhou, Long and Ren, Shuo and Qian, Yanmin and Qian, Yao and Wu, Jian and Zeng, Michael and Yu, Xiangzhan and Wei, Furu},\n   year={2022},\n   month=oct, pages={1505–1518} }\n","memoryUsageMb":361.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["GigaSpeechA2TRetrieval","Libri-Light","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.35428808110516935,"AudioReranking":0.667839,"AudioClustering":0.02491533333333333,"AudioPairClassification":0.5139736666666667,"AudioMultilabelClassification":0.229082},"scoresByTask":{"VoxCelebSA":0.303036,"GTZANGenre":0.629,"VehicleSoundClustering":0.017932,"VoxPopuliGenderClustering":0.000291,"MridinghamTonic":0.264728,"VoxPopuliAccentPairClassification":0.509135,"NMSQAPairClassification":0.484168,"IEMOCAPGender":0.549956,"VoxPopuliLanguageID":0.644,"BirdCLEF":0.12,"SIBFLEURS":0.11857322549019608,"JamAltArtistA2ARetrieval":0.7217525,"CREMADPairClassification":0.548618,"FSD2019Kaggle":0.229082,"CREMA_DClustering":0.056523,"MInDS14":0.11786466666666667,"GTZANAudioReranking":0.667839,"BeijingOpera":0.627571,"CommonLanguageAgeDetection":0.1853,"CREMA_D":0.33714},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3633382837977296}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":53,"model":{"name":"microsoft/wavlm-base-plus-sd","url":"https://huggingface.co/microsoft/wavlm-base-plus-sd","zeroShotPct":96,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-07-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{Chen_2022,\n   title={WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing},\n   volume={16},\n   ISSN={1941-0484},\n   url={http://dx.doi.org/10.1109/JSTSP.2022.3188113},\n   DOI={10.1109/jstsp.2022.3188113},\n   number={6},\n   journal={IEEE Journal of Selected Topics in Signal Processing},\n   publisher={Institute of Electrical and Electronics Engineers (IEEE)},\n   author={Chen, Sanyuan and Wang, Chengyi and Chen, Zhengyang and Wu, Yu and Liu, Shujie and Chen, Zhuo and Li, Jinyu and Kanda, Naoyuki and Yoshioka, Takuya and Xiao, Xiong and Wu, Jian and Zhou, Long and Ren, Shuo and Qian, Yanmin and Qian, Yao and Wu, Jian and Zeng, Michael and Yu, Xiangzhan and Wei, Furu},\n   year={2022},\n   month=oct, pages={1505–1518} }\n","memoryUsageMb":361.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["GigaSpeechA2TRetrieval","Libri-Light","LibriMix","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.35428808110516935,"AudioReranking":0.667839,"AudioClustering":0.02491533333333333,"AudioPairClassification":0.5139736666666667,"AudioMultilabelClassification":0.229082},"scoresByTask":{"VoxCelebSA":0.303036,"GTZANGenre":0.629,"VehicleSoundClustering":0.017932,"VoxPopuliGenderClustering":0.000291,"MridinghamTonic":0.264728,"VoxPopuliAccentPairClassification":0.509135,"NMSQAPairClassification":0.484168,"IEMOCAPGender":0.549956,"VoxPopuliLanguageID":0.644,"BirdCLEF":0.12,"SIBFLEURS":0.11857322549019608,"JamAltArtistA2ARetrieval":0.7217525,"CREMADPairClassification":0.548618,"FSD2019Kaggle":0.229082,"CREMA_DClustering":0.056523,"MInDS14":0.11786466666666667,"GTZANAudioReranking":0.667839,"BeijingOpera":0.627571,"CommonLanguageAgeDetection":0.1853,"CREMA_D":0.33714},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3633382837977296}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":53,"model":{"name":"microsoft/wavlm-base-plus-sv","url":"https://huggingface.co/microsoft/wavlm-base-plus-sv","zeroShotPct":96,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-07-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{Chen_2022,\n   title={WavLM: Large-Scale Self-Supervised Pre-Training for Full Stack Speech Processing},\n   volume={16},\n   ISSN={1941-0484},\n   url={http://dx.doi.org/10.1109/JSTSP.2022.3188113},\n   DOI={10.1109/jstsp.2022.3188113},\n   number={6},\n   journal={IEEE Journal of Selected Topics in Signal Processing},\n   publisher={Institute of Electrical and Electronics Engineers (IEEE)},\n   author={Chen, Sanyuan and Wang, Chengyi and Chen, Zhengyang and Wu, Yu and Liu, Shujie and Chen, Zhuo and Li, Jinyu and Kanda, Naoyuki and Yoshioka, Takuya and Xiao, Xiong and Wu, Jian and Zhou, Long and Ren, Shuo and Qian, Yanmin and Qian, Yao and Wu, Jian and Zeng, Michael and Yu, Xiangzhan and Wei, Furu},\n   year={2022},\n   month=oct, pages={1505–1518} }\n","memoryUsageMb":361.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["GigaSpeechA2TRetrieval","Libri-Light","VoxCeleb1","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.35428808110516935,"AudioReranking":0.667839,"AudioClustering":0.02491533333333333,"AudioPairClassification":0.5139736666666667,"AudioMultilabelClassification":0.229082},"scoresByTask":{"VoxCelebSA":0.303036,"GTZANGenre":0.629,"VehicleSoundClustering":0.017932,"VoxPopuliGenderClustering":0.000291,"MridinghamTonic":0.264728,"VoxPopuliAccentPairClassification":0.509135,"NMSQAPairClassification":0.484168,"IEMOCAPGender":0.549956,"VoxPopuliLanguageID":0.644,"BirdCLEF":0.12,"SIBFLEURS":0.11857322549019608,"JamAltArtistA2ARetrieval":0.7217525,"CREMADPairClassification":0.548618,"FSD2019Kaggle":0.229082,"CREMA_DClustering":0.056523,"MInDS14":0.11786466666666667,"GTZANAudioReranking":0.667839,"BeijingOpera":0.627571,"CommonLanguageAgeDetection":0.1853,"CREMA_D":0.33714},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3633382837977296}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":56,"model":{"name":"speechbrain/m-ctc-t-large","url":"https://huggingface.co/speechbrain/m-ctc-t-large","zeroShotPct":96,"activeParamsB":1.047,"totalParamsB":1.059,"embeddingDim":1536,"maxTokens":null,"releaseDate":"2022-01-10","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Abkhazian","Arabic","Assamese","Basque","Breton","Catalan","Chinese","Chinese (Traditional)","Chuvash","Czech","Dhivehi","Dutch","English","Esperanto","Estonian","Falam Chin (Latin)","Finnish","French","Ganda","Georgian","German","Hindi","Hungarian","Indonesian","Interlingua (International Auxiliary Language Association)","Irish","Italian","Japanese","Kabyle","Kinyarwanda","Kirghiz","Latvian","Lithuanian","Maltese","Modern Greek (1453-)","Mongolian","Oriya (macrolanguage)","Panjabi","Persian","Polish","Portuguese","Romanian","Romansh","Russian","Slovenian","Spanish","Swedish","Tamil","Tatar","Thai","Turkish","Ukrainian","Upper Sorbian","Vietnamese","Votic (Latin)","Welsh","Western Frisian","Yakut"],"citation":"\n@inproceedings{lugosch2022pseudolabelingmassivelymultilingualspeech,\n  author = {Lugosch, Loren and Likhomanenko, Tatiana and Synnaeve, Gabriel and Collobert, Ronan},\n  booktitle = {ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp43922.2022.9746832},\n  month = {May},\n  pages = {7687–7691},\n  publisher = {IEEE},\n  title = {Pseudo-Labeling for Massively Multilingual Speech Recognition},\n  url = {https://doi.org/10.1109/icassp43922.2022.9746832},\n  year = {2022},\n}","memoryUsageMb":4039.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/speechbrain/speechbrain","publicTrainingData":"https://github.com/speechbrain/speechbrain","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["transformers-v4"],"trainingDatasets":["CommonVoiceMini17A2TRetrieval","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":1.047,"totalParamsB":1.059,"embeddingDim":1536,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3728343810160428,"AudioReranking":0.56183,"AudioClustering":0.011220666666666665,"AudioPairClassification":0.545674,"AudioMultilabelClassification":0.188187},"scoresByTask":{"VoxCelebSA":0.303561,"GTZANGenre":0.392,"VehicleSoundClustering":0.00967,"VoxPopuliGenderClustering":0.000756,"MridinghamTonic":0.242657,"VoxPopuliAccentPairClassification":0.519805,"NMSQAPairClassification":0.589789,"IEMOCAPGender":0.50961,"VoxPopuliLanguageID":0.994,"BirdCLEF":0.123,"SIBFLEURS":0.21695727450980393,"JamAltArtistA2ARetrieval":0.46851,"CREMADPairClassification":0.527428,"FSD2019Kaggle":0.188187,"CREMA_DClustering":0.023236,"MInDS14":0.4457749166666667,"GTZANAudioReranking":0.56183,"BeijingOpera":0.432447,"CommonLanguageAgeDetection":0.1735,"CREMA_D":0.267671},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3580106416408669}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":57,"model":{"name":"facebook/wav2vec2-large","url":"https://huggingface.co/facebook/wav2vec2-large","zeroShotPct":100,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2020-10-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{baevski2020wav2vec20frameworkselfsupervised,\n  archiveprefix = {arXiv},\n  author = {Alexei Baevski and Yuhao Zhou and Abdelrahman Mohamed and Michael Auli},\n  booktitle = {NeurIPS},\n  eprint = {2006.11477},\n  title = {wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations},\n  url = {https://proceedings.neurips.cc/paper/2020/hash/92d1e1eb1cd6f9fba3227870bb6d7f07-Abstract.html},\n  year = {2020},\n}","memoryUsageMb":1209.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.2977296073975044,"AudioReranking":0.523548,"AudioClustering":0.03074333333333333,"AudioPairClassification":0.537416,"AudioMultilabelClassification":0.15816},"scoresByTask":{"VoxCelebSA":0.311725,"GTZANGenre":0.543,"VehicleSoundClustering":0.01411,"VoxPopuliGenderClustering":0.000243,"MridinghamTonic":0.266444,"VoxPopuliAccentPairClassification":0.514357,"NMSQAPairClassification":0.541561,"IEMOCAPGender":0.542782,"VoxPopuliLanguageID":0.3,"BirdCLEF":0.078,"SIBFLEURS":0.11303959803921569,"JamAltArtistA2ARetrieval":0.6354850000000001,"CREMADPairClassification":0.55633,"FSD2019Kaggle":0.15816,"CREMA_DClustering":0.077877,"MInDS14":0.09902908333333334,"GTZANAudioReranking":0.523548,"BeijingOpera":0.4875,"CommonLanguageAgeDetection":0.1789,"CREMA_D":0.354606},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3230808779669763}},"trainedOnTasks":[],"experiments":null},{"rank":58,"model":{"name":"facebook/hubert-large-ls960-ft","url":"https://huggingface.co/facebook/hubert-large-ls960-ft","zeroShotPct":100,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2021-06-14","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{hsu2021hubertselfsupervisedspeechrepresentation,\n  author = {Hsu, Wei-Ning and Bolte, Benjamin and Tsai, Yao-Hung Hubert and Lakhotia, Kushal and Salakhutdinov, Ruslan and Mohamed, Abdelrahman},\n  doi = {10.1109/taslp.2021.3122291},\n  issn = {2329-9304},\n  journal = {IEEE/ACM Transactions on Audio, Speech, and Language Processing},\n  pages = {3451–3460},\n  publisher = {Institute of Electrical and Electronics Engineers (IEEE)},\n  title = {HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units},\n  url = {https://doi.org/10.1109/taslp.2021.3122291},\n  volume = {29},\n  year = {2021},\n}","memoryUsageMb":1203.0,"license":"mit","publicTrainingCode":"https://github.com/pytorch/fairseq/tree/master/examples/hubert","publicTrainingData":"https://www.openslr.org/12","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.36723983155080214,"AudioReranking":0.534807,"AudioClustering":0.012164666666666666,"AudioPairClassification":0.5151036666666666,"AudioMultilabelClassification":0.315233},"scoresByTask":{"VoxCelebSA":0.336369,"GTZANGenre":0.506,"VehicleSoundClustering":0.005867,"VoxPopuliGenderClustering":0.000397,"MridinghamTonic":0.242233,"VoxPopuliAccentPairClassification":0.506582,"NMSQAPairClassification":0.499955,"IEMOCAPGender":0.542484,"VoxPopuliLanguageID":0.868,"BirdCLEF":0.149,"SIBFLEURS":0.13514298039215686,"JamAltArtistA2ARetrieval":0.52258,"CREMADPairClassification":0.538774,"FSD2019Kaggle":0.315233,"CREMA_DClustering":0.03023,"MInDS14":0.16452916666666667,"GTZANAudioReranking":0.534807,"BeijingOpera":0.585461,"CommonLanguageAgeDetection":0.1937,"CREMA_D":0.316719},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3515173761609907}},"trainedOnTasks":[],"experiments":null},{"rank":59,"model":{"name":"EximiusLabs/fusion-embedding-1-2b-preview","url":"https://huggingface.co/EximiusLabs/fusion-embedding-1-2b-preview","zeroShotPct":90,"activeParamsB":2.489,"totalParamsB":2.8,"embeddingDim":1024,"maxTokens":512.0,"releaseDate":"2026-07-06","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio","image","text","video"],"languages":["English"],"citation":"\n@article{tonmoy2026fusion,\n  title   = {Fusion Embedding: A Unified Embedding Space for Text, Image, Video, and Audio},\n  author  = {Tonmoy, Abdul Basit and Hoque, Kazi Fardinul and Arham, Md. Shahrier Islam and Luthra, Arman},\n  journal = {arXiv preprint arXiv:2607.18666},\n  year    = {2026}\n}\n","memoryUsageMb":10681.0,"license":"cc-by-nc-4.0","publicTrainingCode":"https://github.com/Eximius-Labs/fusion-embedding","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["fusion-embedding"],"trainingDatasets":["AudioCapsA2TRetrieval","AudioCapsAVA2VRetrieval","AudioCapsAVAT2VRetrieval","AudioCapsAVT2VARetrieval","AudioCapsAVT2VRetrieval","AudioCapsAVV2ARetrieval","AudioCapsAVV2TRetrieval","AudioCapsAVVA2TRetrieval","AudioCapsAVVT2ARetrieval","AudioCapsT2ARetrieval","AudioSet","AudioSetMini","CommonLanguageAgeDetection","CommonVoiceMini21T2ARetrieval","FSD2019Kaggle","FSD50K"]},"zeroShotPct":90,"activeParamsB":2.489,"totalParamsB":2.8,"embeddingDim":1024,"maxTokens":512,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioReranking":0.70663,"AudioZeroshotClassification":0.25585800000000003},"scoresByTask":{"GTZANGenre":0.639,"VehicleSoundClustering":0.03495,"MACST2ARetrieval":0.13232,"ClothoT2ARetrieval":0.27842,"SpeechCommandsZeroshotv0.02":0.189494,"UrbanSound8KT2ARetrieval":0.00943,"GTZANAudioReranking":0.70663,"BeijingOpera":0.927926,"RavdessZeroshot":0.322222},"scoresByCustomGroup":{"Benchmark Variant":{}},"trainedOnTasks":[],"experiments":null},{"rank":60,"model":{"name":"facebook/wav2vec2-xls-r-300m","url":"https://huggingface.co/facebook/wav2vec2-xls-r-300m","zeroShotPct":96,"activeParamsB":0.3,"totalParamsB":0.3,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2021-10-13","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Abkhazian","Afrikaans","Albanian","Amharic (Latin)","Arabic (Latin)","Armenian (Latin)","Assamese (Latin)","Azerbaijani","Bashkir","Basque","Belarusian (Latin)","Bengali","Bosnian","Breton","Bulgarian (Latin)","Burmese (Latin)","Catalan","Cebuano","Chinese (Latin)","Chuvash","Croatian","Czech","Danish","Dhivehi","Dutch","English","Esperanto","Estonian","Faroese","Finnish","French","Galician","Georgian (Latin)","German","Guarani","Gujarati (Latin)","Haitian","Hausa","Hawaiian (Latin)","Hebrew (Latin)","Hindi","Hungarian","Icelandic","Indonesian","Interlingua (International Auxiliary Language Association)","Irish","Italian","Japanese (Latin)","Javanese","Kannada (Latin)","Kazakh (Latin)","Khmer (Latin)","Kinyarwanda","Kirghiz (Latin)","Modern Greek (1453-) (Latin)"],"citation":"\n@inproceedings{babu2021xlsrselfsupervisedcrosslingualspeech,\n  author = {Babu, Arun and Wang, Changhan and Tjandra, Andros and Lakhotia, Kushal and Xu, Qiantong and Goyal, Naman and Singh, Kritika and von Platen, Patrick and Saraf, Yatharth and Pino, Juan and Baevski, Alexei and Conneau, Alexis and Auli, Michael},\n  booktitle = {Interspeech 2022},\n  collection = {interspeech_2022},\n  doi = {10.21437/interspeech.2022-143},\n  month = {Sept},\n  pages = {2278–2282},\n  publisher = {ISCA},\n  series = {interspeech_2022},\n  title = {XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale},\n  url = {https://doi.org/10.21437/interspeech.2022-143},\n  year = {2022},\n}","memoryUsageMb":1200.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/pytorch/fairseq","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["CommonVoiceMini17A2TRetrieval","VoxLingua107_Top10","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":0.3,"totalParamsB":0.3,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.31137314839572194,"AudioReranking":0.46778,"AudioClustering":0.032058333333333335,"AudioPairClassification":0.5121473333333334,"AudioMultilabelClassification":0.259309},"scoresByTask":{"VoxCelebSA":0.176873,"GTZANGenre":0.426,"VehicleSoundClustering":0.023022,"VoxPopuliGenderClustering":0.002116,"MridinghamTonic":0.358315,"VoxPopuliAccentPairClassification":0.502436,"NMSQAPairClassification":0.493155,"IEMOCAPGender":0.531227,"VoxPopuliLanguageID":0.294,"BirdCLEF":0.078,"SIBFLEURS":0.1427458823529412,"JamAltArtistA2ARetrieval":0.630505,"CREMADPairClassification":0.540851,"FSD2019Kaggle":0.259309,"CREMA_DClustering":0.071037,"MInDS14":0.09282675,"GTZANAudioReranking":0.46778,"BeijingOpera":0.817819,"CommonLanguageAgeDetection":0.1539,"CREMA_D":0.353398},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.32400034907120745}},"trainedOnTasks":["VoxPopuliLanguageID"],"experiments":null},{"rank":61,"model":{"name":"asapp/sew-d-tiny-100k-ft-ls100h","url":"https://huggingface.co/asapp/sew-d-tiny-100k-ft-ls100h","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.02,"embeddingDim":384,"maxTokens":null,"releaseDate":"2021-09-14","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{wu2021performanceefficiencytradeoffsunsupervisedpretraining,\n  author = {Wu, Felix and Kim, Kwangyoun and Pan, Jing and Han, Kyu J. and Weinberger, Kilian Q. and Artzi, Yoav},\n  booktitle = {ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp43922.2022.9747432},\n  month = {May},\n  pages = {7667–7671},\n  publisher = {IEEE},\n  title = {Performance-Efficiency Trade-Offs in Unsupervised Pre-Training for Speech Recognition},\n  url = {https://doi.org/10.1109/icassp43922.2022.9747432},\n  year = {2022},\n}","memoryUsageMb":92.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["LibriSpeech"]},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.02,"embeddingDim":384,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3359342272727273,"AudioReranking":0.513635,"AudioClustering":0.020172666666666665,"AudioPairClassification":0.5114666666666667,"AudioMultilabelClassification":0.231705},"scoresByTask":{"VoxCelebSA":0.297505,"GTZANGenre":0.511,"VehicleSoundClustering":0.003376,"VoxPopuliGenderClustering":0.001235,"MridinghamTonic":0.236496,"VoxPopuliAccentPairClassification":0.512142,"NMSQAPairClassification":0.482146,"IEMOCAPGender":0.525749,"VoxPopuliLanguageID":0.662,"BirdCLEF":0.09,"SIBFLEURS":0.14000233333333334,"JamAltArtistA2ARetrieval":0.46875,"CREMADPairClassification":0.540112,"FSD2019Kaggle":0.231705,"CREMA_DClustering":0.055907,"MInDS14":0.1717911666666667,"GTZANAudioReranking":0.513635,"BeijingOpera":0.567908,"CommonLanguageAgeDetection":0.1925,"CREMA_D":0.300325},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3301357631578947}},"trainedOnTasks":[],"experiments":null},{"rank":62,"model":{"name":"facebook/data2vec-audio-large-960h","url":"https://huggingface.co/facebook/data2vec-audio-large-960h","zeroShotPct":100,"activeParamsB":0.313,"totalParamsB":0.313,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2022-02-07","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{baevski2022data2vecgeneralframeworkselfsupervised,\n  archiveprefix = {arXiv},\n  author = {Alexei Baevski and Wei-Ning Hsu and Qiantong Xu and Arun Babu and Jiatao Gu and Michael Auli},\n  booktitle = {ICML},\n  eprint = {2202.03555},\n  pages = {1298-1312},\n  title = {data2vec: A General Framework for Self-supervised Learning in Speech, Vision and Language},\n  url = {https://proceedings.mlr.press/v162/baevski22a.html},\n  year = {2022},\n}","memoryUsageMb":1195.0,"license":"mit","publicTrainingCode":"https://github.com/facebookresearch/fairseq/tree/main/examples/data2vec","publicTrainingData":"https://www.openslr.org/12","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.313,"totalParamsB":0.313,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.36437052985739754,"AudioReranking":0.569126,"AudioClustering":0.008263000000000001,"AudioPairClassification":0.523602,"AudioMultilabelClassification":0.222502},"scoresByTask":{"VoxCelebSA":0.305614,"GTZANGenre":0.436,"VehicleSoundClustering":0.001542,"VoxPopuliGenderClustering":0.000136,"MridinghamTonic":0.212131,"VoxPopuliAccentPairClassification":0.507439,"NMSQAPairClassification":0.523806,"IEMOCAPGender":0.545871,"VoxPopuliLanguageID":0.876,"BirdCLEF":0.095,"SIBFLEURS":0.17503291176470587,"JamAltArtistA2ARetrieval":0.4246575,"CREMADPairClassification":0.539561,"FSD2019Kaggle":0.222502,"CREMA_DClustering":0.023111,"MInDS14":0.22066591666666668,"GTZANAudioReranking":0.569126,"BeijingOpera":0.708067,"CommonLanguageAgeDetection":0.1718,"CREMA_D":0.261894},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.34723443833849327}},"trainedOnTasks":[],"experiments":null},{"rank":63,"model":{"name":"microsoft/unispeech-sat-base-100h-libri-ft","url":"https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft","zeroShotPct":100,"activeParamsB":0.094,"totalParamsB":0.094,"embeddingDim":768,"maxTokens":null,"releaseDate":"2021-10-12","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{chen2021unispeechsatuniversalspeechrepresentation,\n  author = {Chen, Sanyuan and Wu, Yu and Wang, Chengyi and Chen, Zhengyang and Chen, Zhuo and Liu, Shujie and Wu, Jian and Qian, Yao and Wei, Furu and Li, Jinyu and Yu, Xiangzhan},\n  booktitle = {ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp43922.2022.9747077},\n  month = {May},\n  pages = {6152–6156},\n  publisher = {IEEE},\n  title = {Unispeech-Sat: Universal Speech Representation Learning With Speaker Aware Pre-Training},\n  url = {https://doi.org/10.1109/icassp43922.2022.9747077},\n  year = {2022},\n}","memoryUsageMb":359.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["LibriSpeech"]},"zeroShotPct":100,"activeParamsB":0.094,"totalParamsB":0.094,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.33235326559714795,"AudioReranking":0.502527,"AudioClustering":0.016774333333333332,"AudioPairClassification":0.5248683333333334,"AudioMultilabelClassification":0.206146},"scoresByTask":{"VoxCelebSA":0.301276,"GTZANGenre":0.516,"VehicleSoundClustering":0.003279,"VoxPopuliGenderClustering":0.005668,"MridinghamTonic":0.253408,"VoxPopuliAccentPairClassification":0.512725,"NMSQAPairClassification":0.518344,"IEMOCAPGender":0.604543,"VoxPopuliLanguageID":0.586,"BirdCLEF":0.092,"SIBFLEURS":0.12490892156862746,"JamAltArtistA2ARetrieval":0.51478,"CREMADPairClassification":0.543536,"FSD2019Kaggle":0.206146,"CREMA_DClustering":0.041376,"MInDS14":0.099603,"GTZANAudioReranking":0.502527,"BeijingOpera":0.584929,"CommonLanguageAgeDetection":0.1589,"CREMA_D":0.334318},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.33148004850361196}},"trainedOnTasks":[],"experiments":null},{"rank":64,"model":{"name":"facebook/data2vec-audio-base-960h","url":"https://huggingface.co/facebook/data2vec-audio-base-960h","zeroShotPct":100,"activeParamsB":0.093,"totalParamsB":0.093,"embeddingDim":768,"maxTokens":null,"releaseDate":"2022-02-07","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{baevski2022data2vecgeneralframeworkselfsupervised,\n  archiveprefix = {arXiv},\n  author = {Alexei Baevski and Wei-Ning Hsu and Qiantong Xu and Arun Babu and Jiatao Gu and Michael Auli},\n  booktitle = {ICML},\n  eprint = {2202.03555},\n  pages = {1298-1312},\n  title = {data2vec: A General Framework for Self-supervised Learning in Speech, Vision and Language},\n  url = {https://proceedings.mlr.press/v162/baevski22a.html},\n  year = {2022},\n}","memoryUsageMb":355.0,"license":"mit","publicTrainingCode":"https://github.com/facebookresearch/fairseq/tree/main/examples/data2vec","publicTrainingData":"https://www.openslr.org/12","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.093,"totalParamsB":0.093,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3285791568627451,"AudioReranking":0.48995,"AudioClustering":0.011658,"AudioPairClassification":0.5122136666666667,"AudioMultilabelClassification":0.210911},"scoresByTask":{"VoxCelebSA":0.320702,"GTZANGenre":0.403,"VehicleSoundClustering":0.013347,"VoxPopuliGenderClustering":0.000422,"MridinghamTonic":0.207688,"VoxPopuliAccentPairClassification":0.509685,"NMSQAPairClassification":0.503471,"IEMOCAPGender":0.558423,"VoxPopuliLanguageID":0.736,"BirdCLEF":0.074,"SIBFLEURS":0.16025339215686277,"JamAltArtistA2ARetrieval":0.4649875,"CREMADPairClassification":0.523485,"FSD2019Kaggle":0.210911,"CREMA_DClustering":0.021205,"MInDS14":0.15918133333333334,"GTZANAudioReranking":0.48995,"BeijingOpera":0.530053,"CommonLanguageAgeDetection":0.1853,"CREMA_D":0.27977},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.32320648555211556}},"trainedOnTasks":[],"experiments":null},{"rank":64,"model":{"name":"asapp/sew-d-base-plus-400k-ft-ls100h","url":"https://huggingface.co/asapp/sew-d-base-plus-400k-ft-ls100h","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2021-09-14","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{wu2021performanceefficiencytradeoffsunsupervisedpretraining,\n  author = {Wu, Felix and Kim, Kwangyoun and Pan, Jing and Han, Kyu J. and Weinberger, Kilian Q. and Artzi, Yoav},\n  booktitle = {ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp43922.2022.9747432},\n  month = {May},\n  pages = {7667–7671},\n  publisher = {IEEE},\n  title = {Performance-Efficiency Trade-Offs in Unsupervised Pre-Training for Speech Recognition},\n  url = {https://doi.org/10.1109/icassp43922.2022.9747432},\n  year = {2022},\n}","memoryUsageMb":675.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["LibriSpeech"]},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3122093159536542,"AudioReranking":0.466712,"AudioClustering":0.017469,"AudioPairClassification":0.5130683333333333,"AudioMultilabelClassification":0.149305},"scoresByTask":{"VoxCelebSA":0.321291,"GTZANGenre":0.401,"VehicleSoundClustering":0.006319,"VoxPopuliGenderClustering":0.000997,"MridinghamTonic":0.194928,"VoxPopuliAccentPairClassification":0.501061,"NMSQAPairClassification":0.494138,"IEMOCAPGender":0.557333,"VoxPopuliLanguageID":0.716,"BirdCLEF":0.045,"SIBFLEURS":0.13702239215686274,"JamAltArtistA2ARetrieval":0.3761625,"CREMADPairClassification":0.544006,"FSD2019Kaggle":0.149305,"CREMA_DClustering":0.045091,"MInDS14":0.11470408333333333,"GTZANAudioReranking":0.466712,"BeijingOpera":0.402926,"CommonLanguageAgeDetection":0.1997,"CREMA_D":0.344398},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3088836302889577}},"trainedOnTasks":[],"experiments":null},{"rank":66,"model":{"name":"facebook/encodec_24khz","url":"https://huggingface.co/facebook/encodec_24khz","zeroShotPct":100,"activeParamsB":0.023,"totalParamsB":0.023,"embeddingDim":128,"maxTokens":null,"releaseDate":"2022-10-25","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@article{défossez2022highfidelityneuralaudio,\n  archiveprefix = {arXiv},\n  author = {Alexandre D{\\'e}fossez and Jade Copet and Gabriel Synnaeve and Yossi Adi},\n  eprint = {2210.13438},\n  issn = {2835-8856},\n  journal = {Transactions on Machine Learning Research},\n  note = {Featured Certification, Reproducibility Certification},\n  title = {High Fidelity Neural Audio Compression},\n  url = {https://openreview.net/forum?id=ivCd8z8zR2},\n  year = {2023},\n}","memoryUsageMb":88.0,"license":"cc-by-nc-4.0","publicTrainingCode":"https://github.com/facebookresearch/encodec","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["AudioSetMini","CommonVoiceMini17A2TRetrieval","FSD50K"]},"zeroShotPct":100,"activeParamsB":0.023,"totalParamsB":0.023,"embeddingDim":128,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.2481786483957219,"AudioReranking":0.51643,"AudioClustering":0.05139433333333333,"AudioPairClassification":0.5129363333333333,"AudioMultilabelClassification":0.080195},"scoresByTask":{"VoxCelebSA":0.235473,"GTZANGenre":0.299,"VehicleSoundClustering":0.038041,"VoxPopuliGenderClustering":0.000076,"MridinghamTonic":0.240357,"VoxPopuliAccentPairClassification":0.509294,"NMSQAPairClassification":0.475383,"IEMOCAPGender":0.534612,"VoxPopuliLanguageID":0.256,"BirdCLEF":0.018,"SIBFLEURS":0.14640004901960785,"JamAltArtistA2ARetrieval":0.8391525,"CREMADPairClassification":0.554132,"FSD2019Kaggle":0.080195,"CREMA_DClustering":0.116066,"MInDS14":0.08792608333333334,"GTZANAudioReranking":0.51643,"BeijingOpera":0.423493,"CommonLanguageAgeDetection":0.1904,"CREMA_D":0.298304},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.30413366486068116}},"trainedOnTasks":[],"experiments":null},{"rank":67,"model":{"name":"facebook/wav2vec2-base-960h","url":"https://huggingface.co/facebook/wav2vec2-base-960h","zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"releaseDate":"2020-10-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{baevski2020wav2vec20frameworkselfsupervised,\n  archiveprefix = {arXiv},\n  author = {Alexei Baevski and Yuhao Zhou and Abdelrahman Mohamed and Michael Auli},\n  booktitle = {NeurIPS},\n  eprint = {2006.11477},\n  title = {wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations},\n  url = {https://proceedings.neurips.cc/paper/2020/hash/92d1e1eb1cd6f9fba3227870bb6d7f07-Abstract.html},\n  year = {2020},\n}","memoryUsageMb":360.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.095,"totalParamsB":0.095,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.30619742112299464,"AudioReranking":0.516716,"AudioClustering":0.011722666666666666,"AudioPairClassification":0.5209006666666666,"AudioMultilabelClassification":0.154295},"scoresByTask":{"VoxCelebSA":0.304158,"GTZANGenre":0.427,"VehicleSoundClustering":0.006122,"VoxPopuliGenderClustering":0.00059,"MridinghamTonic":0.243233,"VoxPopuliAccentPairClassification":0.510938,"NMSQAPairClassification":0.5192,"IEMOCAPGender":0.518181,"VoxPopuliLanguageID":0.588,"BirdCLEF":0.071,"SIBFLEURS":0.12482754901960784,"JamAltArtistA2ARetrieval":0.456595,"CREMADPairClassification":0.532564,"FSD2019Kaggle":0.154295,"CREMA_DClustering":0.028456,"MInDS14":0.14199308333333335,"GTZANAudioReranking":0.516716,"BeijingOpera":0.491401,"CommonLanguageAgeDetection":0.1637,"CREMA_D":0.294678},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.3125975069659443}},"trainedOnTasks":[],"experiments":null},{"rank":68,"model":{"name":"asapp/sew-d-mid-400k-ft-ls100h","url":"https://huggingface.co/asapp/sew-d-mid-400k-ft-ls100h","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.139,"embeddingDim":512,"maxTokens":null,"releaseDate":"2021-09-14","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{wu2021performanceefficiencytradeoffsunsupervisedpretraining,\n  author = {Wu, Felix and Kim, Kwangyoun and Pan, Jing and Han, Kyu J. and Weinberger, Kilian Q. and Artzi, Yoav},\n  booktitle = {ICASSP 2022 - 2022 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},\n  doi = {10.1109/icassp43922.2022.9747432},\n  month = {May},\n  pages = {7667–7671},\n  publisher = {IEEE},\n  title = {Performance-Efficiency Trade-Offs in Unsupervised Pre-Training for Speech Recognition},\n  url = {https://doi.org/10.1109/icassp43922.2022.9747432},\n  year = {2022},\n}","memoryUsageMb":530.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["LibriSpeech"]},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.139,"embeddingDim":512,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.3033110895721925,"AudioReranking":0.478074,"AudioClustering":0.020763,"AudioPairClassification":0.5110613333333334,"AudioMultilabelClassification":0.185782},"scoresByTask":{"VoxCelebSA":0.306491,"GTZANGenre":0.434,"VehicleSoundClustering":0.006276,"VoxPopuliGenderClustering":0.000364,"MridinghamTonic":0.193777,"VoxPopuliAccentPairClassification":0.50699,"NMSQAPairClassification":0.484208,"IEMOCAPGender":0.562608,"VoxPopuliLanguageID":0.632,"BirdCLEF":0.055,"SIBFLEURS":0.12634273529411763,"JamAltArtistA2ARetrieval":0.3914075,"CREMADPairClassification":0.541986,"FSD2019Kaggle":0.185782,"CREMA_DClustering":0.055649,"MInDS14":0.11170025,"GTZANAudioReranking":0.478074,"BeijingOpera":0.437234,"CommonLanguageAgeDetection":0.1627,"CREMA_D":0.314569},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.30533560448916414}},"trainedOnTasks":[],"experiments":null},{"rank":69,"model":{"name":"facebook/wav2vec2-large-xlsr-53","url":"https://huggingface.co/facebook/wav2vec2-large-xlsr-53","zeroShotPct":100,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2020-10-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["English"],"citation":"\n@inproceedings{conneau2020unsupervisedcrosslingualrepresentationlearning,\n  author = {Conneau, Alexis and Baevski, Alexei and Collobert, Ronan and Mohamed, Abdelrahman and Auli, Michael},\n  booktitle = {Interspeech 2021},\n  collection = {interspeech_2021},\n  doi = {10.21437/interspeech.2021-329},\n  month = {Aug},\n  pages = {2426–2430},\n  publisher = {ISCA},\n  series = {interspeech_2021},\n  title = {Unsupervised Cross-Lingual Representation Learning for Speech Recognition},\n  url = {https://doi.org/10.21437/interspeech.2021-329},\n  year = {2021},\n}","memoryUsageMb":1209.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["CommonVoiceMini17A2TRetrieval"]},"zeroShotPct":100,"activeParamsB":0.317,"totalParamsB":0.317,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioClassification":0.21570182531194296,"AudioReranking":0.461548,"AudioClustering":0.03490733333333334,"AudioPairClassification":0.5318506666666667,"AudioMultilabelClassification":0.078681},"scoresByTask":{"VoxCelebSA":0.1209,"GTZANGenre":0.221,"VehicleSoundClustering":0.017241,"VoxPopuliGenderClustering":0.009407,"MridinghamTonic":0.191335,"VoxPopuliAccentPairClassification":0.516479,"NMSQAPairClassification":0.537846,"IEMOCAPGender":0.500951,"VoxPopuliLanguageID":0.194,"BirdCLEF":0.023,"SIBFLEURS":0.05477791176470589,"JamAltArtistA2ARetrieval":0.4817025,"CREMADPairClassification":0.541227,"FSD2019Kaggle":0.078681,"CREMA_DClustering":0.078074,"MInDS14":0.08014616666666667,"GTZANAudioReranking":0.461548,"BeijingOpera":0.546454,"CommonLanguageAgeDetection":0.1569,"CREMA_D":0.283256},"scoresByCustomGroup":{"Benchmark Variant":{"Audio-Only":0.2640128725490196}},"trainedOnTasks":[],"experiments":null},{"rank":70,"model":{"name":"facebook/wav2vec2-xls-r-2b-21-to-en","url":"https://huggingface.co/facebook/wav2vec2-xls-r-2b-21-to-en","zeroShotPct":96,"activeParamsB":2.0,"totalParamsB":2.0,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2024-09-10","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio"],"languages":["Abkhazian","Afrikaans","Albanian","Amharic (Latin)","Arabic (Latin)","Armenian (Latin)","Assamese (Latin)","Azerbaijani","Bashkir","Basque","Belarusian (Latin)","Bengali","Bosnian","Breton","Bulgarian (Latin)","Burmese (Latin)","Catalan","Cebuano","Chinese (Latin)","Chuvash","Croatian","Czech","Danish","Dhivehi","Dutch","English","Esperanto","Estonian","Faroese","Finnish","French","Galician","Georgian (Latin)","German","Guarani","Gujarati (Latin)","Haitian","Hausa","Hawaiian (Latin)","Hebrew (Latin)","Hindi","Hungarian","Icelandic","Indonesian","Interlingua (International Auxiliary Language Association)","Irish","Italian","Japanese (Latin)","Javanese","Kannada (Latin)","Kazakh (Latin)","Khmer (Latin)","Kinyarwanda","Kirghiz (Latin)","Modern Greek (1453-) (Latin)"],"citation":"\n@inproceedings{babu2021xlsrselfsupervisedcrosslingualspeech,\n  author = {Babu, Arun and Wang, Changhan and Tjandra, Andros and Lakhotia, Kushal and Xu, Qiantong and Goyal, Naman and Singh, Kritika and von Platen, Patrick and Saraf, Yatharth and Pino, Juan and Baevski, Alexei and Conneau, Alexis and Auli, Michael},\n  booktitle = {Interspeech 2022},\n  collection = {interspeech_2022},\n  doi = {10.21437/interspeech.2022-143},\n  month = {Sept},\n  pages = {2278–2282},\n  publisher = {ISCA},\n  series = {interspeech_2022},\n  title = {XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale},\n  url = {https://doi.org/10.21437/interspeech.2022-143},\n  year = {2022},\n}","memoryUsageMb":9200.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["CommonVoiceMini17A2TRetrieval","VoxLingua107_Top10","VoxPopuliLanguageID"]},"zeroShotPct":96,"activeParamsB":2.0,"totalParamsB":2.0,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"AudioMultilabelClassification":0.046774},"scoresByTask":{"SIBFLEURS":0.142292,"JamAltArtistA2ARetrieval":0.0694575,"FSD2019Kaggle":0.046774,"MInDS14":0.08318375},"scoresByCustomGroup":{"Benchmark Variant":{}},"trainedOnTasks":[],"experiments":null},{"rank":71,"model":{"name":"LanguageBind/LanguageBind_Audio_FT","url":"https://github.com/PKU-YuanGroup/LanguageBind","zeroShotPct":100,"activeParamsB":0.307,"totalParamsB":0.345,"embeddingDim":768,"maxTokens":77.0,"releaseDate":"2023-10-03","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["audio","text"],"languages":["English"],"citation":"\n@inproceedings{zhu2024languagebind,\n  archiveprefix = {arXiv},\n  author = {Bin Zhu and Bin Lin and Munan Ning and Yang Yan and Jiaxi Cui and WANG HongFa and Yatian Pang and Wenhao Jiang and Junwu Zhang and Zongwei Li and Cai Wan Zhang and Zhifeng Li and Wei Liu and Li Yuan},\n  booktitle = {The Twelfth International Conference on Learning Representations},\n  eprint = {2310.01852},\n  title = {LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment},\n  url = {https://openreview.net/forum?id=QmZKc7UZCy},\n  year = {2024},\n}\n","memoryUsageMb":1316.0,"license":"mit","publicTrainingCode":"https://github.com/PKU-YuanGroup/LanguageBind","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["languagebind"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.307,"totalParamsB":0.345,"embeddingDim":768,"maxTokens":77,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{},"scoresByTask":{"CREMA_D":0.334452},"scoresByCustomGroup":{"Benchmark Variant":{}},"trainedOnTasks":[],"experiments":null}],"aggregations":["mean_task","mean_task_type","task_types","custom_groups"],"customGroupings":[{"name":"Benchmark Variant","groups":[{"label":"Audio-Only","description":"This model's score restricted to audio only tasks. It is equivalent to MAEB(v1, audio-only).","tasks":["JamAltArtistA2ARetrieval","BeijingOpera","BirdCLEF","CREMA_D","CommonLanguageAgeDetection","GTZANGenre","IEMOCAPGender","MInDS14","MridinghamTonic","SIBFLEURS","VoxCelebSA","VoxPopuliLanguageID","CREMA_DClustering","VehicleSoundClustering","VoxPopuliGenderClustering","CREMADPairClassification","NMSQAPairClassification","VoxPopuliAccentPairClassification","GTZANAudioReranking"],"tasksComplete":true}]}],"showZeroShot":true}