{"benchmarkName":"MIEB(Multilingual)","taskTypes":["Any2AnyRetrieval","Compositionality","ImageClassification","DocumentUnderstanding","ZeroShotClassification","ImageClustering","VisionCentricQA","Any2AnyMultilingualRetrieval","VisualSTS(eng)","VisualSTS(multi)"],"tasks":["GTSRBZeroShot","CVBenchDistance","CVBenchRelation","OxfordPetsZeroShot","InfoSeekIT2ITRetrieval","VidoreTabfquadRetrieval","CIFAR10Clustering","VidoreDocVQARetrieval","VisualNewsI2TRetrieval","VidoreArxivQARetrieval","STL10","CLEVRZeroShot","MemotionI2TRetrieval","PatchCamelyon","VidoreSyntheticDocQAHealthcareIndustryRetrieval","TinyImageNetClustering","WebQAT2ITRetrieval","SUN397","METI2IRetrieval","STS13VisualSTS","CVBenchCount","ROxfordMediumI2IRetrieval","NIGHTSI2IRetrieval","Flickr30kI2TRetrieval","BLINKIT2TRetrieval","VidoreTatdqaRetrieval","ImageCoDe","ImageCoDeT2IRetrieval","Fashion200kI2TRetrieval","FGVCAircraftZeroShot","HatefulMemesT2IRetrieval","XM3600T2IRetrieval","UCF101","MSCOCOI2TRetrieval","OxfordPets","SugarCrepe","FER2013ZeroShot","OVENIT2TRetrieval","Caltech101ZeroShot","MSCOCOT2IRetrieval","RESISC45ZeroShot","Food101Classification","RParisEasyI2IRetrieval","CVBenchDepth","Imagenet1k","VisualSTS-b-Eng","VOC2007","EDIST2ITRetrieval","ROxfordEasyI2IRetrieval","VidoreShiftProjectRetrieval","WITT2IRetrieval","RP2kI2IRetrieval","ROxfordHardI2IRetrieval","DTD","VidoreSyntheticDocQAGovernmentReportsRetrieval","AROVisualAttribution","Country211","AROVisualRelation","BLINKIT2IRetrieval","CIFAR10","InfoSeekIT2TRetrieval","VidoreInfoVQARetrieval","BLINKIT2IMultiChoice","VidoreSyntheticDocQAEnergyRetrieval","CIFAR100","VisualSTS-b-Multilingual","EuroSATZeroShot","VisualSTS17Eng","RESISC45","BirdsnapZeroShot","CIFAR10ZeroShot","STL10ZeroShot","GTSRB","Country211ZeroShot","FORBI2IRetrieval","CIFAR100ZeroShot","OVENIT2ITRetrieval","MNIST","GLDv2I2TRetrieval","CIFAR100Clustering","RParisHardI2IRetrieval","SOPI2IRetrieval","VidoreSyntheticDocQAAIRetrieval","ImageNet10Clustering","StanfordCarsI2IRetrieval","StanfordCarsZeroShot","FER2013","OxfordFlowersClassification","STS12VisualSTS","Imagenet1kZeroShot","STS14VisualSTS","AROCocoOrder","VQA2IT2TRetrieval","CLEVRCountZeroShot","GLDv2I2IRetrieval","RenderedSST2","STS15VisualSTS","SciMMIRT2IRetrieval","WebQAT2TRetrieval","VizWizIT2TRetrieval","SketchyI2IRetrieval","Caltech101","VisualSTS17Multilingual","BLINKIT2TMultiChoice","EuroSAT","MemotionT2IRetrieval","CIRRIT2IRetrieval","DTDZeroShot","Food101ZeroShot","Birdsnap","Flickr30kT2IRetrieval","Winoground","StanfordCars","VisualNewsT2IRetrieval","STS16VisualSTS","MNISTZeroShot","PatchCamelyonZeroShot","HatefulMemesI2TRetrieval","SciMMIRI2TRetrieval","RParisMediumI2IRetrieval","UCF101ZeroShot","AROFlickrOrder","ImageNetDog15Clustering","Fashion200kT2IRetrieval","FashionIQIT2IRetrieval","TUBerlinT2IRetrieval","SUN397ZeroShot","FGVCAircraft","CUB200I2IRetrieval","XFlickr30kCoT2IRetrieval"],"tasksMeta":[{"name":"Birdsnap","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying bird images from 500 species.","reference":"https://openaccess.thecvf.com/content_cvpr_2014/html/Berg_Birdsnap_Large-scale_Fine-grained_2014_CVPR_paper.html","citation":"\n@inproceedings{Berg_2014_CVPR,\n  author = {Berg, Thomas and Liu, Jiongxin and Woo Lee, Seung and Alexander, Michelle L. and Jacobs, David W. and Belhumeur, Peter N.},\n  booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n  month = {June},\n  title = {Birdsnap: Large-scale Fine-grained Visual Categorization of Birds},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/birdsnap","license":"not specified","dateFrom":"2013-01-01","dateTo":"2014-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"Caltech101","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying images of 101 widely varied objects.","reference":"https://ieeexplore.ieee.org/document/1384978","citation":"\n@inproceedings{1384978,\n  author = {Li Fei-Fei and Fergus, R. and Perona, P.},\n  booktitle = {2004 Conference on Computer Vision and Pattern Recognition Workshop},\n  doi = {10.1109/CVPR.2004.383},\n  keywords = {Bayesian methods;Testing;Humans;Maximum likelihood estimation;Assembly;Shape;Machine vision;Image recognition;Parameter estimation;Image databases},\n  number = {},\n  pages = {178-178},\n  title = {Learning Generative Visual Models from Few Training Examples: An Incremental Bayesian Approach Tested on 101 Object Categories},\n  volume = {},\n  year = {2004},\n}\n","isPublic":true,"sourceDataset":"mteb/Caltech101","license":"not specified","dateFrom":"2003-01-01","dateTo":"2004-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"CIFAR10","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Classifying images from 10 classes.","reference":"https://huggingface.co/datasets/uoft-cs/cifar10","citation":"\n@techreport{Krizhevsky09learningmultiple,\n  author = {Alex Krizhevsky},\n  institution = {},\n  title = {Learning multiple layers of features from tiny images},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/cifar10","license":"not specified","dateFrom":"2008-01-01","dateTo":"2009-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"CIFAR100","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Classifying images from 100 classes.","reference":"https://huggingface.co/datasets/uoft-cs/cifar100","citation":"\n@techreport{Krizhevsky09learningmultiple,\n  author = {Alex Krizhevsky},\n  institution = {},\n  title = {Learning multiple layers of features from tiny images},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/cifar100","license":"not specified","dateFrom":"2008-01-01","dateTo":"2009-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"Country211","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Scene"],"modalities":["image"],"description":"Classifying images of 211 countries.","reference":"https://huggingface.co/datasets/clip-benchmark/wds_country211","citation":"\n@inproceedings{radford2021learning,\n  archiveprefix = {arXiv},\n  author = {Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},\n  booktitle = {Proceedings of the 38th International Conference on Machine Learning},\n  eprint = {2103.00020},\n  pages = {8748--8763},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Learning Transferable Visual Models From Natural Language Supervision},\n  url = {https://proceedings.mlr.press/v139/radford21a.html},\n  volume = {139},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_country211","license":"cc-by-sa-4.0","dateFrom":"2020-01-01","dateTo":"2021-02-26","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"DTD","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Describable Textures Dataset in 47 categories.","reference":"https://www.robots.ox.ac.uk/~vgg/data/dtd/","citation":"\n@inproceedings{cimpoi14describing,\n  author = {M. Cimpoi and S. Maji and I. Kokkinos and S. Mohamed and and A. Vedaldi},\n  booktitle = {Proceedings of the {IEEE} Conf. on Computer Vision and Pattern Recognition ({CVPR})},\n  title = {Describing Textures in the Wild},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/dtd","license":"not specified","dateFrom":"2014-01-01","dateTo":"2014-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"EuroSAT","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying satellite images.","reference":"https://ieeexplore.ieee.org/document/8736785","citation":"\n@article{8736785,\n  author = {Helber, Patrick and Bischke, Benjamin and Dengel, Andreas and Borth, Damian},\n  doi = {10.1109/JSTARS.2019.2918242},\n  journal = {IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing},\n  keywords = {Satellites;Earth;Remote sensing;Machine learning;Spatial resolution;Feature extraction;Benchmark testing;Dataset;deep convolutional neural network;deep learning;earth observation;land cover classification;land use classification;machine learning;remote sensing;satellite image classification;satellite images},\n  number = {7},\n  pages = {2217-2226},\n  title = {EuroSAT: A Novel Dataset and Deep Learning Benchmark for Land Use and Land Cover Classification},\n  volume = {12},\n  year = {2019},\n}\n","isPublic":true,"sourceDataset":"mteb/eurosat-rgb","license":"not specified","dateFrom":"2019-01-01","dateTo":"2019-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"FER2013","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying facial emotions.","reference":"https://arxiv.org/abs/1412.6572","citation":"\n@inproceedings{goodfellow2015explainingharnessingadversarialexamples,\n  archiveprefix = {arXiv},\n  author = {Ian J. Goodfellow and Jonathon Shlens and Christian Szegedy},\n  booktitle = {3rd International Conference on Learning Representations, {ICLR} 2015},\n  eprint = {1412.6572},\n  primaryclass = {stat.ML},\n  title = {Explaining and Harnessing Adversarial Examples},\n  year = {2015},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_fer2013","license":"not specified","dateFrom":"2014-01-01","dateTo":"2014-12-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"FGVCAircraft","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying aircraft images from 41 manufacturers and 102 variants.","reference":"https://arxiv.org/abs/1306.5151","citation":"\n@misc{maji2013finegrainedvisualclassificationaircraft,\n  archiveprefix = {arXiv},\n  author = {Subhransu Maji and Esa Rahtu and Juho Kannala and Matthew Blaschko and Andrea Vedaldi},\n  eprint = {1306.5151},\n  primaryclass = {cs.CV},\n  title = {Fine-Grained Visual Classification of Aircraft},\n  url = {https://arxiv.org/abs/1306.5151},\n  year = {2013},\n}\n","isPublic":true,"sourceDataset":"mteb/FGVCAircraft","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"Food101Classification","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Classifying food.","reference":"https://huggingface.co/datasets/ethz/food101","citation":"\n@inproceedings{bossard14,\n  author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc},\n  booktitle = {European Conference on Computer Vision},\n  title = {Food-101 -- Mining Discriminative Components with Random Forests},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/food101","license":"not specified","dateFrom":"2013-01-01","dateTo":"2014-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"GTSRB","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Scene"],"modalities":["image"],"description":"The German Traffic Sign Recognition Benchmark (GTSRB) is a multi-class classification dataset for traffic signs. It consists of dataset of more than 50,000 traffic sign images. The dataset comprises 43 classes with unbalanced class frequencies.","reference":"https://benchmark.ini.rub.de/","citation":"\n@inproceedings{6033395,\n  author = {Stallkamp, Johannes and Schlipsing, Marc and Salmen, Jan and Igel, Christian},\n  booktitle = {The 2011 International Joint Conference on Neural Networks},\n  doi = {10.1109/IJCNN.2011.6033395},\n  keywords = {Humans;Training;Image color analysis;Benchmark testing;Lead;Histograms;Image resolution},\n  number = {},\n  pages = {1453-1460},\n  title = {The German Traffic Sign Recognition Benchmark: A multi-class classification competition},\n  volume = {},\n  year = {2011},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_gtsrb","license":"not specified","dateFrom":"2011-01-01","dateTo":"2011-12-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"Imagenet1k","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Scene"],"modalities":["image"],"description":"ImageNet, a large-scale ontology of images built upon the backbone of the WordNet structure.","reference":"https://ieeexplore.ieee.org/document/5206848","citation":"\n@inproceedings{deng2009imagenet,\n  author = {Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Kai Li and Li Fei-Fei},\n  booktitle = {2009 IEEE Conference on Computer Vision and Pattern Recognition},\n  doi = {10.1109/CVPR.2009.5206848},\n  keywords = {Large-scale systems;Image databases;Explosions;Internet;Robustness;Information retrieval;Image retrieval;Multimedia databases;Ontologies;Spine},\n  number = {},\n  pages = {248-255},\n  title = {ImageNet: A large-scale hierarchical image database},\n  volume = {},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_imagenet1k","license":"not specified","dateFrom":"2010-01-01","dateTo":"2012-01-01","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"MNIST","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying handwritten digits.","reference":"https://en.wikipedia.org/wiki/MNIST_database","citation":"\n@article{lecun1998gradient,\n  author = {LeCun, Yann and Bottou, L{\\'e}on and Bengio, Yoshua and Haffner, Patrick},\n  journal = {Proceedings of the IEEE},\n  number = {11},\n  pages = {2278--2324},\n  publisher = {Ieee},\n  title = {Gradient-based learning applied to document recognition},\n  volume = {86},\n  year = {1998},\n}\n","isPublic":true,"sourceDataset":"mteb/mnist","license":"not specified","dateFrom":"2010-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"OxfordFlowersClassification","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Reviews"],"modalities":["image"],"description":"Classifying flowers","reference":"https://huggingface.co/datasets/nelorth/oxford-flowers/viewer/default/train","citation":"\n@inproceedings{4756141,\n  author = {Nilsback, Maria-Elena and Zisserman, Andrew},\n  booktitle = {2008 Sixth Indian Conference on Computer Vision, Graphics & Image Processing},\n  doi = {10.1109/ICVGIP.2008.47},\n  keywords = {Shape;Kernel;Distributed computing;Support vector machines;Support vector machine classification;object classification;segmentation},\n  number = {},\n  pages = {722-729},\n  title = {Automated Flower Classification over a Large Number of Classes},\n  volume = {},\n  year = {2008},\n}\n","isPublic":true,"sourceDataset":"mteb/oxford-flowers","license":"not specified","dateFrom":"2012-01-01","dateTo":"2015-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"OxfordPets","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying animal images.","reference":"https://ieeexplore.ieee.org/abstract/document/6248092","citation":"\n@inproceedings{6248092,\n  author = {Parkhi, Omkar M and Vedaldi, Andrea and Zisserman, Andrew and Jawahar, C. V.},\n  booktitle = {2012 IEEE Conference on Computer Vision and Pattern Recognition},\n  doi = {10.1109/CVPR.2012.6248092},\n  keywords = {Positron emission tomography;Image segmentation;Cats;Dogs;Layout;Deformable models;Head},\n  number = {},\n  pages = {3498-3505},\n  title = {Cats and dogs},\n  volume = {},\n  year = {2012},\n}\n","isPublic":true,"sourceDataset":"mteb/OxfordPets","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"PatchCamelyon","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Medical"],"modalities":["image"],"description":"Histopathology diagnosis classification dataset.","reference":"https://link.springer.com/chapter/10.1007/978-3-030-00934-2_24","citation":"\n@inproceedings{10.1007/978-3-030-00934-2_24,\n  address = {Cham},\n  author = {Veeling, Bastiaan S.\nand Linmans, Jasper\nand Winkens, Jim\nand Cohen, Taco\nand Welling, Max},\n  booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2018},\n  editor = {Frangi, Alejandro F.\nand Schnabel, Julia A.\nand Davatzikos, Christos\nand Alberola-L{\\'o}pez, Carlos\nand Fichtinger, Gabor},\n  isbn = {978-3-030-00934-2},\n  pages = {210--218},\n  publisher = {Springer International Publishing},\n  title = {Rotation Equivariant CNNs for Digital Pathology},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_vtab-pcam","license":"not specified","dateFrom":"2018-01-01","dateTo":"2018-12-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"RESISC45","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Remote Sensing Image Scene Classification by Northwestern Polytechnical University (NWPU).","reference":"https://ieeexplore.ieee.org/abstract/document/7891544","citation":"\n@article{7891544,\n  author = {Cheng, Gong and Han, Junwei and Lu, Xiaoqiang},\n  doi = {10.1109/JPROC.2017.2675998},\n  journal = {Proceedings of the IEEE},\n  keywords = {Remote sensing;Benchmark testing;Spatial resolution;Social network services;Satellites;Image analysis;Machine learning;Unsupervised learning;Classification;Benchmark data set;deep learning;handcrafted features;remote sensing image;scene classification;unsupervised feature learning},\n  number = {10},\n  pages = {1865-1883},\n  title = {Remote Sensing Image Scene Classification: Benchmark and State of the Art},\n  volume = {105},\n  year = {2017},\n}\n","isPublic":true,"sourceDataset":"mteb/resisc45","license":"not specified","dateFrom":"2017-01-01","dateTo":"2017-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"StanfordCars","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying car images from 196 makes.","reference":"https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content","citation":"\n@inproceedings{Krause2013CollectingAL,\n  author = {Jonathan Krause and Jia Deng and Michael Stark and Li Fei-Fei},\n  title = {Collecting a Large-scale Dataset of Fine-grained Cars},\n  url = {https://api.semanticscholar.org/CorpusID:16632981},\n  year = {2013},\n}\n","isPublic":true,"sourceDataset":"mteb/StanfordCars","license":"not specified","dateFrom":"2013-01-01","dateTo":"2013-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"STL10","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying 96x96 images from 10 classes.","reference":"https://cs.stanford.edu/~acoates/stl10/","citation":"\n@inproceedings{pmlr-v15-coates11a,\n  address = {Fort Lauderdale, FL, USA},\n  author = {Coates, Adam and Ng, Andrew and Lee, Honglak},\n  booktitle = {Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},\n  editor = {Gordon, Geoffrey and Dunson, David and Dudík, Miroslav},\n  month = {11--13 Apr},\n  pages = {215--223},\n  pdf = {http://proceedings.mlr.press/v15/coates11a/coates11a.pdf},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {An Analysis of Single-Layer Networks in Unsupervised Feature Learning},\n  url = {https://proceedings.mlr.press/v15/coates11a.html},\n  volume = {15},\n  year = {2011},\n}\n","isPublic":true,"sourceDataset":"mteb/stl10","license":"not specified","dateFrom":"2011-01-01","dateTo":"2011-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"SUN397","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Large scale scene recognition in 397 categories.","reference":"https://ieeexplore.ieee.org/abstract/document/5539970","citation":"\n@inproceedings{5539970,\n  author = {Xiao, Jianxiong and Hays, James and Ehinger, Krista A. and Oliva, Aude and Torralba, Antonio},\n  booktitle = {2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition},\n  doi = {10.1109/CVPR.2010.5539970},\n  number = {},\n  pages = {3485-3492},\n  title = {SUN database: Large-scale scene recognition from abbey to zoo},\n  volume = {},\n  year = {2010},\n}\n","isPublic":true,"sourceDataset":"mteb/sun397","license":"not specified","dateFrom":"2017-01-01","dateTo":"2017-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"UCF101","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Activity","Sport","Web"],"modalities":["image"],"description":"UCF101 is an action recognition data set of realistic action videos collected from YouTube, having 101 action categories. This version of the dataset does not contain images but images saved frame by frame. Train and test splits are generated based on the authors' first version train/test list.","reference":"https://huggingface.co/datasets/flwrlabs/ucf101","citation":"\n@misc{soomro2012ucf101dataset101human,\n  archiveprefix = {arXiv},\n  author = {Khurram Soomro and Amir Roshan Zamir and Mubarak Shah},\n  eprint = {1212.0402},\n  primaryclass = {cs.CV},\n  title = {UCF101: A Dataset of 101 Human Actions Classes From Videos in The Wild},\n  url = {https://arxiv.org/abs/1212.0402},\n  year = {2012},\n}\n","isPublic":true,"sourceDataset":"mteb/ucf101","license":"not specified","dateFrom":"2012-01-01","dateTo":"2012-12-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"VOC2007","type":"ImageClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Classifying bird images from 500 species.","reference":"http://host.robots.ox.ac.uk/pascal/VOC/","citation":"\n@article{Everingham10,\n  author = {Everingham, M. and Van~Gool, L. and Williams, C. K. I. and Winn, J. and Zisserman, A.},\n  journal = {International Journal of Computer Vision},\n  month = jun,\n  number = {2},\n  pages = {303--338},\n  title = {The Pascal Visual Object Classes (VOC) Challenge},\n  volume = {88},\n  year = {2010},\n}\n","isPublic":true,"sourceDataset":"mteb/VOC2007","license":"not specified","dateFrom":"2005-01-01","dateTo":"2014-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"lrap","numModels":0},{"name":"CIFAR10Clustering","type":"ImageClustering","simplifiedType":"clustering","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Clustering images from 10 classes.","reference":"https://huggingface.co/datasets/uoft-cs/cifar10","citation":"\n@techreport{Krizhevsky09learningmultiple,\n  author = {Alex Krizhevsky},\n  institution = {},\n  title = {Learning multiple layers of features from tiny images},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/cifar10","license":"not specified","dateFrom":"2008-01-01","dateTo":"2009-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"nmi","numModels":0},{"name":"CIFAR100Clustering","type":"ImageClustering","simplifiedType":"clustering","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Clustering images from 100 classes.","reference":"https://huggingface.co/datasets/uoft-cs/cifar100","citation":"\n@techreport{Krizhevsky09learningmultiple,\n  author = {Alex Krizhevsky},\n  institution = {},\n  title = {Learning multiple layers of features from tiny images},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/cifar100","license":"not specified","dateFrom":"2008-01-01","dateTo":"2009-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"nmi","numModels":0},{"name":"ImageNetDog15Clustering","type":"ImageClustering","simplifiedType":"clustering","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Clustering images from a 15-class dogs-only subset of the dog classes in ImageNet.","reference":"http://vision.stanford.edu/aditya86/ImageNetDogs/main.html","citation":"\n@inproceedings{deng2009imagenet,\n  author = {Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Kai Li and Li Fei-Fei},\n  booktitle = {2009 IEEE Conference on Computer Vision and Pattern Recognition},\n  doi = {10.1109/CVPR.2009.5206848},\n  keywords = {Large-scale systems;Image databases;Explosions;Internet;Robustness;Information retrieval;Image retrieval;Multimedia databases;Ontologies;Spine},\n  number = {},\n  pages = {248-255},\n  title = {ImageNet: A large-scale hierarchical image database},\n  volume = {},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/imagenet-dog-15","license":"not specified","dateFrom":"2009-06-20","dateTo":"2009-06-20","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"nmi","numModels":0},{"name":"ImageNet10Clustering","type":"ImageClustering","simplifiedType":"clustering","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Clustering images from an 10-class subset of ImageNet which are generally easy to distinguish.","reference":"https://www.kaggle.com/datasets/liusha249/imagenet10","citation":"\n@inproceedings{deng2009imagenet,\n  author = {Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Kai Li and Li Fei-Fei},\n  booktitle = {2009 IEEE Conference on Computer Vision and Pattern Recognition},\n  doi = {10.1109/CVPR.2009.5206848},\n  keywords = {Large-scale systems;Image databases;Explosions;Internet;Robustness;Information retrieval;Image retrieval;Multimedia databases;Ontologies;Spine},\n  number = {},\n  pages = {248-255},\n  title = {ImageNet: A large-scale hierarchical image database},\n  volume = {},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/imagenet-10","license":"not specified","dateFrom":"2009-06-20","dateTo":"2009-06-20","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"nmi","numModels":0},{"name":"TinyImageNetClustering","type":"ImageClustering","simplifiedType":"clustering","languages":["English"],"domains":["Reviews"],"modalities":["image"],"description":"Clustering over 200 classes.","reference":"https://huggingface.co/datasets/zh-plus/tiny-imagenet/viewer/default/valid","citation":"","isPublic":true,"sourceDataset":"mteb/tiny-imagenet","license":"not specified","dateFrom":"2012-01-01","dateTo":"2015-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"nmi","numModels":0},{"name":"BirdsnapZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Classifying bird images from 500 species. ","reference":"https://openaccess.thecvf.com/content_cvpr_2014/html/Berg_Birdsnap_Large-scale_Fine-grained_2014_CVPR_paper.html","citation":"\n@inproceedings{Berg_2014_CVPR,\n  author = {Berg, Thomas and Liu, Jiongxin and Woo Lee, Seung and Alexander, Michelle L. and Jacobs, David W. and Belhumeur, Peter N.},\n  booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n  month = {June},\n  title = {Birdsnap: Large-scale Fine-grained Visual Categorization of Birds},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/birdsnap","license":"not specified","dateFrom":"2013-01-01","dateTo":"2014-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"Caltech101ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Classifying images of 101 widely varied objects.","reference":"https://ieeexplore.ieee.org/document/1384978","citation":"\n@inproceedings{1384978,\n  author = {Li Fei-Fei and Fergus, R. and Perona, P.},\n  booktitle = {2004 Conference on Computer Vision and Pattern Recognition Workshop},\n  doi = {10.1109/CVPR.2004.383},\n  keywords = {Bayesian methods;Testing;Humans;Maximum likelihood estimation;Assembly;Shape;Machine vision;Image recognition;Parameter estimation;Image databases},\n  number = {},\n  pages = {178-178},\n  title = {Learning Generative Visual Models from Few Training Examples: An Incremental Bayesian Approach Tested on 101 Object Categories},\n  volume = {},\n  year = {2004},\n}\n","isPublic":true,"sourceDataset":"mteb/Caltech101","license":"not specified","dateFrom":"2003-01-01","dateTo":"2004-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"CIFAR10ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Web"],"modalities":["text","image"],"description":"Classifying images from 10 classes.","reference":"https://huggingface.co/datasets/uoft-cs/cifar10","citation":"\n@techreport{Krizhevsky09learningmultiple,\n  author = {Alex Krizhevsky},\n  institution = {},\n  title = {Learning multiple layers of features from tiny images},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/cifar10","license":"not specified","dateFrom":"2008-01-01","dateTo":"2009-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"CIFAR100ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Web"],"modalities":["text","image"],"description":"Classifying images from 100 classes.","reference":"https://huggingface.co/datasets/uoft-cs/cifar100","citation":"\n@techreport{Krizhevsky09learningmultiple,\n  author = {Alex Krizhevsky},\n  institution = {},\n  title = {Learning multiple layers of features from tiny images},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/cifar100","license":"not specified","dateFrom":"2008-01-01","dateTo":"2009-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"CLEVRZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Constructed"],"modalities":["text","image"],"description":"CLEVR closest object distance identification task.","reference":"https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html","citation":"\n@inproceedings{Johnson_2017_CVPR,\n  author = {Johnson, Justin and Hariharan, Bharath and van der Maaten, Laurens and Fei-Fei, Li and Lawrence Zitnick, C. and Girshick, Ross},\n  booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n  month = {July},\n  title = {CLEVR: A Diagnostic Dataset for Compositional Language and Elementary Visual Reasoning},\n  year = {2017},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_vtab-clevr_closest_object_distance","license":"cc-by-4.0","dateFrom":"2016-01-01","dateTo":"2016-12-20","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"CLEVRCountZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Constructed"],"modalities":["text","image"],"description":"CLEVR count objects task.","reference":"https://openaccess.thecvf.com/content_cvpr_2017/html/Johnson_CLEVR_A_Diagnostic_CVPR_2017_paper.html","citation":"\n@inproceedings{Johnson_2017_CVPR,\n  author = {Johnson, Justin and Hariharan, Bharath and van der Maaten, Laurens and Fei-Fei, Li and Lawrence Zitnick, C. and Girshick, Ross},\n  booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n  month = {July},\n  title = {CLEVR: A Diagnostic Dataset for Compositional Language and Elementary Visual Reasoning},\n  year = {2017},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_vtab-clevr_count_all","license":"not specified","dateFrom":"2016-01-01","dateTo":"2016-12-20","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"Country211ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Scene"],"modalities":["image","text"],"description":"Classifying images of 211 countries.","reference":"https://huggingface.co/datasets/clip-benchmark/wds_country211","citation":"\n@inproceedings{radford2021learning,\n  archiveprefix = {arXiv},\n  author = {Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},\n  booktitle = {Proceedings of the 38th International Conference on Machine Learning},\n  eprint = {2103.00020},\n  pages = {8748--8763},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Learning Transferable Visual Models From Natural Language Supervision},\n  url = {https://proceedings.mlr.press/v139/radford21a.html},\n  volume = {139},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_country211","license":"cc-by-sa-4.0","dateFrom":"2020-01-01","dateTo":"2021-02-26","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"DTDZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Describable Textures Dataset in 47 categories.","reference":"https://www.robots.ox.ac.uk/~vgg/data/dtd/","citation":"\n@inproceedings{cimpoi14describing,\n  author = {M. Cimpoi and S. Maji and I. Kokkinos and S. Mohamed and and A. Vedaldi},\n  booktitle = {Proceedings of the {IEEE} Conf. on Computer Vision and Pattern Recognition ({CVPR})},\n  title = {Describing Textures in the Wild},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/dtd","license":"not specified","dateFrom":"2014-01-01","dateTo":"2014-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"EuroSATZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Classifying satellite images.","reference":"https://ieeexplore.ieee.org/document/8736785","citation":"\n@article{8736785,\n  author = {Helber, Patrick and Bischke, Benjamin and Dengel, Andreas and Borth, Damian},\n  doi = {10.1109/JSTARS.2019.2918242},\n  journal = {IEEE Journal of Selected Topics in Applied Earth Observations and Remote Sensing},\n  keywords = {Satellites;Earth;Remote sensing;Machine learning;Spatial resolution;Feature extraction;Benchmark testing;Dataset;deep convolutional neural network;deep learning;earth observation;land cover classification;land use classification;machine learning;remote sensing;satellite image classification;satellite images},\n  number = {7},\n  pages = {2217-2226},\n  title = {EuroSAT: A Novel Dataset and Deep Learning Benchmark for Land Use and Land Cover Classification},\n  volume = {12},\n  year = {2019},\n}\n","isPublic":true,"sourceDataset":"mteb/eurosat-rgb","license":"not specified","dateFrom":"2019-01-01","dateTo":"2019-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"FER2013ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Classifying facial emotions.","reference":"https://arxiv.org/abs/1412.6572","citation":"\n@inproceedings{goodfellow2015explainingharnessingadversarialexamples,\n  archiveprefix = {arXiv},\n  author = {Ian J. Goodfellow and Jonathon Shlens and Christian Szegedy},\n  booktitle = {3rd International Conference on Learning Representations, {ICLR} 2015},\n  eprint = {1412.6572},\n  primaryclass = {stat.ML},\n  title = {Explaining and Harnessing Adversarial Examples},\n  year = {2015},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_fer2013","license":"not specified","dateFrom":"2014-01-01","dateTo":"2014-12-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"FGVCAircraftZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Classifying aircraft images from 41 manufacturers and 102 variants.","reference":"https://arxiv.org/abs/1306.5151","citation":"\n@misc{maji2013finegrainedvisualclassificationaircraft,\n  archiveprefix = {arXiv},\n  author = {Subhransu Maji and Esa Rahtu and Juho Kannala and Matthew Blaschko and Andrea Vedaldi},\n  eprint = {1306.5151},\n  primaryclass = {cs.CV},\n  title = {Fine-Grained Visual Classification of Aircraft},\n  url = {https://arxiv.org/abs/1306.5151},\n  year = {2013},\n}\n","isPublic":true,"sourceDataset":"mteb/FGVCAircraftZeroShot","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"Food101ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Web"],"modalities":["text","image"],"description":"Classifying food.","reference":"https://huggingface.co/datasets/ethz/food101","citation":"\n@inproceedings{bossard14,\n  author = {Bossard, Lukas and Guillaumin, Matthieu and Van Gool, Luc},\n  booktitle = {European Conference on Computer Vision},\n  title = {Food-101 -- Mining Discriminative Components with Random Forests},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/food101","license":"not specified","dateFrom":"2013-01-01","dateTo":"2014-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"GTSRBZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Scene"],"modalities":["image","text"],"description":"The German Traffic Sign Recognition Benchmark (GTSRB) is a multi-class classification dataset for traffic signs. It consists of dataset of more than 50,000 traffic sign images. The dataset comprises 43 classes with unbalanced class frequencies.","reference":"https://benchmark.ini.rub.de/","citation":"\n@inproceedings{6033395,\n  author = {Stallkamp, Johannes and Schlipsing, Marc and Salmen, Jan and Igel, Christian},\n  booktitle = {The 2011 International Joint Conference on Neural Networks},\n  doi = {10.1109/IJCNN.2011.6033395},\n  keywords = {Humans;Training;Image color analysis;Benchmark testing;Lead;Histograms;Image resolution},\n  number = {},\n  pages = {1453-1460},\n  title = {The German Traffic Sign Recognition Benchmark: A multi-class classification competition},\n  volume = {},\n  year = {2011},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_gtsrb","license":"not specified","dateFrom":"2011-01-01","dateTo":"2011-12-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"Imagenet1kZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Scene"],"modalities":["image","text"],"description":"ImageNet, a large-scale ontology of images built upon the backbone of the WordNet structure.","reference":"https://ieeexplore.ieee.org/document/5206848","citation":"\n@inproceedings{deng2009imagenet,\n  author = {Deng, Jia and Dong, Wei and Socher, Richard and Li, Li-Jia and Kai Li and Li Fei-Fei},\n  booktitle = {2009 IEEE Conference on Computer Vision and Pattern Recognition},\n  doi = {10.1109/CVPR.2009.5206848},\n  keywords = {Large-scale systems;Image databases;Explosions;Internet;Robustness;Information retrieval;Image retrieval;Multimedia databases;Ontologies;Spine},\n  number = {},\n  pages = {248-255},\n  title = {ImageNet: A large-scale hierarchical image database},\n  volume = {},\n  year = {2009},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_imagenet1k","license":"not specified","dateFrom":"2010-01-01","dateTo":"2012-01-01","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"MNISTZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Classifying handwritten digits.","reference":"https://en.wikipedia.org/wiki/MNIST_database","citation":"\n@article{lecun1998gradient,\n  author = {LeCun, Yann and Bottou, L{\\'e}on and Bengio, Yoshua and Haffner, Patrick},\n  journal = {Proceedings of the IEEE},\n  number = {11},\n  pages = {2278--2324},\n  publisher = {Ieee},\n  title = {Gradient-based learning applied to document recognition},\n  volume = {86},\n  year = {1998},\n}\n","isPublic":true,"sourceDataset":"mteb/mnist","license":"not specified","dateFrom":"2010-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"OxfordPetsZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Classifying animal images.","reference":"https://arxiv.org/abs/1306.5151","citation":"\n@misc{maji2013finegrainedvisualclassificationaircraft,\n  archiveprefix = {arXiv},\n  author = {Subhransu Maji and Esa Rahtu and Juho Kannala and Matthew Blaschko and Andrea Vedaldi},\n  eprint = {1306.5151},\n  primaryclass = {cs.CV},\n  title = {Fine-Grained Visual Classification of Aircraft},\n  url = {https://arxiv.org/abs/1306.5151},\n  year = {2013},\n}\n","isPublic":true,"sourceDataset":"mteb/OxfordPets","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"PatchCamelyonZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Medical"],"modalities":["image","text"],"description":"Histopathology diagnosis classification dataset.","reference":"https://link.springer.com/chapter/10.1007/978-3-030-00934-2_24","citation":"\n@inproceedings{10.1007/978-3-030-00934-2_24,\n  address = {Cham},\n  author = {Veeling, Bastiaan S.\nand Linmans, Jasper\nand Winkens, Jim\nand Cohen, Taco\nand Welling, Max},\n  booktitle = {Medical Image Computing and Computer Assisted Intervention -- MICCAI 2018},\n  editor = {Frangi, Alejandro F.\nand Schnabel, Julia A.\nand Davatzikos, Christos\nand Alberola-L{\\'o}pez, Carlos\nand Fichtinger, Gabor},\n  isbn = {978-3-030-00934-2},\n  pages = {210--218},\n  publisher = {Springer International Publishing},\n  title = {Rotation Equivariant CNNs for Digital Pathology},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/wds_vtab-pcam","license":"not specified","dateFrom":"2018-01-01","dateTo":"2018-12-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"RenderedSST2","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Reviews"],"modalities":["text","image"],"description":"RenderedSST2.","reference":"https://huggingface.co/datasets/clip-benchmark/wds_renderedsst2","citation":"","isPublic":true,"sourceDataset":"mteb/wds_renderedsst2","license":"mit","dateFrom":"2016-01-01","dateTo":"2016-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"RESISC45ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Remote Sensing Image Scene Classification by Northwestern Polytechnical University (NWPU).","reference":"https://ieeexplore.ieee.org/abstract/document/7891544","citation":"\n@article{7891544,\n  author = {Cheng, Gong and Han, Junwei and Lu, Xiaoqiang},\n  doi = {10.1109/JPROC.2017.2675998},\n  journal = {Proceedings of the IEEE},\n  keywords = {Remote sensing;Benchmark testing;Spatial resolution;Social network services;Satellites;Image analysis;Machine learning;Unsupervised learning;Classification;Benchmark data set;deep learning;handcrafted features;remote sensing image;scene classification;unsupervised feature learning},\n  number = {10},\n  pages = {1865-1883},\n  title = {Remote Sensing Image Scene Classification: Benchmark and State of the Art},\n  volume = {105},\n  year = {2017},\n}\n","isPublic":true,"sourceDataset":"mteb/resisc45","license":"not specified","dateFrom":"2017-01-01","dateTo":"2017-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"StanfordCarsZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Scene"],"modalities":["image","text"],"description":"Classifying car images from 96 makes.","reference":"https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content","citation":"\n@inproceedings{Krause2013CollectingAL,\n  author = {Jonathan Krause and Jia Deng and Michael Stark and Li Fei-Fei},\n  title = {Collecting a Large-scale Dataset of Fine-grained Cars},\n  url = {https://api.semanticscholar.org/CorpusID:16632981},\n  year = {2013},\n}\n","isPublic":true,"sourceDataset":"mteb/StanfordCars","license":"not specified","dateFrom":"2013-01-01","dateTo":"2013-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"STL10ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Classifying 96x96 images from 10 classes.","reference":"https://cs.stanford.edu/~acoates/stl10/","citation":"\n@inproceedings{pmlr-v15-coates11a,\n  address = {Fort Lauderdale, FL, USA},\n  author = {Coates, Adam and Ng, Andrew and Lee, Honglak},\n  booktitle = {Proceedings of the Fourteenth International Conference on Artificial Intelligence and Statistics},\n  editor = {Gordon, Geoffrey and Dunson, David and Dudík, Miroslav},\n  month = {11--13 Apr},\n  pages = {215--223},\n  pdf = {http://proceedings.mlr.press/v15/coates11a/coates11a.pdf},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {An Analysis of Single-Layer Networks in Unsupervised Feature Learning},\n  url = {https://proceedings.mlr.press/v15/coates11a.html},\n  volume = {15},\n  year = {2011},\n}\n","isPublic":true,"sourceDataset":"mteb/stl10","license":"not specified","dateFrom":"2011-01-01","dateTo":"2011-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"SUN397ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Large scale scene recognition in 397 categories.","reference":"https://ieeexplore.ieee.org/abstract/document/5539970","citation":"\n@inproceedings{5539970,\n  author = {Xiao, Jianxiong and Hays, James and Ehinger, Krista A. and Oliva, Aude and Torralba, Antonio},\n  booktitle = {2010 IEEE Computer Society Conference on Computer Vision and Pattern Recognition},\n  doi = {10.1109/CVPR.2010.5539970},\n  number = {},\n  pages = {3485-3492},\n  title = {SUN database: Large-scale scene recognition from abbey to zoo},\n  volume = {},\n  year = {2010},\n}\n","isPublic":true,"sourceDataset":"mteb/sun397","license":"not specified","dateFrom":"2017-01-01","dateTo":"2017-03-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"UCF101ZeroShot","type":"ZeroShotClassification","simplifiedType":"classification","languages":["English"],"domains":["Activity","Sport","Web"],"modalities":["image","text"],"description":"UCF101 is an action recognition data set of realistic action videos collected from YouTube, having 101 action categories. This version of the dataset does not contain images but images saved frame by frame. Train and test splits are generated based on the authors' first version train/test list.","reference":"https://huggingface.co/datasets/flwrlabs/ucf101","citation":"\n@misc{soomro2012ucf101dataset101human,\n  archiveprefix = {arXiv},\n  author = {Khurram Soomro and Amir Roshan Zamir and Mubarak Shah},\n  eprint = {1212.0402},\n  primaryclass = {cs.CV},\n  title = {UCF101: A Dataset of 101 Human Actions Classes From Videos in The Wild},\n  url = {https://arxiv.org/abs/1212.0402},\n  year = {2012},\n}\n","isPublic":true,"sourceDataset":"mteb/ucf101","license":"not specified","dateFrom":"2012-01-01","dateTo":"2012-12-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"BLINKIT2IMultiChoice","type":"VisionCentricQA","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve images based on images and specific retrieval instructions.","reference":"https://doi.org/10.1007/978-3-031-73337-6_9","citation":"\n@inbook{fu2024blink,\n  author = {Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth, Dan and Smith, Noah A. and Ma, Wei-Chiu and Krishna, Ranjay},\n  booktitle = {Computer Vision – ECCV 2024},\n  doi = {10.1007/978-3-031-73337-6_9},\n  isbn = {9783031733376},\n  issn = {1611-3349},\n  month = {Oct},\n  pages = {148–166},\n  publisher = {Springer Nature Switzerland},\n  title = {BLINK: Multimodal Large Language Models Can See but Not Perceive},\n  url = {https://doi.org/10.1007/978-3-031-73337-6_9},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/blink-it2i-multi","license":"not specified","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"BLINKIT2TMultiChoice","type":"VisionCentricQA","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve the correct text answer based on images and specific retrieval instructions.","reference":"https://doi.org/10.1007/978-3-031-73337-6_9","citation":"\n@inbook{fu2024blink,\n  author = {Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth, Dan and Smith, Noah A. and Ma, Wei-Chiu and Krishna, Ranjay},\n  booktitle = {Computer Vision – ECCV 2024},\n  doi = {10.1007/978-3-031-73337-6_9},\n  isbn = {9783031733376},\n  issn = {1611-3349},\n  month = {Oct},\n  pages = {148–166},\n  publisher = {Springer Nature Switzerland},\n  title = {BLINK: Multimodal Large Language Models Can See but Not Perceive},\n  url = {https://doi.org/10.1007/978-3-031-73337-6_9},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/blink-it2t-multi","license":"not specified","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"CVBenchCount","type":"VisionCentricQA","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["image","text"],"description":"count the number of objects in the image.","reference":"https://papers.nips.cc/paper_files/paper/2024/hash/9ee3a664ccfeabc0da16ac6f1f1cfe59-Abstract-Conference.html","citation":"\n@inproceedings{tong2024cambrian,\n  archiveprefix = {arXiv},\n  author = {Peter Tong and Ellis Brown and Penghao Wu and Sanghyun Woo and Adithya Iyer and Sai Charitha Akula and Shusheng Yang and Jihan Yang and Manoj Middepogu and Ziteng Wang and Xichen Pan and Rob Fergus and Yann LeCun and Saining Xie},\n  booktitle = {NeurIPS},\n  eprint = {2406.16860},\n  title = {Cambrian-1: A Fully Open, Vision-Centric Exploration of Multimodal LLMs},\n  url = {http://papers.nips.cc/paper_files/paper/2024/hash/9ee3a664ccfeabc0da16ac6f1f1cfe59-Abstract-Conference.html},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/CV-Bench","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-06-24","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"CVBenchRelation","type":"VisionCentricQA","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"decide the relation of the objects in the image.","reference":"https://papers.nips.cc/paper_files/paper/2024/hash/9ee3a664ccfeabc0da16ac6f1f1cfe59-Abstract-Conference.html","citation":"\n@inproceedings{tong2024cambrian,\n  archiveprefix = {arXiv},\n  author = {Peter Tong and Ellis Brown and Penghao Wu and Sanghyun Woo and Adithya Iyer and Sai Charitha Akula and Shusheng Yang and Jihan Yang and Manoj Middepogu and Ziteng Wang and Xichen Pan and Rob Fergus and Yann LeCun and Saining Xie},\n  booktitle = {NeurIPS},\n  eprint = {2406.16860},\n  title = {Cambrian-1: A Fully Open, Vision-Centric Exploration of Multimodal LLMs},\n  url = {http://papers.nips.cc/paper_files/paper/2024/hash/9ee3a664ccfeabc0da16ac6f1f1cfe59-Abstract-Conference.html},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/CV-Bench","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-06-24","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"CVBenchDepth","type":"VisionCentricQA","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"judge the depth of the objects in the image with similarity matching.","reference":"https://papers.nips.cc/paper_files/paper/2024/hash/9ee3a664ccfeabc0da16ac6f1f1cfe59-Abstract-Conference.html","citation":"\n@inproceedings{tong2024cambrian,\n  archiveprefix = {arXiv},\n  author = {Peter Tong and Ellis Brown and Penghao Wu and Sanghyun Woo and Adithya Iyer and Sai Charitha Akula and Shusheng Yang and Jihan Yang and Manoj Middepogu and Ziteng Wang and Xichen Pan and Rob Fergus and Yann LeCun and Saining Xie},\n  booktitle = {NeurIPS},\n  eprint = {2406.16860},\n  title = {Cambrian-1: A Fully Open, Vision-Centric Exploration of Multimodal LLMs},\n  url = {http://papers.nips.cc/paper_files/paper/2024/hash/9ee3a664ccfeabc0da16ac6f1f1cfe59-Abstract-Conference.html},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/CV-Bench","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-06-24","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"CVBenchDistance","type":"VisionCentricQA","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"judge the distance of the objects in the image with similarity matching.","reference":"https://papers.nips.cc/paper_files/paper/2024/hash/9ee3a664ccfeabc0da16ac6f1f1cfe59-Abstract-Conference.html","citation":"\n@inproceedings{tong2024cambrian,\n  archiveprefix = {arXiv},\n  author = {Peter Tong and Ellis Brown and Penghao Wu and Sanghyun Woo and Adithya Iyer and Sai Charitha Akula and Shusheng Yang and Jihan Yang and Manoj Middepogu and Ziteng Wang and Xichen Pan and Rob Fergus and Yann LeCun and Saining Xie},\n  booktitle = {NeurIPS},\n  eprint = {2406.16860},\n  title = {Cambrian-1: A Fully Open, Vision-Centric Exploration of Multimodal LLMs},\n  url = {http://papers.nips.cc/paper_files/paper/2024/hash/9ee3a664ccfeabc0da16ac6f1f1cfe59-Abstract-Conference.html},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/CV-Bench","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-06-24","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"accuracy","numModels":0},{"name":"AROCocoOrder","type":"Compositionality","simplifiedType":"pair-classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Compositionality Evaluation of images to their captions.Each capation has four hard negatives created by order permutations.","reference":"https://openreview.net/forum?id=KRLUvxh8uaX","citation":"\n@inproceedings{yuksekgonul2023and,\n  author = {Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James},\n  booktitle = {The Eleventh International Conference on Learning Representations},\n  title = {When and why vision-language models behave like bags-of-words, and what to do about it?},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/ARO-COCO-order","license":"mit","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"expert-annotated","dialect":[],"sampleCreation":"created","mainScore":"text_acc","numModels":0},{"name":"AROFlickrOrder","type":"Compositionality","simplifiedType":"pair-classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Compositionality Evaluation of images to their captions.Each capation has four hard negatives created by order permutations.","reference":"https://openreview.net/forum?id=KRLUvxh8uaX","citation":"\n@inproceedings{yuksekgonul2023and,\n  author = {Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James},\n  booktitle = {The Eleventh International Conference on Learning Representations},\n  title = {When and why vision-language models behave like bags-of-words, and what to do about it?},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/ARO-Flickr-Order","license":"mit","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"expert-annotated","dialect":[],"sampleCreation":"created","mainScore":"text_acc","numModels":0},{"name":"AROVisualAttribution","type":"Compositionality","simplifiedType":"pair-classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Compositionality Evaluation of images to their captions.","reference":"https://openreview.net/forum?id=KRLUvxh8uaX","citation":"\n@inproceedings{yuksekgonul2023and,\n  author = {Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James},\n  booktitle = {The Eleventh International Conference on Learning Representations},\n  title = {When and why vision-language models behave like bags-of-words, and what to do about it?},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/ARO-Visual-Attribution","license":"mit","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"expert-annotated","dialect":[],"sampleCreation":"created","mainScore":"text_acc","numModels":0},{"name":"AROVisualRelation","type":"Compositionality","simplifiedType":"pair-classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Compositionality Evaluation of images to their captions.","reference":"https://openreview.net/forum?id=KRLUvxh8uaX","citation":"\n@inproceedings{yuksekgonul2023and,\n  author = {Yuksekgonul, Mert and Bianchi, Federico and Kalluri, Pratyusha and Jurafsky, Dan and Zou, James},\n  booktitle = {The Eleventh International Conference on Learning Representations},\n  title = {When and why vision-language models behave like bags-of-words, and what to do about it?},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/ARO-Visual-Relation","license":"mit","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"expert-annotated","dialect":[],"sampleCreation":"created","mainScore":"text_acc","numModels":0},{"name":"SugarCrepe","type":"Compositionality","simplifiedType":"pair-classification","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Compositionality Evaluation of images to their captions. Note that 572 of the 7511 pairs (7.6%) have a caption and negative_caption that are the same words in a different order (the swap_obj and swap_att subsets of the source benchmark). A model whose text representation is order-insensitive scores chance on those pairs by construction, capping its text_acc at 0.962.","reference":"https://proceedings.neurips.cc/paper_files/paper/2023/hash/63461de0b4cb760fc498e85b18a7fe81-Abstract-Datasets_and_Benchmarks.html","citation":"\n@article{hsieh2024sugarcrepe,\n  author = {Hsieh, Cheng-Yu and Zhang, Jieyu and Ma, Zixian and Kembhavi, Aniruddha and Krishna, Ranjay},\n  journal = {Advances in neural information processing systems},\n  title = {Sugarcrepe: Fixing hackable benchmarks for vision-language compositionality},\n  volume = {36},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/SUGARCREPE_fmt","license":"mit","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"expert-annotated","dialect":[],"sampleCreation":"created","mainScore":"text_acc","numModels":0},{"name":"Winoground","type":"Compositionality","simplifiedType":"pair-classification","languages":["English"],"domains":["Social"],"modalities":["text","image"],"description":"Compositionality Evaluation of images to their captions.","reference":"https://openaccess.thecvf.com/content/CVPR2022/html/Thrush_Winoground_Probing_Vision_and_Language_Models_for_Visio-Linguistic_Compositionality_CVPR_2022_paper","citation":"\n@inproceedings{thrush2022winogroundprobingvisionlanguage,\n  author = {Thrush, Tristan and Jiang, Ryan and Bartolo, Max and Singh, Amanpreet and Williams, Adina and Kiela, Douwe and Ross, Candace},\n  booktitle = {2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n  doi = {10.1109/cvpr52688.2022.00517},\n  month = {June},\n  pages = {5228–5238},\n  publisher = {IEEE},\n  title = {Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality},\n  url = {https://doi.org/10.1109/cvpr52688.2022.00517},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"facebook/winoground","license":"https://huggingface.co/datasets/facebook/winoground/blob/main/license_agreement.txt","dateFrom":"2022-01-01","dateTo":"2022-04-07","annotationsCreators":"expert-annotated","dialect":[],"sampleCreation":"created","mainScore":"accuracy","numModels":0},{"name":"ImageCoDe","type":"Compositionality","simplifiedType":"pair-classification","languages":["English"],"domains":["Web","Written"],"modalities":["text","image"],"description":"Identify the correct image from a set of similar images based on a precise caption.","reference":"https://aclanthology.org/2022.acl-long.241.pdf","citation":"\n@inproceedings{krojer2022image,\n  address = {Dublin, Ireland},\n  author = {Krojer, Benno and Adlakha, Vaibhav and Vineet, Vibhav and Goyal, Yash and Ponti, Edoardo and Reddy, Siva},\n  booktitle = {Proceedings of the 60th Annual Meeting of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)},\n  doi = {10.18653/v1/2022.acl-long.241},\n  editor = {Muresan, Smaranda and Nakov, Preslav and Villavicencio, Aline},\n  month = {may},\n  pages = {3426--3440},\n  publisher = {Association for Computational Linguistics},\n  title = {Image Retrieval from Contextual Descriptions},\n  url = {https://aclanthology.org/2022.acl-long.241/},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/imagecode-multi","license":"cc-by-sa-4.0","dateFrom":"2022-05-22","dateTo":"2022-05-27","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"image_acc","numModels":0},{"name":"STS12VisualSTS","type":"VisualSTS(eng)","simplifiedType":"semantic-similarity","languages":["English"],"domains":["Encyclopaedic","News","Written"],"modalities":["image"],"description":"SemEval-2012 Task 6.then rendered into images.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/rendered-sts12","license":"not specified","dateFrom":"2005-01-01","dateTo":"2012-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0},{"name":"STS13VisualSTS","type":"VisualSTS(eng)","simplifiedType":"semantic-similarity","languages":["English"],"domains":["Web","News","Non-fiction","Written"],"modalities":["image"],"description":"SemEval STS 2013 dataset.then rendered into images.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/rendered-sts13","license":"not specified","dateFrom":"2012-01-01","dateTo":"2012-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0},{"name":"STS14VisualSTS","type":"VisualSTS(eng)","simplifiedType":"semantic-similarity","languages":["English"],"domains":["Blog","Web","Spoken"],"modalities":["image"],"description":"SemEval STS 2014 dataset. Currently only the English dataset.rendered into images.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/rendered-sts14","license":"not specified","dateFrom":"2012-01-01","dateTo":"2012-08-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0},{"name":"STS15VisualSTS","type":"VisualSTS(eng)","simplifiedType":"semantic-similarity","languages":["English"],"domains":["Blog","News","Web","Written","Spoken"],"modalities":["image"],"description":"SemEval STS 2015 datasetrendered into images.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/rendered-sts15","license":"not specified","dateFrom":"2008-01-01","dateTo":"2014-07-28","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0},{"name":"STS16VisualSTS","type":"VisualSTS(eng)","simplifiedType":"semantic-similarity","languages":["English"],"domains":["Blog","Web","Spoken"],"modalities":["image"],"description":"SemEval STS 2016 datasetrendered into images.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/rendered-sts16","license":"not specified","dateFrom":"2015-10-01","dateTo":"2015-12-31","annotationsCreators":"human-annotated","dialect":[],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0},{"name":"BLINKIT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve images based on images and specific retrieval instructions.","reference":"https://doi.org/10.1007/978-3-031-73337-6_9","citation":"\n@inbook{fu2024blink,\n  author = {Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth, Dan and Smith, Noah A. and Ma, Wei-Chiu and Krishna, Ranjay},\n  booktitle = {Computer Vision – ECCV 2024},\n  doi = {10.1007/978-3-031-73337-6_9},\n  isbn = {9783031733376},\n  issn = {1611-3349},\n  month = {Oct},\n  pages = {148–166},\n  publisher = {Springer Nature Switzerland},\n  title = {BLINK: Multimodal Large Language Models Can See but Not Perceive},\n  url = {https://doi.org/10.1007/978-3-031-73337-6_9},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/blink-it2i","license":"cc-by-sa-4.0","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_1","numModels":0},{"name":"BLINKIT2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve images based on images and specific retrieval instructions.","reference":"https://doi.org/10.1007/978-3-031-73337-6_9","citation":"\n@inbook{fu2024blink,\n  author = {Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth, Dan and Smith, Noah A. and Ma, Wei-Chiu and Krishna, Ranjay},\n  booktitle = {Computer Vision – ECCV 2024},\n  doi = {10.1007/978-3-031-73337-6_9},\n  isbn = {9783031733376},\n  issn = {1611-3349},\n  month = {Oct},\n  pages = {148–166},\n  publisher = {Springer Nature Switzerland},\n  title = {BLINK: Multimodal Large Language Models Can See but Not Perceive},\n  url = {https://doi.org/10.1007/978-3-031-73337-6_9},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/blink-it2t","license":"cc-by-sa-4.0","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_1","numModels":0},{"name":"CIRRIT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve images based on texts and images.","reference":"https://openaccess.thecvf.com/content/ICCV2021/html/Liu_Image_Retrieval_on_Real-Life_Images_With_Pre-Trained_Vision-and-Language_Models_ICCV_2021_paper.html","citation":"\n@inproceedings{liu2021image,\n  author = {Liu, Zheyuan and Rodriguez-Opazo, Cristian and Teney, Damien and Gould, Stephen},\n  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision},\n  pages = {2125--2134},\n  title = {Image retrieval on real-life images with pre-trained vision-and-language models},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_cirr_task7","license":"cc-by-sa-4.0","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"CUB200I2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Retrieve bird images from 200 classes.","reference":"https://www.florian-schroff.de/publications/CUB-200.pdf","citation":"\n@article{welinder2010caltech,\n  author = {Welinder, Peter and Branson, Steve and Mita, Takeshi and Wah, Catherine and Schroff, Florian and Belongie, Serge and Perona, Pietro},\n  month = {09},\n  pages = {},\n  title = {Caltech-UCSD Birds 200},\n  year = {2010},\n}\n","isPublic":true,"sourceDataset":"mteb/cub200_retrieval","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"hit_rate_at_1","numModels":0},{"name":"EDIST2ITRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["News"],"modalities":["text","image"],"description":"Retrieve news images and titles based on news content.","reference":"https://aclanthology.org/2023.emnlp-main.297/","citation":"\n@inproceedings{liu2023edis,\n  author = {Liu, Siqi and Feng, Weixi and Fu, Tsu-Jui and Chen, Wenhu and Wang, William},\n  booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},\n  pages = {4877--4894},\n  title = {EDIS: Entity-Driven Image Search over Multimodal Web Content},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_edis_task2","license":"apache-2.0","dateFrom":"2023-01-01","dateTo":"2023-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"Fashion200kI2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve clothes based on descriptions.","reference":"https://openaccess.thecvf.com/content_iccv_2017/html/Han_Automatic_Spatially-Aware_Fashion_ICCV_2017_paper.html","citation":"\n@inproceedings{han2017automatic,\n  author = {Han, Xintong and Wu, Zuxuan and Huang, Phoenix X and Zhang, Xiao and Zhu, Menglong and Li, Yuan and Zhao, Yang and Davis, Larry S},\n  booktitle = {Proceedings of the IEEE international conference on computer vision},\n  pages = {1463--1471},\n  title = {Automatic spatially-aware fashion concept discovery},\n  year = {2017},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_fashion200k_task3","license":"apache-2.0","dateFrom":"2017-01-01","dateTo":"2017-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"Fashion200kT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve clothes based on descriptions.","reference":"https://openaccess.thecvf.com/content_iccv_2017/html/Han_Automatic_Spatially-Aware_Fashion_ICCV_2017_paper.html","citation":"\n@inproceedings{han2017automatic,\n  author = {Han, Xintong and Wu, Zuxuan and Huang, Phoenix X and Zhang, Xiao and Zhu, Menglong and Li, Yuan and Zhao, Yang and Davis, Larry S},\n  booktitle = {Proceedings of the IEEE international conference on computer vision},\n  pages = {1463--1471},\n  title = {Automatic spatially-aware fashion concept discovery},\n  year = {2017},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_fashion200k_task0","license":"apache-2.0","dateFrom":"2017-01-01","dateTo":"2017-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"FashionIQIT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve clothes based on descriptions.","reference":"https://openaccess.thecvf.com/content/CVPR2021/html/Wu_Fashion_IQ_A_New_Dataset_Towards_Retrieving_Images_by_Natural_CVPR_2021_paper.html","citation":"\n@inproceedings{wu2021fashion,\n  author = {Wu, Hui and Gao, Yupeng and Guo, Xiaoxiao and Al-Halah, Ziad and Rennie, Steven and Grauman, Kristen and Feris, Rogerio},\n  booktitle = {Proceedings of the IEEE/CVF Conference on computer vision and pattern recognition},\n  pages = {11307--11317},\n  title = {Fashion iq: A new dataset towards retrieving images by natural language feedback},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_fashioniq_task7","license":"apache-2.0","dateFrom":"2021-01-01","dateTo":"2021-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"Flickr30kI2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web","Written"],"modalities":["text","image"],"description":"Retrieve captions based on images.","reference":"https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31","citation":"\n@article{Young2014FromID,\n  author = {Peter Young and Alice Lai and Micah Hodosh and J. Hockenmaier},\n  journal = {Transactions of the Association for Computational Linguistics},\n  pages = {67-78},\n  title = {From image descriptions to visual denotations: New similarity metrics for semantic inference over event descriptions},\n  url = {https://api.semanticscholar.org/CorpusID:3104920},\n  volume = {2},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/flickr30ki2t","license":"cc-by-sa-4.0","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"Flickr30kT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web","Written"],"modalities":["text","image"],"description":"Retrieve images based on captions.","reference":"https://www.semanticscholar.org/paper/From-image-descriptions-to-visual-denotations%3A-New-Young-Lai/44040913380206991b1991daf1192942e038fe31","citation":"\n@article{Young2014FromID,\n  author = {Peter Young and Alice Lai and Micah Hodosh and J. Hockenmaier},\n  journal = {Transactions of the Association for Computational Linguistics},\n  pages = {67-78},\n  title = {From image descriptions to visual denotations: New similarity metrics for semantic inference over event descriptions},\n  url = {https://api.semanticscholar.org/CorpusID:3104920},\n  volume = {2},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/flickr30kt2i","license":"cc-by-sa-4.0","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"FORBI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Retrieve flat object images from 8 classes.","reference":"https://github.com/pxiangwu/FORB","citation":"\n@inproceedings{wu2023forbflatobjectretrieval,\n  archiveprefix = {arXiv},\n  author = {Pengxiang Wu and Siman Wang and Kevin S Dela Rosa and Derek Hao Hu},\n  booktitle = {Thirty-seventh Conference on Neural Information Processing Systems Datasets and Benchmarks Track},\n  eprint = {2309.16249},\n  title = {{FORB}: A Flat Object Retrieval Benchmark for Universal Image Embedding},\n  url = {https://openreview.net/forum?id=c3kuX7ltzr},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/forb_retrieval","license":"not specified","dateFrom":"2022-01-01","dateTo":"2023-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"hit_rate_at_1","numModels":0},{"name":"GLDv2I2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Retrieve names of landmarks based on their image.","reference":"https://openaccess.thecvf.com/content_CVPR_2020/html/Weyand_Google_Landmarks_Dataset_v2_-_A_Large-Scale_Benchmark_for_Instance-Level_CVPR_2020_paper.html","citation":"\n@inproceedings{Weyand_2020_CVPR,\n  author = {Weyand, Tobias and Araujo, Andre and Cao, Bingyi and Sim, Jack},\n  booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n  month = {June},\n  title = {Google Landmarks Dataset v2 - A Large-Scale Benchmark for Instance-Level Recognition and Retrieval},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/gld-v2","license":"apache-2.0","dateFrom":"2017-01-01","dateTo":"2017-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"GLDv2I2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve names of landmarks based on their image.","reference":"https://openaccess.thecvf.com/content_CVPR_2020/html/Weyand_Google_Landmarks_Dataset_v2_-_A_Large-Scale_Benchmark_for_Instance-Level_CVPR_2020_paper.html","citation":"\n@inproceedings{Weyand_2020_CVPR,\n  author = {Weyand, Tobias and Araujo, Andre and Cao, Bingyi and Sim, Jack},\n  booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},\n  month = {June},\n  title = {Google Landmarks Dataset v2 - A Large-Scale Benchmark for Instance-Level Recognition and Retrieval},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/gld-v2-i2t","license":"apache-2.0","dateFrom":"2017-01-01","dateTo":"2017-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"HatefulMemesI2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve captions based on memes to assess OCR abilities.","reference":"https://arxiv.org/pdf/2005.04790","citation":"\n@article{kiela2020hateful,\n  author = {Kiela, Douwe and Firooz, Hamed and Mohan, Aravind and Goswami, Vedanuj and Singh, Amanpreet and Ringshia, Pratik and Testuggine, Davide},\n  journal = {Advances in neural information processing systems},\n  pages = {2611--2624},\n  title = {The hateful memes challenge: Detecting hate speech in multimodal memes},\n  volume = {33},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/MMSoc_HatefulMemes","license":"mit","dateFrom":"2020-01-01","dateTo":"2020-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"HatefulMemesT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve captions based on memes to assess OCR abilities.","reference":"https://arxiv.org/pdf/2005.04790","citation":"\n@article{kiela2020hateful,\n  author = {Kiela, Douwe and Firooz, Hamed and Mohan, Aravind and Goswami, Vedanuj and Singh, Amanpreet and Ringshia, Pratik and Testuggine, Davide},\n  journal = {Advances in neural information processing systems},\n  pages = {2611--2624},\n  title = {The hateful memes challenge: Detecting hate speech in multimodal memes},\n  volume = {33},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/MMSoc_HatefulMemes","license":"mit","dateFrom":"2020-01-01","dateTo":"2020-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"ImageCoDeT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web","Written"],"modalities":["text","image"],"description":"Retrieve a specific video frame based on a precise caption.","reference":"https://aclanthology.org/2022.acl-long.241.pdf","citation":"\n@inproceedings{krojer2022image,\n  address = {Dublin, Ireland},\n  author = {Krojer, Benno and Adlakha, Vaibhav and Vineet, Vibhav and Goyal, Yash and Ponti, Edoardo and Reddy, Siva},\n  booktitle = {Proceedings of the 60th Annual Meeting of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)},\n  doi = {10.18653/v1/2022.acl-long.241},\n  editor = {Muresan, Smaranda and Nakov, Preslav and Villavicencio, Aline},\n  month = {may},\n  pages = {3426--3440},\n  publisher = {Association for Computational Linguistics},\n  title = {Image Retrieval from Contextual Descriptions},\n  url = {https://aclanthology.org/2022.acl-long.241/},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/imagecode","license":"cc-by-sa-4.0","dateFrom":"2022-05-22","dateTo":"2022-05-27","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"hit_rate_at_3","numModels":0},{"name":"InfoSeekIT2ITRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve source text and image information to answer questions about images.","reference":"https://aclanthology.org/2023.emnlp-main.925","citation":"\n@inproceedings{chen2023can,\n  author = {Chen, Yang and Hu, Hexiang and Luan, Yi and Sun, Haitian and Changpinyo, Soravit and Ritter, Alan and Chang, Ming-Wei},\n  booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},\n  pages = {14948--14968},\n  title = {Can Pre-trained Vision and Language Models Answer Visual Information-Seeking Questions?},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/InfoSeekIT2ITRetrieval","license":"cc-by-sa-4.0","dateFrom":"2023-01-01","dateTo":"2023-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"InfoSeekIT2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve source information to answer questions about images.","reference":"https://aclanthology.org/2023.emnlp-main.925","citation":"\n@inproceedings{chen2023can,\n  author = {Chen, Yang and Hu, Hexiang and Luan, Yi and Sun, Haitian and Changpinyo, Soravit and Ritter, Alan and Chang, Ming-Wei},\n  booktitle = {Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing},\n  pages = {14948--14968},\n  title = {Can Pre-trained Vision and Language Models Answer Visual Information-Seeking Questions?},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_infoseek_task6","license":"cc-by-sa-4.0","dateFrom":"2023-01-01","dateTo":"2023-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"MemotionI2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve captions based on memes.","reference":"https://aclanthology.org/2020.semeval-1.99/","citation":"\n@inproceedings{sharma2020semeval,\n  author = {Sharma, Chhavi and Bhageria, Deepesh and Scott, William and Pykl, Srinivas and Das, Amitava and Chakraborty, Tanmoy and Pulabaigari, Viswanath and Gamb{\\\"a}ck, Bj{\\\"o}rn},\n  booktitle = {Proceedings of the Fourteenth Workshop on Semantic Evaluation},\n  pages = {759--773},\n  title = {SemEval-2020 Task 8: Memotion Analysis-the Visuo-Lingual Metaphor!},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/MMSoc_Memotion_corrected","license":"mit","dateFrom":"2020-01-01","dateTo":"2020-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"MemotionT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve memes based on captions.","reference":"https://aclanthology.org/2020.semeval-1.99/","citation":"\n@inproceedings{sharma2020semeval,\n  author = {Sharma, Chhavi and Bhageria, Deepesh and Scott, William and Pykl, Srinivas and Das, Amitava and Chakraborty, Tanmoy and Pulabaigari, Viswanath and Gamb{\\\"a}ck, Bj{\\\"o}rn},\n  booktitle = {Proceedings of the Fourteenth Workshop on Semantic Evaluation},\n  pages = {759--773},\n  title = {SemEval-2020 Task 8: Memotion Analysis-the Visuo-Lingual Metaphor!},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/MMSoc_Memotion_corrected","license":"mit","dateFrom":"2020-01-01","dateTo":"2020-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"METI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Retrieve photos of more than 224k artworks.","reference":"https://arxiv.org/abs/2202.01747","citation":"\n@inproceedings{ypsilantis2021met,\n  author = {Ypsilantis, Nikolaos-Antonios and Garcia, Noa and Han, Guangxing and Ibrahimi, Sarah and Van Noord, Nanne and Tolias, Giorgos},\n  booktitle = {Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)},\n  title = {The met dataset: Instance-level recognition for artworks},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/met","license":"not specified","dateFrom":"2021-12-06","dateTo":"2021-12-14","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"hit_rate_at_1","numModels":0},{"name":"MSCOCOI2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve captions based on images.","reference":"https://link.springer.com/chapter/10.1007/978-3-319-10602-1_48","citation":"\n@inproceedings{lin2014microsoft,\n  author = {Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\\'a}r, Piotr and Zitnick, C Lawrence},\n  booktitle = {Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13},\n  organization = {Springer},\n  pages = {740--755},\n  title = {Microsoft coco: Common objects in context},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_mscoco_task3","license":"cc-by-sa-4.0","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"MSCOCOT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve images based on captions.","reference":"https://link.springer.com/chapter/10.1007/978-3-319-10602-1_48","citation":"\n@inproceedings{lin2014microsoft,\n  author = {Lin, Tsung-Yi and Maire, Michael and Belongie, Serge and Hays, James and Perona, Pietro and Ramanan, Deva and Doll{\\'a}r, Piotr and Zitnick, C Lawrence},\n  booktitle = {Computer Vision--ECCV 2014: 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part V 13},\n  organization = {Springer},\n  pages = {740--755},\n  title = {Microsoft coco: Common objects in context},\n  year = {2014},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_mscoco_task0","license":"cc-by-sa-4.0","dateFrom":"2018-01-01","dateTo":"2018-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"NIGHTSI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Retrieval identical image to the given image.","reference":"https://proceedings.neurips.cc/paper_files/paper/2023/hash/9f09f316a3eaf59d9ced5ffaefe97e0f-Abstract-Conference.html","citation":"\n@article{fu2024dreamsim,\n  author = {Fu, Stephanie and Tamir, Netanel and Sundaram, Shobhita and Chai, Lucy and Zhang, Richard and Dekel, Tali and Isola, Phillip},\n  journal = {Advances in Neural Information Processing Systems},\n  title = {DreamSim: Learning New Dimensions of Human Visual Similarity using Synthetic Data},\n  volume = {36},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_nights_task4","license":"cc-by-sa-4.0","dateFrom":"2023-01-01","dateTo":"2023-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"OVENIT2ITRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Retrieval a Wiki image and passage to answer query about an image.","reference":"https://openaccess.thecvf.com/content/ICCV2023/html/Hu_Open-domain_Visual_Entity_Recognition_Towards_Recognizing_Millions_of_Wikipedia_Entities_ICCV_2023_paper.html","citation":"\n@inproceedings{hu2023open,\n  author = {Hu, Hexiang and Luan, Yi and Chen, Yang and Khandelwal, Urvashi and Joshi, Mandar and Lee, Kenton and Toutanova, Kristina and Chang, Ming-Wei},\n  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision},\n  pages = {12065--12075},\n  title = {Open-domain visual entity recognition: Towards recognizing millions of wikipedia entities},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_oven_task8","license":"cc-by-sa-4.0","dateFrom":"2023-01-01","dateTo":"2023-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"OVENIT2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieval a Wiki passage to answer query about an image.","reference":"https://openaccess.thecvf.com/content/ICCV2023/html/Hu_Open-domain_Visual_Entity_Recognition_Towards_Recognizing_Millions_of_Wikipedia_Entities_ICCV_2023_paper.html","citation":"\n@inproceedings{hu2023open,\n  author = {Hu, Hexiang and Luan, Yi and Chen, Yang and Khandelwal, Urvashi and Joshi, Mandar and Lee, Kenton and Toutanova, Kristina and Chang, Ming-Wei},\n  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision},\n  pages = {12065--12075},\n  title = {Open-domain visual entity recognition: Towards recognizing millions of wikipedia entities},\n  year = {2023},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_oven_task6","license":"cc-by-sa-4.0","dateFrom":"2023-01-01","dateTo":"2023-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"ROxfordEasyI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Retrieve photos of landmarks in Oxford, UK.","reference":"https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html","citation":"\n@inproceedings{radenovic2018revisiting,\n  author = {Radenovi{\\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\\v{r}}ej},\n  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},\n  pages = {5706--5715},\n  title = {Revisiting oxford and paris: Large-scale image retrieval benchmarking},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/r-oxford-easy-multi","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"map_at_5","numModels":0},{"name":"ROxfordMediumI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Retrieve photos of landmarks in Oxford, UK.","reference":"https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html","citation":"\n@inproceedings{radenovic2018revisiting,\n  author = {Radenovi{\\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\\v{r}}ej},\n  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},\n  pages = {5706--5715},\n  title = {Revisiting oxford and paris: Large-scale image retrieval benchmarking},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/r-oxford-medium-multi","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"map_at_5","numModels":0},{"name":"ROxfordHardI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Retrieve photos of landmarks in Oxford, UK.","reference":"https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Oxford_and_CVPR_2018_paper.html","citation":"\n@inproceedings{radenovic2018revisiting,\n  author = {Radenovi{\\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\\v{r}}ej},\n  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},\n  pages = {5706--5715},\n  title = {Revisiting oxford and paris: Large-scale image retrieval benchmarking},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/r-oxford-hard-multi","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"map_at_5","numModels":0},{"name":"RP2kI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Retrieve photos of 39457 products.","reference":"https://arxiv.org/abs/2006.12634","citation":"\n@article{peng2020rp2k,\n  author = {Peng, Jingtian and Xiao, Chang and Li, Yifan},\n  journal = {arXiv preprint arXiv:2006.12634},\n  title = {RP2K: A large-scale retail product dataset for fine-grained image classification},\n  year = {2020},\n}\n","isPublic":true,"sourceDataset":"mteb/rp2k","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"hit_rate_at_1","numModels":0},{"name":"RParisEasyI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Retrieve photos of landmarks in Paris, UK.","reference":"https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html","citation":"\n@inproceedings{radenovic2018revisiting,\n  author = {Radenovi{\\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\\v{r}}ej},\n  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},\n  pages = {5706--5715},\n  title = {Revisiting oxford and paris: Large-scale image retrieval benchmarking},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/r-paris-easy-multi","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"map_at_5","numModels":0},{"name":"RParisMediumI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Retrieve photos of landmarks in Paris, UK.","reference":"https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html","citation":"\n@inproceedings{radenovic2018revisiting,\n  author = {Radenovi{\\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\\v{r}}ej},\n  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},\n  pages = {5706--5715},\n  title = {Revisiting oxford and paris: Large-scale image retrieval benchmarking},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/r-paris-medium-multi","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"map_at_5","numModels":0},{"name":"RParisHardI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["image"],"description":"Retrieve photos of landmarks in Paris, UK.","reference":"https://openaccess.thecvf.com/content_cvpr_2018/html/Radenovic_Revisiting_Paris_and_CVPR_2018_paper.html","citation":"\n@inproceedings{radenovic2018revisiting,\n  author = {Radenovi{\\'c}, Filip and Iscen, Ahmet and Tolias, Giorgos and Avrithis, Yannis and Chum, Ond{\\v{r}}ej},\n  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},\n  pages = {5706--5715},\n  title = {Revisiting oxford and paris: Large-scale image retrieval benchmarking},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/r-paris-hard-multi","license":"not specified","dateFrom":"2009-01-01","dateTo":"2010-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"map_at_5","numModels":0},{"name":"SciMMIRI2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve captions based on figures and tables.","reference":"https://aclanthology.org/2024.findings-acl.746/","citation":"\n@inproceedings{wu2024scimmirbenchmarkingscientificmultimodal,\n  address = {Bangkok, Thailand},\n  author = {Wu, Siwei and Li, Yizhi and Zhu, Kang and Zhang, Ge and Liang, Yiming and Ma, Kaijing and Xiao, Chenghao and Zhang, Haoran and Yang, Bohao and Chen, Wenhu and Huang, Wenhao and Al Moubayed, Noura and Fu, Jie and Lin, Chenghua},\n  booktitle = {Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2024},\n  doi = {10.18653/v1/2024.findings-acl.746},\n  editor = {Ku, Lun-Wei and Martins, Andre and Srikumar, Vivek},\n  month = {aug},\n  pages = {12560--12574},\n  publisher = {Association for Computational Linguistics},\n  title = {{S}ci{MMIR}: Benchmarking Scientific Multi-modal Information Retrieval},\n  url = {https://aclanthology.org/2024.findings-acl.746/},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/SciMMIR","license":"mit","dateFrom":"2023-01-01","dateTo":"2023-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"SciMMIRT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve figures and tables based on captions.","reference":"https://aclanthology.org/2024.findings-acl.746/","citation":"\n@inproceedings{wu2024scimmirbenchmarkingscientificmultimodal,\n  address = {Bangkok, Thailand},\n  author = {Wu, Siwei and Li, Yizhi and Zhu, Kang and Zhang, Ge and Liang, Yiming and Ma, Kaijing and Xiao, Chenghao and Zhang, Haoran and Yang, Bohao and Chen, Wenhu and Huang, Wenhao and Al Moubayed, Noura and Fu, Jie and Lin, Chenghua},\n  booktitle = {Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2024},\n  doi = {10.18653/v1/2024.findings-acl.746},\n  editor = {Ku, Lun-Wei and Martins, Andre and Srikumar, Vivek},\n  month = {aug},\n  pages = {12560--12574},\n  publisher = {Association for Computational Linguistics},\n  title = {{S}ci{MMIR}: Benchmarking Scientific Multi-modal Information Retrieval},\n  url = {https://aclanthology.org/2024.findings-acl.746/},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"mteb/SciMMIR","license":"mit","dateFrom":"2023-01-01","dateTo":"2023-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"SketchyI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Retrieve photos from sketches.","reference":"https://arxiv.org/abs/2202.01747","citation":"\n@inproceedings{ypsilantis2021met,\n  author = {Ypsilantis, Nikolaos-Antonios and Garcia, Noa and Han, Guangxing and Ibrahimi, Sarah and Van Noord, Nanne and Tolias, Giorgos},\n  booktitle = {Thirty-fifth Conference on Neural Information Processing Systems Datasets and Benchmarks Track (Round 2)},\n  title = {The met dataset: Instance-level recognition for artworks},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/sketchy","license":"not specified","dateFrom":"2021-12-06","dateTo":"2021-12-14","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"hit_rate_at_1","numModels":0},{"name":"SOPI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Retrieve product photos of 22634 online products.","reference":"https://paperswithcode.com/dataset/stanford-online-products","citation":"\n@inproceedings{oh2016deep,\n  author = {Oh Song, Hyun and Xiang, Yu and Jegelka, Stefanie and Savarese, Silvio},\n  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},\n  pages = {4004--4012},\n  title = {Deep metric learning via lifted structured feature embedding},\n  year = {2016},\n}\n","isPublic":true,"sourceDataset":"mteb/stanford-online-products","license":"not specified","dateFrom":"2019-07-17","dateTo":"2019-07-17","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"hit_rate_at_1","numModels":0},{"name":"StanfordCarsI2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image"],"description":"Retrieve car images from 196 makes.","reference":"https://pure.mpg.de/rest/items/item_2029263/component/file_2029262/content","citation":"\n@inproceedings{Krause2013CollectingAL,\n  author = {Jonathan Krause and Jia Deng and Michael Stark and Li Fei-Fei},\n  title = {Collecting a Large-scale Dataset of Fine-grained Cars},\n  url = {https://api.semanticscholar.org/CorpusID:16632981},\n  year = {2013},\n}\n","isPublic":true,"sourceDataset":"mteb/stanford_cars_retrieval","license":"not specified","dateFrom":"2012-01-01","dateTo":"2013-04-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"hit_rate_at_1","numModels":0},{"name":"TUBerlinT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text","image"],"description":"Retrieve sketch images based on text descriptions.","reference":"https://dl.acm.org/doi/pdf/10.1145/2185520.2185540","citation":"\n@article{eitz2012humans,\n  author = {Eitz, Mathias and Hays, James and Alexa, Marc},\n  journal = {ACM Transactions on graphics (TOG)},\n  number = {4},\n  pages = {1--10},\n  publisher = {Acm New York, NY, USA},\n  title = {How do humans sketch objects?},\n  volume = {31},\n  year = {2012},\n}\n","isPublic":true,"sourceDataset":"mteb/tu-berlin","license":"cc-by-sa-4.0","dateFrom":"2012-01-01","dateTo":"2012-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"VidoreArxivQARetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/arxivqa_test_subsampled_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreDocVQARetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/docvqa_test_subsampled_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreInfoVQARetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/infovqa_test_subsampled_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreTabfquadRetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/tabfquad_test_subsampled_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreTatdqaRetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/tatdqa_test_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreShiftProjectRetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/shiftproject_test_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreSyntheticDocQAAIRetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/syntheticDocQA_artificial_intelligence_test_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreSyntheticDocQAEnergyRetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/syntheticDocQA_energy_test_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreSyntheticDocQAGovernmentReportsRetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/syntheticDocQA_government_reports_test_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VidoreSyntheticDocQAHealthcareIndustryRetrieval","type":"DocumentUnderstanding","simplifiedType":"retrieval","languages":["English"],"domains":["Academic"],"modalities":["text","image"],"description":"Retrieve associated pages according to questions.","reference":"https://openreview.net/forum?id=ogjBpZ8uSi","citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}\n","isPublic":true,"sourceDataset":"mteb/syntheticDocQA_healthcare_industry_test_beir","license":"mit","dateFrom":"2024-01-01","dateTo":"2024-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_5","numModels":0},{"name":"VisualNewsI2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Retrieval entity-rich captions for news images.","reference":"https://aclanthology.org/2021.emnlp-main.542/","citation":"\n@inproceedings{liu2021visual,\n  author = {Liu, Fuxiao and Wang, Yinghan and Wang, Tianlu and Ordonez, Vicente},\n  booktitle = {Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},\n  pages = {6761--6771},\n  title = {Visual News: Benchmark and Challenges in News Image Captioning},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_visualnews_task3","license":"cc-by-sa-4.0","dateFrom":"2020-01-01","dateTo":"2020-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"VisualNewsT2IRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Retrieve news images with captions.","reference":"https://aclanthology.org/2021.emnlp-main.542/","citation":"\n@inproceedings{liu2021visual,\n  author = {Liu, Fuxiao and Wang, Yinghan and Wang, Tianlu and Ordonez, Vicente},\n  booktitle = {Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing},\n  pages = {6761--6771},\n  title = {Visual News: Benchmark and Challenges in News Image Captioning},\n  year = {2021},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_visualnews_task0","license":"cc-by-sa-4.0","dateFrom":"2020-01-01","dateTo":"2020-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"VizWizIT2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["text","image"],"description":"Retrieve the correct answer for a question about an image.","reference":"https://openaccess.thecvf.com/content_cvpr_2018/papers/Gurari_VizWiz_Grand_Challenge_CVPR_2018_paper.pdf","citation":"\n@inproceedings{gurari2018vizwiz,\n  author = {Gurari, Danna and Li, Qing and Stangl, Abigale J and Guo, Anhong and Lin, Chi and Grauman, Kristen and Luo, Jiebo and Bigham, Jeffrey P},\n  booktitle = {Proceedings of the IEEE conference on computer vision and pattern recognition},\n  pages = {3608--3617},\n  title = {Vizwiz grand challenge: Answering visual questions from blind people},\n  year = {2018},\n}\n","isPublic":true,"sourceDataset":"mteb/vizwiz","license":"cc-by-sa-4.0","dateFrom":"2018-01-01","dateTo":"2018-01-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"VQA2IT2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Web"],"modalities":["text","image"],"description":"Retrieve the correct answer for a question about an image.","reference":"https://openaccess.thecvf.com/content_cvpr_2017/html/Goyal_Making_the_v_CVPR_2017_paper.html","citation":"\n@inproceedings{Goyal_2017_CVPR,\n  author = {Goyal, Yash and Khot, Tejas and Summers-Stay, Douglas and Batra, Dhruv and Parikh, Devi},\n  booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)},\n  month = {July},\n  title = {Making the v in VQA Matter: Elevating the Role of Image Understanding in Visual Question Answering},\n  year = {2017},\n}\n","isPublic":true,"sourceDataset":"mteb/vqa-2","license":"cc-by-sa-4.0","dateFrom":"2017-07-01","dateTo":"2017-07-01","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"WebQAT2ITRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["image","text"],"description":"Retrieve sources of information based on questions.","reference":"https://openaccess.thecvf.com/content/CVPR2022/html/Chang_WebQA_Multihop_and_Multimodal_QA_CVPR_2022_paper.html","citation":"\n@inproceedings{chang2022webqa,\n  author = {Chang, Yingshan and Narang, Mridu and Suzuki, Hisami and Cao, Guihong and Gao, Jianfeng and Bisk, Yonatan},\n  booktitle = {Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},\n  pages = {16495--16504},\n  title = {Webqa: Multihop and multimodal qa},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_webqa_task2","license":"cc-by-sa-4.0","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"WebQAT2TRetrieval","type":"Any2AnyRetrieval","simplifiedType":"retrieval","languages":["English"],"domains":["Encyclopaedic"],"modalities":["text"],"description":"Retrieve sources of information based on questions.","reference":"https://openaccess.thecvf.com/content/CVPR2022/html/Chang_WebQA_Multihop_and_Multimodal_QA_CVPR_2022_paper.html","citation":"\n@inproceedings{chang2022webqa,\n  author = {Chang, Yingshan and Narang, Mridu and Suzuki, Hisami and Cao, Guihong and Gao, Jianfeng and Bisk, Yonatan},\n  booktitle = {Proceedings of the IEEE/CVF conference on computer vision and pattern recognition},\n  pages = {16495--16504},\n  title = {Webqa: Multihop and multimodal qa},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/mbeir_webqa_task1","license":"cc-by-sa-4.0","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"created","mainScore":"ndcg_at_10","numModels":0},{"name":"WITT2IRetrieval","type":"Any2AnyMultilingualRetrieval","simplifiedType":"retrieval","languages":["Arabic","Bulgarian","Danish","English","Estonian","Indonesian","Japanese","Korean","Modern Greek (1453-)","Turkish","Vietnamese"],"domains":["Encyclopaedic","Written"],"modalities":["text","image"],"description":"Retrieve images based on multilingual descriptions.","reference":"https://proceedings.mlr.press/v162/bugliarello22a/bugliarello22a.pdf","citation":"\n@inproceedings{bugliarello2022iglue,\n  author = {Bugliarello, Emanuele and Liu, Fangyu and Pfeiffer, Jonas and Reddy, Siva and Elliott, Desmond and Ponti, Edoardo Maria and Vuli{\\'c}, Ivan},\n  booktitle = {International Conference on Machine Learning},\n  organization = {PMLR},\n  pages = {2370--2392},\n  title = {IGLUE: A benchmark for transfer learning across modalities, tasks, and languages},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/wit","license":"cc-by-sa-4.0","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"XFlickr30kCoT2IRetrieval","type":"Any2AnyMultilingualRetrieval","simplifiedType":"retrieval","languages":["Chinese","English","German","Indonesian","Japanese","Russian","Spanish","Turkish"],"domains":["Encyclopaedic","Written"],"modalities":["text","image"],"description":"Retrieve images based on multilingual descriptions.","reference":"https://proceedings.mlr.press/v162/bugliarello22a/bugliarello22a.pdf","citation":"\n@inproceedings{bugliarello2022iglue,\n  author = {Bugliarello, Emanuele and Liu, Fangyu and Pfeiffer, Jonas and Reddy, Siva and Elliott, Desmond and Ponti, Edoardo Maria and Vuli{\\'c}, Ivan},\n  booktitle = {International Conference on Machine Learning},\n  organization = {PMLR},\n  pages = {2370--2392},\n  title = {IGLUE: A benchmark for transfer learning across modalities, tasks, and languages},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/xflickrco","license":"cc-by-sa-4.0","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"XM3600T2IRetrieval","type":"Any2AnyMultilingualRetrieval","simplifiedType":"retrieval","languages":["Arabic","Bengali","Chinese","Croatian","Cusco Quechua","Czech","Danish","Dutch","English","Filipino","Finnish","French","German","Hebrew","Hindi","Hungarian","Indonesian","Italian","Japanese","Korean","Maori","Modern Greek (1453-)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Persian","Polish","Portuguese","Romanian","Russian","Spanish","Swahili (macrolanguage)","Swedish","Telugu","Thai","Turkish","Ukrainian","Vietnamese"],"domains":["Encyclopaedic","Written"],"modalities":["text","image"],"description":"Retrieve images based on multilingual descriptions.","reference":"https://aclanthology.org/2022.emnlp-main.45/","citation":"\n@inproceedings{thapliyal2022crossmodal,\n  author = {Thapliyal, Ashish V and Tuset, Jordi Pont and Chen, Xi and Soricut, Radu},\n  booktitle = {Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing},\n  pages = {715--729},\n  title = {Crossmodal-3600: A Massively Multilingual Multimodal Evaluation Dataset},\n  year = {2022},\n}\n","isPublic":true,"sourceDataset":"mteb/xm3600","license":"cc-by-sa-4.0","dateFrom":"2022-01-01","dateTo":"2022-12-31","annotationsCreators":"derived","dialect":[],"sampleCreation":"found","mainScore":"ndcg_at_10","numModels":0},{"name":"VisualSTS17Eng","type":"VisualSTS(eng)","simplifiedType":"semantic-similarity","languages":["Arabic","Dutch","English","French","German","Italian","Korean","Spanish","Turkish"],"domains":["Spoken","Written","Web","News","Social"],"modalities":["image"],"description":"STS17MultilingualVisualSTS English only.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"aggregate tasks do not have a path","license":"not specified","dateFrom":"2012-01-01T00:00:00","dateTo":"2017-12-31T00:00:00","annotationsCreators":"human-annotated","dialect":[""],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0},{"name":"VisualSTS-b-Eng","type":"VisualSTS(eng)","simplifiedType":"semantic-similarity","languages":["Dutch","English","French","German","Italian","Mandarin Chinese","Polish","Portuguese","Russian","Spanish"],"domains":["Spoken","Written","Web","News","Social"],"modalities":["image"],"description":"STSBenchmarkMultilingualVisualSTS English only.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"aggregate tasks do not have a path","license":"not specified","dateFrom":"2012-01-01T00:00:00","dateTo":"2017-12-31T00:00:00","annotationsCreators":"human-annotated","dialect":[""],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0},{"name":"VisualSTS17Multilingual","type":"VisualSTS(multi)","simplifiedType":"semantic-similarity","languages":["Arabic","Dutch","English","French","German","Italian","Korean","Spanish","Turkish"],"domains":["Spoken","Written","Web","News","Social"],"modalities":["image"],"description":"STS17MultilingualVisualSTS multilingual.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"aggregate tasks do not have a path","license":"not specified","dateFrom":"2012-01-01T00:00:00","dateTo":"2017-12-31T00:00:00","annotationsCreators":"human-annotated","dialect":[""],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0},{"name":"VisualSTS-b-Multilingual","type":"VisualSTS(multi)","simplifiedType":"semantic-similarity","languages":["Dutch","English","French","German","Italian","Mandarin Chinese","Polish","Portuguese","Russian","Spanish"],"domains":["Spoken","Written","Web","News","Social"],"modalities":["image"],"description":"STSBenchmarkMultilingualVisualSTS multilingual.","reference":"https://arxiv.org/abs/2402.08183/","citation":"\n@article{xiao2024pixel,\n  author = {Xiao, Chenghao and Huang, Zhuoxu and Chen, Danlu and Hudson, G Thomas and Li, Yizhi and Duan, Haoran and Lin, Chenghua and Fu, Jie and Han, Jungong and Moubayed, Noura Al},\n  journal = {arXiv preprint arXiv:2402.08183},\n  title = {Pixel Sentence Representation Learning},\n  year = {2024},\n}\n","isPublic":true,"sourceDataset":"aggregate tasks do not have a path","license":"not specified","dateFrom":"2012-01-01T00:00:00","dateTo":"2017-12-31T00:00:00","annotationsCreators":"human-annotated","dialect":[""],"sampleCreation":"rendered","mainScore":"cosine_spearman","numModels":0}],"rows":[{"rank":1,"model":{"name":"dnotitia/DNA-VL-STEER-2B","url":"https://huggingface.co/dnotitia/DNA-VL-STEER-2B","zeroShotPct":-1,"activeParamsB":1.816,"totalParamsB":2.128,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2026-08-27","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["image","text","video"],"languages":["Arabic","Bengali","Chinese","Croatian","Cusco Quechua","Czech","Danish","Dutch","English","Filipino","Finnish","French","German","Hebrew","Hindi","Hungarian","Indonesian","Italian","Japanese","Korean","Maori","Modern Greek (1453-)","Norwegian","Persian","Polish","Portuguese","Romanian","Russian","Spanish","Swahili (macrolanguage)","Swedish","Telugu","Thai","Turkish","Ukrainian","Vietnamese"],"citation":null,"memoryUsageMb":7629.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3-VL-Embedding-2B","supersededBy":null,"extraRequirementsGroups":["multimodal-sbert"],"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":1.816,"totalParamsB":2.128,"embeddingDim":2048,"maxTokens":32768,"meanTask":0.6752734339871802,"meanTaskType":0.6752734339871802,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.43202488888888885,"Compositionality":0.449389,"ImageClassification":0.7258650909090908,"DocumentUnderstanding":0.840048,"ZeroShotClassification":0.5755087826086956,"ImageClustering":0.7887872,"VisionCentricQA":0.7136516666666667,"Any2AnyMultilingualRetrieval":0.709942925084175,"VisualSTS(eng)":0.8073472857142858,"VisualSTS(multi)":0.7101694999999999},"scoresByTask":{"GTSRBZeroShot":0.211085,"CVBenchDistance":0.71,"CVBenchRelation":0.82923,"OxfordPetsZeroShot":0.766149,"InfoSeekIT2ITRetrieval":0.27312,"VidoreTabfquadRetrieval":0.96342,"CIFAR10Clustering":0.806716,"VidoreDocVQARetrieval":0.50472,"VisualNewsI2TRetrieval":0.23528,"VidoreArxivQARetrieval":0.822,"STL10":0.972062,"CLEVRZeroShot":0.158,"MemotionI2TRetrieval":0.94109,"PatchCamelyon":0.638696,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.95978,"TinyImageNetClustering":0.639792,"WebQAT2ITRetrieval":0.63116,"SUN397":0.782005,"METI2IRetrieval":0.40621,"STS13VisualSTS":0.792375,"CVBenchCount":0.68655,"ROxfordMediumI2IRetrieval":0.12423,"NIGHTSI2IRetrieval":0.26591,"Flickr30kI2TRetrieval":0.89549,"BLINKIT2TRetrieval":0.21185,"VidoreTatdqaRetrieval":0.61865,"ImageCoDe":0.143788,"ImageCoDeT2IRetrieval":0.12815,"Fashion200kI2TRetrieval":0.09104,"FGVCAircraftZeroShot":0.182718,"HatefulMemesT2IRetrieval":0.77721,"XM3600T2IRetrieval":0.7089963888888888,"UCF101":0.883707,"MSCOCOI2TRetrieval":0.62849,"OxfordPets":0.944835,"SugarCrepe":0.911596,"FER2013ZeroShot":0.278768,"OVENIT2TRetrieval":0.1945,"Caltech101ZeroShot":0.817554,"MSCOCOT2IRetrieval":0.71233,"RESISC45ZeroShot":0.675714,"Food101Classification":0.923715,"RParisEasyI2IRetrieval":0.08853,"CVBenchDepth":0.90167,"Imagenet1k":0.001,"VisualSTS-b-Eng":0.807896,"VOC2007":0.734299,"EDIST2ITRetrieval":0.3357,"ROxfordEasyI2IRetrieval":0.201,"VidoreShiftProjectRetrieval":0.81292,"WITT2IRetrieval":0.5658436363636364,"RP2kI2IRetrieval":0.71521,"ROxfordHardI2IRetrieval":0.04162,"DTD":0.763404,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.95917,"AROVisualAttribution":0.692083,"Country211":0.146701,"AROVisualRelation":0.563563,"BLINKIT2IRetrieval":0.45522,"CIFAR10":0.9034,"InfoSeekIT2TRetrieval":0.25744,"VidoreInfoVQARetrieval":0.88439,"BLINKIT2IMultiChoice":0.77363,"VidoreSyntheticDocQAEnergyRetrieval":0.91365,"CIFAR100":0.68498,"VisualSTS-b-Multilingual":0.733748,"EuroSATZeroShot":0.353704,"VisualSTS17Eng":0.874265,"RESISC45":0.886095,"BirdsnapZeroShot":0.303079,"CIFAR10ZeroShot":0.8623,"STL10ZeroShot":0.968375,"GTSRB":0.718321,"Country211ZeroShot":0.116588,"FORBI2IRetrieval":0.6966,"CIFAR100ZeroShot":0.5938,"OVENIT2ITRetrieval":0.1792,"MNIST":0.96212,"GLDv2I2TRetrieval":0.46978,"CIFAR100Clustering":0.688761,"RParisHardI2IRetrieval":0.00088,"SOPI2IRetrieval":0.56118,"VidoreSyntheticDocQAAIRetrieval":0.96178,"ImageNet10Clustering":0.940576,"StanfordCarsI2IRetrieval":0.8045,"StanfordCarsZeroShot":0.620943,"FER2013":0.353943,"OxfordFlowersClassification":0.990196,"STS12VisualSTS":0.743418,"Imagenet1kZeroShot":0.70802,"STS14VisualSTS":0.756631,"AROCocoOrder":0.368693,"VQA2IT2TRetrieval":0.03599,"CLEVRCountZeroShot":0.470133,"GLDv2I2IRetrieval":0.29408,"RenderedSST2":0.861614,"STS15VisualSTS":0.852159,"SciMMIRT2IRetrieval":0.54545,"WebQAT2TRetrieval":0.70543,"VizWizIT2TRetrieval":0.29535,"SketchyI2IRetrieval":0.77627,"Caltech101":0.898586,"VisualSTS17Multilingual":0.686591,"BLINKIT2TMultiChoice":0.38083,"EuroSAT":0.738593,"MemotionT2IRetrieval":0.97166,"CIRRIT2IRetrieval":0.18567,"DTDZeroShot":0.583511,"Food101ZeroShot":0.859327,"Birdsnap":0.639114,"Flickr30kT2IRetrieval":0.91766,"Winoground":0.095,"StanfordCars":0.880488,"VisualNewsT2IRetrieval":0.23396,"STS16VisualSTS":0.824687,"MNISTZeroShot":0.8891,"PatchCamelyonZeroShot":0.53717,"HatefulMemesI2TRetrieval":0.77163,"SciMMIRI2TRetrieval":0.5673,"RParisMediumI2IRetrieval":0.02646,"UCF101ZeroShot":0.699464,"AROFlickrOrder":0.371,"ImageNetDog15Clustering":0.868091,"Fashion200kT2IRetrieval":0.07787,"FashionIQIT2IRetrieval":0.08838,"TUBerlinT2IRetrieval":0.91551,"SUN397ZeroShot":0.719586,"FGVCAircraft":0.522772,"CUB200I2IRetrieval":0.70953,"XFlickr30kCoT2IRetrieval":0.85498875},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.592562672},"MIEB(lite) Task Mean":{"Task Mean":0.5854703142206378}},"trainedOnTasks":[],"experiments":null},{"rank":2,"model":{"name":"jinaai/jina-embeddings-v5-omni-small","url":"https://huggingface.co/jinaai/jina-embeddings-v5-omni-small","zeroShotPct":-1,"activeParamsB":1.471,"totalParamsB":1.626,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2026-04-01","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["Afrikaans","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Latvian","Lithuanian","Macedonian","Malay (macrolanguage)","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Panjabi","Persian","Polish","Portuguese","Quechua (Latin)","Romanian","Russian","Sinhala","Slovak","Slovenian","Spanish","Swahili (macrolanguage)","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Yoruba"],"citation":"@misc{akram2026jinaembeddingsv5texttasktargetedembeddingdistillation,\n      title={jina-embeddings-v5-text: Task-Targeted Embedding Distillation},\n      author={Mohammad Kalim Akram and Saba Sturua and Nastia Havriushenko and Quentin Herreros and Michael Günther and Maximilian Werk and Han Xiao},\n      year={2026},\n      eprint={2602.15547},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2602.15547},\n}","memoryUsageMb":3102.0,"license":"cc-by-nc-4.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"jinaai/jina-embeddings-v5-text-small","supersededBy":null,"extraRequirementsGroups":["peft"],"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":1.471,"totalParamsB":1.626,"embeddingDim":1024,"maxTokens":32768,"meanTask":0.6506953887644249,"meanTaskType":0.6506953887644249,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3794804444444444,"Compositionality":0.46115400000000006,"ImageClassification":0.7529609545454545,"DocumentUnderstanding":0.792526,"ZeroShotClassification":0.6296076521739131,"ImageClustering":0.8601152556723559,"VisionCentricQA":0.534155,"Any2AnyMultilingualRetrieval":0.6283380808080807,"VisualSTS(eng)":0.817448,"VisualSTS(multi)":0.6511685},"scoresByTask":{"GTSRBZeroShot":0.743468,"CVBenchDistance":0.51833,"CVBenchRelation":0.54769,"OxfordPetsZeroShot":0.855001,"InfoSeekIT2ITRetrieval":0.17383,"VidoreTabfquadRetrieval":0.90974,"CIFAR10Clustering":0.880031745144035,"VidoreDocVQARetrieval":0.48269,"VisualNewsI2TRetrieval":0.08757,"VidoreArxivQARetrieval":0.71797,"STL10":0.991187,"CLEVRZeroShot":0.163733,"MemotionI2TRetrieval":0.98,"PatchCamelyon":0.728229,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.94909,"TinyImageNetClustering":0.7688682051024764,"WebQAT2ITRetrieval":0.15662,"SUN397":0.756938,"METI2IRetrieval":0.39095,"STS13VisualSTS":0.792512,"CVBenchCount":0.40609,"ROxfordMediumI2IRetrieval":0.09617,"NIGHTSI2IRetrieval":0.25609,"Flickr30kI2TRetrieval":0.8542,"BLINKIT2TRetrieval":0.16646,"VidoreTatdqaRetrieval":0.57472,"ImageCoDe":0.132493,"ImageCoDeT2IRetrieval":0.1073,"Fashion200kI2TRetrieval":0.05285,"FGVCAircraftZeroShot":0.133813,"HatefulMemesT2IRetrieval":0.79251,"XM3600T2IRetrieval":0.5571758333333333,"UCF101":0.865595,"MSCOCOI2TRetrieval":0.50868,"OxfordPets":0.930063,"SugarCrepe":0.800426,"FER2013ZeroShot":0.480357,"OVENIT2TRetrieval":0.10416,"Caltech101ZeroShot":0.813774,"MSCOCOT2IRetrieval":0.63681,"RESISC45ZeroShot":0.774286,"Food101Classification":0.88941,"RParisEasyI2IRetrieval":0.08588,"CVBenchDepth":0.54333,"Imagenet1k":0.152688,"VisualSTS-b-Eng":0.835838,"VOC2007":0.682028,"EDIST2ITRetrieval":0.1331,"ROxfordEasyI2IRetrieval":0.18734,"VidoreShiftProjectRetrieval":0.68453,"WITT2IRetrieval":0.516480909090909,"RP2kI2IRetrieval":0.56696,"ROxfordHardI2IRetrieval":0.0236,"DTD":0.760957,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.9094,"AROVisualAttribution":0.607068,"Country211":0.107924,"AROVisualRelation":0.538037,"BLINKIT2IRetrieval":0.43781,"CIFAR10":0.96918,"InfoSeekIT2TRetrieval":0.10733,"VidoreInfoVQARetrieval":0.84045,"BLINKIT2IMultiChoice":0.79353,"VidoreSyntheticDocQAEnergyRetrieval":0.89389,"CIFAR100":0.80592,"VisualSTS-b-Multilingual":0.722479,"EuroSATZeroShot":0.763148,"VisualSTS17Eng":0.87057,"RESISC45":0.903873,"BirdsnapZeroShot":0.114533,"CIFAR10ZeroShot":0.9564,"STL10ZeroShot":0.976875,"GTSRB":0.852162,"Country211ZeroShot":0.102038,"FORBI2IRetrieval":0.65479,"CIFAR100ZeroShot":0.7264,"OVENIT2ITRetrieval":0.10982,"MNIST":0.9674,"GLDv2I2TRetrieval":0.31178,"CIFAR100Clustering":0.8153332728824985,"RParisHardI2IRetrieval":0.00103,"SOPI2IRetrieval":0.51134,"VidoreSyntheticDocQAAIRetrieval":0.96278,"ImageNet10Clustering":0.9416167142603237,"StanfordCarsI2IRetrieval":0.90635,"StanfordCarsZeroShot":0.655889,"FER2013":0.430816,"OxfordFlowersClassification":0.991373,"STS12VisualSTS":0.760892,"Imagenet1kZeroShot":0.59032,"STS14VisualSTS":0.775646,"AROCocoOrder":0.514154,"VQA2IT2TRetrieval":0.02921,"CLEVRCountZeroShot":0.8012,"GLDv2I2IRetrieval":0.27101,"RenderedSST2":0.680945,"STS15VisualSTS":0.87504,"SciMMIRT2IRetrieval":0.47357,"WebQAT2TRetrieval":0.71434,"VizWizIT2TRetrieval":0.26896,"SketchyI2IRetrieval":0.81395,"Caltech101":0.884221,"VisualSTS17Multilingual":0.579858,"BLINKIT2TMultiChoice":0.39596,"EuroSAT":0.884556,"MemotionT2IRetrieval":0.97407,"CIRRIT2IRetrieval":0.08069,"DTDZeroShot":0.71383,"Food101ZeroShot":0.740317,"Birdsnap":0.620097,"Flickr30kT2IRetrieval":0.89123,"Winoground":0.0575,"StanfordCars":0.895535,"VisualNewsT2IRetrieval":0.12201,"STS16VisualSTS":0.811638,"MNISTZeroShot":0.9077,"PatchCamelyonZeroShot":0.49823,"HatefulMemesI2TRetrieval":0.79565,"SciMMIRI2TRetrieval":0.43406,"RParisMediumI2IRetrieval":0.02626,"UCF101ZeroShot":0.62265,"AROFlickrOrder":0.5784,"ImageNetDog15Clustering":0.8947263409724453,"Fashion200kT2IRetrieval":0.03285,"FashionIQIT2IRetrieval":0.01033,"TUBerlinT2IRetrieval":0.9361,"SUN397ZeroShot":0.666069,"FGVCAircraft":0.494989,"CUB200I2IRetrieval":0.801,"XFlickr30kCoT2IRetrieval":0.8113575},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5800297382268943},"MIEB(lite) Task Mean":{"Task Mean":0.5501042997744934}},"trainedOnTasks":[],"experiments":null},{"rank":3,"model":{"name":"mongodb/voyage-multimodal-3","url":"https://huggingface.co/voyageai/voyage-multimodal-3","zeroShotPct":100,"activeParamsB":null,"totalParamsB":null,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2024-11-10","modelType":"dense","instructionTuned":false,"openWeights":false,"openness":{"open weights":false,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":2,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":[],"citation":null,"memoryUsageMb":null,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["voyage_v"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":null,"embeddingDim":1024,"maxTokens":32768,"meanTask":0.6379485539084537,"meanTaskType":0.6379485539084537,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3883937777777777,"Compositionality":0.4351089616366795,"ImageClassification":0.7130656225819005,"DocumentUnderstanding":0.8393900000000001,"ZeroShotClassification":0.5824420714282983,"ImageClustering":0.8241341604680107,"VisionCentricQA":0.485605112152371,"Any2AnyMultilingualRetrieval":0.5887249074074073,"VisualSTS(eng)":0.8184429256320929,"VisualSTS(multi)":0.704178},"scoresByTask":{"GTSRBZeroShot":0.5511480601741885,"CVBenchDistance":0.475,"CVBenchRelation":0.5353846153846153,"OxfordPetsZeroShot":0.7050967566094304,"InfoSeekIT2ITRetrieval":0.33953,"VidoreTabfquadRetrieval":0.93416,"CIFAR10Clustering":0.8622309717386565,"VidoreDocVQARetrieval":0.52892,"VisualNewsI2TRetrieval":0.12474,"VidoreArxivQARetrieval":0.83176,"STL10":0.9878500000000001,"CLEVRZeroShot":0.13906666666666667,"MemotionI2TRetrieval":0.87061,"PatchCamelyon":0.677899169921875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.94702,"TinyImageNetClustering":0.6929420627528728,"WebQAT2ITRetrieval":0.65641,"SUN397":0.7631080459770115,"METI2IRetrieval":0.31974,"STS13VisualSTS":0.8160271504089486,"CVBenchCount":0.2639593908629442,"ROxfordMediumI2IRetrieval":0.09256,"NIGHTSI2IRetrieval":0.24863,"Flickr30kI2TRetrieval":0.92171,"BLINKIT2TRetrieval":0.33796,"VidoreTatdqaRetrieval":0.68349,"ImageCoDe":0.12815,"ImageCoDeT2IRetrieval":0.11512,"Fashion200kI2TRetrieval":0.07318,"FGVCAircraftZeroShot":0.11671167116711671,"HatefulMemesT2IRetrieval":0.7198,"XM3600T2IRetrieval":0.49775972222222215,"UCF101":0.8432668504436176,"MSCOCOI2TRetrieval":0.62155,"OxfordPets":0.8655764513491414,"SugarCrepe":0.8961523175239563,"FER2013ZeroShot":0.5135135135135135,"OVENIT2TRetrieval":0.16397,"Caltech101ZeroShot":0.7817225509533202,"MSCOCOT2IRetrieval":0.70362,"RESISC45ZeroShot":0.6173015873015874,"Food101Classification":0.8738376237623763,"RParisEasyI2IRetrieval":0.10271,"CVBenchDepth":0.5316666666666666,"Imagenet1k":0.58506,"VisualSTS-b-Eng":0.82555,"VOC2007":0.7850586099274607,"EDIST2ITRetrieval":0.30749,"ROxfordEasyI2IRetrieval":0.16469,"VidoreShiftProjectRetrieval":0.82692,"WITT2IRetrieval":0.48257999999999995,"RP2kI2IRetrieval":0.5352,"ROxfordHardI2IRetrieval":0.06662,"DTD":0.7256382978723404,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.94002,"AROVisualAttribution":0.6587588787078857,"Country211":0.08897630331753555,"AROVisualRelation":0.5194051265716553,"BLINKIT2IRetrieval":0.37313,"CIFAR10":0.9553800000000001,"InfoSeekIT2TRetrieval":0.22647,"VidoreInfoVQARetrieval":0.87236,"BLINKIT2IMultiChoice":0.69652,"VidoreSyntheticDocQAEnergyRetrieval":0.88639,"CIFAR100":0.7813399999999999,"VisualSTS-b-Multilingual":0.730498,"EuroSATZeroShot":0.5235185185185185,"VisualSTS17Eng":0.896835,"RESISC45":0.8733650793650793,"BirdsnapZeroShot":0.13614262560777957,"CIFAR10ZeroShot":0.9456,"STL10ZeroShot":0.987125,"GTSRB":0.755249406175772,"Country211ZeroShot":0.11071090047393364,"FORBI2IRetrieval":0.51585,"CIFAR100ZeroShot":0.7192,"OVENIT2ITRetrieval":0.20454,"MNIST":0.94478,"GLDv2I2TRetrieval":0.46656,"CIFAR100Clustering":0.7515067668506068,"RParisHardI2IRetrieval":0.04256,"SOPI2IRetrieval":0.4718,"VidoreSyntheticDocQAAIRetrieval":0.94286,"ImageNet10Clustering":0.9758070118084365,"StanfordCarsI2IRetrieval":0.46785,"StanfordCarsZeroShot":0.49396841188906854,"FER2013":0.4668988576205072,"OxfordFlowersClassification":0.9264705882352942,"STS12VisualSTS":0.7161988759864429,"Imagenet1kZeroShot":0.60912,"STS14VisualSTS":0.7798398073449294,"AROCocoOrder":0.4089964032173157,"VQA2IT2TRetrieval":0.07691,"CLEVRCountZeroShot":0.45653333333333335,"GLDv2I2IRetrieval":0.11831,"RenderedSST2":0.8544755628775398,"STS15VisualSTS":0.8684963727234637,"SciMMIRT2IRetrieval":0.58646,"WebQAT2TRetrieval":0.6262,"VizWizIT2TRetrieval":0.24914,"SketchyI2IRetrieval":0.60225,"Caltech101":0.913905325443787,"VisualSTS17Multilingual":0.677858,"BLINKIT2TMultiChoice":0.4111,"EuroSAT":0.7926666666666666,"MemotionT2IRetrieval":0.90829,"CIRRIT2IRetrieval":0.35005,"DTDZeroShot":0.5904255319148937,"Food101ZeroShot":0.7442376237623762,"Birdsnap":0.3268503511615343,"Flickr30kT2IRetrieval":0.93746,"Winoground":0.0625,"StanfordCars":0.5290635493097874,"VisualNewsT2IRetrieval":0.16907,"STS16VisualSTS":0.8261532729608647,"MNISTZeroShot":0.8883,"PatchCamelyonZeroShot":0.611114501953125,"HatefulMemesI2TRetrieval":0.55506,"SciMMIRI2TRetrieval":0.5616,"RParisMediumI2IRetrieval":0.02618,"UCF101ZeroShot":0.6146980445252731,"AROFlickrOrder":0.3718000054359436,"ImageNetDog15Clustering":0.8381839891894807,"Fashion200kT2IRetrieval":0.05474,"FashionIQIT2IRetrieval":0.15465,"TUBerlinT2IRetrieval":0.60246,"SUN397ZeroShot":0.6864367816091954,"FGVCAircraft":0.2252025202520252,"CUB200I2IRetrieval":0.64429,"XFlickr30kCoT2IRetrieval":0.7858350000000001},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5661151682063068},"MIEB(lite) Task Mean":{"Task Mean":0.5420789153373505}},"trainedOnTasks":[],"experiments":null},{"rank":4,"model":{"name":"royokong/e5-v","url":"https://huggingface.co/royokong/e5-v","zeroShotPct":100,"activeParamsB":7.83,"totalParamsB":8.355,"embeddingDim":4096,"maxTokens":8192.0,"releaseDate":"2024-07-17","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@article{jiang2024e5v,\n      title={E5-V: Universal Embeddings with Multimodal Large Language Models},\n      author={Jiang, Ting and Song, Minghui and Zhang, Zihan and Huang, Haizhen and Deng, Weiwei and Sun, Feng and Zhang, Qi and Wang, Deqing and Zhuang, Fuzhen},\n      journal={arXiv preprint arXiv:2407.12580},\n      year={2024},\n      eprint={2407.12580},\n      archivePrefix={arXiv},\n      url={https://arxiv.org/abs/2407.12580}\n}","memoryUsageMb":15936.0,"license":null,"publicTrainingCode":"https://github.com/kongds/E5-V","publicTrainingData":"https://huggingface.co/datasets/princeton-nlp/datasets-for-simcse","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["e5-v"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":7.83,"totalParamsB":8.355,"embeddingDim":4096,"maxTokens":8192,"meanTask":0.581611653021081,"meanTaskType":0.581611653021081,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3403891111111111,"Compositionality":0.4630392493517058,"ImageClassification":0.7449219120385842,"DocumentUnderstanding":0.626856,"ZeroShotClassification":0.5001421062095525,"ImageClustering":0.700450361800557,"VisionCentricQA":0.519046398325307,"Any2AnyMultilingualRetrieval":0.6657381060606061,"VisualSTS(eng)":0.793028785313386,"VisualSTS(multi)":0.4625045},"scoresByTask":{"GTSRBZeroShot":0.46167854315122725,"CVBenchDistance":0.43833333333333335,"CVBenchRelation":0.5969230769230769,"OxfordPetsZeroShot":0.32597437994003814,"InfoSeekIT2ITRetrieval":0.08037,"VidoreTabfquadRetrieval":0.81369,"CIFAR10Clustering":0.8257571183470694,"VidoreDocVQARetrieval":0.34731,"VisualNewsI2TRetrieval":0.08401,"VidoreArxivQARetrieval":0.48266,"STL10":0.9881249999999999,"CLEVRZeroShot":0.158,"MemotionI2TRetrieval":0.80314,"PatchCamelyon":0.72386474609375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.82313,"TinyImageNetClustering":0.6664375439236414,"WebQAT2ITRetrieval":0.49629,"SUN397":0.7264367816091953,"METI2IRetrieval":0.35145,"STS13VisualSTS":0.781757974370646,"CVBenchCount":0.39213197969543145,"ROxfordMediumI2IRetrieval":0.07043,"NIGHTSI2IRetrieval":0.22375,"Flickr30kI2TRetrieval":0.84742,"BLINKIT2TRetrieval":0.36066,"VidoreTatdqaRetrieval":0.29317,"ImageCoDe":0.13206,"ImageCoDeT2IRetrieval":0.12163,"Fashion200kI2TRetrieval":0.02037,"FGVCAircraftZeroShot":0.078007800780078,"HatefulMemesT2IRetrieval":0.69389,"XM3600T2IRetrieval":0.6659425,"UCF101":0.8606828241220157,"MSCOCOI2TRetrieval":0.55189,"OxfordPets":0.7991278277459799,"SugarCrepe":0.880175769329071,"FER2013ZeroShot":0.5844246308163834,"OVENIT2TRetrieval":0.06897,"Caltech101ZeroShot":0.8034188034188035,"MSCOCOT2IRetrieval":0.68142,"RESISC45ZeroShot":0.5795238095238096,"Food101Classification":0.8531247524752474,"RParisEasyI2IRetrieval":0.09649,"CVBenchDepth":0.485,"Imagenet1k":0.6656639999999999,"VisualSTS-b-Eng":0.794002,"VOC2007":0.9610881560825192,"EDIST2ITRetrieval":0.23426,"ROxfordEasyI2IRetrieval":0.13223,"VidoreShiftProjectRetrieval":0.42466,"WITT2IRetrieval":0.4995818181818182,"RP2kI2IRetrieval":0.59221,"ROxfordHardI2IRetrieval":0.03018,"DTD":0.7224468085106384,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.82161,"AROVisualAttribution":0.7453736066818237,"Country211":0.10636966824644549,"AROVisualRelation":0.5919705629348755,"BLINKIT2IRetrieval":0.31343,"CIFAR10":0.9414,"InfoSeekIT2TRetrieval":0.0642,"VidoreInfoVQARetrieval":0.69216,"BLINKIT2IMultiChoice":0.71891,"VidoreSyntheticDocQAEnergyRetrieval":0.78105,"CIFAR100":0.71896,"VisualSTS-b-Multilingual":0.519773,"EuroSATZeroShot":0.5048148148148148,"VisualSTS17Eng":0.858398,"RESISC45":0.8909523809523808,"BirdsnapZeroShot":0.07293354943273905,"CIFAR10ZeroShot":0.8966,"STL10ZeroShot":0.96575,"GTSRB":0.8002216943784639,"Country211ZeroShot":0.07208530805687204,"FORBI2IRetrieval":0.55283,"CIFAR100ZeroShot":0.6063,"OVENIT2ITRetrieval":0.07306,"MNIST":0.91602,"GLDv2I2TRetrieval":0.28552,"CIFAR100Clustering":0.7043272313963993,"RParisHardI2IRetrieval":0.02771,"SOPI2IRetrieval":0.46764,"VidoreSyntheticDocQAAIRetrieval":0.78912,"ImageNet10Clustering":0.9384590450130137,"StanfordCarsI2IRetrieval":0.42569,"StanfordCarsZeroShot":0.26078845914687226,"FER2013":0.5396489272777932,"OxfordFlowersClassification":0.9415686274509805,"STS12VisualSTS":0.7315165345482618,"Imagenet1kZeroShot":0.4866,"STS14VisualSTS":0.7487574638135666,"AROCocoOrder":0.4129948019981384,"VQA2IT2TRetrieval":0.06964,"CLEVRCountZeroShot":0.2072,"GLDv2I2IRetrieval":0.10823,"RenderedSST2":0.7605711147721033,"STS15VisualSTS":0.8422433764690826,"SciMMIRT2IRetrieval":0.36423,"WebQAT2TRetrieval":0.60473,"VizWizIT2TRetrieval":0.16298,"SketchyI2IRetrieval":0.68758,"Caltech101":0.9182774490466799,"VisualSTS17Multilingual":0.405236,"BLINKIT2TMultiChoice":0.48298,"EuroSAT":0.875074074074074,"MemotionT2IRetrieval":0.92122,"CIRRIT2IRetrieval":0.28326,"DTDZeroShot":0.5452127659574468,"Food101ZeroShot":0.6068514851485148,"Birdsnap":0.44624527282549975,"Flickr30kT2IRetrieval":0.8975,"Winoground":0.11749999970197678,"StanfordCars":0.6155453301828131,"VisualNewsT2IRetrieval":0.12973,"STS16VisualSTS":0.7945261479921448,"MNISTZeroShot":0.7355,"PatchCamelyonZeroShot":0.538360595703125,"HatefulMemesI2TRetrieval":0.62497,"SciMMIRI2TRetrieval":0.30071,"RParisMediumI2IRetrieval":0.02604,"UCF101ZeroShot":0.6047643361798681,"AROFlickrOrder":0.3612000048160553,"ImageNetDog15Clustering":0.367270870322661,"Fashion200kT2IRetrieval":0.01004,"FashionIQIT2IRetrieval":0.05762,"TUBerlinT2IRetrieval":0.88917,"SUN397ZeroShot":0.6479080459770115,"FGVCAircraft":0.3774377437743774,"CUB200I2IRetrieval":0.42872,"XFlickr30kCoT2IRetrieval":0.83169},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5190930155942307},"MIEB(lite) Task Mean":{"Task Mean":0.47521699009514207}},"trainedOnTasks":[],"experiments":null},{"rank":5,"model":{"name":"google/siglip-so400m-patch14-384","url":"https://huggingface.co/google/siglip-so400m-patch14-384","zeroShotPct":100,"activeParamsB":0.841,"totalParamsB":0.878,"embeddingDim":1152,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":3349.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.841,"totalParamsB":0.878,"embeddingDim":1152,"maxTokens":64,"meanTask":0.5708827827654444,"meanTaskType":0.5708827827654444,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.407792,"Compositionality":0.40355344554492406,"ImageClassification":0.845752660485849,"DocumentUnderstanding":0.563812,"ZeroShotClassification":0.7083751919051211,"ImageClustering":0.8210856335054398,"VisionCentricQA":0.46248576055360324,"Any2AnyMultilingualRetrieval":0.4018784553872054,"VisualSTS(eng)":0.6801711802723013,"VisualSTS(multi)":0.41392150000000005},"scoresByTask":{"GTSRBZeroShot":0.6445764053840063,"CVBenchDistance":0.4,"CVBenchRelation":0.5338461538461539,"OxfordPetsZeroShot":0.9653856636685746,"InfoSeekIT2ITRetrieval":0.04379,"VidoreTabfquadRetrieval":0.60285,"CIFAR10Clustering":0.8379096167856044,"VidoreDocVQARetrieval":0.31278,"VisualNewsI2TRetrieval":0.36504,"VidoreArxivQARetrieval":0.50206,"STL10":0.9950750000000002,"CLEVRZeroShot":0.22426666666666667,"MemotionI2TRetrieval":0.97212,"PatchCamelyon":0.7540771484375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.83103,"TinyImageNetClustering":0.6831321129349398,"WebQAT2ITRetrieval":0.26172,"SUN397":0.8016275862068966,"METI2IRetrieval":0.42686,"STS13VisualSTS":0.6295103035945632,"CVBenchCount":0.217005076142132,"ROxfordMediumI2IRetrieval":0.18108,"NIGHTSI2IRetrieval":0.23627,"Flickr30kI2TRetrieval":0.89943,"BLINKIT2TRetrieval":0.20177,"VidoreTatdqaRetrieval":0.27515,"ImageCoDe":0.13076,"ImageCoDeT2IRetrieval":0.11946,"Fashion200kI2TRetrieval":0.23807,"FGVCAircraftZeroShot":0.6024602460246025,"HatefulMemesT2IRetrieval":0.84392,"XM3600T2IRetrieval":0.29812388888888885,"UCF101":0.9165442283806305,"MSCOCOI2TRetrieval":0.63534,"OxfordPets":0.94854183701281,"SugarCrepe":0.8592730760574341,"FER2013ZeroShot":0.5420730008358874,"OVENIT2TRetrieval":0.00574,"Caltech101ZeroShot":0.856508875739645,"MSCOCOT2IRetrieval":0.68942,"RESISC45ZeroShot":0.6957142857142857,"Food101Classification":0.9547247524752475,"RParisEasyI2IRetrieval":0.10943,"CVBenchDepth":0.48333333333333334,"Imagenet1k":0.821572,"VisualSTS-b-Eng":0.626256,"VOC2007":0.9240438060487968,"EDIST2ITRetrieval":0.12415,"ROxfordEasyI2IRetrieval":0.30745,"VidoreShiftProjectRetrieval":0.25044,"WITT2IRetrieval":0.41077272727272723,"RP2kI2IRetrieval":0.61949,"ROxfordHardI2IRetrieval":0.13124,"DTD":0.8080851063829787,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.75349,"AROVisualAttribution":0.5967023968696594,"Country211":0.32469194312796207,"AROVisualRelation":0.4656389653682709,"BLINKIT2IRetrieval":0.36816,"CIFAR10":0.9692200000000002,"InfoSeekIT2TRetrieval":0.00242,"VidoreInfoVQARetrieval":0.69728,"BLINKIT2IMultiChoice":0.76368,"VidoreSyntheticDocQAEnergyRetrieval":0.73524,"CIFAR100":0.8458600000000001,"VisualSTS-b-Multilingual":0.439051,"EuroSATZeroShot":0.5868518518518518,"VisualSTS17Eng":0.784132,"RESISC45":0.9164444444444445,"BirdsnapZeroShot":0.6250675310642896,"CIFAR10ZeroShot":0.9689,"STL10ZeroShot":0.9875,"GTSRB":0.8639113222486146,"Country211ZeroShot":0.3380568720379147,"FORBI2IRetrieval":0.66898,"CIFAR100ZeroShot":0.8151,"OVENIT2ITRetrieval":0.02538,"MNIST":0.96114,"GLDv2I2TRetrieval":0.81792,"CIFAR100Clustering":0.7667230969900574,"RParisHardI2IRetrieval":0.04702,"SOPI2IRetrieval":0.64303,"VidoreSyntheticDocQAAIRetrieval":0.6778,"ImageNet10Clustering":0.9819186962940006,"StanfordCarsI2IRetrieval":0.93222,"StanfordCarsZeroShot":0.946275338888198,"FER2013":0.4741432153803288,"OxfordFlowersClassification":0.9952941176470589,"STS12VisualSTS":0.6189891413604087,"Imagenet1kZeroShot":0.80886,"STS14VisualSTS":0.605816502412127,"AROCocoOrder":0.3007996678352356,"VQA2IT2TRetrieval":0.00762,"CLEVRCountZeroShot":0.40573333333333333,"GLDv2I2IRetrieval":0.33817,"RenderedSST2":0.700713893465129,"STS15VisualSTS":0.7616519013355569,"SciMMIRT2IRetrieval":0.36049,"WebQAT2TRetrieval":0.2004,"VizWizIT2TRetrieval":0.03133,"SketchyI2IRetrieval":0.78573,"Caltech101":0.9702827087442472,"VisualSTS17Multilingual":0.388792,"BLINKIT2TMultiChoice":0.37705,"EuroSAT":0.8897037037037038,"MemotionT2IRetrieval":0.98063,"CIRRIT2IRetrieval":0.20275,"DTDZeroShot":0.6691489361702128,"Food101ZeroShot":0.9546138613861386,"Birdsnap":0.729119394921664,"Flickr30kT2IRetrieval":0.91312,"Winoground":0.12250000238418579,"StanfordCars":0.9575177216764084,"VisualNewsT2IRetrieval":0.3646,"STS16VisualSTS":0.7348424132034526,"MNISTZeroShot":0.8856,"PatchCamelyonZeroShot":0.54815673828125,"HatefulMemesI2TRetrieval":0.84156,"SciMMIRI2TRetrieval":0.33377,"RParisMediumI2IRetrieval":0.02716,"UCF101ZeroShot":0.7685141891678691,"AROFlickrOrder":0.3492000102996826,"ImageNetDog15Clustering":0.8357446445225972,"Fashion200kT2IRetrieval":0.19726,"FashionIQIT2IRetrieval":0.11057,"TUBerlinT2IRetrieval":0.95344,"SUN397ZeroShot":0.752551724137931,"FGVCAircraft":0.784938493849385,"CUB200I2IRetrieval":0.78512,"XFlickr30kCoT2IRetrieval":0.49673875},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.586834904448607},"MIEB(lite) Task Mean":{"Task Mean":0.5228116129222382}},"trainedOnTasks":[],"experiments":null},{"rank":6,"model":{"name":"google/siglip-large-patch16-384","url":"https://huggingface.co/google/siglip-large-patch16-384","zeroShotPct":100,"activeParamsB":0.62,"totalParamsB":0.652,"embeddingDim":1024,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":2489.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.62,"totalParamsB":0.652,"embeddingDim":1024,"maxTokens":64,"meanTask":0.5703075873079977,"meanTaskType":0.5703075873079977,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3990811111111111,"Compositionality":0.396930785508837,"ImageClassification":0.8370194191246033,"DocumentUnderstanding":0.533215,"ZeroShotClassification":0.68000759674088,"ImageClustering":0.7994043138149161,"VisionCentricQA":0.4535773701679031,"Any2AnyMultilingualRetrieval":0.5110923148148148,"VisualSTS(eng)":0.695127461796911,"VisualSTS(multi)":0.39762050000000004},"scoresByTask":{"GTSRBZeroShot":0.6370546318289786,"CVBenchDistance":0.4583333333333333,"CVBenchRelation":0.5092307692307693,"OxfordPetsZeroShot":0.9678386481330062,"InfoSeekIT2ITRetrieval":0.09365,"VidoreTabfquadRetrieval":0.61086,"CIFAR10Clustering":0.8161189776138361,"VidoreDocVQARetrieval":0.28534,"VisualNewsI2TRetrieval":0.28689,"VidoreArxivQARetrieval":0.47451,"STL10":0.9933249999999999,"CLEVRZeroShot":0.20733333333333334,"MemotionI2TRetrieval":0.96199,"PatchCamelyon":0.773077392578125,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.74674,"TinyImageNetClustering":0.6621236987081185,"WebQAT2ITRetrieval":0.30458,"SUN397":0.7902804597701149,"METI2IRetrieval":0.42842,"STS13VisualSTS":0.620781299589911,"CVBenchCount":0.08756345177664974,"ROxfordMediumI2IRetrieval":0.18332,"NIGHTSI2IRetrieval":0.24858,"Flickr30kI2TRetrieval":0.88319,"BLINKIT2TRetrieval":0.19798,"VidoreTatdqaRetrieval":0.25258,"ImageCoDe":0.13467,"ImageCoDeT2IRetrieval":0.11381,"Fashion200kI2TRetrieval":0.24062,"FGVCAircraftZeroShot":0.5283528352835284,"HatefulMemesT2IRetrieval":0.84696,"XM3600T2IRetrieval":0.44148944444444443,"UCF101":0.9044875233426369,"MSCOCOI2TRetrieval":0.62354,"OxfordPets":0.9508857999454893,"SugarCrepe":0.8518173098564148,"FER2013ZeroShot":0.5827528559487323,"OVENIT2TRetrieval":0.01599,"Caltech101ZeroShot":0.8438527284681131,"MSCOCOT2IRetrieval":0.68465,"RESISC45ZeroShot":0.6865079365079365,"Food101Classification":0.9499564356435644,"RParisEasyI2IRetrieval":0.10193,"CVBenchDepth":0.5466666666666666,"Imagenet1k":0.8120480000000001,"VisualSTS-b-Eng":0.665927,"VOC2007":0.9222635435633721,"EDIST2ITRetrieval":0.14146,"ROxfordEasyI2IRetrieval":0.34515,"VidoreShiftProjectRetrieval":0.25369,"WITT2IRetrieval":0.42827,"RP2kI2IRetrieval":0.62182,"ROxfordHardI2IRetrieval":0.12816,"DTD":0.8046808510638298,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.74524,"AROVisualAttribution":0.5779880285263062,"Country211":0.23000947867298577,"AROVisualRelation":0.45247942209243774,"BLINKIT2IRetrieval":0.33582,"CIFAR10":0.96738,"InfoSeekIT2TRetrieval":0.01093,"VidoreInfoVQARetrieval":0.64111,"BLINKIT2IMultiChoice":0.73632,"VidoreSyntheticDocQAEnergyRetrieval":0.67343,"CIFAR100":0.8349399999999999,"VisualSTS-b-Multilingual":0.453448,"EuroSATZeroShot":0.5511111111111111,"VisualSTS17Eng":0.795784,"RESISC45":0.9093968253968254,"BirdsnapZeroShot":0.6353322528363047,"CIFAR10ZeroShot":0.9567,"STL10ZeroShot":0.993,"GTSRB":0.8576247030878859,"Country211ZeroShot":0.2304739336492891,"FORBI2IRetrieval":0.65253,"CIFAR100ZeroShot":0.7923,"OVENIT2ITRetrieval":0.06516,"MNIST":0.96212,"GLDv2I2TRetrieval":0.67119,"CIFAR100Clustering":0.744251520586655,"RParisHardI2IRetrieval":0.04563,"SOPI2IRetrieval":0.63796,"VidoreSyntheticDocQAAIRetrieval":0.64865,"ImageNet10Clustering":0.9328403871766044,"StanfordCarsI2IRetrieval":0.93359,"StanfordCarsZeroShot":0.9419226464370103,"FER2013":0.4620228475898579,"OxfordFlowersClassification":0.995686274509804,"STS12VisualSTS":0.663030444076807,"Imagenet1kZeroShot":0.79928,"STS14VisualSTS":0.6165727065792361,"AROCocoOrder":0.29816073179244995,"VQA2IT2TRetrieval":0.01005,"CLEVRCountZeroShot":0.32986666666666664,"GLDv2I2IRetrieval":0.27534,"RenderedSST2":0.5639758374519495,"STS15VisualSTS":0.7710720201085501,"SciMMIRT2IRetrieval":0.31879,"WebQAT2TRetrieval":0.21475,"VizWizIT2TRetrieval":0.04697,"SketchyI2IRetrieval":0.7944,"Caltech101":0.9683431952662721,"VisualSTS17Multilingual":0.341793,"BLINKIT2TMultiChoice":0.38335,"EuroSAT":0.8962222222222221,"MemotionT2IRetrieval":0.97867,"CIRRIT2IRetrieval":0.15841,"DTDZeroShot":0.6377659574468085,"Food101ZeroShot":0.9495049504950495,"Birdsnap":0.7201512695840087,"Flickr30kT2IRetrieval":0.90118,"Winoground":0.12999999523162842,"StanfordCars":0.9551299589603284,"VisualNewsT2IRetrieval":0.27904,"STS16VisualSTS":0.7327247622238717,"MNISTZeroShot":0.8517,"PatchCamelyonZeroShot":0.523773193359375,"HatefulMemesI2TRetrieval":0.83255,"SciMMIRI2TRetrieval":0.29921,"RParisMediumI2IRetrieval":0.02699,"UCF101ZeroShot":0.6975912980370671,"AROFlickrOrder":0.33340001106262207,"ImageNetDog15Clustering":0.8416869849893666,"Fashion200kT2IRetrieval":0.21056,"FashionIQIT2IRetrieval":0.07639,"TUBerlinT2IRetrieval":0.94852,"SUN397ZeroShot":0.732183908045977,"FGVCAircraft":0.7543954395439544,"CUB200I2IRetrieval":0.78133,"XFlickr30kCoT2IRetrieval":0.6635175},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.57366636373603},"MIEB(lite) Task Mean":{"Task Mean":0.5144074600046539}},"trainedOnTasks":[],"experiments":null},{"rank":7,"model":{"name":"google/siglip-large-patch16-256","url":"https://huggingface.co/google/siglip-large-patch16-256","zeroShotPct":100,"activeParamsB":0.619,"totalParamsB":0.652,"embeddingDim":1024,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":2488.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.619,"totalParamsB":0.652,"embeddingDim":1024,"maxTokens":64,"meanTask":0.5515133762670158,"meanTaskType":0.5515133762670158,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.38827488888888884,"Compositionality":0.4074759468255724,"ImageClassification":0.8245893224087759,"DocumentUnderstanding":0.393656,"ZeroShotClassification":0.6774271564872647,"ImageClustering":0.8212868640570831,"VisionCentricQA":0.4486673803635733,"Any2AnyMultilingualRetrieval":0.49836712962962965,"VisualSTS(eng)":0.6743155740093696,"VisualSTS(multi)":0.3810735},"scoresByTask":{"GTSRBZeroShot":0.61298495645289,"CVBenchDistance":0.46166666666666667,"CVBenchRelation":0.4815384615384615,"OxfordPetsZeroShot":0.9615699100572364,"InfoSeekIT2ITRetrieval":0.09999,"VidoreTabfquadRetrieval":0.49814,"CIFAR10Clustering":0.8361153564096051,"VidoreDocVQARetrieval":0.22386,"VisualNewsI2TRetrieval":0.26052,"VidoreArxivQARetrieval":0.40189,"STL10":0.992075,"CLEVRZeroShot":0.21466666666666667,"MemotionI2TRetrieval":0.96054,"PatchCamelyon":0.75478515625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.56029,"TinyImageNetClustering":0.6652218912348475,"WebQAT2ITRetrieval":0.3261,"SUN397":0.7786206896551724,"METI2IRetrieval":0.41796,"STS13VisualSTS":0.5944008322154556,"CVBenchCount":0.08883248730964467,"ROxfordMediumI2IRetrieval":0.16836,"NIGHTSI2IRetrieval":0.23975,"Flickr30kI2TRetrieval":0.86584,"BLINKIT2TRetrieval":0.1942,"VidoreTatdqaRetrieval":0.12383,"ImageCoDe":0.13293,"ImageCoDeT2IRetrieval":0.10817,"Fashion200kI2TRetrieval":0.22545,"FGVCAircraftZeroShot":0.5244524452445245,"HatefulMemesT2IRetrieval":0.83826,"XM3600T2IRetrieval":0.4377363888888889,"UCF101":0.8887407454153771,"MSCOCOI2TRetrieval":0.60783,"OxfordPets":0.9461433633142546,"SugarCrepe":0.8475568890571594,"FER2013ZeroShot":0.5658957926999164,"OVENIT2TRetrieval":0.01596,"Caltech101ZeroShot":0.8441814595660749,"MSCOCOT2IRetrieval":0.67359,"RESISC45ZeroShot":0.6774603174603174,"Food101Classification":0.9364039603960397,"RParisEasyI2IRetrieval":0.10007,"CVBenchDepth":0.5616666666666666,"Imagenet1k":0.793436,"VisualSTS-b-Eng":0.625034,"VOC2007":0.9212277932076152,"EDIST2ITRetrieval":0.14734,"ROxfordEasyI2IRetrieval":0.34436,"VidoreShiftProjectRetrieval":0.09127,"WITT2IRetrieval":0.41408,"RP2kI2IRetrieval":0.62201,"ROxfordHardI2IRetrieval":0.08947,"DTD":0.8,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.55453,"AROVisualAttribution":0.5796924829483032,"Country211":0.19399052132701422,"AROVisualRelation":0.4582027792930603,"BLINKIT2IRetrieval":0.35572,"CIFAR10":0.9671800000000002,"InfoSeekIT2TRetrieval":0.0109,"VidoreInfoVQARetrieval":0.54087,"BLINKIT2IMultiChoice":0.73134,"VidoreSyntheticDocQAEnergyRetrieval":0.50787,"CIFAR100":0.8348399999999998,"VisualSTS-b-Multilingual":0.428035,"EuroSATZeroShot":0.537962962962963,"VisualSTS17Eng":0.792072,"RESISC45":0.8993015873015873,"BirdsnapZeroShot":0.5753646677471637,"CIFAR10ZeroShot":0.9614,"STL10ZeroShot":0.991125,"GTSRB":0.8424228028503563,"Country211ZeroShot":0.19919431279620853,"FORBI2IRetrieval":0.59894,"CIFAR100ZeroShot":0.7992,"OVENIT2ITRetrieval":0.07298,"MNIST":0.96052,"GLDv2I2TRetrieval":0.64813,"CIFAR100Clustering":0.7623386500620861,"RParisHardI2IRetrieval":0.04551,"SOPI2IRetrieval":0.62465,"VidoreSyntheticDocQAAIRetrieval":0.43401,"ImageNet10Clustering":0.978710064780804,"StanfordCarsI2IRetrieval":0.92874,"StanfordCarsZeroShot":0.9360776022882726,"FER2013":0.45447199777096686,"OxfordFlowersClassification":0.9947058823529412,"STS12VisualSTS":0.6394252276626233,"Imagenet1kZeroShot":0.78774,"STS14VisualSTS":0.5935130294322843,"AROCocoOrder":0.3263494670391083,"VQA2IT2TRetrieval":0.00942,"CLEVRCountZeroShot":0.40646666666666664,"GLDv2I2IRetrieval":0.23751,"RenderedSST2":0.6166941241076331,"STS15VisualSTS":0.7574159543123563,"SciMMIRT2IRetrieval":0.2645,"WebQAT2TRetrieval":0.23583,"VizWizIT2TRetrieval":0.0452,"SketchyI2IRetrieval":0.77617,"Caltech101":0.9677186061801446,"VisualSTS17Multilingual":0.334112,"BLINKIT2TMultiChoice":0.36696,"EuroSAT":0.8927777777777777,"MemotionT2IRetrieval":0.9802,"CIRRIT2IRetrieval":0.15304,"DTDZeroShot":0.6473404255319148,"Food101ZeroShot":0.9318019801980199,"Birdsnap":0.6588870880605079,"Flickr30kT2IRetrieval":0.88185,"Winoground":0.12999999523162842,"StanfordCars":0.9512249720184055,"VisualNewsT2IRetrieval":0.25363,"STS16VisualSTS":0.7183479744428668,"MNISTZeroShot":0.85,"PatchCamelyonZeroShot":0.522979736328125,"HatefulMemesI2TRetrieval":0.82535,"SciMMIRI2TRetrieval":0.25337,"RParisMediumI2IRetrieval":0.02722,"UCF101ZeroShot":0.6875299402485865,"AROFlickrOrder":0.3776000142097473,"ImageNetDog15Clustering":0.8640483577980729,"Fashion200kT2IRetrieval":0.19689,"FashionIQIT2IRetrieval":0.06753,"TUBerlinT2IRetrieval":0.95102,"SUN397ZeroShot":0.728735632183908,"FGVCAircraft":0.7114911491149115,"CUB200I2IRetrieval":0.7223,"XFlickr30kCoT2IRetrieval":0.643285},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5560135915240929},"MIEB(lite) Task Mean":{"Task Mean":0.4936055566100637}},"trainedOnTasks":[],"experiments":null},{"rank":8,"model":{"name":"google/siglip-base-patch16-512","url":"https://huggingface.co/google/siglip-base-patch16-512","zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.204,"embeddingDim":768,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":777.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.204,"embeddingDim":768,"maxTokens":64,"meanTask":0.5495045002204987,"meanTaskType":0.5495045002204987,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3810984444444445,"Compositionality":0.3750890924208505,"ImageClassification":0.8089032071034999,"DocumentUnderstanding":0.5206419999999999,"ZeroShotClassification":0.640563947404732,"ImageClustering":0.7465324127589212,"VisionCentricQA":0.5320094351164909,"Any2AnyMultilingualRetrieval":0.43209689393939393,"VisualSTS(eng)":0.6769195690166534,"VisualSTS(multi)":0.38119000000000003},"scoresByTask":{"GTSRBZeroShot":0.5117181314330959,"CVBenchDistance":0.42833333333333334,"CVBenchRelation":0.5138461538461538,"OxfordPetsZeroShot":0.949577541564459,"InfoSeekIT2ITRetrieval":0.05731,"VidoreTabfquadRetrieval":0.50705,"CIFAR10Clustering":0.7394905452905783,"VidoreDocVQARetrieval":0.28375,"VisualNewsI2TRetrieval":0.22731,"VidoreArxivQARetrieval":0.46017,"STL10":0.9872,"CLEVRZeroShot":0.22206666666666666,"MemotionI2TRetrieval":0.96611,"PatchCamelyon":0.735211181640625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.79698,"TinyImageNetClustering":0.5982454851027604,"WebQAT2ITRetrieval":0.31434,"SUN397":0.7715678160919539,"METI2IRetrieval":0.42086,"STS13VisualSTS":0.590952140143663,"CVBenchCount":0.5520304568527918,"ROxfordMediumI2IRetrieval":0.17131,"NIGHTSI2IRetrieval":0.25587,"Flickr30kI2TRetrieval":0.8666,"BLINKIT2TRetrieval":0.13745,"VidoreTatdqaRetrieval":0.25297,"ImageCoDe":0.13597,"ImageCoDeT2IRetrieval":0.10773,"Fashion200kI2TRetrieval":0.21748,"FGVCAircraftZeroShot":0.45724572457245727,"HatefulMemesT2IRetrieval":0.83899,"XM3600T2IRetrieval":0.3570625,"UCF101":0.8773162636864585,"MSCOCOI2TRetrieval":0.59791,"OxfordPets":0.9291905151267376,"SugarCrepe":0.8435627818107605,"FER2013ZeroShot":0.5140707718027305,"OVENIT2TRetrieval":0.00626,"Caltech101ZeroShot":0.8410585141354372,"MSCOCOT2IRetrieval":0.65601,"RESISC45ZeroShot":0.6296825396825397,"Food101Classification":0.9267089108910891,"RParisEasyI2IRetrieval":0.10208,"CVBenchDepth":0.5366666666666666,"Imagenet1k":0.77542,"VisualSTS-b-Eng":0.622053,"VOC2007":0.9263036964510842,"EDIST2ITRetrieval":0.12308,"ROxfordEasyI2IRetrieval":0.31923,"VidoreShiftProjectRetrieval":0.2295,"WITT2IRetrieval":0.3397881818181818,"RP2kI2IRetrieval":0.61798,"ROxfordHardI2IRetrieval":0.10359,"DTD":0.7947872340425532,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.72786,"AROVisualAttribution":0.5413246154785156,"Country211":0.18048341232227488,"AROVisualRelation":0.46091824769973755,"BLINKIT2IRetrieval":0.38308,"CIFAR10":0.92656,"InfoSeekIT2TRetrieval":0.00371,"VidoreInfoVQARetrieval":0.6451,"BLINKIT2IMultiChoice":0.74378,"VidoreSyntheticDocQAEnergyRetrieval":0.66792,"CIFAR100":0.7447,"VisualSTS-b-Multilingual":0.423171,"EuroSATZeroShot":0.3837037037037037,"VisualSTS17Eng":0.800878,"RESISC45":0.8835555555555554,"BirdsnapZeroShot":0.5402485143165856,"CIFAR10ZeroShot":0.9296,"STL10ZeroShot":0.9855,"GTSRB":0.8076009501187649,"Country211ZeroShot":0.18028436018957347,"FORBI2IRetrieval":0.66355,"CIFAR100ZeroShot":0.6972,"OVENIT2ITRetrieval":0.03858,"MNIST":0.9548,"GLDv2I2TRetrieval":0.58309,"CIFAR100Clustering":0.6656220961258387,"RParisHardI2IRetrieval":0.04291,"SOPI2IRetrieval":0.60697,"VidoreSyntheticDocQAAIRetrieval":0.63512,"ImageNet10Clustering":0.9331865477583325,"StanfordCarsI2IRetrieval":0.91966,"StanfordCarsZeroShot":0.9251336898395722,"FER2013":0.42917247144051274,"OxfordFlowersClassification":0.9909803921568627,"STS12VisualSTS":0.6497069361194111,"Imagenet1kZeroShot":0.78126,"STS14VisualSTS":0.6112904117135436,"AROCocoOrder":0.2299480140209198,"VQA2IT2TRetrieval":0.00774,"CLEVRCountZeroShot":0.24093333333333333,"GLDv2I2IRetrieval":0.23611,"RenderedSST2":0.5722130697419,"STS15VisualSTS":0.7508475172719615,"SciMMIRT2IRetrieval":0.28226,"WebQAT2TRetrieval":0.22037,"VizWizIT2TRetrieval":0.03519,"SketchyI2IRetrieval":0.70187,"Caltech101":0.9693293885601578,"VisualSTS17Multilingual":0.339209,"BLINKIT2TMultiChoice":0.4174,"EuroSAT":0.8640000000000001,"MemotionT2IRetrieval":0.97765,"CIRRIT2IRetrieval":0.15367,"DTDZeroShot":0.6553191489361702,"Food101ZeroShot":0.9188910891089109,"Birdsnap":0.6745542949756889,"Flickr30kT2IRetrieval":0.88691,"Winoground":0.0925000011920929,"StanfordCars":0.945678398209178,"VisualNewsT2IRetrieval":0.21637,"STS16VisualSTS":0.7127089778679953,"MNISTZeroShot":0.8285,"PatchCamelyonZeroShot":0.609130859375,"HatefulMemesI2TRetrieval":0.83652,"SciMMIRI2TRetrieval":0.27555,"RParisMediumI2IRetrieval":0.027,"UCF101ZeroShot":0.6467595686883088,"AROFlickrOrder":0.321399986743927,"ImageNetDog15Clustering":0.7961173895170968,"Fashion200kT2IRetrieval":0.1769,"FashionIQIT2IRetrieval":0.06856,"TUBerlinT2IRetrieval":0.94918,"SUN397ZeroShot":0.7128735632183908,"FGVCAircraft":0.7007500750075006,"CUB200I2IRetrieval":0.74853,"XFlickr30kCoT2IRetrieval":0.59944},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5533877652091352},"MIEB(lite) Task Mean":{"Task Mean":0.4961531476796857}},"trainedOnTasks":[],"experiments":null},{"rank":9,"model":{"name":"jinaai/jina-embeddings-v5-omni-nano","url":"https://huggingface.co/jinaai/jina-embeddings-v5-omni-nano","zeroShotPct":-1,"activeParamsB":0.887,"totalParamsB":0.986,"embeddingDim":768,"maxTokens":8192.0,"releaseDate":"2026-04-01","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["Afrikaans","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Latvian","Lithuanian","Macedonian","Malay (macrolanguage)","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Panjabi","Persian","Polish","Portuguese","Quechua (Latin)","Romanian","Russian","Sinhala","Slovak","Slovenian","Spanish","Swahili (macrolanguage)","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Yoruba"],"citation":"@misc{akram2026jinaembeddingsv5texttasktargetedembeddingdistillation,\n      title={jina-embeddings-v5-text: Task-Targeted Embedding Distillation},\n      author={Mohammad Kalim Akram and Saba Sturua and Nastia Havriushenko and Quentin Herreros and Michael Günther and Maximilian Werk and Han Xiao},\n      year={2026},\n      eprint={2602.15547},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2602.15547},\n}","memoryUsageMb":1881.0,"license":"cc-by-nc-4.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"jinaai/jina-embeddings-v5-text-nano","supersededBy":null,"extraRequirementsGroups":["peft"],"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":0.887,"totalParamsB":0.986,"embeddingDim":768,"maxTokens":8192,"meanTask":0.5449638892139032,"meanTaskType":0.5449638892139032,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3066308888888889,"Compositionality":0.33589071428571426,"ImageClassification":0.6657134545454546,"DocumentUnderstanding":0.792457,"ZeroShotClassification":0.40519056521739133,"ImageClustering":0.7227688187446318,"VisionCentricQA":0.4787333333333333,"Any2AnyMultilingualRetrieval":0.44349718855218856,"VisualSTS(eng)":0.7592444285714286,"VisualSTS(multi)":0.5395125000000001},"scoresByTask":{"GTSRBZeroShot":0.385986,"CVBenchDistance":0.495,"CVBenchRelation":0.53231,"OxfordPetsZeroShot":0.174162,"InfoSeekIT2ITRetrieval":0.04992,"VidoreTabfquadRetrieval":0.90849,"CIFAR10Clustering":0.8435441889091453,"VidoreDocVQARetrieval":0.50653,"VisualNewsI2TRetrieval":0.04547,"VidoreArxivQARetrieval":0.74937,"STL10":0.958612,"CLEVRZeroShot":0.224133,"MemotionI2TRetrieval":0.939,"PatchCamelyon":0.746588,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.9444,"TinyImageNetClustering":0.6486864723623645,"WebQAT2ITRetrieval":0.39997,"SUN397":0.707172,"METI2IRetrieval":0.3283,"STS13VisualSTS":0.737605,"CVBenchCount":0.18147,"ROxfordMediumI2IRetrieval":0.11457,"NIGHTSI2IRetrieval":0.23466,"Flickr30kI2TRetrieval":0.51682,"BLINKIT2TRetrieval":0.11475,"VidoreTatdqaRetrieval":0.58439,"ImageCoDe":0.117724,"ImageCoDeT2IRetrieval":0.0656,"Fashion200kI2TRetrieval":0.00332,"FGVCAircraftZeroShot":0.063306,"HatefulMemesT2IRetrieval":0.77411,"XM3600T2IRetrieval":0.3699861111111111,"UCF101":0.786765,"MSCOCOI2TRetrieval":0.25475,"OxfordPets":0.628673,"SugarCrepe":0.577287,"FER2013ZeroShot":0.336305,"OVENIT2TRetrieval":0.04609,"Caltech101ZeroShot":0.702827,"MSCOCOT2IRetrieval":0.3833,"RESISC45ZeroShot":0.503016,"Food101Classification":0.695216,"RParisEasyI2IRetrieval":0.07693,"CVBenchDepth":0.52667,"Imagenet1k":0.574552,"VisualSTS-b-Eng":0.753532,"VOC2007":0.681159,"EDIST2ITRetrieval":0.13824,"ROxfordEasyI2IRetrieval":0.2394,"VidoreShiftProjectRetrieval":0.67427,"WITT2IRetrieval":0.40390545454545457,"RP2kI2IRetrieval":0.48874,"ROxfordHardI2IRetrieval":0.0091,"DTD":0.670745,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.89406,"AROVisualAttribution":0.541359,"Country211":0.084512,"AROVisualRelation":0.547395,"BLINKIT2IRetrieval":0.36318,"CIFAR10":0.91756,"InfoSeekIT2TRetrieval":0.0471,"VidoreInfoVQARetrieval":0.8257,"BLINKIT2IMultiChoice":0.76368,"VidoreSyntheticDocQAEnergyRetrieval":0.88433,"CIFAR100":0.66312,"VisualSTS-b-Multilingual":0.625113,"EuroSATZeroShot":0.344444,"VisualSTS17Eng":0.822445,"RESISC45":0.812286,"BirdsnapZeroShot":0.031334,"CIFAR10ZeroShot":0.8927,"STL10ZeroShot":0.915375,"GTSRB":0.691338,"Country211ZeroShot":0.057915,"FORBI2IRetrieval":0.60657,"CIFAR100ZeroShot":0.5183,"OVENIT2ITRetrieval":0.04182,"MNIST":0.91822,"GLDv2I2TRetrieval":0.19668,"CIFAR100Clustering":0.697043337510701,"RParisHardI2IRetrieval":0.00114,"SOPI2IRetrieval":0.41123,"VidoreSyntheticDocQAAIRetrieval":0.95303,"ImageNet10Clustering":0.8640304080597893,"StanfordCarsI2IRetrieval":0.54259,"StanfordCarsZeroShot":0.15023,"FER2013":0.448091,"OxfordFlowersClassification":0.883725,"STS12VisualSTS":0.713757,"Imagenet1kZeroShot":0.32162,"STS14VisualSTS":0.708694,"AROCocoOrder":0.27597,"VQA2IT2TRetrieval":0.02222,"CLEVRCountZeroShot":0.2794,"GLDv2I2IRetrieval":0.16226,"RenderedSST2":0.681494,"STS15VisualSTS":0.82025,"SciMMIRT2IRetrieval":0.44066,"WebQAT2TRetrieval":0.70013,"VizWizIT2TRetrieval":0.20488,"SketchyI2IRetrieval":0.63588,"Caltech101":0.914004,"VisualSTS17Multilingual":0.453912,"BLINKIT2TMultiChoice":0.37327,"EuroSAT":0.751481,"MemotionT2IRetrieval":0.97223,"CIRRIT2IRetrieval":0.07566,"DTDZeroShot":0.30266,"Food101ZeroShot":0.224634,"Birdsnap":0.300486,"Flickr30kT2IRetrieval":0.66291,"Winoground":0.0225,"StanfordCars":0.512921,"VisualNewsT2IRetrieval":0.06316,"STS16VisualSTS":0.758428,"MNISTZeroShot":0.7962,"PatchCamelyonZeroShot":0.499298,"HatefulMemesI2TRetrieval":0.76376,"SciMMIRI2TRetrieval":0.42381,"RParisMediumI2IRetrieval":0.02496,"UCF101ZeroShot":0.424159,"AROFlickrOrder":0.269,"ImageNetDog15Clustering":0.5605396868811591,"Fashion200kT2IRetrieval":0.00214,"FashionIQIT2IRetrieval":0.01319,"TUBerlinT2IRetrieval":0.72498,"SUN397ZeroShot":0.489885,"FGVCAircraft":0.29847,"CUB200I2IRetrieval":0.47221,"XFlickr30kCoT2IRetrieval":0.5566},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4787218327497853},"MIEB(lite) Task Mean":{"Task Mean":0.44239868088039386}},"trainedOnTasks":[],"experiments":null},{"rank":10,"model":{"name":"laion/CLIP-ViT-bigG-14-laion2B-39B-b160k","url":"https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k","zeroShotPct":100,"activeParamsB":2.476,"totalParamsB":2.54,"embeddingDim":1280,"maxTokens":77.0,"releaseDate":"2023-01-23","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{cherti2023reproducible,\n    title={Reproducible scaling laws for contrastive language-image learning},\n    author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},\n    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},\n    pages={2818--2829},\n    year={2023}\n}","memoryUsageMb":9689.0,"license":"mit","publicTrainingCode":"https://github.com/mlfoundations/open_clip","publicTrainingData":"https://laion.ai/blog/laion-5b/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["open_clip_torch"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":2.476,"totalParamsB":2.54,"embeddingDim":1280,"maxTokens":77,"meanTask":0.5422987768536937,"meanTaskType":0.5422987768536937,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.4150286666666666,"Compositionality":0.4235559678963252,"ImageClassification":0.8364010183067857,"DocumentUnderstanding":0.431926,"ZeroShotClassification":0.6936940061198035,"ImageClustering":0.8559280002636523,"VisionCentricQA":0.43157499457677123,"Any2AnyMultilingualRetrieval":0.28012117003367004,"VisualSTS(eng)":0.709340944673262,"VisualSTS(multi)":0.345417},"scoresByTask":{"GTSRBZeroShot":0.6114806017418845,"CVBenchDistance":0.4216666666666667,"CVBenchRelation":0.4815384615384615,"OxfordPetsZeroShot":0.9528481875170346,"InfoSeekIT2ITRetrieval":0.26278,"VidoreTabfquadRetrieval":0.35015,"CIFAR10Clustering":0.8766434169672674,"VidoreDocVQARetrieval":0.20442,"VisualNewsI2TRetrieval":0.44503,"VidoreArxivQARetrieval":0.38839,"STL10":0.99535,"CLEVRZeroShot":0.20046666666666665,"MemotionI2TRetrieval":0.90757,"PatchCamelyon":0.712548828125,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.57857,"TinyImageNetClustering":0.7548918339385243,"WebQAT2ITRetrieval":0.46104,"SUN397":0.7834758620689655,"METI2IRetrieval":0.43821,"STS13VisualSTS":0.6815737460365511,"CVBenchCount":0.04187817258883249,"ROxfordMediumI2IRetrieval":0.17006,"NIGHTSI2IRetrieval":0.26015,"Flickr30kI2TRetrieval":0.8779,"BLINKIT2TRetrieval":0.2976,"VidoreTatdqaRetrieval":0.1621,"ImageCoDe":0.12815,"ImageCoDeT2IRetrieval":0.11381,"Fashion200kI2TRetrieval":0.1539,"FGVCAircraftZeroShot":0.49564956495649565,"HatefulMemesT2IRetrieval":0.67437,"XM3600T2IRetrieval":0.17715805555555555,"UCF101":0.9084429349618917,"MSCOCOI2TRetrieval":0.60177,"OxfordPets":0.9348596347778686,"SugarCrepe":0.8647317290306091,"FER2013ZeroShot":0.5713290610197826,"OVENIT2TRetrieval":0.06536,"Caltech101ZeroShot":0.8427021696252466,"MSCOCOT2IRetrieval":0.65603,"RESISC45ZeroShot":0.6976190476190476,"Food101Classification":0.9251881188118812,"RParisEasyI2IRetrieval":0.11511,"CVBenchDepth":0.4716666666666667,"Imagenet1k":0.7822800000000001,"VisualSTS-b-Eng":0.665396,"VOC2007":0.9180911354436659,"EDIST2ITRetrieval":0.31384,"ROxfordEasyI2IRetrieval":0.30748,"VidoreShiftProjectRetrieval":0.25024,"WITT2IRetrieval":0.34628545454545456,"RP2kI2IRetrieval":0.68944,"ROxfordHardI2IRetrieval":0.10543,"DTD":0.795,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.62274,"AROVisualAttribution":0.6111381649971008,"Country211":0.28567772511848344,"AROVisualRelation":0.5179846882820129,"BLINKIT2IRetrieval":0.39552,"CIFAR10":0.9841600000000001,"InfoSeekIT2TRetrieval":0.05738,"VidoreInfoVQARetrieval":0.60898,"BLINKIT2IMultiChoice":0.73134,"VidoreSyntheticDocQAEnergyRetrieval":0.59948,"CIFAR100":0.8802599999999998,"VisualSTS-b-Multilingual":0.368881,"EuroSATZeroShot":0.6655555555555556,"VisualSTS17Eng":0.799667,"RESISC45":0.9094285714285715,"BirdsnapZeroShot":0.7439222042139384,"CIFAR10ZeroShot":0.9794,"STL10ZeroShot":0.98225,"GTSRB":0.8769279493269991,"Country211ZeroShot":0.3239336492890995,"FORBI2IRetrieval":0.53849,"CIFAR100ZeroShot":0.8535,"OVENIT2ITRetrieval":0.16194,"MNIST":0.9612399999999999,"GLDv2I2TRetrieval":0.80053,"CIFAR100Clustering":0.7996706044579868,"RParisHardI2IRetrieval":0.04877,"SOPI2IRetrieval":0.65432,"VidoreSyntheticDocQAAIRetrieval":0.55419,"ImageNet10Clustering":0.9875395877165348,"StanfordCarsI2IRetrieval":0.90785,"StanfordCarsZeroShot":0.9401815694565353,"FER2013":0.4730286988018947,"OxfordFlowersClassification":0.9933333333333334,"STS12VisualSTS":0.6280955956368836,"Imagenet1kZeroShot":0.77648,"STS14VisualSTS":0.655017193562808,"AROCocoOrder":0.331987202167511,"VQA2IT2TRetrieval":0.0209,"CLEVRCountZeroShot":0.29713333333333336,"GLDv2I2IRetrieval":0.31208,"RenderedSST2":0.6337177375068643,"STS15VisualSTS":0.7866943854545813,"SciMMIRT2IRetrieval":0.29046,"WebQAT2TRetrieval":0.36337,"VizWizIT2TRetrieval":0.09891,"SketchyI2IRetrieval":0.7889,"Caltech101":0.9653188691650231,"VisualSTS17Multilingual":0.321953,"BLINKIT2TMultiChoice":0.44136,"EuroSAT":0.9221851851851852,"MemotionT2IRetrieval":0.93697,"CIRRIT2IRetrieval":0.13758,"DTDZeroShot":0.6420212765957447,"Food101ZeroShot":0.9278019801980198,"Birdsnap":0.7418692598595353,"Flickr30kT2IRetrieval":0.88406,"Winoground":0.12250000238418579,"StanfordCars":0.9519462753388881,"VisualNewsT2IRetrieval":0.44646,"STS16VisualSTS":0.7489426920220097,"MNISTZeroShot":0.7709,"PatchCamelyonZeroShot":0.624481201171875,"HatefulMemesI2TRetrieval":0.6632,"SciMMIRI2TRetrieval":0.29756,"RParisMediumI2IRetrieval":0.02717,"UCF101ZeroShot":0.6861860354377802,"AROFlickrOrder":0.38839998841285706,"ImageNetDog15Clustering":0.8608945582379488,"Fashion200kT2IRetrieval":0.14207,"FashionIQIT2IRetrieval":0.0516,"TUBerlinT2IRetrieval":0.92127,"SUN397ZeroShot":0.7354022988505747,"FGVCAircraft":0.7002100210021001,"CUB200I2IRetrieval":0.81205,"XFlickr30kCoT2IRetrieval":0.31692},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5772056232021662},"MIEB(lite) Task Mean":{"Task Mean":0.5044713689400186}},"trainedOnTasks":[],"experiments":null},{"rank":11,"model":{"name":"google/siglip-base-patch16-384","url":"https://huggingface.co/google/siglip-base-patch16-384","zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.203,"embeddingDim":768,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":776.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.203,"embeddingDim":768,"maxTokens":64,"meanTask":0.5421104035913448,"meanTaskType":0.5421104035913448,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3766171111111112,"Compositionality":0.38542048980781013,"ImageClassification":0.8059676182431462,"DocumentUnderstanding":0.449991,"ZeroShotClassification":0.6410024328602617,"ImageClustering":0.7626675088732788,"VisionCentricQA":0.5280183949412122,"Any2AnyMultilingualRetrieval":0.42546723484848487,"VisualSTS(eng)":0.6705067452281427,"VisualSTS(multi)":0.3754455},"scoresByTask":{"GTSRBZeroShot":0.5186856690419636,"CVBenchDistance":0.4216666666666667,"CVBenchRelation":0.5169230769230769,"OxfordPetsZeroShot":0.9501226492232215,"InfoSeekIT2ITRetrieval":0.05922,"VidoreTabfquadRetrieval":0.47607,"CIFAR10Clustering":0.7161764971905245,"VidoreDocVQARetrieval":0.26433,"VisualNewsI2TRetrieval":0.22575,"VidoreArxivQARetrieval":0.43589,"STL10":0.9862500000000001,"CLEVRZeroShot":0.22366666666666668,"MemotionI2TRetrieval":0.9648,"PatchCamelyon":0.7357421875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.67672,"TinyImageNetClustering":0.5814758046874788,"WebQAT2ITRetrieval":0.31196,"SUN397":0.7692965517241379,"METI2IRetrieval":0.41798,"STS13VisualSTS":0.5937592056504977,"CVBenchCount":0.5342639593908629,"ROxfordMediumI2IRetrieval":0.16288,"NIGHTSI2IRetrieval":0.25524,"Flickr30kI2TRetrieval":0.8596,"BLINKIT2TRetrieval":0.12736,"VidoreTatdqaRetrieval":0.19189,"ImageCoDe":0.12902,"ImageCoDeT2IRetrieval":0.11077,"Fashion200kI2TRetrieval":0.21393,"FGVCAircraftZeroShot":0.45094509450945097,"HatefulMemesT2IRetrieval":0.8377,"XM3600T2IRetrieval":0.3544525,"UCF101":0.8695563249581912,"MSCOCOI2TRetrieval":0.59283,"OxfordPets":0.9270645952575632,"SugarCrepe":0.8399680256843567,"FER2013ZeroShot":0.5122596823627752,"OVENIT2TRetrieval":0.00712,"Caltech101ZeroShot":0.8423734385272846,"MSCOCOT2IRetrieval":0.65219,"RESISC45ZeroShot":0.6282539682539683,"Food101Classification":0.9212910891089109,"RParisEasyI2IRetrieval":0.09507,"CVBenchDepth":0.5216666666666666,"Imagenet1k":0.767684,"VisualSTS-b-Eng":0.602784,"VOC2007":0.925352838076992,"EDIST2ITRetrieval":0.12416,"ROxfordEasyI2IRetrieval":0.2973,"VidoreShiftProjectRetrieval":0.14464,"WITT2IRetrieval":0.3347554545454546,"RP2kI2IRetrieval":0.61651,"ROxfordHardI2IRetrieval":0.09172,"DTD":0.7905319148936171,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.58538,"AROVisualAttribution":0.5438291430473328,"Country211":0.17376303317535544,"AROVisualRelation":0.4599156081676483,"BLINKIT2IRetrieval":0.35821,"CIFAR10":0.9290600000000001,"InfoSeekIT2TRetrieval":0.00379,"VidoreInfoVQARetrieval":0.5928,"BLINKIT2IMultiChoice":0.75871,"VidoreSyntheticDocQAEnergyRetrieval":0.57465,"CIFAR100":0.74814,"VisualSTS-b-Multilingual":0.41655,"EuroSATZeroShot":0.3996296296296296,"VisualSTS17Eng":0.792679,"RESISC45":0.8844126984126983,"BirdsnapZeroShot":0.5343057806591032,"CIFAR10ZeroShot":0.9316,"STL10ZeroShot":0.9865,"GTSRB":0.8014726840855108,"Country211ZeroShot":0.17308056872037914,"FORBI2IRetrieval":0.65064,"CIFAR100ZeroShot":0.7,"OVENIT2ITRetrieval":0.04133,"MNIST":0.9535399999999999,"GLDv2I2TRetrieval":0.58449,"CIFAR100Clustering":0.6777896354867872,"RParisHardI2IRetrieval":0.04385,"SOPI2IRetrieval":0.6092,"VidoreSyntheticDocQAAIRetrieval":0.55754,"ImageNet10Clustering":0.9763177068628376,"StanfordCarsI2IRetrieval":0.92003,"StanfordCarsZeroShot":0.9237657007834846,"FER2013":0.42084146001671774,"OxfordFlowersClassification":0.9901960784313726,"STS12VisualSTS":0.646221570706761,"Imagenet1kZeroShot":0.77332,"STS14VisualSTS":0.6117200902707741,"AROCocoOrder":0.2734106481075287,"VQA2IT2TRetrieval":0.00777,"CLEVRCountZeroShot":0.2202,"GLDv2I2IRetrieval":0.22236,"RenderedSST2":0.5595826468973092,"STS15VisualSTS":0.7434359568126032,"SciMMIRT2IRetrieval":0.26174,"WebQAT2TRetrieval":0.22052,"VizWizIT2TRetrieval":0.03492,"SketchyI2IRetrieval":0.70173,"Caltech101":0.9710716633793558,"VisualSTS17Multilingual":0.334341,"BLINKIT2TMultiChoice":0.41488,"EuroSAT":0.8698888888888888,"MemotionT2IRetrieval":0.97696,"CIRRIT2IRetrieval":0.15544,"DTDZeroShot":0.6526595744680851,"Food101ZeroShot":0.9127128712871287,"Birdsnap":0.6615883306320909,"Flickr30kT2IRetrieval":0.88,"Winoground":0.10999999940395355,"StanfordCars":0.9445342619077228,"VisualNewsT2IRetrieval":0.21287,"STS16VisualSTS":0.7029473931563623,"MNISTZeroShot":0.8088,"PatchCamelyonZeroShot":0.691802978515625,"HatefulMemesI2TRetrieval":0.83118,"SciMMIRI2TRetrieval":0.25495,"RParisMediumI2IRetrieval":0.02703,"UCF101ZeroShot":0.6414327144008651,"AROFlickrOrder":0.3418000042438507,"ImageNetDog15Clustering":0.8615779001387661,"Fashion200kT2IRetrieval":0.17075,"FashionIQIT2IRetrieval":0.06489,"TUBerlinT2IRetrieval":0.95482,"SUN397ZeroShot":0.7073563218390805,"FGVCAircraft":0.6900090009000899,"CUB200I2IRetrieval":0.73421,"XFlickr30kCoT2IRetrieval":0.58719375},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5463596969312046},"MIEB(lite) Task Mean":{"Task Mean":0.4901329944171882}},"trainedOnTasks":[],"experiments":null},{"rank":12,"model":{"name":"QuanSun/EVA02-CLIP-bigE-14-plus","url":"https://huggingface.co/QuanSun/EVA-CLIP","zeroShotPct":100,"activeParamsB":null,"totalParamsB":5.0,"embeddingDim":1024,"maxTokens":77.0,"releaseDate":"2023-04-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@article{EVA-CLIP,\n      title={EVA-CLIP: Improved Training Techniques for CLIP at Scale},\n      author={Sun, Quan and Fang, Yuxin and Wu, Ledell and Wang, Xinlong and Cao, Yue},\n      journal={arXiv preprint arXiv:2303.15389},\n      year={2023}\n}","memoryUsageMb":19073.0,"license":"mit","publicTrainingCode":"https://github.com/baaivision/EVA/tree/master/EVA-CLIP","publicTrainingData":"https://laion.ai/blog/laion-5b/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":5.0,"embeddingDim":1024,"maxTokens":77,"meanTask":0.5346558854654465,"meanTaskType":0.5346558854654465,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.4012162222222222,"Compositionality":0.45674480795519695,"ImageClassification":0.8595402154101222,"DocumentUnderstanding":0.3227,"ZeroShotClassification":0.7079529165624209,"ImageClustering":0.9238095195762004,"VisionCentricQA":0.3943487216365136,"Any2AnyMultilingualRetrieval":0.27823220959595957,"VisualSTS(eng)":0.7198742416958296,"VisualSTS(multi)":0.28214},"scoresByTask":{"GTSRBZeroShot":0.6824228028503563,"CVBenchDistance":0.405,"CVBenchRelation":0.47384615384615386,"OxfordPetsZeroShot":0.9580267102752794,"InfoSeekIT2ITRetrieval":0.00312,"VidoreTabfquadRetrieval":0.30362,"CIFAR10Clustering":0.9865420911013263,"VidoreDocVQARetrieval":0.1684,"VisualNewsI2TRetrieval":0.46362,"VidoreArxivQARetrieval":0.34862,"STL10":0.997625,"CLEVRZeroShot":0.19973333333333335,"MemotionI2TRetrieval":0.90256,"PatchCamelyon":0.764776611328125,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.42474,"TinyImageNetClustering":0.835698291041313,"WebQAT2ITRetrieval":0.47131,"SUN397":0.805232183908046,"METI2IRetrieval":0.44702,"STS13VisualSTS":0.6800368581240961,"CVBenchCount":0.10152284263959391,"ROxfordMediumI2IRetrieval":0.18644,"NIGHTSI2IRetrieval":0.26463,"Flickr30kI2TRetrieval":0.88246,"BLINKIT2TRetrieval":0.28121,"VidoreTatdqaRetrieval":0.07515,"ImageCoDe":0.12685,"ImageCoDeT2IRetrieval":0.1212,"Fashion200kI2TRetrieval":0.15657,"FGVCAircraftZeroShot":0.540954095409541,"HatefulMemesT2IRetrieval":0.66544,"XM3600T2IRetrieval":0.17373833333333333,"UCF101":0.9324800422247147,"MSCOCOI2TRetrieval":0.61183,"OxfordPets":0.9499046061597166,"SugarCrepe":0.8649979829788208,"FER2013ZeroShot":0.5451379214265812,"OVENIT2TRetrieval":0.00175,"Caltech101ZeroShot":0.8441814595660749,"MSCOCOT2IRetrieval":0.666,"RESISC45ZeroShot":0.7246031746031746,"Food101Classification":0.9493148514851484,"RParisEasyI2IRetrieval":0.11544,"CVBenchDepth":0.43833333333333335,"Imagenet1k":0.823968,"VisualSTS-b-Eng":0.685925,"VOC2007":0.9167724328334458,"EDIST2ITRetrieval":0.18357,"ROxfordEasyI2IRetrieval":0.34962,"VidoreShiftProjectRetrieval":0.12762,"WITT2IRetrieval":0.34551454545454546,"RP2kI2IRetrieval":0.71801,"ROxfordHardI2IRetrieval":0.12016,"DTD":0.8113829787234043,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.43139,"AROVisualAttribution":0.6051899194717407,"Country211":0.3164170616113744,"AROVisualRelation":0.5140159726142883,"BLINKIT2IRetrieval":0.00746,"CIFAR10":0.99498,"InfoSeekIT2TRetrieval":0.00118,"VidoreInfoVQARetrieval":0.55193,"BLINKIT2IMultiChoice":0.5199,"VidoreSyntheticDocQAEnergyRetrieval":0.44988,"CIFAR100":0.94148,"VisualSTS-b-Multilingual":0.319124,"EuroSATZeroShot":0.7129629629629629,"VisualSTS17Eng":0.828655,"RESISC45":0.9276507936507936,"BirdsnapZeroShot":0.7920043219881145,"CIFAR10ZeroShot":0.9937,"STL10ZeroShot":0.989625,"GTSRB":0.8898653998416469,"Country211ZeroShot":0.340521327014218,"FORBI2IRetrieval":0.56045,"CIFAR100ZeroShot":0.9109,"OVENIT2ITRetrieval":0.00443,"MNIST":0.92788,"GLDv2I2TRetrieval":0.80734,"CIFAR100Clustering":0.8951287044598072,"RParisHardI2IRetrieval":0.04938,"SOPI2IRetrieval":0.69056,"VidoreSyntheticDocQAAIRetrieval":0.34565,"ImageNet10Clustering":0.9908901739583962,"StanfordCarsI2IRetrieval":0.91058,"StanfordCarsZeroShot":0.9416739211540853,"FER2013":0.5083588743382558,"OxfordFlowersClassification":0.9954901960784314,"STS12VisualSTS":0.6335794050719475,"Imagenet1kZeroShot":0.79132,"STS14VisualSTS":0.6638491087376777,"AROCocoOrder":0.4504598081111908,"VQA2IT2TRetrieval":0.00595,"CLEVRCountZeroShot":0.2972,"GLDv2I2IRetrieval":0.34284,"RenderedSST2":0.615046677649643,"STS15VisualSTS":0.7944502231464021,"SciMMIRT2IRetrieval":0.25305,"WebQAT2TRetrieval":0.39176,"VizWizIT2TRetrieval":0.02587,"SketchyI2IRetrieval":0.85501,"Caltech101":0.9740302432610125,"VisualSTS17Multilingual":0.245156,"BLINKIT2TMultiChoice":0.42749,"EuroSAT":0.9385555555555556,"MemotionT2IRetrieval":0.92731,"CIRRIT2IRetrieval":0.28993,"DTDZeroShot":0.6398936170212766,"Food101ZeroShot":0.9462178217821782,"Birdsnap":0.8076715289032956,"Flickr30kT2IRetrieval":0.88711,"Winoground":0.10750000178813934,"StanfordCars":0.9541101853003358,"VisualNewsT2IRetrieval":0.45254,"STS16VisualSTS":0.7526240967906839,"MNISTZeroShot":0.7393,"PatchCamelyonZeroShot":0.6405029296875,"HatefulMemesI2TRetrieval":0.65286,"SciMMIRI2TRetrieval":0.26702,"RParisMediumI2IRetrieval":0.02746,"UCF101ZeroShot":0.6961154409929693,"AROFlickrOrder":0.5281999707221985,"ImageNetDog15Clustering":0.910788337320159,"Fashion200kT2IRetrieval":0.13922,"FashionIQIT2IRetrieval":0.10877,"TUBerlinT2IRetrieval":0.94106,"SUN397ZeroShot":0.7408735632183908,"FGVCAircraft":0.781938193819382,"CUB200I2IRetrieval":0.84363,"XFlickr30kCoT2IRetrieval":0.31544375},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5735680407617251},"MIEB(lite) Task Mean":{"Task Mean":0.502046109387981}},"trainedOnTasks":[],"experiments":null},{"rank":13,"model":{"name":"laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K","url":"https://huggingface.co/laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K","zeroShotPct":100,"activeParamsB":0.39,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77.0,"releaseDate":"2023-04-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{cherti2023reproducible,\n    title={Reproducible scaling laws for contrastive language-image learning},\n    author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},\n    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},\n    pages={2818--2829},\n    year={2023}\n}","memoryUsageMb":1633.0,"license":"mit","publicTrainingCode":"https://github.com/mlfoundations/open_clip","publicTrainingData":"https://huggingface.co/datasets/mlfoundations/datacomp_1b","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["open_clip_torch"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.39,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77,"meanTask":0.5344171287500619,"meanTaskType":0.5344171287500619,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.381098,"Compositionality":0.391120154602187,"ImageClassification":0.8196308759431269,"DocumentUnderstanding":0.3864,"ZeroShotClassification":0.6839243690216784,"ImageClustering":0.8643514297053934,"VisionCentricQA":0.523338177578203,"Any2AnyMultilingualRetrieval":0.23774849747474747,"VisualSTS(eng)":0.6987337831752819,"VisualSTS(multi)":0.357826},"scoresByTask":{"GTSRBZeroShot":0.5692003167062549,"CVBenchDistance":0.465,"CVBenchRelation":0.5553846153846154,"OxfordPetsZeroShot":0.949577541564459,"InfoSeekIT2ITRetrieval":0.16888,"VidoreTabfquadRetrieval":0.307,"CIFAR10Clustering":0.9364893912089942,"VidoreDocVQARetrieval":0.19682,"VisualNewsI2TRetrieval":0.34674,"VidoreArxivQARetrieval":0.34509,"STL10":0.9943500000000001,"CLEVRZeroShot":0.25193333333333334,"MemotionI2TRetrieval":0.88823,"PatchCamelyon":0.735272216796875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.58038,"TinyImageNetClustering":0.7827893992381822,"WebQAT2ITRetrieval":0.40713,"SUN397":0.7718436781609196,"METI2IRetrieval":0.42693,"STS13VisualSTS":0.6763872355718203,"CVBenchCount":0.43274111675126903,"ROxfordMediumI2IRetrieval":0.17988,"NIGHTSI2IRetrieval":0.2568,"Flickr30kI2TRetrieval":0.83054,"BLINKIT2TRetrieval":0.29256,"VidoreTatdqaRetrieval":0.15274,"ImageCoDe":0.1225,"ImageCoDeT2IRetrieval":0.10252,"Fashion200kI2TRetrieval":0.11121,"FGVCAircraftZeroShot":0.474047404740474,"HatefulMemesT2IRetrieval":0.64649,"XM3600T2IRetrieval":0.13441833333333333,"UCF101":0.8864479319355958,"MSCOCOI2TRetrieval":0.5556,"OxfordPets":0.9241755246661215,"SugarCrepe":0.8386366367340088,"FER2013ZeroShot":0.4097241571468376,"OVENIT2TRetrieval":0.04125,"Caltech101ZeroShot":0.866370808678501,"MSCOCOT2IRetrieval":0.60686,"RESISC45ZeroShot":0.7122222222222222,"Food101Classification":0.9273108910891089,"RParisEasyI2IRetrieval":0.10503,"CVBenchDepth":0.5583333333333333,"Imagenet1k":0.772912,"VisualSTS-b-Eng":0.633829,"VOC2007":0.9168263812460611,"EDIST2ITRetrieval":0.23458,"ROxfordEasyI2IRetrieval":0.33098,"VidoreShiftProjectRetrieval":0.16187,"WITT2IRetrieval":0.3183309090909091,"RP2kI2IRetrieval":0.65516,"ROxfordHardI2IRetrieval":0.09786,"DTD":0.7849999999999999,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.50276,"AROVisualAttribution":0.591762900352478,"Country211":0.2458957345971564,"AROVisualRelation":0.4773363471031189,"BLINKIT2IRetrieval":0.36318,"CIFAR10":0.9854800000000001,"InfoSeekIT2TRetrieval":0.01517,"VidoreInfoVQARetrieval":0.5561,"BLINKIT2IMultiChoice":0.73134,"VidoreSyntheticDocQAEnergyRetrieval":0.58929,"CIFAR100":0.8885400000000001,"VisualSTS-b-Multilingual":0.381937,"EuroSATZeroShot":0.707037037037037,"VisualSTS17Eng":0.806348,"RESISC45":0.9044444444444444,"BirdsnapZeroShot":0.7563479200432199,"CIFAR10ZeroShot":0.981,"STL10ZeroShot":0.99175,"GTSRB":0.8812034837688045,"Country211ZeroShot":0.2991943127962085,"FORBI2IRetrieval":0.48053,"CIFAR100ZeroShot":0.8554,"OVENIT2ITRetrieval":0.11781,"MNIST":0.96158,"GLDv2I2TRetrieval":0.75647,"CIFAR100Clustering":0.8425612973508715,"RParisHardI2IRetrieval":0.04679,"SOPI2IRetrieval":0.62473,"VidoreSyntheticDocQAAIRetrieval":0.47195,"ImageNet10Clustering":0.933926763662648,"StanfordCarsI2IRetrieval":0.88733,"StanfordCarsZeroShot":0.9281183932346723,"FER2013":0.41315129562552244,"OxfordFlowersClassification":0.9909803921568627,"STS12VisualSTS":0.6236290508738301,"Imagenet1kZeroShot":0.7741,"STS14VisualSTS":0.642502710462577,"AROCocoOrder":0.2870051860809326,"VQA2IT2TRetrieval":0.0199,"CLEVRCountZeroShot":0.3556666666666667,"GLDv2I2IRetrieval":0.29035,"RenderedSST2":0.5562877539813289,"STS15VisualSTS":0.7736447716986014,"SciMMIRT2IRetrieval":0.18092,"WebQAT2TRetrieval":0.32441,"VizWizIT2TRetrieval":0.08197,"SketchyI2IRetrieval":0.75639,"Caltech101":0.9643984220907298,"VisualSTS17Multilingual":0.333715,"BLINKIT2TMultiChoice":0.39723,"EuroSAT":0.7757037037037038,"MemotionT2IRetrieval":0.90798,"CIRRIT2IRetrieval":0.12358,"DTDZeroShot":0.6446808510638298,"Food101ZeroShot":0.9424158415841584,"Birdsnap":0.7259859535386278,"Flickr30kT2IRetrieval":0.84196,"Winoground":0.07500000298023224,"StanfordCars":0.9389130705136177,"VisualNewsT2IRetrieval":0.33895,"STS16VisualSTS":0.7347957136201445,"MNISTZeroShot":0.8144,"PatchCamelyonZeroShot":0.519866943359375,"HatefulMemesI2TRetrieval":0.63638,"SciMMIRI2TRetrieval":0.19304,"RParisMediumI2IRetrieval":0.02737,"UCF101ZeroShot":0.6320684086273812,"AROFlickrOrder":0.3456000089645386,"ImageNetDog15Clustering":0.8259902970662699,"Fashion200kT2IRetrieval":0.0764,"FashionIQIT2IRetrieval":0.04392,"TUBerlinT2IRetrieval":0.92886,"SUN397ZeroShot":0.7388505747126437,"FGVCAircraft":0.6414641464146416,"CUB200I2IRetrieval":0.79979,"XFlickr30kCoT2IRetrieval":0.26049625},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5589305082934868},"MIEB(lite) Task Mean":{"Task Mean":0.49040829204224456}},"trainedOnTasks":[],"experiments":null},{"rank":14,"model":{"name":"google/siglip-base-patch16-256-multilingual","url":"https://huggingface.co/google/siglip-base-patch16-256-multilingual","zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.371,"embeddingDim":768,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":1414.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.371,"embeddingDim":768,"maxTokens":64,"meanTask":0.5309441004945046,"meanTaskType":0.5309441004945046,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3561133333333334,"Compositionality":0.38136334499086655,"ImageClassification":0.789318169265475,"DocumentUnderstanding":0.26354,"ZeroShotClassification":0.6116738737633596,"ImageClustering":0.7455713776805223,"VisionCentricQA":0.5124547064948587,"Any2AnyMultilingualRetrieval":0.5920846254208755,"VisualSTS(eng)":0.6545945739957549,"VisualSTS(multi)":0.402727},"scoresByTask":{"GTSRBZeroShot":0.5305621536025337,"CVBenchDistance":0.49,"CVBenchRelation":0.5384615384615384,"OxfordPetsZeroShot":0.937857726901063,"InfoSeekIT2ITRetrieval":0.06804,"VidoreTabfquadRetrieval":0.44159,"CIFAR10Clustering":0.759389701706794,"VidoreDocVQARetrieval":0.16957,"VisualNewsI2TRetrieval":0.18089,"VidoreArxivQARetrieval":0.30329,"STL10":0.9824249999999999,"CLEVRZeroShot":0.2036,"MemotionI2TRetrieval":0.94791,"PatchCamelyon":0.73914794921875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.37167,"TinyImageNetClustering":0.5588374889176846,"WebQAT2ITRetrieval":0.29418,"SUN397":0.7462436781609195,"METI2IRetrieval":0.39632,"STS13VisualSTS":0.54798562786248,"CVBenchCount":0.3464467005076142,"ROxfordMediumI2IRetrieval":0.15549,"NIGHTSI2IRetrieval":0.24647,"Flickr30kI2TRetrieval":0.81577,"BLINKIT2TRetrieval":0.18916,"VidoreTatdqaRetrieval":0.04383,"ImageCoDe":0.12511,"ImageCoDeT2IRetrieval":0.10817,"Fashion200kI2TRetrieval":0.18937,"FGVCAircraftZeroShot":0.3234323432343234,"HatefulMemesT2IRetrieval":0.80902,"XM3600T2IRetrieval":0.5795669444444445,"UCF101":0.8382572552214359,"MSCOCOI2TRetrieval":0.54731,"OxfordPets":0.9197601526301445,"SugarCrepe":0.8279856443405151,"FER2013ZeroShot":0.5118417386458624,"OVENIT2TRetrieval":0.00807,"Caltech101ZeroShot":0.8454963839579224,"MSCOCOT2IRetrieval":0.60433,"RESISC45ZeroShot":0.5847619047619048,"Food101Classification":0.8997465346534653,"RParisEasyI2IRetrieval":0.09482,"CVBenchDepth":0.54,"Imagenet1k":0.73948,"VisualSTS-b-Eng":0.562863,"VOC2007":0.9187960055233763,"EDIST2ITRetrieval":0.10918,"ROxfordEasyI2IRetrieval":0.28827,"VidoreShiftProjectRetrieval":0.03887,"WITT2IRetrieval":0.47865818181818187,"RP2kI2IRetrieval":0.62567,"ROxfordHardI2IRetrieval":0.07752,"DTD":0.7793617021276595,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.28726,"AROVisualAttribution":0.5385417938232422,"Country211":0.1510616113744076,"AROVisualRelation":0.4519363343715668,"BLINKIT2IRetrieval":0.37313,"CIFAR10":0.9289,"InfoSeekIT2TRetrieval":0.00449,"VidoreInfoVQARetrieval":0.4528,"BLINKIT2IMultiChoice":0.75124,"VidoreSyntheticDocQAEnergyRetrieval":0.29932,"CIFAR100":0.7467600000000001,"VisualSTS-b-Multilingual":0.450979,"EuroSATZeroShot":0.3333333333333333,"VisualSTS17Eng":0.805337,"RESISC45":0.856031746031746,"BirdsnapZeroShot":0.4105888708806051,"CIFAR10ZeroShot":0.9123,"STL10ZeroShot":0.97625,"GTSRB":0.8031353919239905,"Country211ZeroShot":0.15203791469194314,"FORBI2IRetrieval":0.56845,"CIFAR100ZeroShot":0.7101,"OVENIT2ITRetrieval":0.05406,"MNIST":0.94978,"GLDv2I2TRetrieval":0.57956,"CIFAR100Clustering":0.67891209574897,"RParisHardI2IRetrieval":0.0418,"SOPI2IRetrieval":0.60078,"VidoreSyntheticDocQAAIRetrieval":0.2272,"ImageNet10Clustering":0.9262849153658925,"StanfordCarsI2IRetrieval":0.89292,"StanfordCarsZeroShot":0.8913070513617709,"FER2013":0.4273056561716356,"OxfordFlowersClassification":0.9933333333333334,"STS12VisualSTS":0.6662235379502067,"Imagenet1kZeroShot":0.74362,"STS14VisualSTS":0.5899690086767073,"AROCocoOrder":0.27596962451934814,"VQA2IT2TRetrieval":0.0072,"CLEVRCountZeroShot":0.254,"GLDv2I2IRetrieval":0.18498,"RenderedSST2":0.5683690280065898,"STS15VisualSTS":0.7264591008054552,"SciMMIRT2IRetrieval":0.15972,"WebQAT2TRetrieval":0.21094,"VizWizIT2TRetrieval":0.03957,"SketchyI2IRetrieval":0.68685,"Caltech101":0.9698224852071006,"VisualSTS17Multilingual":0.354475,"BLINKIT2TMultiChoice":0.40858,"EuroSAT":0.8744444444444444,"MemotionT2IRetrieval":0.96966,"CIRRIT2IRetrieval":0.13104,"DTDZeroShot":0.6106382978723405,"Food101ZeroShot":0.8948118811881188,"Birdsnap":0.5722312263641275,"Flickr30kT2IRetrieval":0.83383,"Winoground":0.09000000357627869,"StanfordCars":0.9296356174605149,"VisualNewsT2IRetrieval":0.1631,"STS16VisualSTS":0.6833247426754354,"MNISTZeroShot":0.8309,"PatchCamelyonZeroShot":0.5194091796875,"HatefulMemesI2TRetrieval":0.79844,"SciMMIRI2TRetrieval":0.15689,"RParisMediumI2IRetrieval":0.02688,"UCF101ZeroShot":0.6286605987762864,"AROFlickrOrder":0.36000001430511475,"ImageNetDog15Clustering":0.8044326866632702,"Fashion200kT2IRetrieval":0.1622,"FashionIQIT2IRetrieval":0.05525,"TUBerlinT2IRetrieval":0.95591,"SUN397ZeroShot":0.6946206896551724,"FGVCAircraft":0.5993399339933994,"CUB200I2IRetrieval":0.61149,"XFlickr30kCoT2IRetrieval":0.71802875},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5131863150454067},"MIEB(lite) Task Mean":{"Task Mean":0.4660950072943405}},"trainedOnTasks":[],"experiments":null},{"rank":15,"model":{"name":"laion/CLIP-ViT-H-14-laion2B-s32B-b79K","url":"https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K","zeroShotPct":100,"activeParamsB":0.936,"totalParamsB":0.986,"embeddingDim":1024,"maxTokens":77.0,"releaseDate":"2022-09-15","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{cherti2023reproducible,\n    title={Reproducible scaling laws for contrastive language-image learning},\n    author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},\n    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},\n    pages={2818--2829},\n    year={2023}\n}","memoryUsageMb":3762.0,"license":"mit","publicTrainingCode":"https://github.com/mlfoundations/open_clip","publicTrainingData":"https://laion.ai/blog/laion-5b/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["open_clip_torch"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.936,"totalParamsB":0.986,"embeddingDim":1024,"maxTokens":77,"meanTask":0.5266703064343481,"meanTaskType":0.5266703064343481,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.39733466666666667,"Compositionality":0.42027482071774347,"ImageClassification":0.8247603463046329,"DocumentUnderstanding":0.404063,"ZeroShotClassification":0.674687285428144,"ImageClustering":0.8386477455219925,"VisionCentricQA":0.45797560761855177,"Any2AnyMultilingualRetrieval":0.2553991750841751,"VisualSTS(eng)":0.655034417001575,"VisualSTS(multi)":0.338526},"scoresByTask":{"GTSRBZeroShot":0.5749010292953286,"CVBenchDistance":0.40166666666666667,"CVBenchRelation":0.5092307692307693,"OxfordPetsZeroShot":0.9454892341237394,"InfoSeekIT2ITRetrieval":0.2363,"VidoreTabfquadRetrieval":0.33107,"CIFAR10Clustering":0.8809994999730191,"VidoreDocVQARetrieval":0.19141,"VisualNewsI2TRetrieval":0.40155,"VidoreArxivQARetrieval":0.3303,"STL10":0.9938499999999999,"CLEVRZeroShot":0.16846666666666665,"MemotionI2TRetrieval":0.88761,"PatchCamelyon":0.730230712890625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.55502,"TinyImageNetClustering":0.7266929114696439,"WebQAT2ITRetrieval":0.42937,"SUN397":0.7811126436781609,"METI2IRetrieval":0.43431,"STS13VisualSTS":0.6225009463870684,"CVBenchCount":0.19796954314720813,"ROxfordMediumI2IRetrieval":0.18694,"NIGHTSI2IRetrieval":0.26043,"Flickr30kI2TRetrieval":0.86018,"BLINKIT2TRetrieval":0.21059,"VidoreTatdqaRetrieval":0.15413,"ImageCoDe":0.1338,"ImageCoDeT2IRetrieval":0.11555,"Fashion200kI2TRetrieval":0.14196,"FGVCAircraftZeroShot":0.42514251425142513,"HatefulMemesT2IRetrieval":0.65314,"XM3600T2IRetrieval":0.14973638888888888,"UCF101":0.897615967367639,"MSCOCOI2TRetrieval":0.59134,"OxfordPets":0.918615426546743,"SugarCrepe":0.8552789092063904,"FER2013ZeroShot":0.5082195597659516,"OVENIT2TRetrieval":0.07013,"Caltech101ZeroShot":0.8295529257067719,"MSCOCOT2IRetrieval":0.64363,"RESISC45ZeroShot":0.7065079365079365,"Food101Classification":0.9103762376237624,"RParisEasyI2IRetrieval":0.1086,"CVBenchDepth":0.4866666666666667,"Imagenet1k":0.7653679999999999,"VisualSTS-b-Eng":0.589896,"VOC2007":0.9210853686130369,"EDIST2ITRetrieval":0.28346,"ROxfordEasyI2IRetrieval":0.32934,"VidoreShiftProjectRetrieval":0.21808,"WITT2IRetrieval":0.3273336363636364,"RP2kI2IRetrieval":0.67328,"ROxfordHardI2IRetrieval":0.13076,"DTD":0.7925531914893618,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.52923,"AROVisualAttribution":0.6218867301940918,"Country211":0.2511184834123223,"AROVisualRelation":0.5016501545906067,"BLINKIT2IRetrieval":0.37811,"CIFAR10":0.9764199999999998,"InfoSeekIT2TRetrieval":0.05735,"VidoreInfoVQARetrieval":0.5882,"BLINKIT2IMultiChoice":0.74627,"VidoreSyntheticDocQAEnergyRetrieval":0.60234,"CIFAR100":0.8628599999999998,"VisualSTS-b-Multilingual":0.348508,"EuroSATZeroShot":0.7229629629629629,"VisualSTS17Eng":0.766932,"RESISC45":0.9087936507936508,"BirdsnapZeroShot":0.7174500270124257,"CIFAR10ZeroShot":0.9712,"STL10ZeroShot":0.983125,"GTSRB":0.8388598574821853,"Country211ZeroShot":0.2885781990521327,"FORBI2IRetrieval":0.4846,"CIFAR100ZeroShot":0.8357,"OVENIT2ITRetrieval":0.14913,"MNIST":0.96174,"GLDv2I2TRetrieval":0.75027,"CIFAR100Clustering":0.7869412456360588,"RParisHardI2IRetrieval":0.04855,"SOPI2IRetrieval":0.63973,"VidoreSyntheticDocQAAIRetrieval":0.54085,"ImageNet10Clustering":0.9393220525662637,"StanfordCarsI2IRetrieval":0.89678,"StanfordCarsZeroShot":0.9302325581395349,"FER2013":0.4428531624407913,"OxfordFlowersClassification":0.9898039215686275,"STS12VisualSTS":0.5700385758480732,"Imagenet1kZeroShot":0.7611,"STS14VisualSTS":0.58623183571467,"AROCocoOrder":0.3301079571247101,"VQA2IT2TRetrieval":0.01694,"CLEVRCountZeroShot":0.26566666666666666,"GLDv2I2IRetrieval":0.27029,"RenderedSST2":0.6199890170236134,"STS15VisualSTS":0.7439593454478063,"SciMMIRT2IRetrieval":0.24302,"WebQAT2TRetrieval":0.33992,"VizWizIT2TRetrieval":0.10541,"SketchyI2IRetrieval":0.74486,"Caltech101":0.9574621959237344,"VisualSTS17Multilingual":0.328544,"BLINKIT2TMultiChoice":0.40605,"EuroSAT":0.9236296296296296,"MemotionT2IRetrieval":0.91378,"CIRRIT2IRetrieval":0.123,"DTDZeroShot":0.625,"Food101ZeroShot":0.9218613861386139,"Birdsnap":0.7226364127498649,"Flickr30kT2IRetrieval":0.86947,"Winoground":0.10499999672174454,"StanfordCars":0.9467976619823405,"VisualNewsT2IRetrieval":0.40642,"STS16VisualSTS":0.7056822156134077,"MNISTZeroShot":0.781,"PatchCamelyonZeroShot":0.5186767578125,"HatefulMemesI2TRetrieval":0.64228,"SciMMIRI2TRetrieval":0.25699,"RParisMediumI2IRetrieval":0.02733,"UCF101ZeroShot":0.6705023651003554,"AROFlickrOrder":0.39419999718666077,"ImageNetDog15Clustering":0.8592830179649765,"Fashion200kT2IRetrieval":0.12512,"FashionIQIT2IRetrieval":0.04189,"TUBerlinT2IRetrieval":0.91229,"SUN397ZeroShot":0.7464827586206897,"FGVCAircraft":0.6509450945094509,"CUB200I2IRetrieval":0.78806,"XFlickr30kCoT2IRetrieval":0.2891275},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5604118577672459},"MIEB(lite) Task Mean":{"Task Mean":0.49339674069500433}},"trainedOnTasks":[],"experiments":null},{"rank":16,"model":{"name":"laion/CLIP-ViT-g-14-laion2B-s34B-b88K","url":"https://huggingface.co/laion/CLIP-ViT-g-14-laion2B-s34B-b88K","zeroShotPct":100,"activeParamsB":null,"totalParamsB":1.367,"embeddingDim":1024,"maxTokens":77.0,"releaseDate":"2023-03-06","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{cherti2023reproducible,\n    title={Reproducible scaling laws for contrastive language-image learning},\n    author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},\n    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},\n    pages={2818--2829},\n    year={2023}\n}","memoryUsageMb":5215.0,"license":"mit","publicTrainingCode":"https://github.com/mlfoundations/open_clip","publicTrainingData":"https://laion.ai/blog/laion-5b/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["open_clip_torch"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":1.367,"embeddingDim":1024,"maxTokens":77,"meanTask":0.5237253158889082,"meanTaskType":0.5237253158889082,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.39845933333333333,"Compositionality":0.4194004230894361,"ImageClassification":0.8277129227519229,"DocumentUnderstanding":0.37628000000000006,"ZeroShotClassification":0.6787089024601279,"ImageClustering":0.8273889643828294,"VisionCentricQA":0.44164243958523147,"Any2AnyMultilingualRetrieval":0.2592436195286196,"VisualSTS(eng)":0.6914435537575818,"VisualSTS(multi)":0.316973},"scoresByTask":{"GTSRBZeroShot":0.49778305621536023,"CVBenchDistance":0.41333333333333333,"CVBenchRelation":0.5015384615384615,"OxfordPetsZeroShot":0.9394930498773508,"InfoSeekIT2ITRetrieval":0.24531,"VidoreTabfquadRetrieval":0.32507,"CIFAR10Clustering":0.8762505990099256,"VidoreDocVQARetrieval":0.18104,"VisualNewsI2TRetrieval":0.41005,"VidoreArxivQARetrieval":0.32818,"STL10":0.9939250000000002,"CLEVRZeroShot":0.18586666666666668,"MemotionI2TRetrieval":0.8872,"PatchCamelyon":0.74080810546875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.52085,"TinyImageNetClustering":0.7209905077549027,"WebQAT2ITRetrieval":0.42413,"SUN397":0.7819494252873562,"METI2IRetrieval":0.43349,"STS13VisualSTS":0.6642943546468935,"CVBenchCount":0.10152284263959391,"ROxfordMediumI2IRetrieval":0.15093,"NIGHTSI2IRetrieval":0.26445,"Flickr30kI2TRetrieval":0.85548,"BLINKIT2TRetrieval":0.2169,"VidoreTatdqaRetrieval":0.14799,"ImageCoDe":0.15204,"ImageCoDeT2IRetrieval":0.11164,"Fashion200kI2TRetrieval":0.14109,"FGVCAircraftZeroShot":0.44434443444344435,"HatefulMemesT2IRetrieval":0.65995,"XM3600T2IRetrieval":0.15553972222222223,"UCF101":0.8909994234261112,"MSCOCOI2TRetrieval":0.58825,"OxfordPets":0.9271191060234395,"SugarCrepe":0.8522167205810547,"FER2013ZeroShot":0.5565617163555308,"OVENIT2TRetrieval":0.09876,"Caltech101ZeroShot":0.8318540433925049,"MSCOCOT2IRetrieval":0.64261,"RESISC45ZeroShot":0.7120634920634921,"Food101Classification":0.9113188118811882,"RParisEasyI2IRetrieval":0.10818,"CVBenchDepth":0.47,"Imagenet1k":0.7732439999999999,"VisualSTS-b-Eng":0.641254,"VOC2007":0.9199267102496694,"EDIST2ITRetrieval":0.3023,"ROxfordEasyI2IRetrieval":0.28515,"VidoreShiftProjectRetrieval":0.1672,"WITT2IRetrieval":0.3287136363636364,"RP2kI2IRetrieval":0.683,"ROxfordHardI2IRetrieval":0.09131,"DTD":0.7887234042553191,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.5221,"AROVisualAttribution":0.6289133429527283,"Country211":0.2610047393364929,"AROVisualRelation":0.5090445876121521,"BLINKIT2IRetrieval":0.38557,"CIFAR10":0.97876,"InfoSeekIT2TRetrieval":0.08743,"VidoreInfoVQARetrieval":0.56853,"BLINKIT2IMultiChoice":0.76119,"VidoreSyntheticDocQAEnergyRetrieval":0.60065,"CIFAR100":0.86622,"VisualSTS-b-Multilingual":0.368712,"EuroSATZeroShot":0.6309259259259259,"VisualSTS17Eng":0.798843,"RESISC45":0.9074285714285715,"BirdsnapZeroShot":0.690977849810913,"CIFAR10ZeroShot":0.9781,"STL10ZeroShot":0.98825,"GTSRB":0.8548218527315914,"Country211ZeroShot":0.2952132701421801,"FORBI2IRetrieval":0.44913,"CIFAR100ZeroShot":0.7793,"OVENIT2ITRetrieval":0.15334,"MNIST":0.96526,"GLDv2I2TRetrieval":0.7592,"CIFAR100Clustering":0.7813119850173428,"RParisHardI2IRetrieval":0.04758,"SOPI2IRetrieval":0.6339,"VidoreSyntheticDocQAAIRetrieval":0.40119,"ImageNet10Clustering":0.9438277153472088,"StanfordCarsI2IRetrieval":0.89516,"StanfordCarsZeroShot":0.9375699539858227,"FER2013":0.4563945388687657,"OxfordFlowersClassification":0.991764705882353,"STS12VisualSTS":0.6184937878563733,"Imagenet1kZeroShot":0.7644,"STS14VisualSTS":0.6231968760550424,"AROCocoOrder":0.3291883170604706,"VQA2IT2TRetrieval":0.019,"CLEVRCountZeroShot":0.3742666666666667,"GLDv2I2IRetrieval":0.28293,"RenderedSST2":0.6556836902800659,"STS15VisualSTS":0.7673293357599065,"SciMMIRT2IRetrieval":0.24233,"WebQAT2TRetrieval":0.34339,"VizWizIT2TRetrieval":0.13704,"SketchyI2IRetrieval":0.7712,"Caltech101":0.9535502958579881,"VisualSTS17Multilingual":0.265234,"BLINKIT2TMultiChoice":0.40227,"EuroSAT":0.9184074074074073,"MemotionT2IRetrieval":0.91569,"CIRRIT2IRetrieval":0.12235,"DTDZeroShot":0.6617021276595745,"Food101ZeroShot":0.9190891089108911,"Birdsnap":0.7186385737439223,"Flickr30kT2IRetrieval":0.8677,"Winoground":0.09000000357627869,"StanfordCars":0.9495336400945156,"VisualNewsT2IRetrieval":0.41439,"STS16VisualSTS":0.7266935219848576,"MNISTZeroShot":0.7874,"PatchCamelyonZeroShot":0.546661376953125,"HatefulMemesI2TRetrieval":0.64787,"SciMMIRI2TRetrieval":0.25762,"RParisMediumI2IRetrieval":0.02686,"UCF101ZeroShot":0.699970741026531,"AROFlickrOrder":0.37439998984336853,"ImageNetDog15Clustering":0.8145640147847679,"Fashion200kT2IRetrieval":0.12568,"FashionIQIT2IRetrieval":0.03924,"TUBerlinT2IRetrieval":0.91852,"SUN397ZeroShot":0.7328275862068966,"FGVCAircraft":0.6598859885988599,"CUB200I2IRetrieval":0.78737,"XFlickr30kCoT2IRetrieval":0.2934775},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5606093308358393},"MIEB(lite) Task Mean":{"Task Mean":0.48911573717142864}},"trainedOnTasks":[],"experiments":null},{"rank":17,"model":{"name":"QuanSun/EVA02-CLIP-bigE-14","url":"https://huggingface.co/QuanSun/EVA-CLIP","zeroShotPct":100,"activeParamsB":null,"totalParamsB":4.7,"embeddingDim":1024,"maxTokens":77.0,"releaseDate":"2023-04-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@article{EVA-CLIP,\n      title={EVA-CLIP: Improved Training Techniques for CLIP at Scale},\n      author={Sun, Quan and Fang, Yuxin and Wu, Ledell and Wang, Xinlong and Cao, Yue},\n      journal={arXiv preprint arXiv:2303.15389},\n      year={2023}\n}","memoryUsageMb":17929.0,"license":"mit","publicTrainingCode":"https://github.com/baaivision/EVA/tree/master/EVA-CLIP","publicTrainingData":"https://laion.ai/blog/laion-5b/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":4.7,"embeddingDim":1024,"maxTokens":77,"meanTask":0.5224557272333639,"meanTaskType":0.5224557272333639,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3902331111111111,"Compositionality":0.4234513615771703,"ImageClassification":0.8451367826751303,"DocumentUnderstanding":0.316225,"ZeroShotClassification":0.6933194359152799,"ImageClustering":0.8941540812079289,"VisionCentricQA":0.4357100110633867,"Any2AnyMultilingualRetrieval":0.2553592255892256,"VisualSTS(eng)":0.688023763194407,"VisualSTS(multi)":0.28294450000000004},"scoresByTask":{"GTSRBZeroShot":0.6531274742676167,"CVBenchDistance":0.4816666666666667,"CVBenchRelation":0.49846153846153846,"OxfordPetsZeroShot":0.9569364949577541,"InfoSeekIT2ITRetrieval":0.00212,"VidoreTabfquadRetrieval":0.28795,"CIFAR10Clustering":0.9030060729175552,"VidoreDocVQARetrieval":0.16354,"VisualNewsI2TRetrieval":0.44513,"VidoreArxivQARetrieval":0.32722,"STL10":0.9973500000000002,"CLEVRZeroShot":0.1618,"MemotionI2TRetrieval":0.90153,"PatchCamelyon":0.728228759765625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.4298,"TinyImageNetClustering":0.8358252054206314,"WebQAT2ITRetrieval":0.42336,"SUN397":0.7993011494252874,"METI2IRetrieval":0.44479,"STS13VisualSTS":0.6235632838242521,"CVBenchCount":0.30456852791878175,"ROxfordMediumI2IRetrieval":0.17951,"NIGHTSI2IRetrieval":0.25926,"Flickr30kI2TRetrieval":0.8765,"BLINKIT2TRetrieval":0.21816,"VidoreTatdqaRetrieval":0.0709,"ImageCoDe":0.14292,"ImageCoDeT2IRetrieval":0.12728,"Fashion200kI2TRetrieval":0.16105,"FGVCAircraftZeroShot":0.48064806480648065,"HatefulMemesT2IRetrieval":0.66687,"XM3600T2IRetrieval":0.1496472222222222,"UCF101":0.9211536641127216,"MSCOCOI2TRetrieval":0.6068,"OxfordPets":0.9430907604251839,"SugarCrepe":0.8591399192810059,"FER2013ZeroShot":0.5441627194204514,"OVENIT2TRetrieval":0.00143,"Caltech101ZeroShot":0.8504273504273504,"MSCOCOT2IRetrieval":0.65825,"RESISC45ZeroShot":0.7350793650793651,"Food101Classification":0.9470019801980196,"RParisEasyI2IRetrieval":0.11672,"CVBenchDepth":0.48833333333333334,"Imagenet1k":0.8147080000000001,"VisualSTS-b-Eng":0.628504,"VOC2007":0.8598185634089974,"EDIST2ITRetrieval":0.15466,"ROxfordEasyI2IRetrieval":0.32667,"VidoreShiftProjectRetrieval":0.10135,"WITT2IRetrieval":0.3285254545454545,"RP2kI2IRetrieval":0.70117,"ROxfordHardI2IRetrieval":0.14032,"DTD":0.7974468085106384,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.41323,"AROVisualAttribution":0.6201474666595459,"Country211":0.28838862559241707,"AROVisualRelation":0.49400508403778076,"BLINKIT2IRetrieval":0.00249,"CIFAR10":0.99468,"InfoSeekIT2TRetrieval":0.00158,"VidoreInfoVQARetrieval":0.54804,"BLINKIT2IMultiChoice":0.44527,"VidoreSyntheticDocQAEnergyRetrieval":0.48496,"CIFAR100":0.9376000000000001,"VisualSTS-b-Multilingual":0.299848,"EuroSATZeroShot":0.7475925925925926,"VisualSTS17Eng":0.809551,"RESISC45":0.9195873015873015,"BirdsnapZeroShot":0.7671528903295516,"CIFAR10ZeroShot":0.9921,"STL10ZeroShot":0.99075,"GTSRB":0.8584006334125098,"Country211ZeroShot":0.3372511848341232,"FORBI2IRetrieval":0.51728,"CIFAR100ZeroShot":0.9081,"OVENIT2ITRetrieval":0.00317,"MNIST":0.92598,"GLDv2I2TRetrieval":0.78268,"CIFAR100Clustering":0.8902576456083566,"RParisHardI2IRetrieval":0.04809,"SOPI2IRetrieval":0.68167,"VidoreSyntheticDocQAAIRetrieval":0.33526,"ImageNet10Clustering":0.9431708954857504,"StanfordCarsI2IRetrieval":0.9066,"StanfordCarsZeroShot":0.9398084815321477,"FER2013":0.49189189189189186,"OxfordFlowersClassification":0.9947058823529412,"STS12VisualSTS":0.622428848800624,"Imagenet1kZeroShot":0.7993,"STS14VisualSTS":0.621680681160459,"AROCocoOrder":0.3323470652103424,"VQA2IT2TRetrieval":0.00589,"CLEVRCountZeroShot":0.1624,"GLDv2I2IRetrieval":0.31978,"RenderedSST2":0.5820977484898407,"STS15VisualSTS":0.7741473848456984,"SciMMIRT2IRetrieval":0.23126,"WebQAT2TRetrieval":0.35733,"VizWizIT2TRetrieval":0.02321,"SketchyI2IRetrieval":0.81426,"Caltech101":0.9640039447731755,"VisualSTS17Multilingual":0.266041,"BLINKIT2TMultiChoice":0.39596,"EuroSAT":0.9336296296296297,"MemotionT2IRetrieval":0.91685,"CIRRIT2IRetrieval":0.27635,"DTDZeroShot":0.6329787234042553,"Food101ZeroShot":0.9464158415841584,"Birdsnap":0.7821717990275526,"Flickr30kT2IRetrieval":0.88276,"Winoground":0.125,"StanfordCars":0.950155453301828,"VisualNewsT2IRetrieval":0.43789,"STS16VisualSTS":0.7362911437298152,"MNISTZeroShot":0.7946,"PatchCamelyonZeroShot":0.492034912109375,"HatefulMemesI2TRetrieval":0.65368,"SciMMIRI2TRetrieval":0.24027,"RParisMediumI2IRetrieval":0.02742,"UCF101ZeroShot":0.7112153661244195,"AROFlickrOrder":0.3905999958515167,"ImageNetDog15Clustering":0.8985105866073517,"Fashion200kT2IRetrieval":0.13864,"FashionIQIT2IRetrieval":0.09921,"TUBerlinT2IRetrieval":0.94003,"SUN397ZeroShot":0.760367816091954,"FGVCAircraft":0.7437143714371437,"CUB200I2IRetrieval":0.84052,"XFlickr30kCoT2IRetrieval":0.287905},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5610196207258025},"MIEB(lite) Task Mean":{"Task Mean":0.49328008374683974}},"trainedOnTasks":[],"experiments":null},{"rank":18,"model":{"name":"google/siglip-base-patch16-256","url":"https://huggingface.co/google/siglip-base-patch16-256","zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.203,"embeddingDim":768,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":775.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.203,"embeddingDim":768,"maxTokens":64,"meanTask":0.5197942325188819,"meanTaskType":0.5197942325188819,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3658831111111111,"Compositionality":0.39524329369204386,"ImageClassification":0.796807300259393,"DocumentUnderstanding":0.31661,"ZeroShotClassification":0.6307484126526209,"ImageClustering":0.7523565136521126,"VisionCentricQA":0.5217553568484533,"Any2AnyMultilingualRetrieval":0.4125868476430976,"VisualSTS(eng)":0.6615899893299871,"VisualSTS(multi)":0.3443615},"scoresByTask":{"GTSRBZeroShot":0.5077593032462391,"CVBenchDistance":0.4066666666666667,"CVBenchRelation":0.5107692307692308,"OxfordPetsZeroShot":0.9424911420005451,"InfoSeekIT2ITRetrieval":0.06326,"VidoreTabfquadRetrieval":0.36998,"CIFAR10Clustering":0.7682276447161108,"VidoreDocVQARetrieval":0.19416,"VisualNewsI2TRetrieval":0.20016,"VidoreArxivQARetrieval":0.35166,"STL10":0.983475,"CLEVRZeroShot":0.23393333333333333,"MemotionI2TRetrieval":0.96248,"PatchCamelyon":0.7309814453125,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.49943,"TinyImageNetClustering":0.5873477874964238,"WebQAT2ITRetrieval":0.31498,"SUN397":0.756551724137931,"METI2IRetrieval":0.41163,"STS13VisualSTS":0.5801659710836782,"CVBenchCount":0.5444162436548223,"ROxfordMediumI2IRetrieval":0.17387,"NIGHTSI2IRetrieval":0.25664,"Flickr30kI2TRetrieval":0.83257,"BLINKIT2TRetrieval":0.12736,"VidoreTatdqaRetrieval":0.08501,"ImageCoDe":0.13336,"ImageCoDeT2IRetrieval":0.11077,"Fashion200kI2TRetrieval":0.19225,"FGVCAircraftZeroShot":0.44764476447644763,"HatefulMemesT2IRetrieval":0.82235,"XM3600T2IRetrieval":0.3471061111111111,"UCF101":0.8524317936037589,"MSCOCOI2TRetrieval":0.57294,"OxfordPets":0.9168165712728265,"SugarCrepe":0.8355745077133179,"FER2013ZeroShot":0.5300919476177208,"OVENIT2TRetrieval":0.00766,"Caltech101ZeroShot":0.8374424720578567,"MSCOCOT2IRetrieval":0.63618,"RESISC45ZeroShot":0.611904761904762,"Food101Classification":0.9025029702970297,"RParisEasyI2IRetrieval":0.09964,"CVBenchDepth":0.52,"Imagenet1k":0.7473240000000001,"VisualSTS-b-Eng":0.576511,"VOC2007":0.9229300783256813,"EDIST2ITRetrieval":0.12563,"ROxfordEasyI2IRetrieval":0.31659,"VidoreShiftProjectRetrieval":0.05448,"WITT2IRetrieval":0.3234181818181818,"RP2kI2IRetrieval":0.61436,"ROxfordHardI2IRetrieval":0.09779,"DTD":0.7863829787234041,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.40072,"AROVisualAttribution":0.5409767627716064,"Country211":0.14983886255924173,"AROVisualRelation":0.45841166377067566,"BLINKIT2IRetrieval":0.34577,"CIFAR10":0.93338,"InfoSeekIT2TRetrieval":0.00479,"VidoreInfoVQARetrieval":0.4873,"BLINKIT2IMultiChoice":0.73632,"VidoreSyntheticDocQAEnergyRetrieval":0.41278,"CIFAR100":0.75672,"VisualSTS-b-Multilingual":0.377991,"EuroSATZeroShot":0.44185185185185183,"VisualSTS17Eng":0.787326,"RESISC45":0.8751746031746033,"BirdsnapZeroShot":0.48946515397082657,"CIFAR10ZeroShot":0.936,"STL10ZeroShot":0.981875,"GTSRB":0.7987331749802058,"Country211ZeroShot":0.15080568720379148,"FORBI2IRetrieval":0.58626,"CIFAR100ZeroShot":0.7191,"OVENIT2ITRetrieval":0.04431,"MNIST":0.9571400000000001,"GLDv2I2TRetrieval":0.56234,"CIFAR100Clustering":0.675783479615035,"RParisHardI2IRetrieval":0.04424,"SOPI2IRetrieval":0.59732,"VidoreSyntheticDocQAAIRetrieval":0.31058,"ImageNet10Clustering":0.9257385207309518,"StanfordCarsI2IRetrieval":0.90635,"StanfordCarsZeroShot":0.9100858102226091,"FER2013":0.424575090554472,"OxfordFlowersClassification":0.99,"STS12VisualSTS":0.6501454084033992,"Imagenet1kZeroShot":0.75644,"STS14VisualSTS":0.6035652887008132,"AROCocoOrder":0.29968011379241943,"VQA2IT2TRetrieval":0.0078,"CLEVRCountZeroShot":0.22693333333333332,"GLDv2I2IRetrieval":0.19211,"RenderedSST2":0.5755079626578803,"STS15VisualSTS":0.742518249075877,"SciMMIRT2IRetrieval":0.20241,"WebQAT2TRetrieval":0.23542,"VizWizIT2TRetrieval":0.03723,"SketchyI2IRetrieval":0.70872,"Caltech101":0.9682445759368836,"VisualSTS17Multilingual":0.310732,"BLINKIT2TMultiChoice":0.41236,"EuroSAT":0.8761481481481482,"MemotionT2IRetrieval":0.97485,"CIRRIT2IRetrieval":0.1529,"DTDZeroShot":0.6537234042553192,"Food101ZeroShot":0.8916435643564357,"Birdsnap":0.6018368449486764,"Flickr30kT2IRetrieval":0.85627,"Winoground":0.11249999701976776,"StanfordCars":0.9372466111180202,"VisualNewsT2IRetrieval":0.18811,"STS16VisualSTS":0.6908980080461422,"MNISTZeroShot":0.8457,"PatchCamelyonZeroShot":0.505615234375,"HatefulMemesI2TRetrieval":0.8165,"SciMMIRI2TRetrieval":0.19785,"RParisMediumI2IRetrieval":0.02723,"UCF101ZeroShot":0.6110148561003526,"AROFlickrOrder":0.3862000107765198,"ImageNetDog15Clustering":0.8046851357020418,"Fashion200kT2IRetrieval":0.15704,"FashionIQIT2IRetrieval":0.05515,"TUBerlinT2IRetrieval":0.95241,"SUN397ZeroShot":0.700183908045977,"FGVCAircraft":0.6613261326132613,"CUB200I2IRetrieval":0.67225,"XFlickr30kCoT2IRetrieval":0.56723625},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5276636942977795},"MIEB(lite) Task Mean":{"Task Mean":0.46955991273912606}},"trainedOnTasks":[],"experiments":null},{"rank":19,"model":{"name":"google/siglip-base-patch16-224","url":"https://huggingface.co/google/siglip-base-patch16-224","zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.203,"embeddingDim":768,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":775.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.179,"totalParamsB":0.203,"embeddingDim":768,"maxTokens":64,"meanTask":0.5088044238255216,"meanTaskType":0.5088044238255216,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3626726666666667,"Compositionality":0.3983665610589299,"ImageClassification":0.7933389630503475,"DocumentUnderstanding":0.26161199999999996,"ZeroShotClassification":0.6263937439227444,"ImageClustering":0.7449784694177052,"VisionCentricQA":0.5105812627445876,"Any2AnyMultilingualRetrieval":0.41227104797979797,"VisualSTS(eng)":0.6424710234144365,"VisualSTS(multi)":0.3353585},"scoresByTask":{"GTSRBZeroShot":0.5197941409342834,"CVBenchDistance":0.4266666666666667,"CVBenchRelation":0.5153846153846153,"OxfordPetsZeroShot":0.9416734805124012,"InfoSeekIT2ITRetrieval":0.05961,"VidoreTabfquadRetrieval":0.29038,"CIFAR10Clustering":0.7610997056538504,"VidoreDocVQARetrieval":0.16035,"VisualNewsI2TRetrieval":0.19539,"VidoreArxivQARetrieval":0.31488,"STL10":0.9832750000000001,"CLEVRZeroShot":0.24,"MemotionI2TRetrieval":0.95961,"PatchCamelyon":0.730731201171875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.37006,"TinyImageNetClustering":0.5858279368142725,"WebQAT2ITRetrieval":0.30843,"SUN397":0.7527080459770115,"METI2IRetrieval":0.40781,"STS13VisualSTS":0.5539716429084766,"CVBenchCount":0.43908629441624364,"ROxfordMediumI2IRetrieval":0.17088,"NIGHTSI2IRetrieval":0.25749,"Flickr30kI2TRetrieval":0.82775,"BLINKIT2TRetrieval":0.13871,"VidoreTatdqaRetrieval":0.05077,"ImageCoDe":0.14509,"ImageCoDeT2IRetrieval":0.11468,"Fashion200kI2TRetrieval":0.19036,"FGVCAircraftZeroShot":0.43864386438643865,"HatefulMemesT2IRetrieval":0.81976,"XM3600T2IRetrieval":0.3484666666666667,"UCF101":0.8480415133200042,"MSCOCOI2TRetrieval":0.57185,"OxfordPets":0.9144180975742711,"SugarCrepe":0.8375715613365173,"FER2013ZeroShot":0.5242407355809418,"OVENIT2TRetrieval":0.00592,"Caltech101ZeroShot":0.836620644312952,"MSCOCOT2IRetrieval":0.6381,"RESISC45ZeroShot":0.6226984126984128,"Food101Classification":0.8988118811881188,"RParisEasyI2IRetrieval":0.10002,"CVBenchDepth":0.515,"Imagenet1k":0.741896,"VisualSTS-b-Eng":0.545036,"VOC2007":0.9202156409462979,"EDIST2ITRetrieval":0.1201,"ROxfordEasyI2IRetrieval":0.31534,"VidoreShiftProjectRetrieval":0.0371,"WITT2IRetrieval":0.32298272727272725,"RP2kI2IRetrieval":0.60459,"ROxfordHardI2IRetrieval":0.0853,"DTD":0.7793617021276595,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.32345,"AROVisualAttribution":0.540350615978241,"Country211":0.14580094786729858,"AROVisualRelation":0.46238040924072266,"BLINKIT2IRetrieval":0.3408,"CIFAR10":0.9259600000000001,"InfoSeekIT2TRetrieval":0.00366,"VidoreInfoVQARetrieval":0.46113,"BLINKIT2IMultiChoice":0.75373,"VidoreSyntheticDocQAEnergyRetrieval":0.3553,"CIFAR100":0.7409600000000001,"VisualSTS-b-Multilingual":0.363009,"EuroSATZeroShot":0.4109259259259259,"VisualSTS17Eng":0.777829,"RESISC45":0.8765396825396825,"BirdsnapZeroShot":0.4829821717990275,"CIFAR10ZeroShot":0.9257,"STL10ZeroShot":0.98225,"GTSRB":0.8006809184481394,"Country211ZeroShot":0.1433175355450237,"FORBI2IRetrieval":0.55751,"CIFAR100ZeroShot":0.7015,"OVENIT2ITRetrieval":0.04045,"MNIST":0.9542400000000001,"GLDv2I2TRetrieval":0.56019,"CIFAR100Clustering":0.6701267819838629,"RParisHardI2IRetrieval":0.04336,"SOPI2IRetrieval":0.5919,"VidoreSyntheticDocQAAIRetrieval":0.2527,"ImageNet10Clustering":0.9260668398384923,"StanfordCarsI2IRetrieval":0.90673,"StanfordCarsZeroShot":0.908344733242134,"FER2013":0.42192811368069105,"OxfordFlowersClassification":0.985686274509804,"STS12VisualSTS":0.6319235606138802,"Imagenet1kZeroShot":0.75116,"STS14VisualSTS":0.5798954279027361,"AROCocoOrder":0.3166733384132385,"VQA2IT2TRetrieval":0.00779,"CLEVRCountZeroShot":0.2366,"GLDv2I2IRetrieval":0.18884,"RenderedSST2":0.5233388248215266,"STS15VisualSTS":0.7307197053881567,"SciMMIRT2IRetrieval":0.18358,"WebQAT2TRetrieval":0.22186,"VizWizIT2TRetrieval":0.03471,"SketchyI2IRetrieval":0.71709,"Caltech101":0.9678829717291256,"VisualSTS17Multilingual":0.307708,"BLINKIT2TMultiChoice":0.41362,"EuroSAT":0.8775185185185187,"MemotionT2IRetrieval":0.97318,"CIRRIT2IRetrieval":0.15399,"DTDZeroShot":0.6351063829787233,"Food101ZeroShot":0.8918811881188119,"Birdsnap":0.589086980010805,"Flickr30kT2IRetrieval":0.85083,"Winoground":0.10249999910593033,"StanfordCars":0.9370476308916802,"VisualNewsT2IRetrieval":0.1841,"STS16VisualSTS":0.6779218270878057,"MNISTZeroShot":0.8358,"PatchCamelyonZeroShot":0.5369873046875,"HatefulMemesI2TRetrieval":0.80723,"SciMMIRI2TRetrieval":0.17916,"RParisMediumI2IRetrieval":0.02706,"UCF101ZeroShot":0.6190080060583286,"AROFlickrOrder":0.3840000033378601,"ImageNetDog15Clustering":0.7817710827980477,"Fashion200kT2IRetrieval":0.15237,"FashionIQIT2IRetrieval":0.06122,"TUBerlinT2IRetrieval":0.95491,"SUN397ZeroShot":0.6984827586206896,"FGVCAircraft":0.6606660666066607,"CUB200I2IRetrieval":0.68605,"XFlickr30kCoT2IRetrieval":0.5653637499999999},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.518969170497603},"MIEB(lite) Task Mean":{"Task Mean":0.46070606580396223}},"trainedOnTasks":[],"experiments":null},{"rank":20,"model":{"name":"laion/CLIP-ViT-L-14-laion2B-s32B-b82K","url":"https://huggingface.co/laion/CLIP-ViT-L-14-laion2B-s32B-b82K","zeroShotPct":100,"activeParamsB":0.39,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77.0,"releaseDate":"2022-09-15","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{cherti2023reproducible,\n    title={Reproducible scaling laws for contrastive language-image learning},\n    author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},\n    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},\n    pages={2818--2829},\n    year={2023}\n}","memoryUsageMb":1631.0,"license":"mit","publicTrainingCode":"https://github.com/mlfoundations/open_clip","publicTrainingData":"https://laion.ai/blog/laion-5b/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["open_clip_torch"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.39,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77,"meanTask":0.5063331300042984,"meanTaskType":0.5063331300042984,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.37963333333333327,"Compositionality":0.40842940741641176,"ImageClassification":0.8123530599378465,"DocumentUnderstanding":0.362624,"ZeroShotClassification":0.6581874898418284,"ImageClustering":0.8350170081304048,"VisionCentricQA":0.45850220118009455,"Any2AnyMultilingualRetrieval":0.23021784090909092,"VisualSTS(eng)":0.6581784592939742,"VisualSTS(multi)":0.2601885},"scoresByTask":{"GTSRBZeroShot":0.5826603325415677,"CVBenchDistance":0.475,"CVBenchRelation":0.5507692307692308,"OxfordPetsZeroShot":0.9321340964840557,"InfoSeekIT2ITRetrieval":0.18207,"VidoreTabfquadRetrieval":0.29131,"CIFAR10Clustering":0.9361831213282183,"VidoreDocVQARetrieval":0.17787,"VisualNewsI2TRetrieval":0.35343,"VidoreArxivQARetrieval":0.30964,"STL10":0.9918250000000001,"CLEVRZeroShot":0.16086666666666666,"MemotionI2TRetrieval":0.87353,"PatchCamelyon":0.723175048828125,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.53149,"TinyImageNetClustering":0.7162475508623897,"WebQAT2ITRetrieval":0.39678,"SUN397":0.7706298850574714,"METI2IRetrieval":0.41571,"STS13VisualSTS":0.6275292122649523,"CVBenchCount":0.08248730964467005,"ROxfordMediumI2IRetrieval":0.15923,"NIGHTSI2IRetrieval":0.25686,"Flickr30kI2TRetrieval":0.84111,"BLINKIT2TRetrieval":0.24086,"VidoreTatdqaRetrieval":0.14681,"ImageCoDe":0.13249,"ImageCoDeT2IRetrieval":0.10599,"Fashion200kI2TRetrieval":0.12879,"FGVCAircraftZeroShot":0.35253525352535253,"HatefulMemesT2IRetrieval":0.64784,"XM3600T2IRetrieval":0.1280175,"UCF101":0.8749115776610606,"MSCOCOI2TRetrieval":0.56111,"OxfordPets":0.9169801035704552,"SugarCrepe":0.8422313928604126,"FER2013ZeroShot":0.5626915575369184,"OVENIT2TRetrieval":0.0555,"Caltech101ZeroShot":0.8376068376068376,"MSCOCOT2IRetrieval":0.61719,"RESISC45ZeroShot":0.71,"Food101Classification":0.8961029702970297,"RParisEasyI2IRetrieval":0.10771,"CVBenchDepth":0.49166666666666664,"Imagenet1k":0.746668,"VisualSTS-b-Eng":0.594284,"VOC2007":0.9193960709838868,"EDIST2ITRetrieval":0.26411,"ROxfordEasyI2IRetrieval":0.30787,"VidoreShiftProjectRetrieval":0.13103,"WITT2IRetrieval":0.2998572727272727,"RP2kI2IRetrieval":0.65267,"ROxfordHardI2IRetrieval":0.10136,"DTD":0.7860638297872341,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.49292,"AROVisualAttribution":0.5893975496292114,"Country211":0.2201800947867299,"AROVisualRelation":0.5096712112426758,"BLINKIT2IRetrieval":0.38557,"CIFAR10":0.9714600000000001,"InfoSeekIT2TRetrieval":0.02737,"VidoreInfoVQARetrieval":0.52463,"BLINKIT2IMultiChoice":0.74378,"VidoreSyntheticDocQAEnergyRetrieval":0.57079,"CIFAR100":0.8472799999999999,"VisualSTS-b-Multilingual":0.297376,"EuroSATZeroShot":0.6535185185185185,"VisualSTS17Eng":0.759169,"RESISC45":0.9014285714285715,"BirdsnapZeroShot":0.6434359805510534,"CIFAR10ZeroShot":0.9692,"STL10ZeroShot":0.9875,"GTSRB":0.8418685669041963,"Country211ZeroShot":0.25028436018957345,"FORBI2IRetrieval":0.45857,"CIFAR100ZeroShot":0.8217,"OVENIT2ITRetrieval":0.12125,"MNIST":0.9503400000000001,"GLDv2I2TRetrieval":0.70259,"CIFAR100Clustering":0.7743196814256803,"RParisHardI2IRetrieval":0.04834,"SOPI2IRetrieval":0.61422,"VidoreSyntheticDocQAAIRetrieval":0.44975,"ImageNet10Clustering":0.9373921977497691,"StanfordCarsI2IRetrieval":0.90014,"StanfordCarsZeroShot":0.9211540853127721,"FER2013":0.44708832543884086,"OxfordFlowersClassification":0.9882352941176471,"STS12VisualSTS":0.5752461378630502,"Imagenet1kZeroShot":0.73226,"STS14VisualSTS":0.5994108916673714,"AROCocoOrder":0.3107157051563263,"VQA2IT2TRetrieval":0.01667,"CLEVRCountZeroShot":0.31306666666666666,"GLDv2I2IRetrieval":0.25282,"RenderedSST2":0.6057111477210324,"STS15VisualSTS":0.7455341348878264,"SciMMIRT2IRetrieval":0.20998,"WebQAT2TRetrieval":0.3188,"VizWizIT2TRetrieval":0.08202,"SketchyI2IRetrieval":0.70298,"Caltech101":0.9378698224852071,"VisualSTS17Multilingual":0.223001,"BLINKIT2TMultiChoice":0.40731,"EuroSAT":0.9125555555555553,"MemotionT2IRetrieval":0.90415,"CIRRIT2IRetrieval":0.12162,"DTDZeroShot":0.5888297872340426,"Food101ZeroShot":0.9041188118811881,"Birdsnap":0.6841707185305241,"Flickr30kT2IRetrieval":0.86291,"Winoground":0.08749999850988388,"StanfordCars":0.9390374331550803,"VisualNewsT2IRetrieval":0.35881,"STS16VisualSTS":0.7060758383746201,"MNISTZeroShot":0.6412,"PatchCamelyonZeroShot":0.560150146484375,"HatefulMemesI2TRetrieval":0.63619,"SciMMIRI2TRetrieval":0.22547,"RParisMediumI2IRetrieval":0.0271,"UCF101ZeroShot":0.6719638795103998,"AROFlickrOrder":0.3869999945163727,"ImageNetDog15Clustering":0.8109424892859662,"Fashion200kT2IRetrieval":0.10108,"FashionIQIT2IRetrieval":0.04125,"TUBerlinT2IRetrieval":0.92101,"SUN397ZeroShot":0.7357241379310345,"FGVCAircraft":0.6045004500450045,"CUB200I2IRetrieval":0.77287,"XFlickr30kCoT2IRetrieval":0.26277875},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5448973831975997},"MIEB(lite) Task Mean":{"Task Mean":0.4738555420982675}},"trainedOnTasks":[],"experiments":null},{"rank":21,"model":{"name":"TIGER-Lab/VLM2Vec-LoRA","url":"https://huggingface.co/TIGER-Lab/VLM2Vec-LoRA","zeroShotPct":100,"activeParamsB":4.063,"totalParamsB":4.162,"embeddingDim":3072,"maxTokens":131072.0,"releaseDate":"2024-10-08","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{jiang2024vlm2vec,\n  archiveprefix = {arXiv},\n  author = {Ziyan Jiang and Rui Meng and Xinyi Yang and Semih Yavuz and Yingbo Zhou and Wenhu Chen},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2410.05160},\n  title = {{VLM}2Vec: Training Vision-Language Models for Massive Multimodal Embedding Tasks},\n  url = {https://openreview.net/forum?id=TE0KOzWYAF},\n  year = {2025},\n}","memoryUsageMb":null,"license":"apache-2.0","publicTrainingCode":"https://github.com/TIGER-AI-Lab/VLM2Vec","publicTrainingData":"https://huggingface.co/datasets/TIGER-Lab/MMEB-train","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["peft"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":4.063,"totalParamsB":4.162,"embeddingDim":3072,"maxTokens":131072,"meanTask":0.5046193645715721,"meanTaskType":0.5046193645715721,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.2770042222222222,"Compositionality":0.3461159820316519,"ImageClassification":0.6196764722972629,"DocumentUnderstanding":0.49705999999999995,"ZeroShotClassification":0.4626956807712557,"ImageClustering":0.7263510450877734,"VisionCentricQA":0.6201534025337324,"Any2AnyMultilingualRetrieval":0.34915201599326595,"VisualSTS(eng)":0.7258868247785563,"VisualSTS(multi)":0.422098},"scoresByTask":{"GTSRBZeroShot":0.41931908155186065,"CVBenchDistance":0.5816666666666667,"CVBenchRelation":0.7107692307692308,"OxfordPetsZeroShot":0.48023984736985553,"InfoSeekIT2ITRetrieval":0.05298,"VidoreTabfquadRetrieval":0.61934,"CIFAR10Clustering":0.7288949633914648,"VidoreDocVQARetrieval":0.26915,"VisualNewsI2TRetrieval":0.11621,"VidoreArxivQARetrieval":0.42586,"STL10":0.976,"CLEVRZeroShot":0.19246666666666667,"MemotionI2TRetrieval":0.20025,"PatchCamelyon":0.68157958984375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.70621,"TinyImageNetClustering":0.6121514992868082,"WebQAT2ITRetrieval":0.20906,"SUN397":0.7106482758620689,"METI2IRetrieval":0.35589,"STS13VisualSTS":0.6587014906484038,"CVBenchCount":0.6256345177664975,"ROxfordMediumI2IRetrieval":0.09607,"NIGHTSI2IRetrieval":0.2567,"Flickr30kI2TRetrieval":0.74482,"BLINKIT2TRetrieval":0.39849,"VidoreTatdqaRetrieval":0.21611,"ImageCoDe":0.13423,"ImageCoDeT2IRetrieval":0.10209,"Fashion200kI2TRetrieval":0.01338,"FGVCAircraftZeroShot":0.0708070807080708,"HatefulMemesT2IRetrieval":0.4359,"XM3600T2IRetrieval":0.2263313888888889,"UCF101":0.7651666183797987,"MSCOCOI2TRetrieval":0.50614,"OxfordPets":0.8179885527391659,"SugarCrepe":0.659832239151001,"FER2013ZeroShot":0.3172192811368069,"OVENIT2TRetrieval":0.0938,"Caltech101ZeroShot":0.7907626561472715,"MSCOCOT2IRetrieval":0.59504,"RESISC45ZeroShot":0.5271428571428571,"Food101Classification":0.7468435643564357,"RParisEasyI2IRetrieval":0.09583,"CVBenchDepth":0.625,"Imagenet1k":0.001,"VisualSTS-b-Eng":0.736437,"VOC2007":0.7096541090271176,"EDIST2ITRetrieval":0.16473,"ROxfordEasyI2IRetrieval":0.14997,"VidoreShiftProjectRetrieval":0.24336,"WITT2IRetrieval":0.26989090909090907,"RP2kI2IRetrieval":0.61714,"ROxfordHardI2IRetrieval":0.05902,"DTD":0.6847872340425532,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.6406,"AROVisualAttribution":0.6140949130058289,"Country211":0.08350710900473934,"AROVisualRelation":0.5502360463142395,"BLINKIT2IRetrieval":0.32338,"CIFAR10":0.8793799999999999,"InfoSeekIT2TRetrieval":0.03383,"VidoreInfoVQARetrieval":0.67636,"BLINKIT2IMultiChoice":0.72388,"VidoreSyntheticDocQAEnergyRetrieval":0.63345,"CIFAR100":0.60202,"VisualSTS-b-Multilingual":0.486789,"EuroSATZeroShot":0.24925925925925926,"VisualSTS17Eng":0.835516,"RESISC45":0.794126984126984,"BirdsnapZeroShot":0.09562398703403566,"CIFAR10ZeroShot":0.8526,"STL10ZeroShot":0.954625,"GTSRB":0.6509263657957244,"Country211ZeroShot":0.07990521327014218,"FORBI2IRetrieval":0.35894,"CIFAR100ZeroShot":0.5029,"OVENIT2ITRetrieval":0.06754,"MNIST":0.86714,"GLDv2I2TRetrieval":0.18498,"CIFAR100Clustering":0.6056017432165507,"RParisHardI2IRetrieval":0.02934,"SOPI2IRetrieval":0.49187,"VidoreSyntheticDocQAAIRetrieval":0.54016,"ImageNet10Clustering":0.9703093853102287,"StanfordCarsI2IRetrieval":0.43291,"StanfordCarsZeroShot":0.22372839199104588,"FER2013":0.5136528280858178,"OxfordFlowersClassification":0.8429411764705883,"STS12VisualSTS":0.7118192489544419,"Imagenet1kZeroShot":0.50382,"STS14VisualSTS":0.6261172281010946,"AROCocoOrder":0.20331867039203644,"VQA2IT2TRetrieval":0.17011,"CLEVRCountZeroShot":0.31693333333333334,"GLDv2I2IRetrieval":0.09301,"RenderedSST2":0.7874794069192751,"STS15VisualSTS":0.7591721936281433,"SciMMIRT2IRetrieval":0.2187,"WebQAT2TRetrieval":0.35499,"VizWizIT2TRetrieval":0.24956,"SketchyI2IRetrieval":0.67172,"Caltech101":0.92396449704142,"VisualSTS17Multilingual":0.357407,"BLINKIT2TMultiChoice":0.45397,"EuroSAT":0.7574444444444446,"MemotionT2IRetrieval":0.59269,"CIRRIT2IRetrieval":0.15691,"DTDZeroShot":0.46914893617021275,"Food101ZeroShot":0.571009900990099,"Birdsnap":0.0021609940572663426,"Flickr30kT2IRetrieval":0.81924,"Winoground":0.057500001043081284,"StanfordCars":0.40136798905608756,"VisualNewsT2IRetrieval":0.12179,"STS16VisualSTS":0.7534446121178103,"MNISTZeroShot":0.5929,"PatchCamelyonZeroShot":0.499786376953125,"HatefulMemesI2TRetrieval":0.20598,"SciMMIRI2TRetrieval":0.24407,"RParisMediumI2IRetrieval":0.02647,"UCF101ZeroShot":0.5392199328191021,"AROFlickrOrder":0.20360000431537628,"ImageNetDog15Clustering":0.7147976342338145,"Fashion200kT2IRetrieval":0.00815,"FashionIQIT2IRetrieval":0.02118,"TUBerlinT2IRetrieval":0.88766,"SUN397ZeroShot":0.605103448275862,"FGVCAircraft":0.22058205820582052,"CUB200I2IRetrieval":0.43666,"XFlickr30kCoT2IRetrieval":0.55123375},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4525389466927311},"MIEB(lite) Task Mean":{"Task Mean":0.41992645955637997}},"trainedOnTasks":[],"experiments":null},{"rank":22,"model":{"name":"TIGER-Lab/VLM2Vec-Full","url":"https://huggingface.co/TIGER-Lab/VLM2Vec-Full","zeroShotPct":100,"activeParamsB":4.048,"totalParamsB":4.147,"embeddingDim":3072,"maxTokens":131072.0,"releaseDate":"2024-10-08","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{jiang2024vlm2vec,\n  archiveprefix = {arXiv},\n  author = {Ziyan Jiang and Rui Meng and Xinyi Yang and Semih Yavuz and Yingbo Zhou and Wenhu Chen},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2410.05160},\n  title = {{VLM}2Vec: Training Vision-Language Models for Massive Multimodal Embedding Tasks},\n  url = {https://openreview.net/forum?id=TE0KOzWYAF},\n  year = {2025},\n}","memoryUsageMb":7909.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/TIGER-AI-Lab/VLM2Vec","publicTrainingData":"https://huggingface.co/TIGER-Lab/VLM2Vec-Full","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["peft"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":4.048,"totalParamsB":4.147,"embeddingDim":3072,"maxTokens":131072,"meanTask":0.50379375329283,"meanTaskType":0.50379375329283,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.27636222222222223,"Compositionality":0.3543438378506047,"ImageClassification":0.6196632049266184,"DocumentUnderstanding":0.49801,"ZeroShotClassification":0.4632749822722029,"ImageClustering":0.7071549174445685,"VisionCentricQA":0.6211828591262093,"Any2AnyMultilingualRetrieval":0.34958877525252524,"VisualSTS(eng)":0.7264162338333483,"VisualSTS(multi)":0.4219405},"scoresByTask":{"GTSRBZeroShot":0.41512272367379255,"CVBenchDistance":0.58,"CVBenchRelation":0.7169230769230769,"OxfordPetsZeroShot":0.48242027800490594,"InfoSeekIT2ITRetrieval":0.05391,"VidoreTabfquadRetrieval":0.63536,"CIFAR10Clustering":0.6942614659820454,"VidoreDocVQARetrieval":0.26743,"VisualNewsI2TRetrieval":0.11428,"VidoreArxivQARetrieval":0.42841,"STL10":0.976,"CLEVRZeroShot":0.1926,"MemotionI2TRetrieval":0.19816,"PatchCamelyon":0.682147216796875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.70728,"TinyImageNetClustering":0.6150695767032207,"WebQAT2ITRetrieval":0.20802,"SUN397":0.7104000000000001,"METI2IRetrieval":0.35397,"STS13VisualSTS":0.658793079313577,"CVBenchCount":0.6218274111675127,"ROxfordMediumI2IRetrieval":0.0953,"NIGHTSI2IRetrieval":0.25222,"Flickr30kI2TRetrieval":0.74474,"BLINKIT2TRetrieval":0.39975,"VidoreTatdqaRetrieval":0.2145,"ImageCoDe":0.13467,"ImageCoDeT2IRetrieval":0.09991,"Fashion200kI2TRetrieval":0.01326,"FGVCAircraftZeroShot":0.0699069906990699,"HatefulMemesT2IRetrieval":0.43833,"XM3600T2IRetrieval":0.22638916666666667,"UCF101":0.7649658215030508,"MSCOCOI2TRetrieval":0.50566,"OxfordPets":0.8179340419732897,"SugarCrepe":0.6699507236480713,"FER2013ZeroShot":0.32223460573976037,"OVENIT2TRetrieval":0.09349,"Caltech101ZeroShot":0.7924063116370809,"MSCOCOT2IRetrieval":0.59477,"RESISC45ZeroShot":0.5258730158730158,"Food101Classification":0.747120792079208,"RParisEasyI2IRetrieval":0.09675,"CVBenchDepth":0.6216666666666667,"Imagenet1k":0.001,"VisualSTS-b-Eng":0.737529,"VOC2007":0.7100471000924815,"EDIST2ITRetrieval":0.1636,"ROxfordEasyI2IRetrieval":0.15002,"VidoreShiftProjectRetrieval":0.25007,"WITT2IRetrieval":0.27066090909090906,"RP2kI2IRetrieval":0.61682,"ROxfordHardI2IRetrieval":0.0581,"DTD":0.6851063829787234,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.6403,"AROVisualAttribution":0.623904287815094,"Country211":0.08351658767772513,"AROVisualRelation":0.5571709275245667,"BLINKIT2IRetrieval":0.3209,"CIFAR10":0.87934,"InfoSeekIT2TRetrieval":0.03431,"VidoreInfoVQARetrieval":0.66675,"BLINKIT2IMultiChoice":0.72388,"VidoreSyntheticDocQAEnergyRetrieval":0.63488,"CIFAR100":0.6019399999999999,"VisualSTS-b-Multilingual":0.486468,"EuroSATZeroShot":0.24796296296296297,"VisualSTS17Eng":0.837176,"RESISC45":0.7942857142857143,"BirdsnapZeroShot":0.09616423554835224,"CIFAR10ZeroShot":0.8521,"STL10ZeroShot":0.95425,"GTSRB":0.6505621536025337,"Country211ZeroShot":0.07914691943127962,"FORBI2IRetrieval":0.35857,"CIFAR100ZeroShot":0.5026,"OVENIT2ITRetrieval":0.06739,"MNIST":0.8675,"GLDv2I2TRetrieval":0.18402,"CIFAR100Clustering":0.6071942332538618,"RParisHardI2IRetrieval":0.02877,"SOPI2IRetrieval":0.4912,"VidoreSyntheticDocQAAIRetrieval":0.53512,"ImageNet10Clustering":0.9263874211314128,"StanfordCarsI2IRetrieval":0.43104,"StanfordCarsZeroShot":0.22260912821788334,"FER2013":0.5140707718027306,"OxfordFlowersClassification":0.8419607843137256,"STS12VisualSTS":0.7114883831652911,"Imagenet1kZeroShot":0.50414,"STS14VisualSTS":0.6263333910655561,"AROCocoOrder":0.22271092236042023,"VQA2IT2TRetrieval":0.16965,"CLEVRCountZeroShot":0.31593333333333334,"GLDv2I2IRetrieval":0.09352,"RenderedSST2":0.7841845140032949,"STS15VisualSTS":0.7599869484936869,"SciMMIRT2IRetrieval":0.21804,"WebQAT2TRetrieval":0.3556,"VizWizIT2TRetrieval":0.25003,"SketchyI2IRetrieval":0.67045,"Caltech101":0.9221564760026298,"VisualSTS17Multilingual":0.357413,"BLINKIT2TMultiChoice":0.4628,"EuroSAT":0.758,"MemotionT2IRetrieval":0.58548,"CIRRIT2IRetrieval":0.15739,"DTDZeroShot":0.4723404255319149,"Food101ZeroShot":0.5715643564356435,"Birdsnap":0.0021609940572663426,"Flickr30kT2IRetrieval":0.8184,"Winoground":0.05000000074505806,"StanfordCars":0.4022136550180326,"VisualNewsT2IRetrieval":0.12003,"STS16VisualSTS":0.7536068347953265,"MNISTZeroShot":0.6098,"PatchCamelyonZeroShot":0.49969482421875,"HatefulMemesI2TRetrieval":0.20866,"SciMMIRI2TRetrieval":0.24363,"RParisMediumI2IRetrieval":0.02647,"UCF101ZeroShot":0.5385458290185909,"AROFlickrOrder":0.22200000286102295,"ImageNetDog15Clustering":0.6928618901523016,"Fashion200kT2IRetrieval":0.00747,"FashionIQIT2IRetrieval":0.0221,"TUBerlinT2IRetrieval":0.89133,"SUN397ZeroShot":0.6037241379310345,"FGVCAircraft":0.22016201620162015,"CUB200I2IRetrieval":0.43079,"XFlickr30kCoT2IRetrieval":0.55171625},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.45226005875531233},"MIEB(lite) Task Mean":{"Task Mean":0.42045000475151345}},"trainedOnTasks":[],"experiments":null},{"rank":23,"model":{"name":"openai/clip-vit-large-patch14","url":"https://huggingface.co/openai/clip-vit-large-patch14","zeroShotPct":-1,"activeParamsB":0.39,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77.0,"releaseDate":"2021-02-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{radford2021learning,\n  archiveprefix = {arXiv},\n  author = {Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},\n  booktitle = {Proceedings of the 38th International Conference on Machine Learning},\n  eprint = {2103.00020},\n  pages = {8748--8763},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Learning Transferable Visual Models From Natural Language Supervision},\n  url = {https://proceedings.mlr.press/v139/radford21a.html},\n  volume = {139},\n  year = {2021},\n}","memoryUsageMb":1631.0,"license":null,"publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":0.39,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77,"meanTask":0.4989098465752064,"meanTaskType":0.4989098465752064,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3367068888888889,"Compositionality":0.4475403241453852,"ImageClassification":0.8013064873231325,"DocumentUnderstanding":0.379704,"ZeroShotClassification":0.6205316579303077,"ImageClustering":0.764140473140213,"VisionCentricQA":0.4409750859039438,"Any2AnyMultilingualRetrieval":0.20242925505050505,"VisualSTS(eng)":0.6445232933696877,"VisualSTS(multi)":0.351241},"scoresByTask":{"GTSRBZeroShot":0.4870942201108472,"CVBenchDistance":0.4683333333333333,"CVBenchRelation":0.5092307692307693,"OxfordPetsZeroShot":0.9340419732897247,"InfoSeekIT2ITRetrieval":0.10287,"VidoreTabfquadRetrieval":0.30945,"CIFAR10Clustering":0.8087299156065013,"VidoreDocVQARetrieval":0.16692,"VisualNewsI2TRetrieval":0.35239,"VidoreArxivQARetrieval":0.28637,"STL10":0.9938750000000001,"CLEVRZeroShot":0.1608,"MemotionI2TRetrieval":0.76087,"PatchCamelyon":0.74744873046875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.60229,"TinyImageNetClustering":0.6982397990205423,"WebQAT2ITRetrieval":0.34303,"SUN397":0.7607724137931033,"METI2IRetrieval":0.37341,"STS13VisualSTS":0.6677601613518561,"CVBenchCount":0.0266497461928934,"ROxfordMediumI2IRetrieval":0.14045,"NIGHTSI2IRetrieval":0.21887,"Flickr30kI2TRetrieval":0.77861,"BLINKIT2TRetrieval":0.09332,"VidoreTatdqaRetrieval":0.11003,"ImageCoDe":0.12858,"ImageCoDeT2IRetrieval":0.10035,"Fashion200kI2TRetrieval":0.04138,"FGVCAircraftZeroShot":0.32553255325532554,"HatefulMemesT2IRetrieval":0.61855,"XM3600T2IRetrieval":0.09329333333333333,"UCF101":0.8777634670162445,"MSCOCOI2TRetrieval":0.48837,"OxfordPets":0.9193785772690106,"SugarCrepe":0.7671415209770203,"FER2013ZeroShot":0.4752020061298412,"OVENIT2TRetrieval":0.00203,"Caltech101ZeroShot":0.8198553583168968,"MSCOCOT2IRetrieval":0.52655,"RESISC45ZeroShot":0.6766666666666666,"Food101Classification":0.9198574257425743,"RParisEasyI2IRetrieval":0.10174,"CVBenchDepth":0.5266666666666666,"Imagenet1k":0.751044,"VisualSTS-b-Eng":0.567353,"VOC2007":0.9093420782442442,"EDIST2ITRetrieval":0.2605,"ROxfordEasyI2IRetrieval":0.21291,"VidoreShiftProjectRetrieval":0.17046,"WITT2IRetrieval":0.30217818181818185,"RP2kI2IRetrieval":0.56534,"ROxfordHardI2IRetrieval":0.10937,"DTD":0.7277659574468085,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.52843,"AROVisualAttribution":0.6162515878677368,"Country211":0.2697061611374407,"AROVisualRelation":0.5327317714691162,"BLINKIT2IRetrieval":0.35821,"CIFAR10":0.9615200000000002,"InfoSeekIT2TRetrieval":0.00136,"VidoreInfoVQARetrieval":0.62444,"BLINKIT2IMultiChoice":0.71144,"VidoreSyntheticDocQAEnergyRetrieval":0.61619,"CIFAR100":0.79988,"VisualSTS-b-Multilingual":0.315204,"EuroSATZeroShot":0.6085185185185186,"VisualSTS17Eng":0.75261,"RESISC45":0.8963809523809523,"BirdsnapZeroShot":0.5316045380875203,"CIFAR10ZeroShot":0.9517,"STL10ZeroShot":0.994625,"GTSRB":0.8359303246239114,"Country211ZeroShot":0.29080568720379146,"FORBI2IRetrieval":0.47947,"CIFAR100ZeroShot":0.75,"OVENIT2ITRetrieval":0.05705,"MNIST":0.9370199999999999,"GLDv2I2TRetrieval":0.73508,"CIFAR100Clustering":0.6453957531832457,"RParisHardI2IRetrieval":0.04261,"SOPI2IRetrieval":0.53658,"VidoreSyntheticDocQAAIRetrieval":0.38246,"ImageNet10Clustering":0.9400021739560036,"StanfordCarsI2IRetrieval":0.80662,"StanfordCarsZeroShot":0.7657007834846412,"FER2013":0.471078294789635,"OxfordFlowersClassification":0.9892156862745098,"STS12VisualSTS":0.538903861182545,"Imagenet1kZeroShot":0.71206,"STS14VisualSTS":0.5598252262190443,"AROCocoOrder":0.4565773606300354,"VQA2IT2TRetrieval":0.01129,"CLEVRCountZeroShot":0.19433333333333333,"GLDv2I2IRetrieval":0.23973,"RenderedSST2":0.6979681493684788,"STS15VisualSTS":0.7202804160281698,"SciMMIRT2IRetrieval":0.15526,"WebQAT2TRetrieval":0.23977,"VizWizIT2TRetrieval":0.04214,"SketchyI2IRetrieval":0.67556,"Caltech101":0.9432281393819857,"VisualSTS17Multilingual":0.387278,"BLINKIT2TMultiChoice":0.40353,"EuroSAT":0.8054444444444444,"MemotionT2IRetrieval":0.86766,"CIRRIT2IRetrieval":0.094,"DTDZeroShot":0.5281914893617021,"Food101ZeroShot":0.9234059405940594,"Birdsnap":0.6758508914100487,"Flickr30kT2IRetrieval":0.79311,"Winoground":0.08250000327825546,"StanfordCars":0.8743439870662855,"VisualNewsT2IRetrieval":0.37394,"STS16VisualSTS":0.7049303888061984,"MNISTZeroShot":0.6291,"PatchCamelyonZeroShot":0.504425048828125,"HatefulMemesI2TRetrieval":0.56962,"SciMMIRI2TRetrieval":0.17056,"RParisMediumI2IRetrieval":0.02655,"UCF101ZeroShot":0.6665968658476067,"AROFlickrOrder":0.5490000247955322,"ImageNetDog15Clustering":0.7283347239347726,"Fashion200kT2IRetrieval":0.02635,"FashionIQIT2IRetrieval":0.03108,"TUBerlinT2IRetrieval":0.8924,"SUN397ZeroShot":0.644,"FGVCAircraft":0.5618961896189619,"CUB200I2IRetrieval":0.7349,"XFlickr30kCoT2IRetrieval":0.21181624999999998},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5196865524578897},"MIEB(lite) Task Mean":{"Task Mean":0.4636473800877059}},"trainedOnTasks":[],"experiments":null},{"rank":24,"model":{"name":"laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K","url":"https://huggingface.co/laion/CLIP-ViT-B-16-DataComp.XL-s13B-b90K","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.15,"embeddingDim":512,"maxTokens":77.0,"releaseDate":"2023-04-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{cherti2023reproducible,\n    title={Reproducible scaling laws for contrastive language-image learning},\n    author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},\n    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},\n    pages={2818--2829},\n    year={2023}\n}","memoryUsageMb":572.0,"license":"mit","publicTrainingCode":"https://github.com/mlfoundations/open_clip","publicTrainingData":"https://huggingface.co/datasets/mlfoundations/datacomp_1b","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["open_clip_torch"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.15,"embeddingDim":512,"maxTokens":77,"meanTask":0.49636476807741586,"meanTaskType":0.49636476807741586,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.34838444444444444,"Compositionality":0.38648044238839835,"ImageClassification":0.7859599986421144,"DocumentUnderstanding":0.270246,"ZeroShotClassification":0.6266038254030564,"ImageClustering":0.8173425476210271,"VisionCentricQA":0.5433778387348692,"Any2AnyMultilingualRetrieval":0.2156637037037037,"VisualSTS(eng)":0.684694379836545,"VisualSTS(multi)":0.2848945},"scoresByTask":{"GTSRBZeroShot":0.5334125098970705,"CVBenchDistance":0.46,"CVBenchRelation":0.49230769230769234,"OxfordPetsZeroShot":0.928045789043336,"InfoSeekIT2ITRetrieval":0.09992,"VidoreTabfquadRetrieval":0.26602,"CIFAR10Clustering":0.8992987776058757,"VidoreDocVQARetrieval":0.13973,"VisualNewsI2TRetrieval":0.26018,"VidoreArxivQARetrieval":0.28876,"STL10":0.985675,"CLEVRZeroShot":0.23566666666666666,"MemotionI2TRetrieval":0.85032,"PatchCamelyon":0.7021240234375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.35825,"TinyImageNetClustering":0.7212934405589102,"WebQAT2ITRetrieval":0.35592,"SUN397":0.7371770114942529,"METI2IRetrieval":0.41219,"STS13VisualSTS":0.6381343221707103,"CVBenchCount":0.6192893401015228,"ROxfordMediumI2IRetrieval":0.17087,"NIGHTSI2IRetrieval":0.25661,"Flickr30kI2TRetrieval":0.78147,"BLINKIT2TRetrieval":0.16898,"VidoreTatdqaRetrieval":0.09072,"ImageCoDe":0.12163,"ImageCoDeT2IRetrieval":0.09904,"Fashion200kI2TRetrieval":0.08634,"FGVCAircraftZeroShot":0.29852985298529855,"HatefulMemesT2IRetrieval":0.61873,"XM3600T2IRetrieval":0.12231861111111113,"UCF101":0.8280518973870589,"MSCOCOI2TRetrieval":0.50965,"OxfordPets":0.8879258653584083,"SugarCrepe":0.8217281103134155,"FER2013ZeroShot":0.3066313736416829,"OVENIT2TRetrieval":0.02033,"Caltech101ZeroShot":0.8479618671926364,"MSCOCOT2IRetrieval":0.56579,"RESISC45ZeroShot":0.6507936507936508,"Food101Classification":0.8807366336633663,"RParisEasyI2IRetrieval":0.10635,"CVBenchDepth":0.525,"Imagenet1k":0.7074520000000001,"VisualSTS-b-Eng":0.635114,"VOC2007":0.9074442958439086,"EDIST2ITRetrieval":0.21408,"ROxfordEasyI2IRetrieval":0.33568,"VidoreShiftProjectRetrieval":0.07254,"WITT2IRetrieval":0.2759,"RP2kI2IRetrieval":0.65419,"ROxfordHardI2IRetrieval":0.10049,"DTD":0.7625531914893617,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.31054,"AROVisualAttribution":0.56177818775177,"Country211":0.16977251184834125,"AROVisualRelation":0.4644274413585663,"BLINKIT2IRetrieval":0.3408,"CIFAR10":0.96838,"InfoSeekIT2TRetrieval":0.0081,"VidoreInfoVQARetrieval":0.46883,"BLINKIT2IMultiChoice":0.74627,"VidoreSyntheticDocQAEnergyRetrieval":0.38534,"CIFAR100":0.8339599999999999,"VisualSTS-b-Multilingual":0.34236,"EuroSATZeroShot":0.5205555555555555,"VisualSTS17Eng":0.800426,"RESISC45":0.8764126984126985,"BirdsnapZeroShot":0.6591031874662344,"CIFAR10ZeroShot":0.9627,"STL10ZeroShot":0.980875,"GTSRB":0.830989707046714,"Country211ZeroShot":0.20516587677725118,"FORBI2IRetrieval":0.4277,"CIFAR100ZeroShot":0.8121,"OVENIT2ITRetrieval":0.08122,"MNIST":0.95142,"GLDv2I2TRetrieval":0.70206,"CIFAR100Clustering":0.782070963088745,"RParisHardI2IRetrieval":0.04915,"SOPI2IRetrieval":0.59456,"VidoreSyntheticDocQAAIRetrieval":0.32173,"ImageNet10Clustering":0.9342007662770039,"StanfordCarsI2IRetrieval":0.86071,"StanfordCarsZeroShot":0.8829747543837831,"FER2013":0.35277235998885487,"OxfordFlowersClassification":0.9892156862745098,"STS12VisualSTS":0.6419251735779055,"Imagenet1kZeroShot":0.72234,"STS14VisualSTS":0.6234433783822209,"AROCocoOrder":0.3015993535518646,"VQA2IT2TRetrieval":0.01936,"CLEVRCountZeroShot":0.3253333333333333,"GLDv2I2IRetrieval":0.2345,"RenderedSST2":0.5244371224601867,"STS15VisualSTS":0.7548307351374316,"SciMMIRT2IRetrieval":0.1288,"WebQAT2TRetrieval":0.29188,"VizWizIT2TRetrieval":0.06955,"SketchyI2IRetrieval":0.64338,"Caltech101":0.9536160420775806,"VisualSTS17Multilingual":0.227429,"BLINKIT2TMultiChoice":0.4174,"EuroSAT":0.8884074074074075,"MemotionT2IRetrieval":0.88163,"CIRRIT2IRetrieval":0.11924,"DTDZeroShot":0.5537234042553192,"Food101ZeroShot":0.9007524752475248,"Birdsnap":0.6254997298757428,"Flickr30kT2IRetrieval":0.79898,"Winoground":0.07500000298023224,"StanfordCars":0.9121999751274716,"VisualNewsT2IRetrieval":0.24823,"STS16VisualSTS":0.6989870495875466,"MNISTZeroShot":0.7534,"PatchCamelyonZeroShot":0.553955078125,"HatefulMemesI2TRetrieval":0.62192,"SciMMIRI2TRetrieval":0.1465,"RParisMediumI2IRetrieval":0.02714,"UCF101ZeroShot":0.5477753140319725,"AROFlickrOrder":0.35920000076293945,"ImageNetDog15Clustering":0.7498487905746006,"Fashion200kT2IRetrieval":0.05545,"FashionIQIT2IRetrieval":0.03436,"TUBerlinT2IRetrieval":0.91715,"SUN397ZeroShot":0.705655172413793,"FGVCAircraft":0.5393339333933393,"CUB200I2IRetrieval":0.7078,"XFlickr30kCoT2IRetrieval":0.2487725},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5194237718438861},"MIEB(lite) Task Mean":{"Task Mean":0.4552497932733512}},"trainedOnTasks":[],"experiments":null},{"rank":25,"model":{"name":"QuanSun/EVA02-CLIP-L-14","url":"https://huggingface.co/QuanSun/EVA-CLIP","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77.0,"releaseDate":"2023-04-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@article{EVA-CLIP,\n      title={EVA-CLIP: Improved Training Techniques for CLIP at Scale},\n      author={Sun, Quan and Fang, Yuxin and Wu, Ledell and Wang, Xinlong and Cao, Yue},\n      journal={arXiv preprint arXiv:2303.15389},\n      year={2023}\n}","memoryUsageMb":1633.0,"license":"mit","publicTrainingCode":"https://github.com/baaivision/EVA/tree/master/EVA-CLIP","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77,"meanTask":0.472858986245393,"meanTaskType":0.472858986245393,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.35592266666666666,"Compositionality":0.44139057397467746,"ImageClassification":0.6815176943104709,"DocumentUnderstanding":0.220778,"ZeroShotClassification":0.6631642982204753,"ImageClustering":0.8826963009260733,"VisionCentricQA":0.39369589353117274,"Any2AnyMultilingualRetrieval":0.2342585900673401,"VisualSTS(eng)":0.6303943447570539,"VisualSTS(multi)":0.2247715},"scoresByTask":{"GTSRBZeroShot":0.5676959619952494,"CVBenchDistance":0.535,"CVBenchRelation":0.45692307692307693,"OxfordPetsZeroShot":0.9394930498773508,"InfoSeekIT2ITRetrieval":0.00048,"VidoreTabfquadRetrieval":0.20638,"CIFAR10Clustering":0.9783318956729448,"VidoreDocVQARetrieval":0.11235,"VisualNewsI2TRetrieval":0.38688,"VidoreArxivQARetrieval":0.30436,"STL10":0.9939250000000002,"CLEVRZeroShot":0.20166666666666666,"MemotionI2TRetrieval":0.82353,"PatchCamelyon":0.690435791015625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.33677,"TinyImageNetClustering":0.7943647953821809,"WebQAT2ITRetrieval":0.38027,"SUN397":0.6846436781609195,"METI2IRetrieval":0.39477,"STS13VisualSTS":0.608206159869511,"CVBenchCount":0.01015228426395939,"ROxfordMediumI2IRetrieval":0.15568,"NIGHTSI2IRetrieval":0.23056,"Flickr30kI2TRetrieval":0.84617,"BLINKIT2TRetrieval":0.27869,"VidoreTatdqaRetrieval":0.0328,"ImageCoDe":0.12641,"ImageCoDeT2IRetrieval":0.12033,"Fashion200kI2TRetrieval":0.11426,"FGVCAircraftZeroShot":0.35673567356735675,"HatefulMemesT2IRetrieval":0.62114,"XM3600T2IRetrieval":0.1262836111111111,"UCF101":0.8192678945873768,"MSCOCOI2TRetrieval":0.56082,"OxfordPets":0.9025347506132462,"SugarCrepe":0.8438290357589722,"FER2013ZeroShot":0.4944274171078295,"OVENIT2TRetrieval":0.00079,"Caltech101ZeroShot":0.8308678500986193,"MSCOCOT2IRetrieval":0.60743,"RESISC45ZeroShot":0.693015873015873,"Food101Classification":0.9021069306930694,"RParisEasyI2IRetrieval":0.11251,"CVBenchDepth":0.495,"Imagenet1k":0.001,"VisualSTS-b-Eng":0.504608,"VOC2007":0.5174186055862888,"EDIST2ITRetrieval":0.16507,"ROxfordEasyI2IRetrieval":0.27442,"VidoreShiftProjectRetrieval":0.04441,"WITT2IRetrieval":0.3088309090909091,"RP2kI2IRetrieval":0.59179,"ROxfordHardI2IRetrieval":0.09288,"DTD":0.6508510638297872,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.18365,"AROVisualAttribution":0.6197648644447327,"Country211":0.18124170616113744,"AROVisualRelation":0.5331077575683594,"BLINKIT2IRetrieval":0.00249,"CIFAR10":0.9898999999999999,"InfoSeekIT2TRetrieval":0.00045,"VidoreInfoVQARetrieval":0.48478,"BLINKIT2IMultiChoice":0.45274,"VidoreSyntheticDocQAEnergyRetrieval":0.29871,"CIFAR100":0.8716800000000001,"VisualSTS-b-Multilingual":0.221235,"EuroSATZeroShot":0.672037037037037,"VisualSTS17Eng":0.801389,"RESISC45":0.8451746031746031,"BirdsnapZeroShot":0.5840086439762291,"CIFAR10ZeroShot":0.9909,"STL10ZeroShot":0.99625,"GTSRB":0.5933808392715756,"Country211ZeroShot":0.2922748815165877,"FORBI2IRetrieval":0.42883,"CIFAR100ZeroShot":0.9007,"OVENIT2ITRetrieval":0.00124,"MNIST":0.748,"GLDv2I2TRetrieval":0.72878,"CIFAR100Clustering":0.8613700720094072,"RParisHardI2IRetrieval":0.04394,"SOPI2IRetrieval":0.5924,"VidoreSyntheticDocQAAIRetrieval":0.20357,"ImageNet10Clustering":0.9436700723411705,"StanfordCarsI2IRetrieval":0.88671,"StanfordCarsZeroShot":0.9002611615470713,"FER2013":0.442351629980496,"OxfordFlowersClassification":0.9801960784313725,"STS12VisualSTS":0.537466109528027,"Imagenet1kZeroShot":0.7806,"STS14VisualSTS":0.5711856961193446,"AROCocoOrder":0.3941223621368408,"VQA2IT2TRetrieval":0.00489,"CLEVRCountZeroShot":0.31466666666666665,"GLDv2I2IRetrieval":0.26311,"RenderedSST2":0.615046677649643,"STS15VisualSTS":0.7152836175619103,"SciMMIRT2IRetrieval":0.15865,"WebQAT2TRetrieval":0.31198,"VizWizIT2TRetrieval":0.01862,"SketchyI2IRetrieval":0.73127,"Caltech101":0.8986850756081525,"VisualSTS17Multilingual":0.228308,"BLINKIT2TMultiChoice":0.41236,"EuroSAT":0.838851851851852,"MemotionT2IRetrieval":0.89105,"CIRRIT2IRetrieval":0.23579,"DTDZeroShot":0.6276595744680851,"Food101ZeroShot":0.9290693069306931,"Birdsnap":0.0021609940572663426,"Flickr30kT2IRetrieval":0.8678,"Winoground":0.10249999910593033,"StanfordCars":0.8930481283422459,"VisualNewsT2IRetrieval":0.37375,"STS16VisualSTS":0.6746218302205851,"MNISTZeroShot":0.6241,"PatchCamelyonZeroShot":0.5096435546875,"HatefulMemesI2TRetrieval":0.59683,"SciMMIRI2TRetrieval":0.16542,"RParisMediumI2IRetrieval":0.02718,"UCF101ZeroShot":0.6918427703084526,"AROFlickrOrder":0.4699999988079071,"ImageNetDog15Clustering":0.8357446692246638,"Fashion200kT2IRetrieval":0.09301,"FashionIQIT2IRetrieval":0.07561,"TUBerlinT2IRetrieval":0.95207,"SUN397ZeroShot":0.739816091954023,"FGVCAircraft":0.5465346534653466,"CUB200I2IRetrieval":0.80618,"XFlickr30kCoT2IRetrieval":0.26766125},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.5019889554467265},"MIEB(lite) Task Mean":{"Task Mean":0.44445162510032804}},"trainedOnTasks":[],"experiments":null},{"rank":26,"model":{"name":"openai/clip-vit-base-patch16","url":"https://huggingface.co/openai/clip-vit-base-patch16","zeroShotPct":-1,"activeParamsB":0.124,"totalParamsB":0.15,"embeddingDim":512,"maxTokens":77.0,"releaseDate":"2021-02-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{radford2021learning,\n  archiveprefix = {arXiv},\n  author = {Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},\n  booktitle = {Proceedings of the 38th International Conference on Machine Learning},\n  eprint = {2103.00020},\n  pages = {8748--8763},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Learning Transferable Visual Models From Natural Language Supervision},\n  url = {https://proceedings.mlr.press/v139/radford21a.html},\n  volume = {139},\n  year = {2021},\n}","memoryUsageMb":576.0,"license":null,"publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":0.124,"totalParamsB":0.15,"embeddingDim":512,"maxTokens":77,"meanTask":0.46202627654525924,"meanTaskType":0.46202627654525924,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3012855555555556,"Compositionality":0.45156658468144284,"ImageClassification":0.7498313176981145,"DocumentUnderstanding":0.254959,"ZeroShotClassification":0.5635670462817025,"ImageClustering":0.6946830862959044,"VisionCentricQA":0.4691594498676732,"Any2AnyMultilingualRetrieval":0.17662896043771045,"VisualSTS(eng)":0.6605747646344889,"VisualSTS(multi)":0.298007},"scoresByTask":{"GTSRBZeroShot":0.4104513064133017,"CVBenchDistance":0.46166666666666667,"CVBenchRelation":0.49846153846153846,"OxfordPetsZeroShot":0.8904333605887162,"InfoSeekIT2ITRetrieval":0.05238,"VidoreTabfquadRetrieval":0.17606,"CIFAR10Clustering":0.6925116709526076,"VidoreDocVQARetrieval":0.14602,"VisualNewsI2TRetrieval":0.28638,"VidoreArxivQARetrieval":0.2654,"STL10":0.9864750000000001,"CLEVRZeroShot":0.15826666666666667,"MemotionI2TRetrieval":0.72614,"PatchCamelyon":0.705950927734375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.37543,"TinyImageNetClustering":0.6290288786644829,"WebQAT2ITRetrieval":0.27637,"SUN397":0.7212873563218392,"METI2IRetrieval":0.35685,"STS13VisualSTS":0.6325573553595002,"CVBenchCount":0.20812182741116753,"ROxfordMediumI2IRetrieval":0.10568,"NIGHTSI2IRetrieval":0.21729,"Flickr30kI2TRetrieval":0.74315,"BLINKIT2TRetrieval":0.0681,"VidoreTatdqaRetrieval":0.04707,"ImageCoDe":0.12163,"ImageCoDeT2IRetrieval":0.09861,"Fashion200kI2TRetrieval":0.02864,"FGVCAircraftZeroShot":0.24902490249024903,"HatefulMemesT2IRetrieval":0.55779,"XM3600T2IRetrieval":0.07988972222222222,"UCF101":0.8193737432266911,"MSCOCOI2TRetrieval":0.45805,"OxfordPets":0.8601798855273918,"SugarCrepe":0.7690054774284363,"FER2013ZeroShot":0.4354973530231262,"OVENIT2TRetrieval":0.00096,"Caltech101ZeroShot":0.7904339250493096,"MSCOCOT2IRetrieval":0.50329,"RESISC45ZeroShot":0.6053968253968254,"Food101Classification":0.8646495049504951,"RParisEasyI2IRetrieval":0.09527,"CVBenchDepth":0.5166666666666667,"Imagenet1k":0.675276,"VisualSTS-b-Eng":0.579333,"VOC2007":0.9045828498841301,"EDIST2ITRetrieval":0.25045,"ROxfordEasyI2IRetrieval":0.19191,"VidoreShiftProjectRetrieval":0.07131,"WITT2IRetrieval":0.2659609090909091,"RP2kI2IRetrieval":0.54756,"ROxfordHardI2IRetrieval":0.07277,"DTD":0.6951063829787234,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.39837,"AROVisualAttribution":0.618373453617096,"Country211":0.1869478672985782,"AROVisualRelation":0.5364498496055603,"BLINKIT2IRetrieval":0.34577,"CIFAR10":0.916,"InfoSeekIT2TRetrieval":0.00147,"VidoreInfoVQARetrieval":0.51703,"BLINKIT2IMultiChoice":0.71642,"VidoreSyntheticDocQAEnergyRetrieval":0.32431,"CIFAR100":0.7123799999999999,"VisualSTS-b-Multilingual":0.262778,"EuroSATZeroShot":0.48592592592592593,"VisualSTS17Eng":0.781793,"RESISC45":0.865079365079365,"BirdsnapZeroShot":0.4403025391680173,"CIFAR10ZeroShot":0.9013,"STL10ZeroShot":0.98375,"GTSRB":0.7268725257323834,"Country211ZeroShot":0.21293838862559242,"FORBI2IRetrieval":0.40136,"CIFAR100ZeroShot":0.6597,"OVENIT2ITRetrieval":0.03859,"MNIST":0.9113999999999999,"GLDv2I2TRetrieval":0.64547,"CIFAR100Clustering":0.5935431031572765,"RParisHardI2IRetrieval":0.04303,"SOPI2IRetrieval":0.50299,"VidoreSyntheticDocQAAIRetrieval":0.22859,"ImageNet10Clustering":0.925785357922303,"StanfordCarsI2IRetrieval":0.71658,"StanfordCarsZeroShot":0.6353687352319364,"FER2013":0.45631095012538314,"OxfordFlowersClassification":0.9727450980392156,"STS12VisualSTS":0.6381736305871083,"Imagenet1kZeroShot":0.63988,"STS14VisualSTS":0.5698644745957532,"AROCocoOrder":0.48180729150772095,"VQA2IT2TRetrieval":0.00941,"CLEVRCountZeroShot":0.21206666666666665,"GLDv2I2IRetrieval":0.20192,"RenderedSST2":0.6051619989017024,"STS15VisualSTS":0.7331855581118903,"SciMMIRT2IRetrieval":0.12783,"WebQAT2TRetrieval":0.21222,"VizWizIT2TRetrieval":0.03905,"SketchyI2IRetrieval":0.42796,"Caltech101":0.9347468770545694,"VisualSTS17Multilingual":0.333236,"BLINKIT2TMultiChoice":0.41362,"EuroSAT":0.7420740740740741,"MemotionT2IRetrieval":0.83203,"CIRRIT2IRetrieval":0.09525,"DTDZeroShot":0.42872340425531913,"Food101ZeroShot":0.8766732673267327,"Birdsnap":0.5728795245813074,"Flickr30kT2IRetrieval":0.76762,"Winoground":0.07249999791383743,"StanfordCars":0.8058450441487377,"VisualNewsT2IRetrieval":0.29685,"STS16VisualSTS":0.6891163337871707,"MNISTZeroShot":0.6233,"PatchCamelyonZeroShot":0.489959716796875,"HatefulMemesI2TRetrieval":0.51573,"SciMMIRI2TRetrieval":0.14412,"RParisMediumI2IRetrieval":0.02716,"UCF101ZeroShot":0.6219238635613907,"AROFlickrOrder":0.5612000226974487,"ImageNetDog15Clustering":0.6325464207828516,"Fashion200kT2IRetrieval":0.01648,"FashionIQIT2IRetrieval":0.02589,"TUBerlinT2IRetrieval":0.86686,"SUN397ZeroShot":0.6055632183908046,"FGVCAircraft":0.4601260126012601,"CUB200I2IRetrieval":0.61857,"XFlickr30kCoT2IRetrieval":0.18403625},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4771130610378781},"MIEB(lite) Task Mean":{"Task Mean":0.42371224609436553}},"trainedOnTasks":[],"experiments":null},{"rank":27,"model":{"name":"Salesforce/blip-itm-large-coco","url":"https://huggingface.co/Salesforce/blip-itm-large-coco","zeroShotPct":100,"activeParamsB":0.423,"totalParamsB":0.446,"embeddingDim":768,"maxTokens":512.0,"releaseDate":"2023-08-01","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{https://doi.org/10.48550/arxiv.2201.12086,\n  archiveprefix = {arXiv},\n  author = {Junnan Li and Dongxu Li and Caiming Xiong and Steven C. H. Hoi},\n  booktitle = {ICML},\n  eprint = {2201.12086},\n  pages = {12888-12900},\n  title = {BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation},\n  url = {https://proceedings.mlr.press/v162/li22n.html},\n  year = {2022},\n}","memoryUsageMb":1793.0,"license":"bsd-3-clause","publicTrainingCode":"https://github.com/salesforce/BLIP","publicTrainingData":"https://github.com/salesforce/BLIP","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.423,"totalParamsB":0.446,"embeddingDim":768,"maxTokens":512,"meanTask":0.45937303277680197,"meanTaskType":0.45937303277680197,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.32795044444444443,"Compositionality":0.4046759011660303,"ImageClassification":0.7077246048604079,"DocumentUnderstanding":0.17700100000000002,"ZeroShotClassification":0.5390862414392076,"ImageClustering":0.775310690743144,"VisionCentricQA":0.510967443273027,"Any2AnyMultilingualRetrieval":0.1852757786195286,"VisualSTS(eng)":0.6287997232222295,"VisualSTS(multi)":0.33693850000000003},"scoresByTask":{"GTSRBZeroShot":0.3482185273159145,"CVBenchDistance":0.49666666666666665,"CVBenchRelation":0.4876923076923077,"OxfordPetsZeroShot":0.7664213682202234,"InfoSeekIT2ITRetrieval":0.114,"VidoreTabfquadRetrieval":0.24592,"CIFAR10Clustering":0.8495305597809512,"VidoreDocVQARetrieval":0.11247,"VisualNewsI2TRetrieval":0.17028,"VidoreArxivQARetrieval":0.22654,"STL10":0.9885000000000002,"CLEVRZeroShot":0.1908,"MemotionI2TRetrieval":0.64109,"PatchCamelyon":0.676904296875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.2445,"TinyImageNetClustering":0.6423898696992246,"WebQAT2ITRetrieval":0.45987,"SUN397":0.7289103448275862,"METI2IRetrieval":0.39774,"STS13VisualSTS":0.5097420388770215,"CVBenchCount":0.4530456852791878,"ROxfordMediumI2IRetrieval":0.11289,"NIGHTSI2IRetrieval":0.25305,"Flickr30kI2TRetrieval":0.87729,"BLINKIT2TRetrieval":0.24338,"VidoreTatdqaRetrieval":0.03499,"ImageCoDe":0.14075,"ImageCoDeT2IRetrieval":0.12989,"Fashion200kI2TRetrieval":0.11248,"FGVCAircraftZeroShot":0.0699069906990699,"HatefulMemesT2IRetrieval":0.462,"XM3600T2IRetrieval":0.11931472222222222,"UCF101":0.8450464844769673,"MSCOCOI2TRetrieval":0.68274,"OxfordPets":0.8162442082311256,"SugarCrepe":0.9106643795967102,"FER2013ZeroShot":0.47353023126219,"OVENIT2TRetrieval":0.01673,"Caltech101ZeroShot":0.8282380013149244,"MSCOCOT2IRetrieval":0.73851,"RESISC45ZeroShot":0.5703174603174603,"Food101Classification":0.8176237623762376,"RParisEasyI2IRetrieval":0.09928,"CVBenchDepth":0.5,"Imagenet1k":0.6790719999999999,"VisualSTS-b-Eng":0.585303,"VOC2007":0.6932318351831023,"EDIST2ITRetrieval":0.20826,"ROxfordEasyI2IRetrieval":0.20442,"VidoreShiftProjectRetrieval":0.04079,"WITT2IRetrieval":0.19330636363636364,"RP2kI2IRetrieval":0.65739,"ROxfordHardI2IRetrieval":0.07418,"DTD":0.7058510638297871,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.19853,"AROVisualAttribution":0.7652010321617126,"Country211":0.09080568720379148,"AROVisualRelation":0.5227890014648438,"BLINKIT2IRetrieval":0.42537,"CIFAR10":0.9574199999999999,"InfoSeekIT2TRetrieval":0.03395,"VidoreInfoVQARetrieval":0.31373,"BLINKIT2IMultiChoice":0.74378,"VidoreSyntheticDocQAEnergyRetrieval":0.19034,"CIFAR100":0.7645000000000002,"VisualSTS-b-Multilingual":0.354974,"EuroSATZeroShot":0.49203703703703705,"VisualSTS17Eng":0.756901,"RESISC45":0.8567619047619047,"BirdsnapZeroShot":0.20097244732576985,"CIFAR10ZeroShot":0.9478,"STL10ZeroShot":0.98325,"GTSRB":0.6453206650831353,"Country211ZeroShot":0.10488151658767772,"FORBI2IRetrieval":0.52445,"CIFAR100ZeroShot":0.7383,"OVENIT2ITRetrieval":0.0707,"MNIST":0.8392200000000001,"GLDv2I2TRetrieval":0.33342,"CIFAR100Clustering":0.7261513809448328,"RParisHardI2IRetrieval":0.03994,"SOPI2IRetrieval":0.57831,"VidoreSyntheticDocQAAIRetrieval":0.1622,"ImageNet10Clustering":0.9828685569099435,"StanfordCarsI2IRetrieval":0.66634,"StanfordCarsZeroShot":0.6970526053973386,"FER2013":0.48094176650877685,"OxfordFlowersClassification":0.9649019607843137,"STS12VisualSTS":0.6232262080887686,"Imagenet1kZeroShot":0.60944,"STS14VisualSTS":0.5516333681463408,"AROCocoOrder":0.1827269047498703,"VQA2IT2TRetrieval":0.01079,"CLEVRCountZeroShot":0.2902666666666667,"GLDv2I2IRetrieval":0.13638,"RenderedSST2":0.500823723228995,"STS15VisualSTS":0.7014670231943553,"SciMMIRT2IRetrieval":0.07934,"WebQAT2TRetrieval":0.3075,"VizWizIT2TRetrieval":0.06409,"SketchyI2IRetrieval":0.73299,"Caltech101":0.891222879684418,"VisualSTS17Multilingual":0.318903,"BLINKIT2TMultiChoice":0.38462,"EuroSAT":0.8090370370370371,"MemotionT2IRetrieval":0.73136,"CIRRIT2IRetrieval":0.19424,"DTDZeroShot":0.5462765957446809,"Food101ZeroShot":0.7605148514851485,"Birdsnap":0.34673149648838464,"Flickr30kT2IRetrieval":0.90308,"Winoground":0.10499999672174454,"StanfordCars":0.7479915433403805,"VisualNewsT2IRetrieval":0.1628,"STS16VisualSTS":0.6733254242491201,"MNISTZeroShot":0.6068,"PatchCamelyonZeroShot":0.526763916015625,"HatefulMemesI2TRetrieval":0.35657,"SciMMIRI2TRetrieval":0.07379,"RParisMediumI2IRetrieval":0.02663,"UCF101ZeroShot":0.4749233386209844,"AROFlickrOrder":0.20559999346733093,"ImageNetDog15Clustering":0.6756130863807676,"Fashion200kT2IRetrieval":0.09365,"FashionIQIT2IRetrieval":0.06652,"TUBerlinT2IRetrieval":0.9466,"SUN397ZeroShot":0.671448275862069,"FGVCAircraft":0.22370237023702372,"CUB200I2IRetrieval":0.54349,"XFlickr30kCoT2IRetrieval":0.24320624999999998},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4693871387528196},"MIEB(lite) Task Mean":{"Task Mean":0.4074185188963654}},"trainedOnTasks":[],"experiments":null},{"rank":28,"model":{"name":"laion/CLIP-ViT-B-32-laion2B-s34B-b79K","url":"https://huggingface.co/laion/CLIP-ViT-B-32-laion2B-s34B-b79K","zeroShotPct":100,"activeParamsB":0.126,"totalParamsB":0.151,"embeddingDim":512,"maxTokens":77.0,"releaseDate":"2022-09-15","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{cherti2023reproducible,\n    title={Reproducible scaling laws for contrastive language-image learning},\n    author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},\n    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},\n    pages={2818--2829},\n    year={2023}\n}","memoryUsageMb":577.0,"license":"mit","publicTrainingCode":"https://github.com/mlfoundations/open_clip","publicTrainingData":"https://laion.ai/blog/laion-5b/","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["open_clip_torch"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.126,"totalParamsB":0.151,"embeddingDim":512,"maxTokens":77,"meanTask":0.45220397931370504,"meanTaskType":0.45220397931370504,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.332748,"Compositionality":0.4065098347970418,"ImageClassification":0.7564150097292575,"DocumentUnderstanding":0.168555,"ZeroShotClassification":0.5900149794537047,"ImageClustering":0.7799271773370214,"VisionCentricQA":0.42965891947589924,"Any2AnyMultilingualRetrieval":0.20129325336700335,"VisualSTS(eng)":0.5952711189771228,"VisualSTS(multi)":0.2616465},"scoresByTask":{"GTSRBZeroShot":0.457403008709422,"CVBenchDistance":0.4083333333333333,"CVBenchRelation":0.4969230769230769,"OxfordPetsZeroShot":0.9062414826928318,"InfoSeekIT2ITRetrieval":0.10643,"VidoreTabfquadRetrieval":0.16094,"CIFAR10Clustering":0.837585936317345,"VidoreDocVQARetrieval":0.10496,"VisualNewsI2TRetrieval":0.28017,"VidoreArxivQARetrieval":0.24008,"STL10":0.9777250000000001,"CLEVRZeroShot":0.18853333333333333,"MemotionI2TRetrieval":0.84128,"PatchCamelyon":0.696783447265625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.18597,"TinyImageNetClustering":0.6376123533169652,"WebQAT2ITRetrieval":0.37585,"SUN397":0.7165057471264367,"METI2IRetrieval":0.39557,"STS13VisualSTS":0.5715981427886446,"CVBenchCount":0.0038071065989847717,"ROxfordMediumI2IRetrieval":0.14179,"NIGHTSI2IRetrieval":0.25785,"Flickr30kI2TRetrieval":0.77294,"BLINKIT2TRetrieval":0.0744,"VidoreTatdqaRetrieval":0.03657,"ImageCoDe":0.1351,"ImageCoDeT2IRetrieval":0.09818,"Fashion200kI2TRetrieval":0.09636,"FGVCAircraftZeroShot":0.23852385238523852,"HatefulMemesT2IRetrieval":0.58534,"XM3600T2IRetrieval":0.11331555555555556,"UCF101":0.8108120512548371,"MSCOCOI2TRetrieval":0.5021,"OxfordPets":0.8505314799672934,"SugarCrepe":0.8136067390441895,"FER2013ZeroShot":0.4721370855391474,"OVENIT2TRetrieval":0.0167,"Caltech101ZeroShot":0.8238001314924391,"MSCOCOT2IRetrieval":0.56229,"RESISC45ZeroShot":0.6331746031746032,"Food101Classification":0.7877544554455446,"RParisEasyI2IRetrieval":0.09587,"CVBenchDepth":0.5,"Imagenet1k":0.641316,"VisualSTS-b-Eng":0.504756,"VOC2007":0.9028102603036778,"EDIST2ITRetrieval":0.24469,"ROxfordEasyI2IRetrieval":0.28263,"VidoreShiftProjectRetrieval":0.04948,"WITT2IRetrieval":0.26094545454545454,"RP2kI2IRetrieval":0.63377,"ROxfordHardI2IRetrieval":0.07375,"DTD":0.74,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.14502,"AROVisualAttribution":0.5805273652076721,"Country211":0.13413270142180095,"AROVisualRelation":0.5039478540420532,"BLINKIT2IRetrieval":0.32836,"CIFAR10":0.94048,"InfoSeekIT2TRetrieval":0.00955,"VidoreInfoVQARetrieval":0.35346,"BLINKIT2IMultiChoice":0.73383,"VidoreSyntheticDocQAEnergyRetrieval":0.22522,"CIFAR100":0.77172,"VisualSTS-b-Multilingual":0.296052,"EuroSATZeroShot":0.48703703703703705,"VisualSTS17Eng":0.746866,"RESISC45":0.8584761904761905,"BirdsnapZeroShot":0.5180983252296056,"CIFAR10ZeroShot":0.937,"STL10ZeroShot":0.963875,"GTSRB":0.7818844022169438,"Country211ZeroShot":0.15265402843601897,"FORBI2IRetrieval":0.38098,"CIFAR100ZeroShot":0.7464,"OVENIT2ITRetrieval":0.07916,"MNIST":0.9500399999999999,"GLDv2I2TRetrieval":0.6127,"CIFAR100Clustering":0.709165943220096,"RParisHardI2IRetrieval":0.04571,"SOPI2IRetrieval":0.56187,"VidoreSyntheticDocQAAIRetrieval":0.18385,"ImageNet10Clustering":0.9521959319740562,"StanfordCarsI2IRetrieval":0.81184,"StanfordCarsZeroShot":0.8558636985449571,"FER2013":0.4097520200612984,"OxfordFlowersClassification":0.9684313725490196,"STS12VisualSTS":0.5369571447142474,"Imagenet1kZeroShot":0.65154,"STS14VisualSTS":0.5273765744553938,"AROCocoOrder":0.33278688788414,"VQA2IT2TRetrieval":0.018,"CLEVRCountZeroShot":0.15293333333333334,"GLDv2I2IRetrieval":0.17994,"RenderedSST2":0.5623283909939594,"STS15VisualSTS":0.6663714171302558,"SciMMIRT2IRetrieval":0.13249,"WebQAT2TRetrieval":0.31123,"VizWizIT2TRetrieval":0.0602,"SketchyI2IRetrieval":0.59146,"Caltech101":0.9488823142669297,"VisualSTS17Multilingual":0.227241,"BLINKIT2TMultiChoice":0.43506,"EuroSAT":0.8893333333333334,"MemotionT2IRetrieval":0.87267,"CIRRIT2IRetrieval":0.1194,"DTDZeroShot":0.5436170212765957,"Food101ZeroShot":0.8166732673267326,"Birdsnap":0.5076175040518639,"Flickr30kT2IRetrieval":0.79601,"Winoground":0.07500000298023224,"StanfordCars":0.8849148115905983,"VisualNewsT2IRetrieval":0.26595,"STS16VisualSTS":0.612972553751318,"MNISTZeroShot":0.6393,"PatchCamelyonZeroShot":0.60137939453125,"HatefulMemesI2TRetrieval":0.57539,"SciMMIRI2TRetrieval":0.14621,"RParisMediumI2IRetrieval":0.02688,"UCF101ZeroShot":0.5394867058124959,"AROFlickrOrder":0.40459999442100525,"ImageNetDog15Clustering":0.7630757218566449,"Fashion200kT2IRetrieval":0.07137,"FashionIQIT2IRetrieval":0.03047,"TUBerlinT2IRetrieval":0.91017,"SUN397ZeroShot":0.6823448275862068,"FGVCAircraft":0.4712271227122712,"CUB200I2IRetrieval":0.59769,"XFlickr30kCoT2IRetrieval":0.22961875},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4828859265715082},"MIEB(lite) Task Mean":{"Task Mean":0.4203372153591484}},"trainedOnTasks":[],"experiments":null},{"rank":29,"model":{"name":"jinaai/jina-clip-v1","url":"https://huggingface.co/jinaai/jina-clip-v1","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.223,"embeddingDim":768,"maxTokens":8192.0,"releaseDate":"2024-05-30","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@article{koukounas2024jinaclip,\n  title={Jina CLIP: Your CLIP Model Is Also Your Text Retriever},\n  author={Koukounas, Andreas and Mastrapas, Georgios and Günther, Michael and Wang, Bo and Martens, Scott and Mohr, Isabelle and Sturua, Saba and Akram, Mohammad Kalim and Martínez, Joan Fontanals and Ognawala, Saahil and Guzman, Susana and Werk, Maximilian and Wang, Nan and Xiao, Han},\n  journal={arXiv preprint arXiv:2405.20204},\n  year={2024}\n}","memoryUsageMb":849.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":"jinaai/jina-clip-v2","extraRequirementsGroups":["jina-clip"],"trainingDatasets":["HotpotQA","MSMARCO","NQ"]},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.223,"embeddingDim":768,"maxTokens":8192,"meanTask":0.4485240836370098,"meanTaskType":0.4485240836370098,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.32349755555555554,"Compositionality":0.45108842191202303,"ImageClassification":0.6895352882383424,"DocumentUnderstanding":0.176416,"ZeroShotClassification":0.5384657048447042,"ImageClustering":0.6995458454941879,"VisionCentricQA":0.4537687997310078,"Any2AnyMultilingualRetrieval":0.18087312710437708,"VisualSTS(eng)":0.6261870934898999,"VisualSTS(multi)":0.34586300000000003},"scoresByTask":{"GTSRBZeroShot":0.38756927949326997,"CVBenchDistance":0.47,"CVBenchRelation":0.5076923076923077,"OxfordPetsZeroShot":0.8083946579449441,"InfoSeekIT2ITRetrieval":0.14217,"VidoreTabfquadRetrieval":0.20142,"CIFAR10Clustering":0.7411512997597015,"VidoreDocVQARetrieval":0.10989,"VisualNewsI2TRetrieval":0.12771,"VidoreArxivQARetrieval":0.25399,"STL10":0.970575,"CLEVRZeroShot":0.1562,"MemotionI2TRetrieval":0.79006,"PatchCamelyon":0.62967529296875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.20841,"TinyImageNetClustering":0.6147087919222165,"WebQAT2ITRetrieval":0.51613,"SUN397":0.6961563218390804,"METI2IRetrieval":0.38765,"STS13VisualSTS":0.557954515449527,"CVBenchCount":0.1484771573604061,"ROxfordMediumI2IRetrieval":0.11707,"NIGHTSI2IRetrieval":0.23872,"Flickr30kI2TRetrieval":0.75648,"BLINKIT2TRetrieval":0.25347,"VidoreTatdqaRetrieval":0.03342,"ImageCoDe":0.13032,"ImageCoDeT2IRetrieval":0.10209,"Fashion200kI2TRetrieval":0.04777,"FGVCAircraftZeroShot":0.1152115211521152,"HatefulMemesT2IRetrieval":0.54032,"XM3600T2IRetrieval":0.11154722222222221,"UCF101":0.7669201488191709,"MSCOCOI2TRetrieval":0.48279,"OxfordPets":0.8017443445080403,"SugarCrepe":0.8181334137916565,"FER2013ZeroShot":0.4721370855391474,"OVENIT2TRetrieval":0.02299,"Caltech101ZeroShot":0.8086785009861933,"MSCOCOT2IRetrieval":0.58366,"RESISC45ZeroShot":0.5563492063492064,"Food101Classification":0.7949940594059408,"RParisEasyI2IRetrieval":0.1113,"CVBenchDepth":0.49333333333333335,"Imagenet1k":0.605692,"VisualSTS-b-Eng":0.521842,"VOC2007":0.6688946830159087,"EDIST2ITRetrieval":0.22465,"ROxfordEasyI2IRetrieval":0.22617,"VidoreShiftProjectRetrieval":0.03835,"WITT2IRetrieval":0.2092009090909091,"RP2kI2IRetrieval":0.66401,"ROxfordHardI2IRetrieval":0.06681,"DTD":0.6806382978723404,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.21834,"AROVisualAttribution":0.6170168519020081,"Country211":0.09594312796208532,"AROVisualRelation":0.5136399865150452,"BLINKIT2IRetrieval":0.33333,"CIFAR10":0.90624,"InfoSeekIT2TRetrieval":0.03465,"VidoreInfoVQARetrieval":0.35121,"BLINKIT2IMultiChoice":0.74876,"VidoreSyntheticDocQAEnergyRetrieval":0.19341,"CIFAR100":0.7019,"VisualSTS-b-Multilingual":0.422972,"EuroSATZeroShot":0.4737037037037037,"VisualSTS17Eng":0.769422,"RESISC45":0.840126984126984,"BirdsnapZeroShot":0.3236088600756348,"CIFAR10ZeroShot":0.9339,"STL10ZeroShot":0.978125,"GTSRB":0.5753760886777514,"Country211ZeroShot":0.1218957345971564,"FORBI2IRetrieval":0.38098,"CIFAR100ZeroShot":0.7178,"OVENIT2ITRetrieval":0.07963,"MNIST":0.84038,"GLDv2I2TRetrieval":0.47099,"CIFAR100Clustering":0.6484035324415143,"RParisHardI2IRetrieval":0.04222,"SOPI2IRetrieval":0.56591,"VidoreSyntheticDocQAAIRetrieval":0.15572,"ImageNet10Clustering":0.9669074402231649,"StanfordCarsI2IRetrieval":0.64855,"StanfordCarsZeroShot":0.680885462007213,"FER2013":0.44499860685427695,"OxfordFlowersClassification":0.9305882352941177,"STS12VisualSTS":0.5795896944601722,"Imagenet1kZeroShot":0.58696,"STS14VisualSTS":0.569520644183804,"AROCocoOrder":0.5283086895942688,"VQA2IT2TRetrieval":0.02372,"CLEVRCountZeroShot":0.22353333333333333,"GLDv2I2IRetrieval":0.15516,"RenderedSST2":0.5897858319604613,"STS15VisualSTS":0.7051741890108933,"SciMMIRT2IRetrieval":0.11104,"WebQAT2TRetrieval":0.62406,"VizWizIT2TRetrieval":0.09449,"SketchyI2IRetrieval":0.34561,"Caltech101":0.8821827744904667,"VisualSTS17Multilingual":0.268754,"BLINKIT2TMultiChoice":0.35435,"EuroSAT":0.8327407407407407,"MemotionT2IRetrieval":0.83665,"CIRRIT2IRetrieval":0.15853,"DTDZeroShot":0.5287234042553192,"Food101ZeroShot":0.7670495049504951,"Birdsnap":0.4636412749864938,"Flickr30kT2IRetrieval":0.80945,"Winoground":0.05999999865889549,"StanfordCars":0.716876010446462,"VisualNewsT2IRetrieval":0.13949,"STS16VisualSTS":0.679806611324902,"MNISTZeroShot":0.4858,"PatchCamelyonZeroShot":0.50732421875,"HatefulMemesI2TRetrieval":0.50313,"SciMMIRI2TRetrieval":0.10648,"RParisMediumI2IRetrieval":0.02706,"UCF101ZeroShot":0.5066851017323033,"AROFlickrOrder":0.490200012922287,"ImageNetDog15Clustering":0.5265581631243419,"Fashion200kT2IRetrieval":0.04177,"FashionIQIT2IRetrieval":0.04507,"TUBerlinT2IRetrieval":0.90809,"SUN397ZeroShot":0.6543908045977012,"FGVCAircraft":0.3234923492349235,"CUB200I2IRetrieval":0.66931,"XFlickr30kCoT2IRetrieval":0.22187125000000002},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.46109846549073746},"MIEB(lite) Task Mean":{"Task Mean":0.40896222144481376}},"trainedOnTasks":[],"experiments":null},{"rank":30,"model":{"name":"Salesforce/blip-itm-base-coco","url":"https://huggingface.co/Salesforce/blip-itm-base-coco","zeroShotPct":100,"activeParamsB":0.2,"totalParamsB":0.224,"embeddingDim":768,"maxTokens":512.0,"releaseDate":"2023-08-01","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{https://doi.org/10.48550/arxiv.2201.12086,\n  archiveprefix = {arXiv},\n  author = {Junnan Li and Dongxu Li and Caiming Xiong and Steven C. H. Hoi},\n  booktitle = {ICML},\n  eprint = {2201.12086},\n  pages = {12888-12900},\n  title = {BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation},\n  url = {https://proceedings.mlr.press/v162/li22n.html},\n  year = {2022},\n}","memoryUsageMb":942.0,"license":"bsd-3-clause","publicTrainingCode":"https://github.com/salesforce/BLIP","publicTrainingData":"https://github.com/salesforce/BLIP","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.2,"totalParamsB":0.224,"embeddingDim":768,"maxTokens":512,"meanTask":0.44651249024006906,"meanTaskType":0.44651249024006906,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.30927111111111116,"Compositionality":0.41920365224974493,"ImageClassification":0.671974943787541,"DocumentUnderstanding":0.18013300000000002,"ZeroShotClassification":0.48567825202398035,"ImageClustering":0.705926570915505,"VisionCentricQA":0.47259914963772837,"Any2AnyMultilingualRetrieval":0.1681160101010101,"VisualSTS(eng)":0.6663687125740697,"VisualSTS(multi)":0.38585349999999996},"scoresByTask":{"GTSRBZeroShot":0.35091053048297705,"CVBenchDistance":0.455,"CVBenchRelation":0.5292307692307693,"OxfordPetsZeroShot":0.6909239574816026,"InfoSeekIT2ITRetrieval":0.08202,"VidoreTabfquadRetrieval":0.19422,"CIFAR10Clustering":0.7083255544749761,"VidoreDocVQARetrieval":0.11676,"VisualNewsI2TRetrieval":0.12332,"VidoreArxivQARetrieval":0.20553,"STL10":0.9787000000000001,"CLEVRZeroShot":0.19926666666666668,"MemotionI2TRetrieval":0.60313,"PatchCamelyon":0.646185302734375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.25582,"TinyImageNetClustering":0.5818750752511879,"WebQAT2ITRetrieval":0.43279,"SUN397":0.7110436781609195,"METI2IRetrieval":0.40581,"STS13VisualSTS":0.5513723606158416,"CVBenchCount":0.26649746192893403,"ROxfordMediumI2IRetrieval":0.11985,"NIGHTSI2IRetrieval":0.2497,"Flickr30kI2TRetrieval":0.85229,"BLINKIT2TRetrieval":0.22573,"VidoreTatdqaRetrieval":0.03423,"ImageCoDe":0.12728,"ImageCoDeT2IRetrieval":0.12207,"Fashion200kI2TRetrieval":0.08803,"FGVCAircraftZeroShot":0.0555055505550555,"HatefulMemesT2IRetrieval":0.46155,"XM3600T2IRetrieval":0.10454416666666666,"UCF101":0.8180794639297095,"MSCOCOI2TRetrieval":0.65619,"OxfordPets":0.7606977378032161,"SugarCrepe":0.9134602546691895,"FER2013ZeroShot":0.4005293953747562,"OVENIT2TRetrieval":0.00683,"Caltech101ZeroShot":0.7989809335963182,"MSCOCOT2IRetrieval":0.71635,"RESISC45ZeroShot":0.503968253968254,"Food101Classification":0.7630653465346535,"RParisEasyI2IRetrieval":0.11051,"CVBenchDepth":0.45166666666666666,"Imagenet1k":0.6060000000000001,"VisualSTS-b-Eng":0.665039,"VOC2007":0.6672036794736718,"EDIST2ITRetrieval":0.18427,"ROxfordEasyI2IRetrieval":0.22195,"VidoreShiftProjectRetrieval":0.05054,"WITT2IRetrieval":0.17561636363636363,"RP2kI2IRetrieval":0.64765,"ROxfordHardI2IRetrieval":0.07333,"DTD":0.6978723404255319,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.24738,"AROVisualAttribution":0.7690969705581665,"Country211":0.08739336492890995,"AROVisualRelation":0.5205748677253723,"BLINKIT2IRetrieval":0.40299,"CIFAR10":0.87658,"InfoSeekIT2TRetrieval":0.01186,"VidoreInfoVQARetrieval":0.32298,"BLINKIT2IMultiChoice":0.76119,"VidoreSyntheticDocQAEnergyRetrieval":0.18684,"CIFAR100":0.5968799999999999,"VisualSTS-b-Multilingual":0.481446,"EuroSATZeroShot":0.4038888888888889,"VisualSTS17Eng":0.773223,"RESISC45":0.8192698412698413,"BirdsnapZeroShot":0.12911939492166397,"CIFAR10ZeroShot":0.8087,"STL10ZeroShot":0.970375,"GTSRB":0.6295328582739509,"Country211ZeroShot":0.06824644549763033,"FORBI2IRetrieval":0.56468,"CIFAR100ZeroShot":0.5239,"OVENIT2ITRetrieval":0.0524,"MNIST":0.8572199999999999,"GLDv2I2TRetrieval":0.25702,"CIFAR100Clustering":0.6044013561487221,"RParisHardI2IRetrieval":0.03819,"SOPI2IRetrieval":0.56894,"VidoreSyntheticDocQAAIRetrieval":0.18703,"ImageNet10Clustering":0.9319454962920947,"StanfordCarsI2IRetrieval":0.70228,"StanfordCarsZeroShot":0.6628528789951499,"FER2013":0.4385901365282809,"OxfordFlowersClassification":0.8754901960784313,"STS12VisualSTS":0.6290982544912301,"Imagenet1kZeroShot":0.53474,"STS14VisualSTS":0.6016774394435956,"AROCocoOrder":0.21631348133087158,"VQA2IT2TRetrieval":0.00837,"CLEVRCountZeroShot":0.2146,"GLDv2I2IRetrieval":0.1314,"RenderedSST2":0.49203734211971445,"STS15VisualSTS":0.7282748086195688,"SciMMIRT2IRetrieval":0.06966,"WebQAT2TRetrieval":0.27318,"VizWizIT2TRetrieval":0.05728,"SketchyI2IRetrieval":0.63265,"Caltech101":0.8863247863247864,"VisualSTS17Multilingual":0.290261,"BLINKIT2TMultiChoice":0.37201,"EuroSAT":0.8086296296296297,"MemotionT2IRetrieval":0.69509,"CIRRIT2IRetrieval":0.17214,"DTDZeroShot":0.5148936170212766,"Food101ZeroShot":0.6803168316831684,"Birdsnap":0.26990815775256616,"Flickr30kT2IRetrieval":0.88956,"Winoground":0.1274999976158142,"StanfordCars":0.7486382290759857,"VisualNewsT2IRetrieval":0.12657,"STS16VisualSTS":0.7158961248482522,"MNISTZeroShot":0.5762,"PatchCamelyonZeroShot":0.500732421875,"HatefulMemesI2TRetrieval":0.35737,"SciMMIRI2TRetrieval":0.05352,"RParisMediumI2IRetrieval":0.02711,"UCF101ZeroShot":0.4422795035153796,"AROFlickrOrder":0.26019999384880066,"ImageNetDog15Clustering":0.7030853724105441,"Fashion200kT2IRetrieval":0.07171,"FashionIQIT2IRetrieval":0.0538,"TUBerlinT2IRetrieval":0.88786,"SUN397ZeroShot":0.647632183908046,"FGVCAircraft":0.24014401440144012,"CUB200I2IRetrieval":0.4282,"XFlickr30kCoT2IRetrieval":0.22418749999999998},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4450945029283844},"MIEB(lite) Task Mean":{"Task Mean":0.38776363076312365}},"trainedOnTasks":[],"experiments":null},{"rank":31,"model":{"name":"Salesforce/blip-itm-large-flickr","url":"https://huggingface.co/Salesforce/blip-itm-large-flickr","zeroShotPct":100,"activeParamsB":0.423,"totalParamsB":0.446,"embeddingDim":768,"maxTokens":512.0,"releaseDate":"2023-08-01","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{https://doi.org/10.48550/arxiv.2201.12086,\n  archiveprefix = {arXiv},\n  author = {Junnan Li and Dongxu Li and Caiming Xiong and Steven C. H. Hoi},\n  booktitle = {ICML},\n  eprint = {2201.12086},\n  pages = {12888-12900},\n  title = {BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation},\n  url = {https://proceedings.mlr.press/v162/li22n.html},\n  year = {2022},\n}","memoryUsageMb":1793.0,"license":"bsd-3-clause","publicTrainingCode":"https://github.com/salesforce/BLIP","publicTrainingData":"https://github.com/salesforce/BLIP","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.423,"totalParamsB":0.446,"embeddingDim":768,"maxTokens":512,"meanTask":0.44433449219106996,"meanTaskType":0.44433449219106996,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.31848177777777775,"Compositionality":0.39392697328567505,"ImageClassification":0.6936457367037703,"DocumentUnderstanding":0.184687,"ZeroShotClassification":0.5304450479003868,"ImageClustering":0.7643267434322155,"VisionCentricQA":0.47007318278450255,"Any2AnyMultilingualRetrieval":0.1811894191919192,"VisualSTS(eng)":0.6015575408344523,"VisualSTS(multi)":0.3050115},"scoresByTask":{"GTSRBZeroShot":0.332541567695962,"CVBenchDistance":0.52,"CVBenchRelation":0.5323076923076923,"OxfordPetsZeroShot":0.7792313982011447,"InfoSeekIT2ITRetrieval":0.09652,"VidoreTabfquadRetrieval":0.21873,"CIFAR10Clustering":0.8793855684243735,"VidoreDocVQARetrieval":0.12111,"VisualNewsI2TRetrieval":0.1615,"VidoreArxivQARetrieval":0.24894,"STL10":0.9837250000000001,"CLEVRZeroShot":0.1732,"MemotionI2TRetrieval":0.63397,"PatchCamelyon":0.660894775390625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.27182,"TinyImageNetClustering":0.6085732183082799,"WebQAT2ITRetrieval":0.42988,"SUN397":0.7141977011494253,"METI2IRetrieval":0.39384,"STS13VisualSTS":0.47460309237943027,"CVBenchCount":0.25253807106598986,"ROxfordMediumI2IRetrieval":0.0984,"NIGHTSI2IRetrieval":0.24038,"Flickr30kI2TRetrieval":0.88712,"BLINKIT2TRetrieval":0.22194,"VidoreTatdqaRetrieval":0.03332,"ImageCoDe":0.12858,"ImageCoDeT2IRetrieval":0.12641,"Fashion200kI2TRetrieval":0.10383,"FGVCAircraftZeroShot":0.05850585058505851,"HatefulMemesT2IRetrieval":0.42348,"XM3600T2IRetrieval":0.11086416666666667,"UCF101":0.8223389393908971,"MSCOCOI2TRetrieval":0.61669,"OxfordPets":0.8177705096756609,"SugarCrepe":0.8897616863250732,"FER2013ZeroShot":0.47993870158818613,"OVENIT2TRetrieval":0.01504,"Caltech101ZeroShot":0.8183760683760684,"MSCOCOT2IRetrieval":0.6977,"RESISC45ZeroShot":0.5714285714285714,"Food101Classification":0.8194693069306931,"RParisEasyI2IRetrieval":0.10199,"CVBenchDepth":0.4683333333333333,"Imagenet1k":0.6697360000000001,"VisualSTS-b-Eng":0.558639,"VOC2007":0.640260737759497,"EDIST2ITRetrieval":0.19291,"ROxfordEasyI2IRetrieval":0.17906,"VidoreShiftProjectRetrieval":0.04655,"WITT2IRetrieval":0.1933890909090909,"RP2kI2IRetrieval":0.65066,"ROxfordHardI2IRetrieval":0.05494,"DTD":0.691595744680851,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.16141,"AROVisualAttribution":0.7534089088439941,"Country211":0.10116587677725117,"AROVisualRelation":0.5344445705413818,"BLINKIT2IRetrieval":0.35821,"CIFAR10":0.9434000000000001,"InfoSeekIT2TRetrieval":0.02803,"VidoreInfoVQARetrieval":0.33945,"BLINKIT2IMultiChoice":0.68408,"VidoreSyntheticDocQAEnergyRetrieval":0.23159,"CIFAR100":0.73498,"VisualSTS-b-Multilingual":0.34171,"EuroSATZeroShot":0.435,"VisualSTS17Eng":0.727736,"RESISC45":0.8305079365079365,"BirdsnapZeroShot":0.20421393841166938,"CIFAR10ZeroShot":0.9388,"STL10ZeroShot":0.982125,"GTSRB":0.5817893903404593,"Country211ZeroShot":0.1166824644549763,"FORBI2IRetrieval":0.54566,"CIFAR100ZeroShot":0.7115,"OVENIT2ITRetrieval":0.05933,"MNIST":0.8356,"GLDv2I2TRetrieval":0.37184,"CIFAR100Clustering":0.70670733023624,"RParisHardI2IRetrieval":0.03986,"SOPI2IRetrieval":0.58208,"VidoreSyntheticDocQAAIRetrieval":0.17395,"ImageNet10Clustering":0.9433536074259914,"StanfordCarsI2IRetrieval":0.6636,"StanfordCarsZeroShot":0.7116030344484516,"FER2013":0.45427695736974083,"OxfordFlowersClassification":0.9623529411764705,"STS12VisualSTS":0.5967545018559427,"Imagenet1kZeroShot":0.60954,"STS14VisualSTS":0.5282332595525374,"AROCocoOrder":0.16589364409446716,"VQA2IT2TRetrieval":0.00863,"CLEVRCountZeroShot":0.1394,"GLDv2I2IRetrieval":0.1517,"RenderedSST2":0.528281164195497,"STS15VisualSTS":0.6829126320427497,"SciMMIRT2IRetrieval":0.08281,"WebQAT2TRetrieval":0.3072,"VizWizIT2TRetrieval":0.04764,"SketchyI2IRetrieval":0.70191,"Caltech101":0.878862590401052,"VisualSTS17Multilingual":0.268313,"BLINKIT2TMultiChoice":0.36318,"EuroSAT":0.7905185185185185,"MemotionT2IRetrieval":0.6988,"CIRRIT2IRetrieval":0.19869,"DTDZeroShot":0.5297872340425532,"Food101ZeroShot":0.7600792079207921,"Birdsnap":0.3632631010264722,"Flickr30kT2IRetrieval":0.9085,"Winoground":0.13500000536441803,"StanfordCars":0.745678398209178,"VisualNewsT2IRetrieval":0.16154,"STS16VisualSTS":0.6420243000105063,"MNISTZeroShot":0.5851,"PatchCamelyonZeroShot":0.51544189453125,"HatefulMemesI2TRetrieval":0.36137,"SciMMIRI2TRetrieval":0.07789,"RParisMediumI2IRetrieval":0.02666,"UCF101ZeroShot":0.5154140288172203,"AROFlickrOrder":0.15039999783039093,"ImageNetDog15Clustering":0.683613992766193,"Fashion200kT2IRetrieval":0.08558,"FashionIQIT2IRetrieval":0.06595,"TUBerlinT2IRetrieval":0.94001,"SUN397ZeroShot":0.7040459770114943,"FGVCAircraft":0.21782178217821785,"CUB200I2IRetrieval":0.53193,"XFlickr30kCoT2IRetrieval":0.239315},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4579956537752066},"MIEB(lite) Task Mean":{"Task Mean":0.39525483385673366}},"trainedOnTasks":[],"experiments":null},{"rank":32,"model":{"name":"laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K","url":"https://huggingface.co/laion/CLIP-ViT-B-32-DataComp.XL-s13B-b90K","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.151,"embeddingDim":512,"maxTokens":77.0,"releaseDate":"2023-04-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{cherti2023reproducible,\n    title={Reproducible scaling laws for contrastive language-image learning},\n    author={Cherti, Mehdi and Beaumont, Romain and Wightman, Ross and Wortsman, Mitchell and Ilharco, Gabriel and Gordon, Cade and Schuhmann, Christoph and Schmidt, Ludwig and Jitsev, Jenia},\n    booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},\n    pages={2818--2829},\n    year={2023}\n}","memoryUsageMb":576.0,"license":"mit","publicTrainingCode":"https://github.com/mlfoundations/open_clip","publicTrainingData":"https://huggingface.co/datasets/mlfoundations/datacomp_1b","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["open_clip_torch"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.151,"embeddingDim":512,"maxTokens":77,"meanTask":0.44262770365182247,"meanTaskType":0.44262770365182247,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3218228888888889,"Compositionality":0.3974564544398444,"ImageClassification":0.7580846135323505,"DocumentUnderstanding":0.1243,"ZeroShotClassification":0.58749624743003,"ImageClustering":0.773728983236445,"VisionCentricQA":0.4605146945637555,"Any2AnyMultilingualRetrieval":0.20133710016835016,"VisualSTS(eng)":0.5651970542585603,"VisualSTS(multi)":0.23633900000000002},"scoresByTask":{"GTSRBZeroShot":0.48891528107680127,"CVBenchDistance":0.43666666666666665,"CVBenchRelation":0.47384615384615386,"OxfordPetsZeroShot":0.9043336058871627,"InfoSeekIT2ITRetrieval":0.07661,"VidoreTabfquadRetrieval":0.14922,"CIFAR10Clustering":0.7981200811089172,"VidoreDocVQARetrieval":0.09029,"VisualNewsI2TRetrieval":0.20758,"VidoreArxivQARetrieval":0.16567,"STL10":0.976675,"CLEVRZeroShot":0.2064,"MemotionI2TRetrieval":0.81169,"PatchCamelyon":0.69918212890625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.1251,"TinyImageNetClustering":0.6698138352885304,"WebQAT2ITRetrieval":0.37218,"SUN397":0.7051310344827585,"METI2IRetrieval":0.39972,"STS13VisualSTS":0.45824961546399057,"CVBenchCount":0.23857868020304568,"ROxfordMediumI2IRetrieval":0.16274,"NIGHTSI2IRetrieval":0.2571,"Flickr30kI2TRetrieval":0.72617,"BLINKIT2TRetrieval":0.17907,"VidoreTatdqaRetrieval":0.02762,"ImageCoDe":0.12207,"ImageCoDeT2IRetrieval":0.08254,"Fashion200kI2TRetrieval":0.08189,"FGVCAircraftZeroShot":0.24392439243924394,"HatefulMemesT2IRetrieval":0.57244,"XM3600T2IRetrieval":0.11885027777777776,"UCF101":0.7889713749709563,"MSCOCOI2TRetrieval":0.47215,"OxfordPets":0.8497138184791495,"SugarCrepe":0.7986952662467957,"FER2013ZeroShot":0.27904708832543884,"OVENIT2TRetrieval":0.01614,"Caltech101ZeroShot":0.8446745562130178,"MSCOCOT2IRetrieval":0.53488,"RESISC45ZeroShot":0.6090476190476191,"Food101Classification":0.8150574257425742,"RParisEasyI2IRetrieval":0.09529,"CVBenchDepth":0.49166666666666664,"Imagenet1k":0.6493840000000001,"VisualSTS-b-Eng":0.491483,"VOC2007":0.8963261223189065,"EDIST2ITRetrieval":0.19946,"ROxfordEasyI2IRetrieval":0.28987,"VidoreShiftProjectRetrieval":0.03062,"WITT2IRetrieval":0.2552472727272727,"RP2kI2IRetrieval":0.64364,"ROxfordHardI2IRetrieval":0.11041,"DTD":0.7419148936170213,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.09079,"AROVisualAttribution":0.5734660029411316,"Country211":0.1317440758293839,"AROVisualRelation":0.49521660804748535,"BLINKIT2IRetrieval":0.30348,"CIFAR10":0.9586399999999999,"InfoSeekIT2TRetrieval":0.0044,"VidoreInfoVQARetrieval":0.27088,"BLINKIT2IMultiChoice":0.72637,"VidoreSyntheticDocQAEnergyRetrieval":0.15666,"CIFAR100":0.8083600000000001,"VisualSTS-b-Multilingual":0.305537,"EuroSATZeroShot":0.5531481481481482,"VisualSTS17Eng":0.733052,"RESISC45":0.8485396825396826,"BirdsnapZeroShot":0.5840086439762291,"CIFAR10ZeroShot":0.953,"STL10ZeroShot":0.963,"GTSRB":0.8140617577197149,"Country211ZeroShot":0.15985781990521328,"FORBI2IRetrieval":0.31872,"CIFAR100ZeroShot":0.7964,"OVENIT2ITRetrieval":0.06611,"MNIST":0.9555,"GLDv2I2TRetrieval":0.65621,"CIFAR100Clustering":0.7485190053831395,"RParisHardI2IRetrieval":0.04638,"SOPI2IRetrieval":0.564,"VidoreSyntheticDocQAAIRetrieval":0.13615,"ImageNet10Clustering":0.9154380405807664,"StanfordCarsI2IRetrieval":0.82129,"StanfordCarsZeroShot":0.8563611491108071,"FER2013":0.3275285594873224,"OxfordFlowersClassification":0.9774509803921567,"STS12VisualSTS":0.5486406873663947,"Imagenet1kZeroShot":0.6787,"STS14VisualSTS":0.4885168221931793,"AROCocoOrder":0.33174729347229004,"VQA2IT2TRetrieval":0.01665,"CLEVRCountZeroShot":0.1294,"GLDv2I2IRetrieval":0.18012,"RenderedSST2":0.4942339373970346,"STS15VisualSTS":0.6402034296939951,"SciMMIRT2IRetrieval":0.07347,"WebQAT2TRetrieval":0.29457,"VizWizIT2TRetrieval":0.06043,"SketchyI2IRetrieval":0.58563,"Caltech101":0.9477317554240632,"VisualSTS17Multilingual":0.167141,"BLINKIT2TMultiChoice":0.39596,"EuroSAT":0.8907407407407408,"MemotionT2IRetrieval":0.84078,"CIRRIT2IRetrieval":0.11909,"DTDZeroShot":0.5441489361702128,"Food101ZeroShot":0.8344554455445544,"Birdsnap":0.5213398163155051,"Flickr30kT2IRetrieval":0.75545,"Winoground":0.07999999821186066,"StanfordCars":0.8811590598184307,"VisualNewsT2IRetrieval":0.19209,"STS16VisualSTS":0.5962338250923621,"MNISTZeroShot":0.7209,"PatchCamelyonZeroShot":0.501495361328125,"HatefulMemesI2TRetrieval":0.55313,"SciMMIRI2TRetrieval":0.0884,"RParisMediumI2IRetrieval":0.02699,"UCF101ZeroShot":0.497260556895795,"AROFlickrOrder":0.38100001215934753,"ImageNetDog15Clustering":0.7367539538208716,"Fashion200kT2IRetrieval":0.05785,"FashionIQIT2IRetrieval":0.02833,"TUBerlinT2IRetrieval":0.914,"SUN397ZeroShot":0.6697011494252874,"FGVCAircraft":0.49270927092709266,"CUB200I2IRetrieval":0.62289,"XFlickr30kCoT2IRetrieval":0.22991375},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4742849026644479},"MIEB(lite) Task Mean":{"Task Mean":0.4128882764806708}},"trainedOnTasks":[],"experiments":null},{"rank":33,"model":{"name":"kakaobrain/align-base","url":"https://huggingface.co/kakaobrain/align-base","zeroShotPct":100,"activeParamsB":0.149,"totalParamsB":0.172,"embeddingDim":768,"maxTokens":64.0,"releaseDate":"2023-02-24","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":true,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":671.0,"license":null,"publicTrainingCode":"https://github.com/kakaobrain/coyo-align","publicTrainingData":"True","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.149,"totalParamsB":0.172,"embeddingDim":768,"maxTokens":64,"meanTask":0.4422523260562108,"meanTaskType":0.4422523260562108,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.3098126666666666,"Compositionality":0.40291042855773657,"ImageClassification":0.7119087198756208,"DocumentUnderstanding":0.31435599999999997,"ZeroShotClassification":0.5068637986685658,"ImageClustering":0.5958778622686886,"VisionCentricQA":0.45690959260705455,"Any2AnyMultilingualRetrieval":0.22357575757575757,"VisualSTS(eng)":0.6274339343420172,"VisualSTS(multi)":0.2728745},"scoresByTask":{"GTSRBZeroShot":0.2684085510688836,"CVBenchDistance":0.5083333333333333,"CVBenchRelation":0.4707692307692308,"OxfordPetsZeroShot":0.8430089942763695,"InfoSeekIT2ITRetrieval":0.00062,"VidoreTabfquadRetrieval":0.28995,"CIFAR10Clustering":0.541303710745477,"VidoreDocVQARetrieval":0.18029,"VisualNewsI2TRetrieval":0.2477,"VidoreArxivQARetrieval":0.23312,"STL10":0.959525,"CLEVRZeroShot":0.2352,"MemotionI2TRetrieval":0.85088,"PatchCamelyon":0.682958984375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.47456,"TinyImageNetClustering":0.5003296440182597,"WebQAT2ITRetrieval":0.35513,"SUN397":0.7409011494252873,"METI2IRetrieval":0.36628,"STS13VisualSTS":0.5749899452481498,"CVBenchCount":0.47588832487309646,"ROxfordMediumI2IRetrieval":0.11044,"NIGHTSI2IRetrieval":0.22567,"Flickr30kI2TRetrieval":0.82282,"BLINKIT2TRetrieval":0.23329,"VidoreTatdqaRetrieval":0.09686,"ImageCoDe":0.13206,"ImageCoDeT2IRetrieval":0.10295,"Fashion200kI2TRetrieval":0.10366,"FGVCAircraftZeroShot":0.11341134113411341,"HatefulMemesT2IRetrieval":0.57253,"XM3600T2IRetrieval":0.1608175,"UCF101":0.8012196975999037,"MSCOCOI2TRetrieval":0.53261,"OxfordPets":0.8070863995639138,"SugarCrepe":0.8120090365409851,"FER2013ZeroShot":0.3780997492337699,"OVENIT2TRetrieval":0.0009,"Caltech101ZeroShot":0.8057199211045365,"MSCOCOT2IRetrieval":0.58344,"RESISC45ZeroShot":0.4888888888888889,"Food101Classification":0.8274138613861386,"RParisEasyI2IRetrieval":0.09051,"CVBenchDepth":0.43166666666666664,"Imagenet1k":0.671752,"VisualSTS-b-Eng":0.594228,"VOC2007":0.9181286580176676,"EDIST2ITRetrieval":0.10407,"ROxfordEasyI2IRetrieval":0.22521,"VidoreShiftProjectRetrieval":0.10472,"WITT2IRetrieval":0.2635072727272727,"RP2kI2IRetrieval":0.51663,"ROxfordHardI2IRetrieval":0.03133,"DTD":0.7404255319148936,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.42068,"AROVisualAttribution":0.6695422530174255,"Country211":0.15162085308056872,"AROVisualRelation":0.5108827352523804,"BLINKIT2IRetrieval":0.0,"CIFAR10":0.81228,"InfoSeekIT2TRetrieval":0.00009,"VidoreInfoVQARetrieval":0.43149,"BLINKIT2IMultiChoice":0.46766,"VidoreSyntheticDocQAEnergyRetrieval":0.49762,"CIFAR100":0.5883,"VisualSTS-b-Multilingual":0.280933,"EuroSATZeroShot":0.3687037037037037,"VisualSTS17Eng":0.775467,"RESISC45":0.8382222222222222,"BirdsnapZeroShot":0.2522960561858455,"CIFAR10ZeroShot":0.7559,"STL10ZeroShot":0.93725,"GTSRB":0.5943784639746635,"Country211ZeroShot":0.16208530805687205,"FORBI2IRetrieval":0.38906,"CIFAR100ZeroShot":0.5146,"OVENIT2ITRetrieval":0.00156,"MNIST":0.86784,"GLDv2I2TRetrieval":0.51221,"CIFAR100Clustering":0.5068445305780905,"RParisHardI2IRetrieval":0.03946,"SOPI2IRetrieval":0.51441,"VidoreSyntheticDocQAAIRetrieval":0.41427,"ImageNet10Clustering":0.8421433336924401,"StanfordCarsI2IRetrieval":0.78038,"StanfordCarsZeroShot":0.7295112548190523,"FER2013":0.3589300640847032,"OxfordFlowersClassification":0.9623529411764705,"STS12VisualSTS":0.5316712185092839,"Imagenet1kZeroShot":0.62696,"STS14VisualSTS":0.5601116504250067,"AROCocoOrder":0.2525789737701416,"VQA2IT2TRetrieval":0.00566,"CLEVRCountZeroShot":0.19473333333333334,"GLDv2I2IRetrieval":0.16529,"RenderedSST2":0.5749588138385503,"STS15VisualSTS":0.6912547281546391,"SciMMIRT2IRetrieval":0.15007,"WebQAT2TRetrieval":0.2615,"VizWizIT2TRetrieval":0.01855,"SketchyI2IRetrieval":0.60199,"Caltech101":0.9692965154503617,"VisualSTS17Multilingual":0.264816,"BLINKIT2TMultiChoice":0.38714,"EuroSAT":0.6528888888888889,"MemotionT2IRetrieval":0.92184,"CIRRIT2IRetrieval":0.21178,"DTDZeroShot":0.5771276595744681,"Food101ZeroShot":0.8098217821782179,"Birdsnap":0.46245272825499734,"Flickr30kT2IRetrieval":0.85211,"Winoground":0.08250000327825546,"StanfordCars":0.8521576918293745,"VisualNewsT2IRetrieval":0.25425,"STS16VisualSTS":0.6643149980570415,"MNISTZeroShot":0.3774,"PatchCamelyonZeroShot":0.484466552734375,"HatefulMemesI2TRetrieval":0.54684,"SciMMIRI2TRetrieval":0.1614,"RParisMediumI2IRetrieval":0.02652,"UCF101ZeroShot":0.4545338500506295,"AROFlickrOrder":0.36079999804496765,"ImageNetDog15Clustering":0.5887680923091753,"Fashion200kT2IRetrieval":0.07833,"FashionIQIT2IRetrieval":0.06727,"TUBerlinT2IRetrieval":0.78621,"SUN397ZeroShot":0.7047816091954023,"FGVCAircraft":0.40186018601860185,"CUB200I2IRetrieval":0.51812,"XFlickr30kCoT2IRetrieval":0.2464025},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4587059729113977},"MIEB(lite) Task Mean":{"Task Mean":0.4058211495020947}},"trainedOnTasks":[],"experiments":null},{"rank":34,"model":{"name":"BAAI/bge-visualized-m3","url":"https://huggingface.co/BAAI/bge-visualized","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.873,"embeddingDim":1024,"maxTokens":8192.0,"releaseDate":"2024-06-06","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhou2024vista,\n  address = {Bangkok, Thailand},\n  author = {Zhou, Junjie and Liu, Zheng and Xiao, Shitao and Zhao, Bo and Xiong, Yongping},\n  booktitle = {Proceedings of the 62nd Annual Meeting of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)},\n  doi = {10.18653/v1/2024.acl-long.175},\n  editor = {Ku, Lun-Wei and Martins, Andre and Srikumar, Vivek},\n  month = {aug},\n  pages = {3185--3200},\n  publisher = {Association for Computational Linguistics},\n  title = {{VISTA}: Visualized Text Embedding For Universal Multi-Modal Retrieval},\n  url = {https://aclanthology.org/2024.acl-long.175/},\n  year = {2024},\n}","memoryUsageMb":4263.0,"license":null,"publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/datasets/JUNJIE99/VISTA_S2","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.873,"embeddingDim":1024,"maxTokens":8192,"meanTask":0.4375003202086537,"meanTaskType":0.4375003202086537,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.2844075555555555,"Compositionality":0.3227170311663832,"ImageClassification":0.6653973269983258,"DocumentUnderstanding":0.12378,"ZeroShotClassification":0.4648580420491169,"ImageClustering":0.7356514174290167,"VisionCentricQA":0.4384921074667014,"Any2AnyMultilingualRetrieval":0.46346267676767666,"VisualSTS(eng)":0.6415985446537616,"VisualSTS(multi)":0.23463850000000003},"scoresByTask":{"GTSRBZeroShot":0.3107680126682502,"CVBenchDistance":0.49333333333333335,"CVBenchRelation":0.5061538461538462,"OxfordPetsZeroShot":0.49741073862087765,"InfoSeekIT2ITRetrieval":0.09191,"VidoreTabfquadRetrieval":0.21611,"CIFAR10Clustering":0.8140662879442825,"VidoreDocVQARetrieval":0.08264,"VisualNewsI2TRetrieval":0.04888,"VidoreArxivQARetrieval":0.181,"STL10":0.9883750000000001,"CLEVRZeroShot":0.21033333333333334,"MemotionI2TRetrieval":0.58407,"PatchCamelyon":0.682928466796875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.12579,"TinyImageNetClustering":0.7171808856612835,"WebQAT2ITRetrieval":0.59381,"SUN397":0.7341333333333333,"METI2IRetrieval":0.32985,"STS13VisualSTS":0.5690700979639078,"CVBenchCount":0.07614213197969544,"ROxfordMediumI2IRetrieval":0.11773,"NIGHTSI2IRetrieval":0.23557,"Flickr30kI2TRetrieval":0.49044,"BLINKIT2TRetrieval":0.19798,"VidoreTatdqaRetrieval":0.01813,"ImageCoDe":0.11555,"ImageCoDeT2IRetrieval":0.05995,"Fashion200kI2TRetrieval":0.01961,"FGVCAircraftZeroShot":0.1188118811881188,"HatefulMemesT2IRetrieval":0.50754,"XM3600T2IRetrieval":0.41222916666666665,"UCF101":0.7622866174618701,"MSCOCOI2TRetrieval":0.25105,"OxfordPets":0.7398746252384847,"SugarCrepe":0.7566236257553101,"FER2013ZeroShot":0.37768180551685704,"OVENIT2TRetrieval":0.06448,"Caltech101ZeroShot":0.7312623274161736,"MSCOCOT2IRetrieval":0.39618,"RESISC45ZeroShot":0.5206349206349207,"Food101Classification":0.8037306930693069,"RParisEasyI2IRetrieval":0.09605,"CVBenchDepth":0.4533333333333333,"Imagenet1k":0.001,"VisualSTS-b-Eng":0.564802,"VOC2007":0.6734804076594354,"EDIST2ITRetrieval":0.2164,"ROxfordEasyI2IRetrieval":0.24273,"VidoreShiftProjectRetrieval":0.01387,"WITT2IRetrieval":0.4179363636363636,"RP2kI2IRetrieval":0.495,"ROxfordHardI2IRetrieval":0.06554,"DTD":0.6297872340425532,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.08411,"AROVisualAttribution":0.5842841267585754,"Country211":0.10371563981042656,"AROVisualRelation":0.4589129686355591,"BLINKIT2IRetrieval":0.32587,"CIFAR10":0.96266,"InfoSeekIT2TRetrieval":0.06584,"VidoreInfoVQARetrieval":0.32793,"BLINKIT2IMultiChoice":0.73881,"VidoreSyntheticDocQAEnergyRetrieval":0.08911,"CIFAR100":0.7814400000000001,"VisualSTS-b-Multilingual":0.253014,"EuroSATZeroShot":0.4111111111111111,"VisualSTS17Eng":0.791828,"RESISC45":0.8094920634920635,"BirdsnapZeroShot":0.18638573743922204,"CIFAR10ZeroShot":0.9471,"STL10ZeroShot":0.967625,"GTSRB":0.5014093428345209,"Country211ZeroShot":0.09175355450236967,"FORBI2IRetrieval":0.24211,"CIFAR100ZeroShot":0.6825,"OVENIT2ITRetrieval":0.06765,"MNIST":0.8083,"GLDv2I2TRetrieval":0.22992,"CIFAR100Clustering":0.7388886657033413,"RParisHardI2IRetrieval":0.03389,"SOPI2IRetrieval":0.45572,"VidoreSyntheticDocQAAIRetrieval":0.09911,"ImageNet10Clustering":0.9774386736778495,"StanfordCarsI2IRetrieval":0.82316,"StanfordCarsZeroShot":0.5634871284666086,"FER2013":0.44630816383393707,"OxfordFlowersClassification":0.9441176470588235,"STS12VisualSTS":0.6393406635300437,"Imagenet1kZeroShot":0.38478,"STS14VisualSTS":0.5818955288957028,"AROCocoOrder":0.22874850034713745,"VQA2IT2TRetrieval":0.01581,"CLEVRCountZeroShot":0.14013333333333333,"GLDv2I2IRetrieval":0.16351,"RenderedSST2":0.5700164744645799,"STS15VisualSTS":0.7093799073077189,"SciMMIRT2IRetrieval":0.07532,"WebQAT2TRetrieval":0.68575,"VizWizIT2TRetrieval":0.07336,"SketchyI2IRetrieval":0.72543,"Caltech101":0.8834976988823142,"VisualSTS17Multilingual":0.216263,"BLINKIT2TMultiChoice":0.36318,"EuroSAT":0.7777037037037038,"MemotionT2IRetrieval":0.79971,"CIRRIT2IRetrieval":0.15457,"DTDZeroShot":0.33404255319148934,"Food101ZeroShot":0.5513267326732674,"Birdsnap":0.4024851431658563,"Flickr30kT2IRetrieval":0.68995,"Winoground":0.027499999850988388,"StanfordCars":0.8133565476930731,"VisualNewsT2IRetrieval":0.08925,"STS16VisualSTS":0.6348736148789584,"MNISTZeroShot":0.5794,"PatchCamelyonZeroShot":0.513641357421875,"HatefulMemesI2TRetrieval":0.3731,"SciMMIRI2TRetrieval":0.04288,"RParisMediumI2IRetrieval":0.02571,"UCF101ZeroShot":0.43849448238867966,"AROFlickrOrder":0.08739999681711197,"ImageNetDog15Clustering":0.4306825741583267,"Fashion200kT2IRetrieval":0.01448,"FashionIQIT2IRetrieval":0.05313,"TUBerlinT2IRetrieval":0.82817,"SUN397ZeroShot":0.5630344827586207,"FGVCAircraft":0.3886588658865886,"CUB200I2IRetrieval":0.63928,"XFlickr30kCoT2IRetrieval":0.5602225},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4194082793902333},"MIEB(lite) Task Mean":{"Task Mean":0.3690525584867433}},"trainedOnTasks":[],"experiments":null},{"rank":35,"model":{"name":"Salesforce/blip-itm-base-flickr","url":"https://huggingface.co/Salesforce/blip-itm-base-flickr","zeroShotPct":100,"activeParamsB":0.2,"totalParamsB":0.224,"embeddingDim":768,"maxTokens":512.0,"releaseDate":"2023-08-01","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{https://doi.org/10.48550/arxiv.2201.12086,\n  archiveprefix = {arXiv},\n  author = {Junnan Li and Dongxu Li and Caiming Xiong and Steven C. H. Hoi},\n  booktitle = {ICML},\n  eprint = {2201.12086},\n  pages = {12888-12900},\n  title = {BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation},\n  url = {https://proceedings.mlr.press/v162/li22n.html},\n  year = {2022},\n}","memoryUsageMb":942.0,"license":"bsd-3-clause","publicTrainingCode":"https://github.com/salesforce/BLIP","publicTrainingData":"https://github.com/salesforce/BLIP","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.2,"totalParamsB":0.224,"embeddingDim":768,"maxTokens":512,"meanTask":0.43020371283062736,"meanTaskType":0.43020371283062736,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.28215155555555554,"Compositionality":0.42735602410793305,"ImageClassification":0.6480308163625111,"DocumentUnderstanding":0.149638,"ZeroShotClassification":0.4414049113101523,"ImageClustering":0.672718509239628,"VisionCentricQA":0.5235230359234674,"Any2AnyMultilingualRetrieval":0.13444229797979798,"VisualSTS(eng)":0.6424714778272281,"VisualSTS(multi)":0.3803005},"scoresByTask":{"GTSRBZeroShot":0.3254156769596199,"CVBenchDistance":0.5033333333333333,"CVBenchRelation":0.5307692307692308,"OxfordPetsZeroShot":0.621967838648133,"InfoSeekIT2ITRetrieval":0.05264,"VidoreTabfquadRetrieval":0.19325,"CIFAR10Clustering":0.6393910164281965,"VidoreDocVQARetrieval":0.10811,"VisualNewsI2TRetrieval":0.09655,"VidoreArxivQARetrieval":0.17055,"STL10":0.96885,"CLEVRZeroShot":0.12206666666666667,"MemotionI2TRetrieval":0.51623,"PatchCamelyon":0.639825439453125,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.20395,"TinyImageNetClustering":0.5506789264657086,"WebQAT2ITRetrieval":0.41999,"SUN397":0.6746942528735633,"METI2IRetrieval":0.4039,"STS13VisualSTS":0.5444726702809044,"CVBenchCount":0.6065989847715736,"ROxfordMediumI2IRetrieval":0.09202,"NIGHTSI2IRetrieval":0.24276,"Flickr30kI2TRetrieval":0.84673,"BLINKIT2TRetrieval":0.09079,"VidoreTatdqaRetrieval":0.02708,"ImageCoDe":0.12772,"ImageCoDeT2IRetrieval":0.10078,"Fashion200kI2TRetrieval":0.07309,"FGVCAircraftZeroShot":0.0594059405940594,"HatefulMemesT2IRetrieval":0.3983,"XM3600T2IRetrieval":0.07846666666666667,"UCF101":0.8013926697665879,"MSCOCOI2TRetrieval":0.54578,"OxfordPets":0.7688743526846553,"SugarCrepe":0.8688589930534363,"FER2013ZeroShot":0.2984118138757314,"OVENIT2TRetrieval":0.00465,"Caltech101ZeroShot":0.693129520052597,"MSCOCOT2IRetrieval":0.61688,"RESISC45ZeroShot":0.4546031746031746,"Food101Classification":0.7450059405940592,"RParisEasyI2IRetrieval":0.10432,"CVBenchDepth":0.44666666666666666,"Imagenet1k":0.59252,"VisualSTS-b-Eng":0.629288,"VOC2007":0.6062350200328473,"EDIST2ITRetrieval":0.1618,"ROxfordEasyI2IRetrieval":0.17015,"VidoreShiftProjectRetrieval":0.045,"WITT2IRetrieval":0.14947272727272728,"RP2kI2IRetrieval":0.63882,"ROxfordHardI2IRetrieval":0.05525,"DTD":0.6671276595744682,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.15234,"AROVisualAttribution":0.7630096077919006,"Country211":0.08127962085308056,"AROVisualRelation":0.5372018218040466,"BLINKIT2IRetrieval":0.3209,"CIFAR10":0.8384600000000001,"InfoSeekIT2TRetrieval":0.01021,"VidoreInfoVQARetrieval":0.29649,"BLINKIT2IMultiChoice":0.66915,"VidoreSyntheticDocQAEnergyRetrieval":0.15228,"CIFAR100":0.5588,"VisualSTS-b-Multilingual":0.458321,"EuroSATZeroShot":0.34425925925925926,"VisualSTS17Eng":0.759667,"RESISC45":0.7851428571428571,"BirdsnapZeroShot":0.09994597514856834,"CIFAR10ZeroShot":0.7772,"STL10ZeroShot":0.97225,"GTSRB":0.5624386381631037,"Country211ZeroShot":0.06004739336492891,"FORBI2IRetrieval":0.52053,"CIFAR100ZeroShot":0.4722,"OVENIT2ITRetrieval":0.02487,"MNIST":0.83782,"GLDv2I2TRetrieval":0.24807,"CIFAR100Clustering":0.5889139810427171,"RParisHardI2IRetrieval":0.03745,"SOPI2IRetrieval":0.56846,"VidoreSyntheticDocQAAIRetrieval":0.14733,"ImageNet10Clustering":0.9245613313840318,"StanfordCarsI2IRetrieval":0.69469,"StanfordCarsZeroShot":0.5412262156448203,"FER2013":0.41925327389244915,"OxfordFlowersClassification":0.8588235294117649,"STS12VisualSTS":0.5924177972611824,"Imagenet1kZeroShot":0.48282,"STS14VisualSTS":0.5787307554278176,"AROCocoOrder":0.2956017553806305,"VQA2IT2TRetrieval":0.00726,"CLEVRCountZeroShot":0.1408,"GLDv2I2IRetrieval":0.11441,"RenderedSST2":0.5118066996155958,"STS15VisualSTS":0.7110314748557974,"SciMMIRT2IRetrieval":0.04654,"WebQAT2TRetrieval":0.2853,"VizWizIT2TRetrieval":0.03132,"SketchyI2IRetrieval":0.55605,"Caltech101":0.874293228139382,"VisualSTS17Multilingual":0.30228,"BLINKIT2TMultiChoice":0.38462,"EuroSAT":0.7880740740740741,"MemotionT2IRetrieval":0.59727,"CIRRIT2IRetrieval":0.16499,"DTDZeroShot":0.4723404255319149,"Food101ZeroShot":0.5957227722772277,"Birdsnap":0.25921123716909783,"Flickr30kT2IRetrieval":0.8762,"Winoground":0.07750000059604645,"StanfordCars":0.7275960701405298,"VisualNewsT2IRetrieval":0.08939,"STS16VisualSTS":0.6816926469648948,"MNISTZeroShot":0.5471,"PatchCamelyonZeroShot":0.500244140625,"HatefulMemesI2TRetrieval":0.35863,"SciMMIRI2TRetrieval":0.05467,"RParisMediumI2IRetrieval":0.02664,"UCF101ZeroShot":0.4482689874960916,"AROFlickrOrder":0.3215999901294708,"ImageNetDog15Clustering":0.6600472908774858,"Fashion200kT2IRetrieval":0.05499,"FashionIQIT2IRetrieval":0.05011,"TUBerlinT2IRetrieval":0.88961,"SUN397ZeroShot":0.6110804597701149,"FGVCAircraft":0.20096009600960096,"CUB200I2IRetrieval":0.43683,"XFlickr30kCoT2IRetrieval":0.1753875},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.42076571356315057},"MIEB(lite) Task Mean":{"Task Mean":0.3768902160173729}},"trainedOnTasks":[],"experiments":null},{"rank":36,"model":{"name":"BAAI/bge-visualized-base","url":"https://huggingface.co/BAAI/bge-visualized","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.196,"embeddingDim":768,"maxTokens":512.0,"releaseDate":"2024-06-06","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhou2024vista,\n  address = {Bangkok, Thailand},\n  author = {Zhou, Junjie and Liu, Zheng and Xiao, Shitao and Zhao, Bo and Xiong, Yongping},\n  booktitle = {Proceedings of the 62nd Annual Meeting of the {A}ssociation for {C}omputational {L}inguistics (Volume 1: Long Papers)},\n  doi = {10.18653/v1/2024.acl-long.175},\n  editor = {Ku, Lun-Wei and Martins, Andre and Srikumar, Vivek},\n  month = {aug},\n  pages = {3185--3200},\n  publisher = {Association for Computational Linguistics},\n  title = {{VISTA}: Visualized Text Embedding For Universal Multi-Modal Retrieval},\n  url = {https://aclanthology.org/2024.acl-long.175/},\n  year = {2024},\n}","memoryUsageMb":1631.0,"license":null,"publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/datasets/JUNJIE99/VISTA_S2","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.196,"embeddingDim":768,"maxTokens":512,"meanTask":0.41653266431699637,"meanTaskType":0.41653266431699637,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.2805371111111111,"Compositionality":0.3193216522182737,"ImageClassification":0.7039978994548,"DocumentUnderstanding":0.103362,"ZeroShotClassification":0.4860886983525337,"ImageClustering":0.7618607720094445,"VisionCentricQA":0.5242899776562974,"Any2AnyMultilingualRetrieval":0.12251347222222221,"VisualSTS(eng)":0.6158195601452815,"VisualSTS(multi)":0.2475355},"scoresByTask":{"GTSRBZeroShot":0.3234362628661916,"CVBenchDistance":0.5633333333333334,"CVBenchRelation":0.4815384615384615,"OxfordPetsZeroShot":0.5622785500136277,"InfoSeekIT2ITRetrieval":0.14577,"VidoreTabfquadRetrieval":0.11827,"CIFAR10Clustering":0.821623898479473,"VidoreDocVQARetrieval":0.07047,"VisualNewsI2TRetrieval":0.03519,"VidoreArxivQARetrieval":0.15204,"STL10":0.991925,"CLEVRZeroShot":0.15786666666666666,"MemotionI2TRetrieval":0.47268,"PatchCamelyon":0.65185546875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.09393,"TinyImageNetClustering":0.7327619670197832,"WebQAT2ITRetrieval":0.59801,"SUN397":0.7361655172413792,"METI2IRetrieval":0.34062,"STS13VisualSTS":0.5730548519126888,"CVBenchCount":0.5025380710659898,"ROxfordMediumI2IRetrieval":0.0853,"NIGHTSI2IRetrieval":0.22229,"Flickr30kI2TRetrieval":0.49451,"BLINKIT2TRetrieval":0.16015,"VidoreTatdqaRetrieval":0.01939,"ImageCoDe":0.1212,"ImageCoDeT2IRetrieval":0.09296,"Fashion200kI2TRetrieval":0.03674,"FGVCAircraftZeroShot":0.10531053105310531,"HatefulMemesT2IRetrieval":0.34614,"XM3600T2IRetrieval":0.060664166666666665,"UCF101":0.7776340964570826,"MSCOCOI2TRetrieval":0.31425,"OxfordPets":0.7820659580267103,"SugarCrepe":0.8359739184379578,"FER2013ZeroShot":0.35594873223739204,"OVENIT2TRetrieval":0.11676,"Caltech101ZeroShot":0.7769559500328731,"MSCOCOT2IRetrieval":0.50044,"RESISC45ZeroShot":0.5923809523809523,"Food101Classification":0.8412118811881187,"RParisEasyI2IRetrieval":0.10427,"CVBenchDepth":0.49,"Imagenet1k":0.648648,"VisualSTS-b-Eng":0.546617,"VOC2007":0.6888536086191103,"EDIST2ITRetrieval":0.23793,"ROxfordEasyI2IRetrieval":0.19879,"VidoreShiftProjectRetrieval":0.03018,"WITT2IRetrieval":0.16188,"RP2kI2IRetrieval":0.53768,"ROxfordHardI2IRetrieval":0.03788,"DTD":0.6825531914893617,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.06905,"AROVisualAttribution":0.5389244556427002,"Country211":0.10119431279620854,"AROVisualRelation":0.4656389653682709,"BLINKIT2IRetrieval":0.33831,"CIFAR10":0.97746,"InfoSeekIT2TRetrieval":0.11027,"VidoreInfoVQARetrieval":0.29635,"BLINKIT2IMultiChoice":0.73632,"VidoreSyntheticDocQAEnergyRetrieval":0.11053,"CIFAR100":0.8295999999999999,"VisualSTS-b-Multilingual":0.290172,"EuroSATZeroShot":0.5137037037037037,"VisualSTS17Eng":0.755206,"RESISC45":0.8309206349206348,"BirdsnapZeroShot":0.25715829281469477,"CIFAR10ZeroShot":0.9741,"STL10ZeroShot":0.981375,"GTSRB":0.5203800475059384,"Country211ZeroShot":0.08630331753554503,"FORBI2IRetrieval":0.21721,"CIFAR100ZeroShot":0.7685,"OVENIT2ITRetrieval":0.09669,"MNIST":0.81548,"GLDv2I2TRetrieval":0.2824,"CIFAR100Clustering":0.7779675259511241,"RParisHardI2IRetrieval":0.03612,"SOPI2IRetrieval":0.48307,"VidoreSyntheticDocQAAIRetrieval":0.07341,"ImageNet10Clustering":0.9832963462828915,"StanfordCarsI2IRetrieval":0.65042,"StanfordCarsZeroShot":0.43104091530904115,"FER2013":0.4465310671496239,"OxfordFlowersClassification":0.9525490196078431,"STS12VisualSTS":0.5534622370119494,"Imagenet1kZeroShot":0.50346,"STS14VisualSTS":0.5757253090530228,"AROCocoOrder":0.16441422700881958,"VQA2IT2TRetrieval":0.03349,"CLEVRCountZeroShot":0.1606,"GLDv2I2IRetrieval":0.1438,"RenderedSST2":0.5403624382207578,"STS15VisualSTS":0.6827319807694596,"SciMMIRT2IRetrieval":0.06838,"WebQAT2TRetrieval":0.6891,"VizWizIT2TRetrieval":0.14533,"SketchyI2IRetrieval":0.71312,"Caltech101":0.9059171597633136,"VisualSTS17Multilingual":0.204899,"BLINKIT2TMultiChoice":0.37201,"EuroSAT":0.8334074074074074,"MemotionT2IRetrieval":0.6435,"CIRRIT2IRetrieval":0.1544,"DTDZeroShot":0.425531914893617,"Food101ZeroShot":0.6372673267326733,"Birdsnap":0.45164775796866563,"Flickr30kT2IRetrieval":0.74117,"Winoground":0.04749999940395355,"StanfordCars":0.6628777515234423,"VisualNewsT2IRetrieval":0.0841,"STS16VisualSTS":0.6239395422698493,"MNISTZeroShot":0.458,"PatchCamelyonZeroShot":0.499969482421875,"HatefulMemesI2TRetrieval":0.26173,"SciMMIRI2TRetrieval":0.04031,"RParisMediumI2IRetrieval":0.02683,"UCF101ZeroShot":0.44839807120257247,"AROFlickrOrder":0.06159999966621399,"ImageNetDog15Clustering":0.49365412231395045,"Fashion200kT2IRetrieval":0.03133,"FashionIQIT2IRetrieval":0.04524,"TUBerlinT2IRetrieval":0.8816,"SUN397ZeroShot":0.6200919540229886,"FGVCAircraft":0.3590759075907591,"CUB200I2IRetrieval":0.62789,"XFlickr30kCoT2IRetrieval":0.14499625},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4306145285011501},"MIEB(lite) Task Mean":{"Task Mean":0.36863799044567286}},"trainedOnTasks":[],"experiments":null},{"rank":37,"model":{"name":"QuanSun/EVA02-CLIP-B-16","url":"https://huggingface.co/QuanSun/EVA-CLIP","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.149,"embeddingDim":512,"maxTokens":77.0,"releaseDate":"2023-04-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@article{EVA-CLIP,\n      title={EVA-CLIP: Improved Training Techniques for CLIP at Scale},\n      author={Sun, Quan and Fang, Yuxin and Wu, Ledell and Wang, Xinlong and Cao, Yue},\n      journal={arXiv preprint arXiv:2303.15389},\n      year={2023}\n}","memoryUsageMb":568.0,"license":"mit","publicTrainingCode":"https://github.com/baaivision/EVA/tree/master/EVA-CLIP","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.149,"embeddingDim":512,"maxTokens":77,"meanTask":0.414500538253839,"meanTaskType":0.414500538253839,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.31061422222222224,"Compositionality":0.4455550370788574,"ImageClassification":0.5594480722046128,"DocumentUnderstanding":0.094191,"ZeroShotClassification":0.5999898158182904,"ImageClustering":0.8254993190336197,"VisionCentricQA":0.45343470888107945,"Any2AnyMultilingualRetrieval":0.2011756186868687,"VisualSTS(eng)":0.43108008861283864,"VisualSTS(multi)":0.22401749999999998},"scoresByTask":{"GTSRBZeroShot":0.42351543942992875,"CVBenchDistance":0.53,"CVBenchRelation":0.49538461538461537,"OxfordPetsZeroShot":0.92259471245571,"InfoSeekIT2ITRetrieval":0.0,"VidoreTabfquadRetrieval":0.10876,"CIFAR10Clustering":0.8923495082634476,"VidoreDocVQARetrieval":0.05844,"VisualNewsI2TRetrieval":0.29276,"VidoreArxivQARetrieval":0.16224,"STL10":0.99,"CLEVRZeroShot":0.15686666666666665,"MemotionI2TRetrieval":0.7026,"PatchCamelyon":0.655706787109375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.10353,"TinyImageNetClustering":0.7596262211425469,"WebQAT2ITRetrieval":0.36211,"SUN397":0.004551724137931035,"METI2IRetrieval":0.37527,"STS13VisualSTS":0.3657436270234431,"CVBenchCount":0.3680203045685279,"ROxfordMediumI2IRetrieval":0.13164,"NIGHTSI2IRetrieval":0.23223,"Flickr30kI2TRetrieval":0.80306,"BLINKIT2TRetrieval":0.15511,"VidoreTatdqaRetrieval":0.01304,"ImageCoDe":0.13467,"ImageCoDeT2IRetrieval":0.11034,"Fashion200kI2TRetrieval":0.07915,"FGVCAircraftZeroShot":0.24812481248124812,"HatefulMemesT2IRetrieval":0.48476,"XM3600T2IRetrieval":0.10685083333333333,"UCF101":0.7288169334874688,"MSCOCOI2TRetrieval":0.51768,"OxfordPets":0.8752248569092395,"SugarCrepe":0.8144055604934692,"FER2013ZeroShot":0.4831429367511842,"OVENIT2TRetrieval":0.00057,"Caltech101ZeroShot":0.8382642998027613,"MSCOCOT2IRetrieval":0.57286,"RESISC45ZeroShot":0.6311111111111111,"Food101Classification":0.8075960396039605,"RParisEasyI2IRetrieval":0.11212,"CVBenchDepth":0.5333333333333333,"Imagenet1k":0.001,"VisualSTS-b-Eng":0.315756,"VOC2007":0.45063545682423795,"EDIST2ITRetrieval":0.15285,"ROxfordEasyI2IRetrieval":0.24823,"VidoreShiftProjectRetrieval":0.01431,"WITT2IRetrieval":0.2692972727272727,"RP2kI2IRetrieval":0.59988,"ROxfordHardI2IRetrieval":0.04909,"DTD":0.613404255319149,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.05262,"AROVisualAttribution":0.617886483669281,"Country211":0.004739336492890996,"AROVisualRelation":0.5391653180122375,"BLINKIT2IRetrieval":0.00249,"CIFAR10":0.9811799999999999,"InfoSeekIT2TRetrieval":0.0,"VidoreInfoVQARetrieval":0.25125,"BLINKIT2IMultiChoice":0.40547,"VidoreSyntheticDocQAEnergyRetrieval":0.09579,"CIFAR100":0.8281799999999999,"VisualSTS-b-Multilingual":0.282408,"EuroSATZeroShot":0.5846296296296296,"VisualSTS17Eng":0.541491,"RESISC45":0.7618730158730159,"BirdsnapZeroShot":0.5207995678011885,"CIFAR10ZeroShot":0.9829,"STL10ZeroShot":0.994625,"GTSRB":0.38387965162311954,"Country211ZeroShot":0.20454976303317535,"FORBI2IRetrieval":0.33268,"CIFAR100ZeroShot":0.8701,"OVENIT2ITRetrieval":0.00164,"MNIST":0.7646,"GLDv2I2TRetrieval":0.62525,"CIFAR100Clustering":0.8351263574604518,"RParisHardI2IRetrieval":0.04306,"SOPI2IRetrieval":0.55388,"VidoreSyntheticDocQAAIRetrieval":0.08193,"ImageNet10Clustering":0.8922388040073979,"StanfordCarsI2IRetrieval":0.82179,"StanfordCarsZeroShot":0.7882104215893545,"FER2013":0.43697408748955135,"OxfordFlowersClassification":0.9566666666666667,"STS12VisualSTS":0.4024512338189616,"Imagenet1kZeroShot":0.73194,"STS14VisualSTS":0.39174259166701003,"AROCocoOrder":0.40535786747932434,"VQA2IT2TRetrieval":0.00437,"CLEVRCountZeroShot":0.21093333333333333,"GLDv2I2IRetrieval":0.19032,"RenderedSST2":0.5392641405820977,"STS15VisualSTS":0.5118212130294997,"SciMMIRT2IRetrieval":0.06827,"WebQAT2TRetrieval":0.26897,"VizWizIT2TRetrieval":0.01702,"SketchyI2IRetrieval":0.60753,"Caltech101":0.8875739644970413,"VisualSTS17Multilingual":0.165627,"BLINKIT2TMultiChoice":0.3884,"EuroSAT":0.7712222222222221,"MemotionT2IRetrieval":0.76799,"CIRRIT2IRetrieval":0.21171,"DTDZeroShot":0.5047872340425532,"Food101ZeroShot":0.8836039603960396,"Birdsnap":0.0021609940572663426,"Flickr30kT2IRetrieval":0.82608,"Winoground":0.09000000357627869,"StanfordCars":0.005471956224350205,"VisualNewsT2IRetrieval":0.27547,"STS16VisualSTS":0.4885549547509559,"MNISTZeroShot":0.4295,"PatchCamelyonZeroShot":0.50048828125,"HatefulMemesI2TRetrieval":0.46958,"SciMMIRI2TRetrieval":0.07567,"RParisMediumI2IRetrieval":0.02718,"UCF101ZeroShot":0.6465960626600996,"AROFlickrOrder":0.5174000263214111,"ImageNetDog15Clustering":0.7481557042942548,"Fashion200kT2IRetrieval":0.06601,"FashionIQIT2IRetrieval":0.06025,"TUBerlinT2IRetrieval":0.91354,"SUN397ZeroShot":0.7032183908045977,"FGVCAircraft":0.3963996399639964,"CUB200I2IRetrieval":0.76458,"XFlickr30kCoT2IRetrieval":0.22737875},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4320937926449489},"MIEB(lite) Task Mean":{"Task Mean":0.38944737804488266}},"trainedOnTasks":[],"experiments":null},{"rank":38,"model":{"name":"openai/clip-vit-base-patch32","url":"https://huggingface.co/openai/clip-vit-base-patch32","zeroShotPct":-1,"activeParamsB":0.126,"totalParamsB":0.151,"embeddingDim":512,"maxTokens":77.0,"releaseDate":"2021-02-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{radford2021learning,\n  archiveprefix = {arXiv},\n  author = {Radford, Alec and Kim, Jong Wook and Hallacy, Chris and Ramesh, Aditya and Goh, Gabriel and Agarwal, Sandhini and Sastry, Girish and Askell, Amanda and Mishkin, Pamela and Clark, Jack and others},\n  booktitle = {Proceedings of the 38th International Conference on Machine Learning},\n  eprint = {2103.00020},\n  pages = {8748--8763},\n  publisher = {PMLR},\n  series = {Proceedings of Machine Learning Research},\n  title = {Learning Transferable Visual Models From Natural Language Supervision},\n  url = {https://proceedings.mlr.press/v139/radford21a.html},\n  volume = {139},\n  year = {2021},\n}","memoryUsageMb":576.0,"license":null,"publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":0.126,"totalParamsB":0.151,"embeddingDim":512,"maxTokens":77,"meanTask":0.4142966329395482,"meanTaskType":0.4142966329395482,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.27488466666666667,"Compositionality":0.4499178541128976,"ImageClassification":0.7112001009940266,"DocumentUnderstanding":0.132337,"ZeroShotClassification":0.5391176776984367,"ImageClustering":0.6790190750482934,"VisionCentricQA":0.4273197609440757,"Any2AnyMultilingualRetrieval":0.16725033249158247,"VisualSTS(eng)":0.5439458614395025,"VisualSTS(multi)":0.217974},"scoresByTask":{"GTSRBZeroShot":0.34418052256532067,"CVBenchDistance":0.46,"CVBenchRelation":0.4846153846153846,"OxfordPetsZeroShot":0.874897792313982,"InfoSeekIT2ITRetrieval":0.03567,"VidoreTabfquadRetrieval":0.14089,"CIFAR10Clustering":0.7384717195071173,"VidoreDocVQARetrieval":0.0948,"VisualNewsI2TRetrieval":0.24839,"VidoreArxivQARetrieval":0.17109,"STL10":0.9773,"CLEVRZeroShot":0.1634,"MemotionI2TRetrieval":0.65148,"PatchCamelyon":0.7096435546875,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.13141,"TinyImageNetClustering":0.6033608940967555,"WebQAT2ITRetrieval":0.23939,"SUN397":0.6952735632183907,"METI2IRetrieval":0.35843,"STS13VisualSTS":0.5250285299601588,"CVBenchCount":0.06598984771573604,"ROxfordMediumI2IRetrieval":0.11215,"NIGHTSI2IRetrieval":0.23051,"Flickr30kI2TRetrieval":0.7149,"BLINKIT2TRetrieval":0.06683,"VidoreTatdqaRetrieval":0.0186,"ImageCoDe":0.13206,"ImageCoDeT2IRetrieval":0.08905,"Fashion200kI2TRetrieval":0.0226,"FGVCAircraftZeroShot":0.18871887188718872,"HatefulMemesT2IRetrieval":0.46492,"XM3600T2IRetrieval":0.07602111111111111,"UCF101":0.7877826574606079,"MSCOCOI2TRetrieval":0.444,"OxfordPets":0.8051240119923685,"SugarCrepe":0.7660764455795288,"FER2013ZeroShot":0.4352187238785177,"OVENIT2TRetrieval":0.00077,"Caltech101ZeroShot":0.7855029585798816,"MSCOCOT2IRetrieval":0.47651,"RESISC45ZeroShot":0.533968253968254,"Food101Classification":0.7944871287128713,"RParisEasyI2IRetrieval":0.09219,"CVBenchDepth":0.4533333333333333,"Imagenet1k":0.611572,"VisualSTS-b-Eng":0.471708,"VOC2007":0.9007263984769359,"EDIST2ITRetrieval":0.24481,"ROxfordEasyI2IRetrieval":0.21546,"VidoreShiftProjectRetrieval":0.01,"WITT2IRetrieval":0.2531636363636363,"RP2kI2IRetrieval":0.55887,"ROxfordHardI2IRetrieval":0.04539,"DTD":0.6660638297872341,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.09141,"AROVisualAttribution":0.6140252947807312,"Country211":0.14733649289099526,"AROVisualRelation":0.5176087021827698,"BLINKIT2IRetrieval":0.31592,"CIFAR10":0.8985199999999999,"InfoSeekIT2TRetrieval":0.00157,"VidoreInfoVQARetrieval":0.37145,"BLINKIT2IMultiChoice":0.70149,"VidoreSyntheticDocQAEnergyRetrieval":0.1831,"CIFAR100":0.67408,"VisualSTS-b-Multilingual":0.21419,"EuroSATZeroShot":0.4974074074074074,"VisualSTS17Eng":0.710059,"RESISC45":0.8281904761904763,"BirdsnapZeroShot":0.4057266342517558,"CIFAR10ZeroShot":0.8831,"STL10ZeroShot":0.973,"GTSRB":0.7011718131433096,"Country211ZeroShot":0.15962085308056873,"FORBI2IRetrieval":0.29962,"CIFAR100ZeroShot":0.6165,"OVENIT2ITRetrieval":0.03439,"MNIST":0.89474,"GLDv2I2TRetrieval":0.59091,"CIFAR100Clustering":0.58070288403175,"RParisHardI2IRetrieval":0.03734,"SOPI2IRetrieval":0.48561,"VidoreSyntheticDocQAAIRetrieval":0.11062,"ImageNet10Clustering":0.9313661478558977,"StanfordCarsI2IRetrieval":0.6243,"StanfordCarsZeroShot":0.5861211292127845,"FER2013":0.42883811646698244,"OxfordFlowersClassification":0.9482352941176471,"STS12VisualSTS":0.5381414093316044,"Imagenet1kZeroShot":0.58844,"STS14VisualSTS":0.43691385357533835,"AROCocoOrder":0.4636545479297638,"VQA2IT2TRetrieval":0.0082,"CLEVRCountZeroShot":0.232,"GLDv2I2IRetrieval":0.15231,"RenderedSST2":0.5848434925864909,"STS15VisualSTS":0.5956435250051597,"SciMMIRT2IRetrieval":0.08144,"WebQAT2TRetrieval":0.21061,"VizWizIT2TRetrieval":0.03809,"SketchyI2IRetrieval":0.3956,"Caltech101":0.9150558842866534,"VisualSTS17Multilingual":0.221758,"BLINKIT2TMultiChoice":0.39849,"EuroSAT":0.6723333333333334,"MemotionT2IRetrieval":0.75608,"CIRRIT2IRetrieval":0.09918,"DTDZeroShot":0.41914893617021276,"Food101ZeroShot":0.827089108910891,"Birdsnap":0.47088060507833607,"Flickr30kT2IRetrieval":0.73724,"Winoground":0.09000000357627869,"StanfordCars":0.7377689342121627,"VisualNewsT2IRetrieval":0.25477,"STS16VisualSTS":0.5301267122042563,"MNISTZeroShot":0.4899,"PatchCamelyonZeroShot":0.618804931640625,"HatefulMemesI2TRetrieval":0.43219,"SciMMIRI2TRetrieval":0.09715,"RParisMediumI2IRetrieval":0.02709,"UCF101ZeroShot":0.58149628095499,"AROFlickrOrder":0.5659999847412109,"ImageNetDog15Clustering":0.5411937297499465,"Fashion200kT2IRetrieval":0.01576,"FashionIQIT2IRetrieval":0.02213,"TUBerlinT2IRetrieval":0.81945,"SUN397ZeroShot":0.6106206896551725,"FGVCAircraft":0.3812781278127813,"CUB200I2IRetrieval":0.52054,"XFlickr30kCoT2IRetrieval":0.17256625},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4372427900696428},"MIEB(lite) Task Mean":{"Task Mean":0.3843335309743042}},"trainedOnTasks":[],"experiments":null},{"rank":39,"model":{"name":"Salesforce/blip2-opt-2.7b","url":"https://huggingface.co/Salesforce/blip2-opt-2.7b","zeroShotPct":100,"activeParamsB":3.611,"totalParamsB":3.74,"embeddingDim":768,"maxTokens":null,"releaseDate":"2024-03-22","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{li2023blip2,\n    title={{BLIP-2:} Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models},\n    author={Junnan Li and Dongxu Li and Silvio Savarese and Steven Hoi},\n    year={2023},\n    booktitle={ICML},\n}","memoryUsageMb":14285.0,"license":"mit","publicTrainingCode":"https://github.com/salesforce/LAVIS/tree/main/projects/blip2","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["blip2"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":3.611,"totalParamsB":3.74,"embeddingDim":768,"maxTokens":null,"meanTask":0.39728772994332945,"meanTaskType":0.39728772994332945,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.2669202222222223,"Compositionality":0.30964799721734865,"ImageClassification":0.7294877766221862,"DocumentUnderstanding":0.12302900000000001,"ZeroShotClassification":0.4884819082028764,"ImageClustering":0.740053507378038,"VisionCentricQA":0.5314165929107554,"Any2AnyMultilingualRetrieval":0.1386326304713805,"VisualSTS(eng)":0.4275491644084871,"VisualSTS(multi)":0.2176585},"scoresByTask":{"GTSRBZeroShot":0.2705463182897862,"CVBenchDistance":0.5016666666666667,"CVBenchRelation":0.47692307692307695,"OxfordPetsZeroShot":0.23684927773235215,"InfoSeekIT2ITRetrieval":0.10919,"VidoreTabfquadRetrieval":0.12527,"CIFAR10Clustering":0.9666909704618464,"VidoreDocVQARetrieval":0.05743,"VisualNewsI2TRetrieval":0.06707,"VidoreArxivQARetrieval":0.11251,"STL10":0.985975,"CLEVRZeroShot":0.24713333333333334,"MemotionI2TRetrieval":0.56773,"PatchCamelyon":0.73167724609375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.14419,"TinyImageNetClustering":0.7386062785364768,"WebQAT2ITRetrieval":0.21116,"SUN397":0.7563954022988505,"METI2IRetrieval":0.3963,"STS13VisualSTS":0.38721652487710545,"CVBenchCount":0.46954314720812185,"ROxfordMediumI2IRetrieval":0.07108,"NIGHTSI2IRetrieval":0.20235,"Flickr30kI2TRetrieval":0.69828,"BLINKIT2TRetrieval":0.19546,"VidoreTatdqaRetrieval":0.02309,"ImageCoDe":0.11903,"ImageCoDeT2IRetrieval":0.10209,"Fashion200kI2TRetrieval":0.06403,"FGVCAircraftZeroShot":0.0318031803180318,"HatefulMemesT2IRetrieval":0.44707,"XM3600T2IRetrieval":0.08015027777777778,"UCF101":0.8833410305469421,"MSCOCOI2TRetrieval":0.48729,"OxfordPets":0.43793949304987734,"SugarCrepe":0.7249367833137512,"FER2013ZeroShot":0.5110058512120368,"OVENIT2TRetrieval":0.05563,"Caltech101ZeroShot":0.7306048652202498,"MSCOCOT2IRetrieval":0.48626,"RESISC45ZeroShot":0.5557142857142857,"Food101Classification":0.882360396039604,"RParisEasyI2IRetrieval":0.09303,"CVBenchDepth":0.5766666666666667,"Imagenet1k":0.5872440000000001,"VisualSTS-b-Eng":0.336786,"VOC2007":0.9061547734488091,"EDIST2ITRetrieval":0.12724,"ROxfordEasyI2IRetrieval":0.12397,"VidoreShiftProjectRetrieval":0.04123,"WITT2IRetrieval":0.18082636363636365,"RP2kI2IRetrieval":0.59265,"ROxfordHardI2IRetrieval":0.03331,"DTD":0.742872340425532,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.10659,"AROVisualAttribution":0.6720815300941467,"Country211":0.1074691943127962,"AROVisualRelation":0.49822449684143066,"BLINKIT2IRetrieval":0.37065,"CIFAR10":0.9861600000000001,"InfoSeekIT2TRetrieval":0.02409,"VidoreInfoVQARetrieval":0.30915,"BLINKIT2IMultiChoice":0.74378,"VidoreSyntheticDocQAEnergyRetrieval":0.15047,"CIFAR100":0.8751200000000001,"VisualSTS-b-Multilingual":0.216806,"EuroSATZeroShot":0.6568518518518518,"VisualSTS17Eng":0.584946,"RESISC45":0.8859047619047619,"BirdsnapZeroShot":0.032955159373311727,"CIFAR10ZeroShot":0.9811,"STL10ZeroShot":0.979875,"GTSRB":0.681931908155186,"Country211ZeroShot":0.09251184834123223,"FORBI2IRetrieval":0.28491,"CIFAR100ZeroShot":0.8075,"OVENIT2ITRetrieval":0.08006,"MNIST":0.92856,"GLDv2I2TRetrieval":0.31286,"CIFAR100Clustering":0.814574240087341,"RParisHardI2IRetrieval":0.03121,"SOPI2IRetrieval":0.50983,"VidoreSyntheticDocQAAIRetrieval":0.16036,"ImageNet10Clustering":0.9776579055691955,"StanfordCarsI2IRetrieval":0.68574,"StanfordCarsZeroShot":0.628901877875886,"FER2013":0.5236556143772638,"OxfordFlowersClassification":0.966078431372549,"STS12VisualSTS":0.3880717321117322,"Imagenet1kZeroShot":0.39038,"STS14VisualSTS":0.355671906358516,"AROCocoOrder":0.04206317290663719,"VQA2IT2TRetrieval":0.00921,"CLEVRCountZeroShot":0.1826,"GLDv2I2IRetrieval":0.09802,"RenderedSST2":0.5178473366282262,"STS15VisualSTS":0.5181232525189658,"SciMMIRT2IRetrieval":0.02751,"WebQAT2TRetrieval":0.19002,"VizWizIT2TRetrieval":0.10404,"SketchyI2IRetrieval":0.80014,"Caltech101":0.9181788297172913,"VisualSTS17Multilingual":0.218511,"BLINKIT2TMultiChoice":0.41992,"EuroSAT":0.7876666666666667,"MemotionT2IRetrieval":0.56393,"CIRRIT2IRetrieval":0.07362,"DTDZeroShot":0.4627659574468085,"Food101ZeroShot":0.7521980198019802,"Birdsnap":0.30329551593733123,"Flickr30kT2IRetrieval":0.7268,"Winoground":0.05999999865889549,"StanfordCars":0.8280562119139411,"VisualNewsT2IRetrieval":0.10566,"STS16VisualSTS":0.42202873499309024,"MNISTZeroShot":0.5049,"PatchCamelyonZeroShot":0.51129150390625,"HatefulMemesI2TRetrieval":0.42928,"SciMMIRI2TRetrieval":0.03716,"RParisMediumI2IRetrieval":0.02498,"UCF101ZeroShot":0.5051045434596154,"AROFlickrOrder":0.05119999870657921,"ImageNetDog15Clustering":0.20273814223533038,"Fashion200kT2IRetrieval":0.04204,"FashionIQIT2IRetrieval":0.01633,"TUBerlinT2IRetrieval":0.9343,"SUN397ZeroShot":0.6446436781609195,"FGVCAircraft":0.3426942694269427,"CUB200I2IRetrieval":0.39783,"XFlickr30kCoT2IRetrieval":0.15492124999999998},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4205972976007186},"MIEB(lite) Task Mean":{"Task Mean":0.34882811102829764}},"trainedOnTasks":[],"experiments":null},{"rank":40,"model":{"name":"Salesforce/blip2-opt-6.7b-coco","url":"https://huggingface.co/Salesforce/blip2-opt-6.7b-coco","zeroShotPct":100,"activeParamsB":7.544,"totalParamsB":7.75,"embeddingDim":768,"maxTokens":null,"releaseDate":"2024-03-31","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{li2023blip2,\n    title={{BLIP-2:} Bootstrapping Language-Image Pre-training with Frozen Image Encoders and Large Language Models},\n    author={Junnan Li and Dongxu Li and Silvio Savarese and Steven Hoi},\n    year={2023},\n    booktitle={ICML},\n}","memoryUsageMb":29577.0,"license":"mit","publicTrainingCode":"https://github.com/salesforce/LAVIS/tree/main/projects/blip2","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["blip2"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":7.544,"totalParamsB":7.75,"embeddingDim":768,"maxTokens":null,"meanTask":0.39648141761365224,"meanTaskType":0.39648141761365224,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.24519444444444444,"Compositionality":0.3408887854884352,"ImageClassification":0.738947499110321,"DocumentUnderstanding":0.15592799999999998,"ZeroShotClassification":0.4619587636014782,"ImageClustering":0.6783719466882573,"VisionCentricQA":0.5271689632955877,"Any2AnyMultilingualRetrieval":0.13049015572390574,"VisualSTS(eng)":0.4571606177840925,"VisualSTS(multi)":0.228705},"scoresByTask":{"GTSRBZeroShot":0.4033254156769596,"CVBenchDistance":0.5233333333333333,"CVBenchRelation":0.4876923076923077,"OxfordPetsZeroShot":0.2992641046606705,"InfoSeekIT2ITRetrieval":0.06065,"VidoreTabfquadRetrieval":0.17595,"CIFAR10Clustering":0.9037162513248874,"VidoreDocVQARetrieval":0.10367,"VisualNewsI2TRetrieval":0.0413,"VidoreArxivQARetrieval":0.14675,"STL10":0.993825,"CLEVRZeroShot":0.25033333333333335,"MemotionI2TRetrieval":0.41073,"PatchCamelyon":0.716802978515625,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.23954,"TinyImageNetClustering":0.7091773309102821,"WebQAT2ITRetrieval":0.19135,"SUN397":0.7234114942528735,"METI2IRetrieval":0.38505,"STS13VisualSTS":0.38111500452195723,"CVBenchCount":0.44543147208121825,"ROxfordMediumI2IRetrieval":0.06802,"NIGHTSI2IRetrieval":0.18696,"Flickr30kI2TRetrieval":0.76533,"BLINKIT2TRetrieval":0.2623,"VidoreTatdqaRetrieval":0.03796,"ImageCoDe":0.13727,"ImageCoDeT2IRetrieval":0.11034,"Fashion200kI2TRetrieval":0.03194,"FGVCAircraftZeroShot":0.05130513051305131,"HatefulMemesT2IRetrieval":0.28544,"XM3600T2IRetrieval":0.07833944444444445,"UCF101":0.8754663507462472,"MSCOCOI2TRetrieval":0.557,"OxfordPets":0.6692286726628509,"SugarCrepe":0.8430302143096924,"FER2013ZeroShot":0.5188074672610755,"OVENIT2TRetrieval":0.06507,"Caltech101ZeroShot":0.7264957264957265,"MSCOCOT2IRetrieval":0.63199,"RESISC45ZeroShot":0.5287301587301587,"Food101Classification":0.8412673267326733,"RParisEasyI2IRetrieval":0.09708,"CVBenchDepth":0.5966666666666667,"Imagenet1k":0.655092,"VisualSTS-b-Eng":0.391815,"VOC2007":0.9441448254215146,"EDIST2ITRetrieval":0.09225,"ROxfordEasyI2IRetrieval":0.1143,"VidoreShiftProjectRetrieval":0.04079,"WITT2IRetrieval":0.13431727272727273,"RP2kI2IRetrieval":0.58173,"ROxfordHardI2IRetrieval":0.04899,"DTD":0.7259574468085106,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.17472,"AROVisualAttribution":0.6832475066184998,"Country211":0.08720379146919431,"AROVisualRelation":0.5154781341552734,"BLINKIT2IRetrieval":0.36318,"CIFAR10":0.97696,"InfoSeekIT2TRetrieval":0.02412,"VidoreInfoVQARetrieval":0.31972,"BLINKIT2IMultiChoice":0.71393,"VidoreSyntheticDocQAEnergyRetrieval":0.18234,"CIFAR100":0.82256,"VisualSTS-b-Multilingual":0.21901,"EuroSATZeroShot":0.5231481481481481,"VisualSTS17Eng":0.626134,"RESISC45":0.876,"BirdsnapZeroShot":0.02701242571582928,"CIFAR10ZeroShot":0.9737,"STL10ZeroShot":0.99025,"GTSRB":0.798701504354711,"Country211ZeroShot":0.045213270142180094,"FORBI2IRetrieval":0.28853,"CIFAR100ZeroShot":0.7358,"OVENIT2ITRetrieval":0.05802,"MNIST":0.9343999999999999,"GLDv2I2TRetrieval":0.11672,"CIFAR100Clustering":0.7580773437901018,"RParisHardI2IRetrieval":0.02369,"SOPI2IRetrieval":0.4608,"VidoreSyntheticDocQAAIRetrieval":0.13784,"ImageNet10Clustering":0.9312290730065116,"StanfordCarsI2IRetrieval":0.49198,"StanfordCarsZeroShot":0.29710235045392364,"FER2013":0.5044859292281972,"OxfordFlowersClassification":0.9405882352941177,"STS12VisualSTS":0.41360933277627304,"Imagenet1kZeroShot":0.44196,"STS14VisualSTS":0.3832664572778191,"AROCocoOrder":0.0608956404030323,"VQA2IT2TRetrieval":0.02198,"CLEVRCountZeroShot":0.1596,"GLDv2I2IRetrieval":0.05214,"RenderedSST2":0.5063152114222954,"STS15VisualSTS":0.5435934052155486,"SciMMIRT2IRetrieval":0.02682,"WebQAT2TRetrieval":0.22392,"VizWizIT2TRetrieval":0.10853,"SketchyI2IRetrieval":0.78115,"Caltech101":0.9025312294543063,"VisualSTS17Multilingual":0.2384,"BLINKIT2TMultiChoice":0.39596,"EuroSAT":0.7688518518518519,"MemotionT2IRetrieval":0.42725,"CIRRIT2IRetrieval":0.07349,"DTDZeroShot":0.44840425531914896,"Food101ZeroShot":0.6185346534653465,"Birdsnap":0.4123176661264181,"Flickr30kT2IRetrieval":0.83712,"Winoground":0.08250000327825546,"StanfordCars":0.7349334659868175,"VisualNewsT2IRetrieval":0.07663,"STS16VisualSTS":0.4605911246970498,"MNISTZeroShot":0.48,"PatchCamelyonZeroShot":0.5296630859375,"HatefulMemesI2TRetrieval":0.25706,"SciMMIRI2TRetrieval":0.02953,"RParisMediumI2IRetrieval":0.02403,"UCF101ZeroShot":0.46691441176554954,"AROFlickrOrder":0.06379999965429306,"ImageNetDog15Clustering":0.0896597344095037,"Fashion200kT2IRetrieval":0.03313,"FashionIQIT2IRetrieval":0.01859,"TUBerlinT2IRetrieval":0.90633,"SUN397ZeroShot":0.6031724137931035,"FGVCAircraft":0.35211521152115216,"CUB200I2IRetrieval":0.32119,"XFlickr30kCoT2IRetrieval":0.17881375},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4129291670350686},"MIEB(lite) Task Mean":{"Task Mean":0.33819925436321085}},"trainedOnTasks":[],"experiments":null},{"rank":41,"model":{"name":"nomic-ai/nomic-embed-vision-v1.5","url":"https://huggingface.co/nomic-ai/nomic-embed-vision-v1.5","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.093,"embeddingDim":768,"maxTokens":2048.0,"releaseDate":"2024-06-08","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@article{nussbaum2024nomicembedvision,\n      title={Nomic Embed Vision: Expanding the Latent Space},\n      author={Nussbaum, Zach and Duderstadt, Brandon and Mulyar, Andriy},\n      journal={arXiv preprint arXiv:2406.18587},\n      year={2024},\n      eprint={2406.18587},\n      archivePrefix={arXiv},\n      url={https://arxiv.org/abs/2406.18587}\n}","memoryUsageMb":355.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/nomic-ai/contrastors","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["nomic"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.093,"embeddingDim":768,"maxTokens":2048,"meanTask":0.3927221498906398,"meanTaskType":0.3927221498906398,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Any2AnyRetrieval":0.30449733333333334,"Compositionality":0.37282780375531743,"ImageClassification":0.6671702597474755,"DocumentUnderstanding":0.11920299999999999,"ZeroShotClassification":0.5812402331138711,"ImageClustering":0.8364070365945164,"VisionCentricQA":0.4672041040826067,"Any2AnyMultilingualRetrieval":0.14477618265993267,"VisualSTS(eng)":0.2928760456193452,"VisualSTS(multi)":0.14101950000000002},"scoresByTask":{"GTSRBZeroShot":0.442596991290578,"CVBenchDistance":0.5033333333333333,"CVBenchRelation":0.48615384615384616,"OxfordPetsZeroShot":0.910874897792314,"InfoSeekIT2ITRetrieval":0.21976,"VidoreTabfquadRetrieval":0.15197,"CIFAR10Clustering":0.8739217573403475,"VidoreDocVQARetrieval":0.09254,"VisualNewsI2TRetrieval":0.11586,"VidoreArxivQARetrieval":0.15859,"STL10":0.9859250000000002,"CLEVRZeroShot":0.15806666666666666,"MemotionI2TRetrieval":0.51204,"PatchCamelyon":0.656878662109375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.15792,"TinyImageNetClustering":0.7265478676622801,"WebQAT2ITRetrieval":0.52666,"SUN397":0.6893057471264368,"METI2IRetrieval":0.29975,"STS13VisualSTS":0.212214414966272,"CVBenchCount":0.2182741116751269,"ROxfordMediumI2IRetrieval":0.16276,"NIGHTSI2IRetrieval":0.23094,"Flickr30kI2TRetrieval":0.59799,"BLINKIT2TRetrieval":0.21185,"VidoreTatdqaRetrieval":0.02609,"ImageCoDe":0.11946,"ImageCoDeT2IRetrieval":0.07602,"Fashion200kI2TRetrieval":0.07882,"FGVCAircraftZeroShot":0.2808280828082808,"HatefulMemesT2IRetrieval":0.44653,"XM3600T2IRetrieval":0.07554638888888889,"UCF101":0.7347220827799468,"MSCOCOI2TRetrieval":0.37108,"OxfordPets":0.9187789588443718,"SugarCrepe":0.7503661513328552,"FER2013ZeroShot":0.2698523265533575,"OVENIT2TRetrieval":0.00981,"Caltech101ZeroShot":0.7237015121630507,"MSCOCOT2IRetrieval":0.46669,"RESISC45ZeroShot":0.5761904761904761,"Food101Classification":0.8686257425742575,"RParisEasyI2IRetrieval":0.10599,"CVBenchDepth":0.4533333333333333,"Imagenet1k":0.001,"VisualSTS-b-Eng":0.231586,"VOC2007":0.6033439513620416,"EDIST2ITRetrieval":0.20621,"ROxfordEasyI2IRetrieval":0.27727,"VidoreShiftProjectRetrieval":0.0,"WITT2IRetrieval":0.2100309090909091,"RP2kI2IRetrieval":0.44323,"ROxfordHardI2IRetrieval":0.10521,"DTD":0.6415957446808511,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.08897,"AROVisualAttribution":0.550994873046875,"Country211":0.11728909952606634,"AROVisualRelation":0.46359193325042725,"BLINKIT2IRetrieval":0.35572,"CIFAR10":0.9724999999999999,"InfoSeekIT2TRetrieval":0.03019,"VidoreInfoVQARetrieval":0.29551,"BLINKIT2IMultiChoice":0.75373,"VidoreSyntheticDocQAEnergyRetrieval":0.10941,"CIFAR100":0.8412,"VisualSTS-b-Multilingual":0.23863,"EuroSATZeroShot":0.4237037037037037,"VisualSTS17Eng":0.404621,"RESISC45":0.7793968253968254,"BirdsnapZeroShot":0.5159373311723393,"CIFAR10ZeroShot":0.9525,"STL10ZeroShot":0.961625,"GTSRB":0.49029295328582745,"Country211ZeroShot":0.15165876777251186,"FORBI2IRetrieval":0.22853,"CIFAR100ZeroShot":0.8179,"OVENIT2ITRetrieval":0.09834,"MNIST":0.76172,"GLDv2I2TRetrieval":0.57136,"CIFAR100Clustering":0.8116364961461954,"RParisHardI2IRetrieval":0.04745,"SOPI2IRetrieval":0.4496,"VidoreSyntheticDocQAAIRetrieval":0.11103,"ImageNet10Clustering":0.9579918504259365,"StanfordCarsI2IRetrieval":0.89006,"StanfordCarsZeroShot":0.8743937321228703,"FER2013":0.32039565338534415,"OxfordFlowersClassification":0.9888235294117648,"STS12VisualSTS":0.4013397020457345,"Imagenet1kZeroShot":0.69818,"STS14VisualSTS":0.21438214993662158,"AROCocoOrder":0.2958816587924957,"VQA2IT2TRetrieval":0.01079,"CLEVRCountZeroShot":0.23386666666666667,"GLDv2I2IRetrieval":0.24381,"RenderedSST2":0.5650741350906096,"STS15VisualSTS":0.27207402287858595,"SciMMIRT2IRetrieval":0.05775,"WebQAT2TRetrieval":0.64903,"VizWizIT2TRetrieval":0.03548,"SketchyI2IRetrieval":0.71631,"Caltech101":0.8750493096646942,"VisualSTS17Multilingual":0.043409,"BLINKIT2TMultiChoice":0.3884,"EuroSAT":0.4901481481481481,"MemotionT2IRetrieval":0.75946,"CIRRIT2IRetrieval":0.09514,"DTDZeroShot":0.4776595744680851,"Food101ZeroShot":0.8598811881188119,"Birdsnap":0.5215559157212317,"Flickr30kT2IRetrieval":0.67624,"Winoground":0.057500001043081284,"StanfordCars":0.879144385026738,"VisualNewsT2IRetrieval":0.12501,"STS16VisualSTS":0.3139150295082023,"MNISTZeroShot":0.6964,"PatchCamelyonZeroShot":0.628387451171875,"HatefulMemesI2TRetrieval":0.37475,"SciMMIRI2TRetrieval":0.04652,"RParisMediumI2IRetrieval":0.027,"UCF101ZeroShot":0.5050629498208605,"AROFlickrOrder":0.3720000088214874,"ImageNetDog15Clustering":0.8119372113978232,"Fashion200kT2IRetrieval":0.06376,"FashionIQIT2IRetrieval":0.02024,"TUBerlinT2IRetrieval":0.92012,"SUN397ZeroShot":0.644183908045977,"FGVCAircraft":0.54005400540054,"CUB200I2IRetrieval":0.74525,"XFlickr30kCoT2IRetrieval":0.14875125},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{"Task Mean":0.4366869426332348},"MIEB(lite) Task Mean":{"Task Mean":0.37553525585531006}},"trainedOnTasks":[],"experiments":null},{"rank":42,"model":{"name":"DataScience-UIBK/Argus-Colqwen3.5-2b-v0","url":"https://huggingface.co/DataScience-UIBK/Argus-Colqwen3.5-2b-v0","zeroShotPct":93,"activeParamsB":1.814,"totalParamsB":2.322,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2026-05-03","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":8859.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":1.814,"totalParamsB":2.322,"embeddingDim":1024,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9148609999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.93356,"VidoreDocVQARetrieval":0.6747,"VidoreArxivQARetrieval":0.90268,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.84026,"VidoreShiftProjectRetrieval":0.91333,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97286,"VidoreInfoVQARetrieval":0.94967,"VidoreSyntheticDocQAEnergyRetrieval":0.97262,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":43,"model":{"name":"DataScience-UIBK/Argus-Colqwen3.5-2b-v0-bf16","url":"https://huggingface.co/DataScience-UIBK/Argus-Colqwen3.5-2b-v0-bf16","zeroShotPct":93,"activeParamsB":1.814,"totalParamsB":2.322,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2026-05-03","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":4430.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"DataScience-UIBK/Argus-Colqwen3.5-2b-v0","supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":1.814,"totalParamsB":2.322,"embeddingDim":1024,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9148609999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.93356,"VidoreDocVQARetrieval":0.6747,"VidoreArxivQARetrieval":0.90268,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.84026,"VidoreShiftProjectRetrieval":0.91333,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97286,"VidoreInfoVQARetrieval":0.94967,"VidoreSyntheticDocQAEnergyRetrieval":0.97262,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":44,"model":{"name":"DataScience-UIBK/Argus-Colqwen3.5-4b-v0","url":"https://huggingface.co/DataScience-UIBK/Argus-Colqwen3.5-4b-v0","zeroShotPct":93,"activeParamsB":4.073,"totalParamsB":4.708,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2026-04-29","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":8981.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":4.073,"totalParamsB":4.708,"embeddingDim":1024,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.92304},"scoresByTask":{"VidoreTabfquadRetrieval":0.95326,"VidoreDocVQARetrieval":0.67696,"VidoreArxivQARetrieval":0.90953,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.84804,"VidoreShiftProjectRetrieval":0.94696,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97786,"VidoreInfoVQARetrieval":0.9463,"VidoreSyntheticDocQAEnergyRetrieval":0.97887,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":45,"model":{"name":"DataScience-UIBK/Argus-Colqwen3.5-4b-v0-bf16","url":"https://huggingface.co/DataScience-UIBK/Argus-Colqwen3.5-4b-v0-bf16","zeroShotPct":93,"activeParamsB":4.073,"totalParamsB":4.708,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2026-05-03","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":8981.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"DataScience-UIBK/Argus-Colqwen3.5-4b-v0","supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":4.073,"totalParamsB":4.708,"embeddingDim":1024,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9214399999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.95442,"VidoreDocVQARetrieval":0.67787,"VidoreArxivQARetrieval":0.91258,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.84849,"VidoreShiftProjectRetrieval":0.93458,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97786,"VidoreInfoVQARetrieval":0.94467,"VidoreSyntheticDocQAEnergyRetrieval":0.975,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":46,"model":{"name":"DataScience-UIBK/Argus-Colqwen3.5-9b-v0","url":"https://huggingface.co/DataScience-UIBK/Argus-Colqwen3.5-9b-v0","zeroShotPct":93,"activeParamsB":7.804,"totalParamsB":8.821,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2026-05-05","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":33690.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":7.804,"totalParamsB":8.821,"embeddingDim":1024,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.926706},"scoresByTask":{"VidoreTabfquadRetrieval":0.97505,"VidoreDocVQARetrieval":0.6809,"VidoreArxivQARetrieval":0.92276,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.85454,"VidoreShiftProjectRetrieval":0.93647,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98893,"VidoreInfoVQARetrieval":0.94256,"VidoreSyntheticDocQAEnergyRetrieval":0.97323,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":47,"model":{"name":"DataScience-UIBK/Argus-Colqwen3.5-9b-v0-bf16","url":"https://huggingface.co/DataScience-UIBK/Argus-Colqwen3.5-9b-v0-bf16","zeroShotPct":93,"activeParamsB":7.804,"totalParamsB":8.821,"embeddingDim":1024,"maxTokens":32768.0,"releaseDate":"2026-05-05","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":16793.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"DataScience-UIBK/Argus-Colqwen3.5-9b-v0","supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":7.804,"totalParamsB":8.821,"embeddingDim":1024,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.925915},"scoresByTask":{"VidoreTabfquadRetrieval":0.97235,"VidoreDocVQARetrieval":0.68263,"VidoreArxivQARetrieval":0.92168,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.85674,"VidoreShiftProjectRetrieval":0.92978,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98893,"VidoreInfoVQARetrieval":0.94487,"VidoreSyntheticDocQAEnergyRetrieval":0.97693,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":48,"model":{"name":"BidirLM/BidirLM-Omni-2.5B-Embedding","url":"https://huggingface.co/BidirLM/BidirLM-Omni-2.5B-Embedding","zeroShotPct":100,"activeParamsB":2.13,"totalParamsB":2.445,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2026-04-07","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bosnian","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hausa","Hebrew","Hindi","Hungarian","Icelandic","Igbo","Indonesian","Irish","Italian","Japanese","Javanese","Kannada","Kazakh","Kirghiz","Korean","Latvian","Lithuanian","Macedonian","Malagasy (Latin)","Malay (macrolanguage)","Malayalam","Maltese","Marathi","Modern Greek (1453-)","Nepali (macrolanguage)","Norwegian Bokmål","Nyanja","Panjabi","Pedi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Serbian","Shona","Sindhi","Sinhala","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Wolof","Xhosa","Yoruba","Zulu"],"citation":"@misc{boizard2026bidirlmtextomnimodalbidirectional,\n      title={BidirLM: From Text to Omnimodal Bidirectional Encoders by Adapting and Composing Causal LLMs},\n      author={Nicolas Boizard and Théo Deschamps-Berger and Hippolyte Gisserot-Boukhlef and Céline Hudelot and Pierre Colombo},\n      year={2026},\n      eprint={2604.02045},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL},\n      url={https://arxiv.org/abs/2604.02045},\n}","memoryUsageMb":4663.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/datasets/BidirLM/BidirLM-Omni-Contrastive","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["bidirlm-omni"],"trainingDatasets":["AFQMC","ATEC","AdvertiseGen","BQ","CAIL2019-SCM","CHEF","CINLID","ChatMed_Consult_Dataset","ChineseSTS","CmedqaRetrieval","Cmnli","CodeFeedback","DRCD","DuRetrieval","ELI5_custom","ESCIReranking","EmotionClassification","Expertqa","FollowIR","GooAQ","InF-IR","JW300","LAION-Audio-300M","LCSTS","MAmmoTH2","MEDI2BGE","MIRACLRetrieval","MSMARCO","MS_COCO","Multi-CPR","MultiLongDocReranking","NFCorpus","NaturalQuestions","OPUS-100","OpenOrca","PAQ","PubMedQA","QBQTC","RefGPT","SQuAD","SearchQA","SimCLUE","SyntheticClassificationData","T2Ranking","TED Talks","THUCNews","TriviaQA","UMETRIP-QA","WebCPM","WikiAnswers","WikiMatrix","cCOVID-News","ccnews","cmrc2018","colpali_train_set","contract-nli","csl","dureader_mrc","law-gpt","lawzhidao","librispeech_asr","lima-chinese","mmarco-chinese","mnli","natcap","nli_zh","nllb","ocnli","rag-dataset-12000","simcse_sup_nli","webgpt_comparisons","webqa","wikipedia-nq","xnli_zh","yahoo-answers"]},"zeroShotPct":100,"activeParamsB":2.13,"totalParamsB":2.445,"embeddingDim":2048,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.45700335},"scoresByTask":{"CVBenchDistance":0.53167,"CVBenchRelation":0.60154,"OxfordPetsZeroShot":0.512946,"VidoreTabfquadRetrieval":0.74226,"VidoreDocVQARetrieval":0.45027,"VisualNewsI2TRetrieval":0.07015,"PatchCamelyon":0.72652,"TinyImageNetClustering":0.761656,"WebQAT2ITRetrieval":0.59411,"SUN397":0.758823,"STS13VisualSTS":0.620614,"CVBenchCount":0.34772,"NIGHTSI2IRetrieval":0.23916,"VidoreTatdqaRetrieval":0.49878,"ImageCoDe":0.143788,"Fashion200kI2TRetrieval":0.05493,"FGVCAircraftZeroShot":0.19682,"XM3600T2IRetrieval":0.6390899999999999,"OxfordPets":0.852821,"FER2013ZeroShot":0.627473,"OVENIT2TRetrieval":0.12366,"CVBenchDepth":0.515,"VidoreShiftProjectRetrieval":0.55427,"WITT2IRetrieval":0.5002527272727272,"RP2kI2IRetrieval":0.61177,"DTD":0.775106,"AROVisualAttribution":0.637192,"Country211":0.116815,"AROVisualRelation":0.53645,"InfoSeekIT2TRetrieval":0.10503,"VidoreInfoVQARetrieval":0.79463,"BLINKIT2IMultiChoice":0.77363,"VisualSTS-b-Multilingual":0.524533,"RESISC45":0.903016,"GTSRB":0.864386,"Country211ZeroShot":0.103981,"CIFAR100ZeroShot":0.7666,"VidoreSyntheticDocQAAIRetrieval":0.86661,"StanfordCarsZeroShot":0.613232,"AROCocoOrder":0.629708,"VQA2IT2TRetrieval":0.04391,"STS15VisualSTS":0.736838,"VisualSTS17Multilingual":0.3894736999999999,"EuroSAT":0.902519,"CIRRIT2IRetrieval":0.16486,"Food101ZeroShot":0.768752,"Winoground":0.095,"HatefulMemesI2TRetrieval":0.829,"AROFlickrOrder":0.718,"ImageNetDog15Clustering":0.56186,"CUB200I2IRetrieval":0.60597},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.531435204456328}},"trainedOnTasks":[],"experiments":null},{"rank":49,"model":{"name":"athrael-soju/BiUltraVBERT","url":"https://huggingface.co/athrael-soju/BiUltraVBERT","zeroShotPct":93,"activeParamsB":0.217,"totalParamsB":0.255,"embeddingDim":768,"maxTokens":32768.0,"releaseDate":"2026-10-18","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":true,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":486.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/datasets/athrael-soju/vdr-train-500k","adaptedFrom":"athrael-soju/UltraVBERT","supersededBy":null,"extraRequirementsGroups":["image"],"trainingDatasets":["NQ","VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","VisRAGRetArxivQA","VisRAGRetChartQA","VisRAGRetInfoVQA","VisRAGRetMPDocVQA","VisRAGRetPlotQA","VisRAGRetSlideVQA","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":0.217,"totalParamsB":0.255,"embeddingDim":768,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.708174},"scoresByTask":{"VidoreTabfquadRetrieval":0.80433,"VidoreDocVQARetrieval":0.36043,"VidoreArxivQARetrieval":0.69616,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.89665,"VidoreTatdqaRetrieval":0.4235,"VidoreShiftProjectRetrieval":0.49185,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.87512,"VidoreInfoVQARetrieval":0.81041,"VidoreSyntheticDocQAEnergyRetrieval":0.86226,"VidoreSyntheticDocQAAIRetrieval":0.86103},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":50,"model":{"name":"Salesforce/blip-image-captioning-base","url":"https://huggingface.co/Salesforce/blip-image-captioning-base","zeroShotPct":100,"activeParamsB":0.201,"totalParamsB":0.225,"embeddingDim":768,"maxTokens":512.0,"releaseDate":"2023-08-01","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{https://doi.org/10.48550/arxiv.2201.12086,\n  archiveprefix = {arXiv},\n  author = {Junnan Li and Dongxu Li and Caiming Xiong and Steven C. H. Hoi},\n  booktitle = {ICML},\n  eprint = {2201.12086},\n  pages = {12888-12900},\n  title = {BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation},\n  url = {https://proceedings.mlr.press/v162/li22n.html},\n  year = {2022},\n}","memoryUsageMb":942.0,"license":"bsd-3-clause","publicTrainingCode":"https://github.com/salesforce/BLIP","publicTrainingData":"https://github.com/salesforce/BLIP","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.201,"totalParamsB":0.225,"embeddingDim":768,"maxTokens":512,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Compositionality":0.27600763573655057,"ImageClassification":0.5824842920909657,"DocumentUnderstanding":0.009052000000000001,"ZeroShotClassification":0.09435650524638231,"ImageClustering":0.6309400658233257,"VisionCentricQA":0.43444833702112895,"Any2AnyMultilingualRetrieval":0.0028807070707070706,"VisualSTS(eng)":0.5538446724489983,"VisualSTS(multi)":0.3121665},"scoresByTask":{"GTSRBZeroShot":0.0386381631037213,"CVBenchDistance":0.5533333333333333,"CVBenchRelation":0.5261538461538462,"OxfordPetsZeroShot":0.020986644862360317,"InfoSeekIT2ITRetrieval":0.08294,"VidoreTabfquadRetrieval":0.04306,"CIFAR10Clustering":0.6418071900720136,"VidoreDocVQARetrieval":0.00541,"VisualNewsI2TRetrieval":0.0,"VidoreArxivQARetrieval":0.00512,"STL10":0.942,"CLEVRZeroShot":0.15373333333333333,"MemotionI2TRetrieval":0.0,"PatchCamelyon":0.634271240234375,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.02,"TinyImageNetClustering":0.47764502938270487,"WebQAT2ITRetrieval":0.0,"SUN397":0.6453609195402299,"METI2IRetrieval":0.37073,"STS13VisualSTS":0.46841415028914113,"CVBenchCount":0.10152284263959391,"NIGHTSI2IRetrieval":0.22812,"Flickr30kI2TRetrieval":0.00004,"BLINKIT2TRetrieval":0.0699,"VidoreTatdqaRetrieval":0.01116,"ImageCoDe":0.09991,"ImageCoDeT2IRetrieval":0.00043,"Fashion200kI2TRetrieval":0.0,"FGVCAircraftZeroShot":0.009600960096009602,"HatefulMemesT2IRetrieval":0.00103,"XM3600T2IRetrieval":0.0012441666666666669,"UCF101":0.7287827980184216,"MSCOCOI2TRetrieval":0.00044,"OxfordPets":0.5900790406105206,"SugarCrepe":0.5100519061088562,"FER2013ZeroShot":0.17915853998328224,"OVENIT2TRetrieval":0.0,"Caltech101ZeroShot":0.002794214332675871,"MSCOCOT2IRetrieval":0.00096,"RESISC45ZeroShot":0.030476190476190476,"Food101Classification":0.6859089108910892,"CVBenchDepth":0.515,"Imagenet1k":0.500284,"VisualSTS-b-Eng":0.495402,"VOC2007":0.342891760904685,"EDIST2ITRetrieval":0.0,"VidoreShiftProjectRetrieval":0.0,"WITT2IRetrieval":0.005225454545454545,"RP2kI2IRetrieval":0.585,"DTD":0.6446808510638299,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.0,"AROVisualAttribution":0.505113422870636,"Country211":0.09531753554502369,"AROVisualRelation":0.48080378770828247,"BLINKIT2IRetrieval":0.27114,"CIFAR10":0.8137000000000001,"InfoSeekIT2TRetrieval":0.0,"VidoreInfoVQARetrieval":0.00577,"BLINKIT2IMultiChoice":0.5824,"VidoreSyntheticDocQAEnergyRetrieval":0.0,"CIFAR100":0.5257,"VisualSTS-b-Multilingual":0.395792,"EuroSATZeroShot":0.1087037037037037,"VisualSTS17Eng":0.725623,"RESISC45":0.7511111111111111,"BirdsnapZeroShot":0.0010804970286331713,"CIFAR10ZeroShot":0.1302,"STL10ZeroShot":0.1785,"GTSRB":0.34902612826603324,"Country211ZeroShot":0.005260663507109005,"FORBI2IRetrieval":0.40823,"CIFAR100ZeroShot":0.0124,"OVENIT2ITRetrieval":0.05206,"MNIST":0.8512599999999999,"GLDv2I2TRetrieval":0.00986,"CIFAR100Clustering":0.5380579934015649,"SOPI2IRetrieval":0.54228,"VidoreSyntheticDocQAAIRetrieval":0.0,"ImageNet10Clustering":0.9094241313041543,"StanfordCarsI2IRetrieval":0.60552,"StanfordCarsZeroShot":0.00373087924387514,"FER2013":0.4123154081916969,"OxfordFlowersClassification":0.8535294117647059,"STS12VisualSTS":0.49344126557348317,"Imagenet1kZeroShot":0.00074,"STS14VisualSTS":0.48291082346970393,"AROCocoOrder":0.16417433321475983,"VQA2IT2TRetrieval":0.00009,"CLEVRCountZeroShot":0.12513333333333335,"GLDv2I2IRetrieval":0.13737,"RenderedSST2":0.4942339373970346,"STS15VisualSTS":0.605699105674713,"SciMMIRT2IRetrieval":0.00024,"WebQAT2TRetrieval":0.02893,"VizWizIT2TRetrieval":0.0013,"SketchyI2IRetrieval":0.36885,"Caltech101":0.8620315581854043,"VisualSTS17Multilingual":0.228541,"BLINKIT2TMultiChoice":0.32828,"EuroSAT":0.5331481481481481,"MemotionT2IRetrieval":0.0,"CIRRIT2IRetrieval":0.05701,"DTDZeroShot":0.019680851063829788,"Food101ZeroShot":0.009069306930693069,"Birdsnap":0.20626688276607238,"Flickr30kT2IRetrieval":0.00013,"Winoground":0.009999999776482582,"StanfordCars":0.6738714090287278,"VisualNewsT2IRetrieval":0.00004,"STS16VisualSTS":0.6054223621359465,"MNISTZeroShot":0.1304,"PatchCamelyonZeroShot":0.504669189453125,"HatefulMemesI2TRetrieval":0.00086,"SciMMIRI2TRetrieval":0.0003,"UCF101ZeroShot":0.008664385231676569,"AROFlickrOrder":0.16200000047683716,"ImageNetDog15Clustering":0.587765984956191,"Fashion200kT2IRetrieval":0.0,"FashionIQIT2IRetrieval":0.00626,"TUBerlinT2IRetrieval":0.00473,"SUN397ZeroShot":0.0023448275862068967,"FGVCAircraft":0.17311731173117312,"CUB200I2IRetrieval":0.30238,"XFlickr30kCoT2IRetrieval":0.0021725},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.23907451817316164}},"trainedOnTasks":[],"experiments":null},{"rank":51,"model":{"name":"Salesforce/blip-image-captioning-large","url":"https://huggingface.co/Salesforce/blip-image-captioning-large","zeroShotPct":100,"activeParamsB":0.423,"totalParamsB":0.446,"embeddingDim":768,"maxTokens":512.0,"releaseDate":"2023-12-07","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{https://doi.org/10.48550/arxiv.2201.12086,\n  archiveprefix = {arXiv},\n  author = {Junnan Li and Dongxu Li and Caiming Xiong and Steven C. H. Hoi},\n  booktitle = {ICML},\n  eprint = {2201.12086},\n  pages = {12888-12900},\n  title = {BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation},\n  url = {https://proceedings.mlr.press/v162/li22n.html},\n  year = {2022},\n}","memoryUsageMb":1792.0,"license":"bsd-3-clause","publicTrainingCode":"https://github.com/salesforce/BLIP","publicTrainingData":"https://github.com/salesforce/BLIP","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.423,"totalParamsB":0.446,"embeddingDim":768,"maxTokens":512,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"Compositionality":0.2706051584114773,"ImageClassification":0.6635551020378638,"DocumentUnderstanding":0.008048,"ZeroShotClassification":0.0920352515121238,"ImageClustering":0.7369578857743282,"VisionCentricQA":0.49108481517636343,"Any2AnyMultilingualRetrieval":0.003319250841750842,"VisualSTS(eng)":0.6221048408836697,"VisualSTS(multi)":0.31948350000000003},"scoresByTask":{"GTSRBZeroShot":0.009026128266033254,"CVBenchDistance":0.5966666666666667,"CVBenchRelation":0.46923076923076923,"OxfordPetsZeroShot":0.018806214227309895,"InfoSeekIT2ITRetrieval":0.08472,"VidoreTabfquadRetrieval":0.03761,"CIFAR10Clustering":0.7763585493503886,"VidoreDocVQARetrieval":0.00404,"VisualNewsI2TRetrieval":0.00005,"VidoreArxivQARetrieval":0.00364,"STL10":0.9823000000000001,"CLEVRZeroShot":0.1952,"MemotionI2TRetrieval":0.0,"PatchCamelyon":0.667218017578125,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.00431,"TinyImageNetClustering":0.6173932580509577,"WebQAT2ITRetrieval":0.0,"SUN397":0.7018298850574712,"METI2IRetrieval":0.37173,"STS13VisualSTS":0.5017807802813792,"CVBenchCount":0.14720812182741116,"NIGHTSI2IRetrieval":0.23982,"Flickr30kI2TRetrieval":0.0001,"BLINKIT2TRetrieval":0.07922,"VidoreTatdqaRetrieval":0.0098,"ImageCoDe":0.09991,"ImageCoDeT2IRetrieval":0.0013,"Fashion200kI2TRetrieval":0.0,"FGVCAircraftZeroShot":0.006900690069006901,"HatefulMemesT2IRetrieval":0.0003,"XM3600T2IRetrieval":0.0012813888888888888,"UCF101":0.8056662010091478,"MSCOCOI2TRetrieval":0.00016,"OxfordPets":0.8045789043336059,"SugarCrepe":0.5017973780632019,"FER2013ZeroShot":0.2163555307885205,"OVENIT2TRetrieval":0.0,"Caltech101ZeroShot":0.04503616042077581,"MSCOCOT2IRetrieval":0.00088,"RESISC45ZeroShot":0.02095238095238095,"Food101Classification":0.7835881188118812,"CVBenchDepth":0.6333333333333333,"Imagenet1k":0.6442400000000001,"VisualSTS-b-Eng":0.564059,"VOC2007":0.452140549273021,"EDIST2ITRetrieval":0.0,"VidoreShiftProjectRetrieval":0.01131,"WITT2IRetrieval":0.006216363636363636,"RP2kI2IRetrieval":0.59886,"DTD":0.6626595744680851,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.0,"AROVisualAttribution":0.5003130435943604,"Country211":0.08441706161137441,"AROVisualRelation":0.49701300263404846,"BLINKIT2IRetrieval":0.37065,"CIFAR10":0.9457599999999999,"InfoSeekIT2TRetrieval":0.0,"VidoreInfoVQARetrieval":0.0059,"BLINKIT2IMultiChoice":0.70398,"VidoreSyntheticDocQAEnergyRetrieval":0.00387,"CIFAR100":0.72952,"VisualSTS-b-Multilingual":0.345104,"EuroSATZeroShot":0.11777777777777777,"VisualSTS17Eng":0.764288,"RESISC45":0.820063492063492,"BirdsnapZeroShot":0.002701242571582928,"CIFAR10ZeroShot":0.0486,"STL10ZeroShot":0.059125,"GTSRB":0.598511480601742,"Country211ZeroShot":0.004312796208530805,"FORBI2IRetrieval":0.39653,"CIFAR100ZeroShot":0.0039,"OVENIT2ITRetrieval":0.0598,"MNIST":0.8069,"GLDv2I2TRetrieval":0.00548,"CIFAR100Clustering":0.6844741687632565,"SOPI2IRetrieval":0.55172,"VidoreSyntheticDocQAAIRetrieval":0.0,"ImageNet10Clustering":0.9327177234793893,"StanfordCarsI2IRetrieval":0.67206,"StanfordCarsZeroShot":0.0032334286780251213,"FER2013":0.4359988854834215,"OxfordFlowersClassification":0.9419607843137255,"STS12VisualSTS":0.6167381896876682,"Imagenet1kZeroShot":0.00068,"STS14VisualSTS":0.5412294887228664,"AROCocoOrder":0.14330267906188965,"VQA2IT2TRetrieval":0.00007,"CLEVRCountZeroShot":0.16006666666666666,"GLDv2I2IRetrieval":0.12413,"RenderedSST2":0.5057660626029654,"STS15VisualSTS":0.7003037770604883,"SciMMIRT2IRetrieval":0.00026,"WebQAT2TRetrieval":0.03401,"VizWizIT2TRetrieval":0.00479,"SketchyI2IRetrieval":0.58827,"Caltech101":0.8777120315581854,"VisualSTS17Multilingual":0.293863,"BLINKIT2TMultiChoice":0.39609,"EuroSAT":0.6057407407407408,"MemotionT2IRetrieval":0.00045,"CIRRIT2IRetrieval":0.06572,"DTDZeroShot":0.014361702127659574,"Food101ZeroShot":0.011524752475247525,"Birdsnap":0.31820637493246895,"Flickr30kT2IRetrieval":0.00013,"Winoground":0.012500000186264515,"StanfordCars":0.7230195249347096,"VisualNewsT2IRetrieval":0.00002,"STS16VisualSTS":0.6663346504332858,"MNISTZeroShot":0.1529,"PatchCamelyonZeroShot":0.5068359375,"HatefulMemesI2TRetrieval":0.00043,"SciMMIRI2TRetrieval":0.00034,"UCF101ZeroShot":0.011598888159008178,"AROFlickrOrder":0.13940000534057617,"ImageNetDog15Clustering":0.6738457292276491,"Fashion200kT2IRetrieval":0.0,"FashionIQIT2IRetrieval":0.00734,"TUBerlinT2IRetrieval":0.00409,"SUN397ZeroShot":0.0011494252873563218,"FGVCAircraft":0.2061806180618062,"CUB200I2IRetrieval":0.52589,"XFlickr30kCoT2IRetrieval":0.00246},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.2704352644690652}},"trainedOnTasks":[],"experiments":null},{"rank":52,"model":{"name":"mteb/baseline-bm25s","url":"https://github.com/xhluca/bm25s","zeroShotPct":-1,"activeParamsB":0.0,"totalParamsB":0.0,"embeddingDim":null,"maxTokens":null,"releaseDate":"2026-05-06","modelType":"sparse","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["text"],"languages":["Chinese","Chinese (Traditional)","Dutch","English","French","German","Italian","Mandarin Chinese","Mandarin Chinese (Traditional)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Portuguese","Spanish","Swedish"],"citation":"@misc{bm25s,\n      title={BM25S: Orders of magnitude faster lexical search via eager sparse scoring},\n      author={Xing Han Lù},\n      year={2024},\n      eprint={2407.03618},\n      archivePrefix={arXiv},\n      primaryClass={cs.IR},\n      url={https://arxiv.org/abs/2407.03618},\n}","memoryUsageMb":null,"license":null,"publicTrainingCode":"https://github.com/xhluca/bm25s","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["bm25s"],"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":0.0,"totalParamsB":0.0,"embeddingDim":null,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{},"scoresByTask":{"CVBenchDistance":0.49833,"CVBenchRelation":0.49692,"CVBenchCount":0.00888,"BLINKIT2TRetrieval":0.36192,"CVBenchDepth":0.51667,"WebQAT2TRetrieval":0.61789,"BLINKIT2TMultiChoice":0.40984},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":53,"model":{"name":"codefuse-ai/F2LLM-v2-80M","url":"https://huggingface.co/codefuse-ai/F2LLM-v2-80M","zeroShotPct":100,"activeParamsB":0.032,"totalParamsB":0.08,"embeddingDim":320,"maxTokens":40960.0,"releaseDate":"2026-03-09","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":true,"modalities":["text"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Esperanto","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Korean","Lao","Latin","Latvian","Lithuanian","Macedonian","Malay (macrolanguage)","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","North Azerbaijani","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Occitan (post 1500)","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Serbian","Sinhala","Slovak","Slovenian","South Azerbaijani","Spanish","Swahili (macrolanguage)","Swedish","Tagalog","Tajik","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Uzbek","Vietnamese","Waray (Philippines)","Welsh","c#-Code (Code)","c++-Code (Code)","c-Code (Code)","go-Code (Code)","java-Code (Code)","javascript-Code (Code)","php-Code (Code)","python-Code (Code)","ruby-Code (Code)","rust-Code (Code)","sql-Code (Code)"],"citation":"\n@misc{f2llm-v2,\n    title={F2LLM-v2: Inclusive, Performant, and Efficient Embeddings for a Multilingual World},\n    author={Ziyin Zhang and Zihan Liao and Hang Yu and Peng Di and Rui Wang},\n    year={2026},\n    eprint={2603.19223},\n    archivePrefix={arXiv},\n    primaryClass={cs.CL},\n    url={https://arxiv.org/abs/2603.19223},\n}","memoryUsageMb":305.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/codefuse-ai/CodeFuse-Embeddings/tree/main/F2LLM","publicTrainingData":"https://huggingface.co/datasets/codefuse-ai/F2LLM-v2","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["ANLI","AmazonCounterfactualClassification","AmazonPolarityClassification","AmazonQA","AmazonReviewClassification","ArguAna","ArxivClusteringP2P","ArxivClusteringS2S","Aya","BQ","BactrianXLanguageClassification","BactrianXTranslation","Banking77Classification","BioASQ","BiorxivClusteringP2P","BiorxivClusteringS2S","CEDR","CLIRMatrix","CMCQA","CMNLI","CNNDM","COIG","COLIEE","CORD19","CSL","CoLA","CodeFeedbackMT","CodeFeedbackST","CodeSearchNet","CodeSearchNetCCR","CosQA","DBPedia","DuReader","ELI5","ESCI","EmotionClassification","Europarl","FEVER","FEVERNL","FiQA2018","GooAQ","HealthCareMagic","HotpotQA","HotpotQANL","HuatuoEncQA","HuatuoKGQA","ImdbClassification","InfinityInstruct","KoAlpaca","KoAlpacaRealQA","KoMagpie","LCQMC","LCSTS","LLMRetrievalData","Lawzhidao","M2Lingual","MEDI2","MIRACLRetrieval","MKQA","MLDR","MLSUMClustering","MLSUMRetrieval","MMARCO","MNLI","MQA","MSMARCO","MSciNLI","MTOPDomainClassification","MTOPIntentClassification","MURI","MailruQA","MassiveIntentClassification","MassiveScenarioClassification","MedInstruct","MedMCQA","MedQA","MedQuAD","MedicalFlashcards","MedicalInstruction","MedicalQARu","MedrxivClusteringP2P","MedrxivClusteringS2S","MrTidyRetrieval","MultiAlpaca","MultiCPRECom","MultiCPRMedical","NFCorpus","NQ","NaturalReasoning","NordicClassification","NordicRetrieval","NordicSTS","NordicTextMatching","OASST2","OCNLI","OpenCodeGeneticInstruct","OpenCodeReasoning2","OpenOrca","PAQ","PQuAD","ParSQuAD","ParaCrawl","PawsX","PersianQA","ProCQA","PubMedQA","QBQTC","QQP","RedditClustering","RedditClusteringP2P","RefGPT","RuInstruct","RuSentimentClustering","S2ORC","SIB200","SNLI","SPECTER","SQuAD","STS12","STS22","STSBenchmark","SciFact","SentenceCompression","SiberianDataset","SimCLUE","StackExchange","StackExchangeClustering","StackExchangeClusteringP2P","StackExchangeDupQuestions","StackOverflowDupQuestions","StackOverflowQA","SyntheticText2SQL","T2Ranking","THUCNews","TNews","ToxicConversationsClassification","TriviaQA","TweetSentimentExtractionClassification","TwentyNewsgroupsClustering","UNPC","Waimai","WebFAQ","WikiOmnia","WildChat","XCodeEvalCodeToCode","XCodeEvalNLToCode","XCodeEvalTranslation","XNLI","XSum","YahooAnswers","cMedQAv2","webMedQA"]},"zeroShotPct":100,"activeParamsB":0.032,"totalParamsB":0.08,"embeddingDim":320,"maxTokens":40960,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{},"scoresByTask":{"WebQAT2TRetrieval":0.56149},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":54,"model":{"name":"nanovdr/ColNanoVDR-Q-Ettin150M-ColVec4B-640-ML","url":"https://huggingface.co/nanovdr/ColNanoVDR-Q-Ettin150M-ColVec4B-640-ML","zeroShotPct":93,"activeParamsB":4.65,"totalParamsB":4.69,"embeddingDim":640,"maxTokens":512.0,"releaseDate":"2026-09-20","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":true,"modalities":["text","image"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"@inproceedings{nanovdr2026,\n  author = {Liu, Zhuchenyang and Zhang, Yao and Xiao, Yu},\n  booktitle = {Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing},\n  doi = {10.18653/2026.emnlp-main.71},\n  pages = {1631–1648},\n  publisher = {Association for Computational Linguistics},\n  title = {NanoVDR: Distilling a 2B Vision-Language Retriever into a 70M Text-Only Encoder for Visual Document Retrieval},\n  url = {https://doi.org/10.18653/2026.emnlp-main.71},\n  year = {2026},\n}","memoryUsageMb":601.0,"license":"https://huggingface.co/webAI-Official/webAI-ColVec1.1-4b/blob/main/LICENSE.md","publicTrainingCode":"https://github.com/Ryenhails/NanoVDR","publicTrainingData":"https://huggingface.co/datasets/nanovdr/NanoVDR-Train","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colnanovdr","colvec1_1"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":4.65,"totalParamsB":4.69,"embeddingDim":640,"maxTokens":512,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8992290000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.95588,"VidoreDocVQARetrieval":0.61961,"VidoreArxivQARetrieval":0.87795,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.77893,"VidoreShiftProjectRetrieval":0.91764,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96762,"VidoreInfoVQARetrieval":0.9318,"VidoreSyntheticDocQAEnergyRetrieval":0.95762,"VidoreSyntheticDocQAAIRetrieval":0.98893},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":55,"model":{"name":"nanovdr/ColNanoVDR-Q-Ettin150M-ColVec8B-640-ML","url":"https://huggingface.co/nanovdr/ColNanoVDR-Q-Ettin150M-ColVec8B-640-ML","zeroShotPct":93,"activeParamsB":8.504,"totalParamsB":8.545,"embeddingDim":640,"maxTokens":512.0,"releaseDate":"2026-09-20","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":true,"modalities":["text","image"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"@inproceedings{nanovdr2026,\n  author = {Liu, Zhuchenyang and Zhang, Yao and Xiao, Yu},\n  booktitle = {Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing},\n  doi = {10.18653/2026.emnlp-main.71},\n  pages = {1631–1648},\n  publisher = {Association for Computational Linguistics},\n  title = {NanoVDR: Distilling a 2B Vision-Language Retriever into a 70M Text-Only Encoder for Visual Document Retrieval},\n  url = {https://doi.org/10.18653/2026.emnlp-main.71},\n  year = {2026},\n}","memoryUsageMb":601.0,"license":"https://huggingface.co/webAI-Official/webAI-ColVec1.1-8b/blob/main/LICENSE.md","publicTrainingCode":"https://github.com/Ryenhails/NanoVDR","publicTrainingData":"https://huggingface.co/datasets/nanovdr/NanoVDR-Train","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colnanovdr","colvec1_1"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":8.504,"totalParamsB":8.545,"embeddingDim":640,"maxTokens":512,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9070129999999998},"scoresByTask":{"VidoreTabfquadRetrieval":0.9641,"VidoreDocVQARetrieval":0.64067,"VidoreArxivQARetrieval":0.90646,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.78661,"VidoreShiftProjectRetrieval":0.92889,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96893,"VidoreInfoVQARetrieval":0.931,"VidoreSyntheticDocQAEnergyRetrieval":0.95454,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":56,"model":{"name":"vidore/colpali-v1.1","url":"https://huggingface.co/vidore/colpali-v1.1","zeroShotPct":93,"activeParamsB":null,"totalParamsB":2.92,"embeddingDim":128,"maxTokens":16384.0,"releaseDate":"2024-08-21","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":4700.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["image","colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":2.92,"embeddingDim":128,"maxTokens":16384,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.816062},"scoresByTask":{"VidoreTabfquadRetrieval":0.81872,"VidoreDocVQARetrieval":0.59066,"VidoreArxivQARetrieval":0.79962,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.94786,"VidoreTatdqaRetrieval":0.66156,"VidoreShiftProjectRetrieval":0.70222,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.92839,"VidoreInfoVQARetrieval":0.82181,"VidoreSyntheticDocQAEnergyRetrieval":0.92085,"VidoreSyntheticDocQAAIRetrieval":0.96893},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":57,"model":{"name":"vidore/colpali-v1.2","url":"https://huggingface.co/vidore/colpali-v1.2","zeroShotPct":93,"activeParamsB":null,"totalParamsB":2.92,"embeddingDim":128,"maxTokens":16384.0,"releaseDate":"2024-08-26","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":4700.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["image","colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":2.92,"embeddingDim":128,"maxTokens":16384,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8315020000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.8884,"VidoreDocVQARetrieval":0.56617,"VidoreArxivQARetrieval":0.77777,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.94893,"VidoreTatdqaRetrieval":0.68086,"VidoreShiftProjectRetrieval":0.77355,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.9444,"VidoreInfoVQARetrieval":0.82184,"VidoreSyntheticDocQAEnergyRetrieval":0.93786,"VidoreSyntheticDocQAAIRetrieval":0.97524},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":58,"model":{"name":"vidore/colpali-v1.3","url":"https://huggingface.co/vidore/colpali-v1.3","zeroShotPct":93,"activeParamsB":null,"totalParamsB":2.92,"embeddingDim":128,"maxTokens":16384.0,"releaseDate":"2024-11-01","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":4700.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["image","colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":2.92,"embeddingDim":128,"maxTokens":16384,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8475439999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.86749,"VidoreDocVQARetrieval":0.58374,"VidoreArxivQARetrieval":0.83251,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.97411,"VidoreTatdqaRetrieval":0.70777,"VidoreShiftProjectRetrieval":0.77288,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96085,"VidoreInfoVQARetrieval":0.85529,"VidoreSyntheticDocQAEnergyRetrieval":0.94649,"VidoreSyntheticDocQAAIRetrieval":0.97431},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":59,"model":{"name":"nomic-ai/colnomic-embed-multimodal-3b","url":"https://huggingface.co/nomic-ai/colnomic-embed-multimodal-3b","zeroShotPct":93,"activeParamsB":null,"totalParamsB":3.0,"embeddingDim":128,"maxTokens":128000.0,"releaseDate":"2025-03-31","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":false,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Spanish"],"citation":null,"memoryUsageMb":7200.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/nomic-ai/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VDRMultilingual","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":3.0,"embeddingDim":128,"maxTokens":128000,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8985569999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.94427,"VidoreDocVQARetrieval":0.61339,"VidoreArxivQARetrieval":0.88088,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98323,"VidoreTatdqaRetrieval":0.83242,"VidoreShiftProjectRetrieval":0.90101,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96603,"VidoreInfoVQARetrieval":0.92779,"VidoreSyntheticDocQAEnergyRetrieval":0.97393,"VidoreSyntheticDocQAAIRetrieval":0.96262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":60,"model":{"name":"nomic-ai/colnomic-embed-multimodal-7b","url":"https://huggingface.co/nomic-ai/colnomic-embed-multimodal-7b","zeroShotPct":93,"activeParamsB":null,"totalParamsB":7.0,"embeddingDim":128,"maxTokens":128000.0,"releaseDate":"2025-03-31","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":false,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Spanish"],"citation":null,"memoryUsageMb":14400.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/nomic-ai/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VDRMultilingual","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":7.0,"embeddingDim":128,"maxTokens":128000,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.897195},"scoresByTask":{"VidoreTabfquadRetrieval":0.95962,"VidoreDocVQARetrieval":0.60113,"VidoreArxivQARetrieval":0.88319,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.81084,"VidoreShiftProjectRetrieval":0.89302,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.95917,"VidoreInfoVQARetrieval":0.92212,"VidoreSyntheticDocQAEnergyRetrieval":0.96262,"VidoreSyntheticDocQAAIRetrieval":0.98762},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":61,"model":{"name":"vidore/colqwen2-v1.0","url":"https://huggingface.co/vidore/colqwen2-v1.0","zeroShotPct":93,"activeParamsB":null,"totalParamsB":2.21,"embeddingDim":128,"maxTokens":32768.0,"releaseDate":"2025-11-03","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":7200.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":2.21,"embeddingDim":128,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.892269},"scoresByTask":{"VidoreTabfquadRetrieval":0.89011,"VidoreDocVQARetrieval":0.61483,"VidoreArxivQARetrieval":0.87983,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98762,"VidoreTatdqaRetrieval":0.82228,"VidoreShiftProjectRetrieval":0.89906,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.95478,"VidoreInfoVQARetrieval":0.92507,"VidoreSyntheticDocQAEnergyRetrieval":0.95893,"VidoreSyntheticDocQAAIRetrieval":0.99018},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":62,"model":{"name":"vidore/colqwen2.5-v0.2","url":"https://huggingface.co/vidore/colqwen2.5-v0.2","zeroShotPct":93,"activeParamsB":null,"totalParamsB":3.0,"embeddingDim":128,"maxTokens":128000.0,"releaseDate":"2025-01-31","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":7200.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":3.0,"embeddingDim":128,"maxTokens":128000,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8954460000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.90821,"VidoreDocVQARetrieval":0.63627,"VidoreArxivQARetrieval":0.88868,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98024,"VidoreTatdqaRetrieval":0.82122,"VidoreShiftProjectRetrieval":0.87909,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.95803,"VidoreInfoVQARetrieval":0.92518,"VidoreSyntheticDocQAEnergyRetrieval":0.96123,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":63,"model":{"name":"TomoroAI/tomoro-colqwen3-embed-4b","url":"https://huggingface.co/TomoroAI/tomoro-colqwen3-embed-4b","zeroShotPct":93,"activeParamsB":null,"totalParamsB":4.0,"embeddingDim":320,"maxTokens":262144.0,"releaseDate":"2025-11-26","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":8466.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colqwen3"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":4.0,"embeddingDim":320,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9057449999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.94329,"VidoreDocVQARetrieval":0.66296,"VidoreArxivQARetrieval":0.90576,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.79867,"VidoreShiftProjectRetrieval":0.87389,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97172,"VidoreInfoVQARetrieval":0.94312,"VidoreSyntheticDocQAEnergyRetrieval":0.96911,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":64,"model":{"name":"athrael-soju/colqwen3.5-4.5B-v3","url":"https://huggingface.co/athrael-soju/colqwen3.5-4.5B-v3","zeroShotPct":93,"activeParamsB":3.964,"totalParamsB":4.6,"embeddingDim":128,"maxTokens":262144.0,"releaseDate":"2026-03-15","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":8660.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data"]},"zeroShotPct":93,"activeParamsB":3.964,"totalParamsB":4.6,"embeddingDim":128,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.915398},"scoresByTask":{"VidoreTabfquadRetrieval":0.95922,"VidoreDocVQARetrieval":0.66578,"VidoreArxivQARetrieval":0.9186,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98893,"VidoreTatdqaRetrieval":0.83951,"VidoreShiftProjectRetrieval":0.90188,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97286,"VidoreInfoVQARetrieval":0.93597,"VidoreSyntheticDocQAEnergyRetrieval":0.97123,"VidoreSyntheticDocQAAIRetrieval":1.0},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":65,"model":{"name":"TomoroAI/tomoro-colqwen3-embed-8b","url":"https://huggingface.co/TomoroAI/tomoro-colqwen3-embed-8b","zeroShotPct":93,"activeParamsB":null,"totalParamsB":8.0,"embeddingDim":320,"maxTokens":262144.0,"releaseDate":"2025-11-26","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":16724.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colqwen3"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":8.0,"embeddingDim":320,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9076489999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.94231,"VidoreDocVQARetrieval":0.66369,"VidoreArxivQARetrieval":0.91151,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99062,"VidoreTatdqaRetrieval":0.80918,"VidoreShiftProjectRetrieval":0.87889,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97579,"VidoreInfoVQARetrieval":0.94478,"VidoreSyntheticDocQAEnergyRetrieval":0.9671,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":66,"model":{"name":"ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-3B-v1","url":"https://huggingface.co/ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-3B-v1","zeroShotPct":93,"activeParamsB":null,"totalParamsB":3.0,"embeddingDim":128,"maxTokens":128000.0,"releaseDate":"2025-11-04","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":false,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":7200.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":3.0,"embeddingDim":128,"maxTokens":128000,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9067160000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.93987,"VidoreDocVQARetrieval":0.63668,"VidoreArxivQARetrieval":0.90456,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.82002,"VidoreShiftProjectRetrieval":0.88595,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98893,"VidoreInfoVQARetrieval":0.92222,"VidoreSyntheticDocQAEnergyRetrieval":0.97631,"VidoreSyntheticDocQAAIRetrieval":1.0},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":67,"model":{"name":"ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-7B-v1","url":"https://huggingface.co/ApsaraStackMaaS/EvoQwen2.5-VL-Retriever-7B-v1","zeroShotPct":93,"activeParamsB":null,"totalParamsB":7.0,"embeddingDim":128,"maxTokens":128000.0,"releaseDate":"2025-11-04","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":false,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":null,"memoryUsageMb":14400.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":7.0,"embeddingDim":128,"maxTokens":128000,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9068039999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.93632,"VidoreDocVQARetrieval":0.65072,"VidoreArxivQARetrieval":0.91485,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98893,"VidoreTatdqaRetrieval":0.82259,"VidoreShiftProjectRetrieval":0.88802,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96286,"VidoreInfoVQARetrieval":0.94113,"VidoreSyntheticDocQAEnergyRetrieval":0.96631,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":68,"model":{"name":"vultr/VultronRetrieverCore-Qwen3.5-4.5B","url":"https://huggingface.co/vultr/VultronRetrieverCore-Qwen3.5-4.5B","zeroShotPct":93,"activeParamsB":3.904,"totalParamsB":4.54,"embeddingDim":320,"maxTokens":262144.0,"releaseDate":"2026-06-22","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":9080.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":3.904,"totalParamsB":4.54,"embeddingDim":320,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9221360000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.9618,"VidoreDocVQARetrieval":0.67317,"VidoreArxivQARetrieval":0.91958,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":1.0,"VidoreTatdqaRetrieval":0.82416,"VidoreShiftProjectRetrieval":0.93802,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98524,"VidoreInfoVQARetrieval":0.94046,"VidoreSyntheticDocQAEnergyRetrieval":0.98262,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":69,"model":{"name":"vultr/VultronRetrieverFlash-Qwen3.5-0.8B","url":"https://huggingface.co/vultr/VultronRetrieverFlash-Qwen3.5-0.8B","zeroShotPct":93,"activeParamsB":0.599,"totalParamsB":0.853,"embeddingDim":320,"maxTokens":262144.0,"releaseDate":"2026-06-21","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":1707.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":0.599,"totalParamsB":0.853,"embeddingDim":320,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.881541},"scoresByTask":{"VidoreTabfquadRetrieval":0.91795,"VidoreDocVQARetrieval":0.62173,"VidoreArxivQARetrieval":0.88569,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98024,"VidoreTatdqaRetrieval":0.77918,"VidoreShiftProjectRetrieval":0.80385,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96155,"VidoreInfoVQARetrieval":0.92157,"VidoreSyntheticDocQAEnergyRetrieval":0.9671,"VidoreSyntheticDocQAAIRetrieval":0.97655},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":70,"model":{"name":"vultr/VultronRetrieverPrime-Qwen3.5-8B","url":"https://huggingface.co/vultr/VultronRetrieverPrime-Qwen3.5-8B","zeroShotPct":93,"activeParamsB":7.377,"totalParamsB":8.394,"embeddingDim":320,"maxTokens":262144.0,"releaseDate":"2026-06-18","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":16788.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":7.377,"totalParamsB":8.394,"embeddingDim":320,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9207919999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.96436,"VidoreDocVQARetrieval":0.67713,"VidoreArxivQARetrieval":0.9321,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":1.0,"VidoreTatdqaRetrieval":0.81477,"VidoreShiftProjectRetrieval":0.94571,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97286,"VidoreInfoVQARetrieval":0.94057,"VidoreSyntheticDocQAEnergyRetrieval":0.9678,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":71,"model":{"name":"vidore/colSmol-256M","url":"https://huggingface.co/vidore/colSmol-256M","zeroShotPct":93,"activeParamsB":null,"totalParamsB":0.256,"embeddingDim":128,"maxTokens":8192.0,"releaseDate":"2025-01-22","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":800.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":0.256,"embeddingDim":128,"maxTokens":8192,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.797358},"scoresByTask":{"VidoreTabfquadRetrieval":0.64649,"VidoreDocVQARetrieval":0.56609,"VidoreArxivQARetrieval":0.7289,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.96417,"VidoreTatdqaRetrieval":0.76626,"VidoreShiftProjectRetrieval":0.62083,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.94516,"VidoreInfoVQARetrieval":0.84203,"VidoreSyntheticDocQAEnergyRetrieval":0.92954,"VidoreSyntheticDocQAAIRetrieval":0.96411},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":72,"model":{"name":"vidore/colSmol-500M","url":"https://huggingface.co/vidore/colSmol-500M","zeroShotPct":93,"activeParamsB":null,"totalParamsB":0.5,"embeddingDim":128,"maxTokens":8192.0,"releaseDate":"2025-01-22","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":1200.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":"https://huggingface.co/datasets/vidore/colpali_train_set","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["colpali_engine"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":0.5,"embeddingDim":128,"maxTokens":8192,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8248529999999998},"scoresByTask":{"VidoreTabfquadRetrieval":0.74712,"VidoreDocVQARetrieval":0.58016,"VidoreArxivQARetrieval":0.74898,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.97992,"VidoreTatdqaRetrieval":0.77469,"VidoreShiftProjectRetrieval":0.66825,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.9474,"VidoreInfoVQARetrieval":0.87223,"VidoreSyntheticDocQAEnergyRetrieval":0.94585,"VidoreSyntheticDocQAAIRetrieval":0.98393},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":73,"model":{"name":"athrael-soju/ColUltraVBERT","url":"https://huggingface.co/athrael-soju/ColUltraVBERT","zeroShotPct":93,"activeParamsB":0.217,"totalParamsB":0.255,"embeddingDim":768,"maxTokens":32768.0,"releaseDate":"2026-10-17","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":true,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":486.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/datasets/athrael-soju/vdr-train-500k","adaptedFrom":"athrael-soju/UltraVBERT","supersededBy":null,"extraRequirementsGroups":["image"],"trainingDatasets":["NQ","VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","VisRAGRetArxivQA","VisRAGRetChartQA","VisRAGRetInfoVQA","VisRAGRetMPDocVQA","VisRAGRetPlotQA","VisRAGRetSlideVQA","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":0.217,"totalParamsB":0.255,"embeddingDim":768,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.864457},"scoresByTask":{"VidoreTabfquadRetrieval":0.92511,"VidoreDocVQARetrieval":0.57724,"VidoreArxivQARetrieval":0.80085,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.97893,"VidoreTatdqaRetrieval":0.75095,"VidoreShiftProjectRetrieval":0.81216,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.95085,"VidoreInfoVQARetrieval":0.89507,"VidoreSyntheticDocQAEnergyRetrieval":0.9571,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":74,"model":{"name":"facebook/dinov2-base","url":"https://huggingface.co/facebook/dinov2-base","zeroShotPct":100,"activeParamsB":0.087,"totalParamsB":0.087,"embeddingDim":768,"maxTokens":null,"releaseDate":"2023-07-18","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image"],"languages":["English"],"citation":"@article{oquab2023dinov2,\n  archiveprefix = {arXiv},\n  author = {Maxime Oquab and Timoth{\\'e}e Darcet and Th{\\'e}o Moutakanni and Huy V. Vo and Marc Szafraniec and Vasil Khalidov and Pierre Fernandez and Daniel HAZIZA and Francisco Massa and Alaaeldin El-Nouby and Mido Assran and Nicolas Ballas and Wojciech Galuba and Russell Howes and Po-Yao Huang and Shang-Wen Li and Ishan Misra and Michael Rabbat and Vasu Sharma and Gabriel Synnaeve and Hu Xu and Herve Jegou and Julien Mairal and Patrick Labatut and Armand Joulin and Piotr Bojanowski},\n  eprint = {2304.07193},\n  issn = {2835-8856},\n  journal = {Transactions on Machine Learning Research},\n  note = {Featured Certification},\n  title = {{DINO}v2: Learning Robust Visual Features without Supervision},\n  url = {https://openreview.net/forum?id=a68SUt6zFt},\n  year = {2024},\n}","memoryUsageMb":330.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/facebookresearch/dinov2","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.087,"totalParamsB":0.087,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"ImageClassification":0.7661925844645403,"ImageClustering":0.8274498652863354,"VisualSTS(eng)":0.16826345550796637,"VisualSTS(multi)":0.1929195},"scoresByTask":{"CIFAR10Clustering":0.8261784220207251,"STL10":0.9772500000000001,"PatchCamelyon":0.7272705078125,"TinyImageNetClustering":0.7430537932480211,"SUN397":0.7102896551724138,"METI2IRetrieval":0.43083,"STS13VisualSTS":0.05816840617235697,"ROxfordMediumI2IRetrieval":0.24859,"NIGHTSI2IRetrieval":0.25939,"UCF101":0.8551072685600856,"OxfordPets":0.9411828836195149,"Food101Classification":0.8443960396039604,"RParisEasyI2IRetrieval":0.11357,"Imagenet1k":0.759196,"VisualSTS-b-Eng":0.126548,"VOC2007":0.8593949384036431,"ROxfordEasyI2IRetrieval":0.41851,"RP2kI2IRetrieval":0.63573,"ROxfordHardI2IRetrieval":0.26888,"DTD":0.7592553191489362,"Country211":0.10728909952606636,"CIFAR10":0.9644600000000001,"CIFAR100":0.8448800000000001,"VisualSTS-b-Multilingual":0.184963,"VisualSTS17Eng":0.430191,"RESISC45":0.813015873015873,"GTSRB":0.5348218527315914,"FORBI2IRetrieval":0.29449,"MNIST":0.8671199999999999,"CIFAR100Clustering":0.7720462516461998,"RParisHardI2IRetrieval":0.05147,"SOPI2IRetrieval":0.4813,"ImageNet10Clustering":0.9392561230332483,"StanfordCarsI2IRetrieval":0.64221,"FER2013":0.399442741710783,"OxfordFlowersClassification":0.995686274509804,"STS12VisualSTS":0.24133576956764222,"STS14VisualSTS":0.0036221200178206,"GLDv2I2IRetrieval":0.34803,"STS15VisualSTS":0.13750465362153932,"SketchyI2IRetrieval":0.54442,"Caltech101":0.8944773175542406,"VisualSTS17Multilingual":0.200876,"EuroSAT":0.8165555555555557,"Birdsnap":0.7724473257698541,"StanfordCars":0.7852754632508395,"STS16VisualSTS":0.18047423917640573,"RParisMediumI2IRetrieval":0.0273,"ImageNetDog15Clustering":0.856714736483482,"FGVCAircraft":0.6274227422742275,"CUB200I2IRetrieval":0.85744},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":75,"model":{"name":"facebook/dinov2-giant","url":"https://huggingface.co/facebook/dinov2-giant","zeroShotPct":100,"activeParamsB":1.136,"totalParamsB":1.136,"embeddingDim":1536,"maxTokens":null,"releaseDate":"2023-07-18","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image"],"languages":["English"],"citation":"@article{oquab2023dinov2,\n  archiveprefix = {arXiv},\n  author = {Maxime Oquab and Timoth{\\'e}e Darcet and Th{\\'e}o Moutakanni and Huy V. Vo and Marc Szafraniec and Vasil Khalidov and Pierre Fernandez and Daniel HAZIZA and Francisco Massa and Alaaeldin El-Nouby and Mido Assran and Nicolas Ballas and Wojciech Galuba and Russell Howes and Po-Yao Huang and Shang-Wen Li and Ishan Misra and Michael Rabbat and Vasu Sharma and Gabriel Synnaeve and Hu Xu and Herve Jegou and Julien Mairal and Patrick Labatut and Armand Joulin and Piotr Bojanowski},\n  eprint = {2304.07193},\n  issn = {2835-8856},\n  journal = {Transactions on Machine Learning Research},\n  note = {Featured Certification},\n  title = {{DINO}v2: Learning Robust Visual Features without Supervision},\n  url = {https://openreview.net/forum?id=a68SUt6zFt},\n  year = {2024},\n}","memoryUsageMb":4335.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/facebookresearch/dinov2","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":1.136,"totalParamsB":1.136,"embeddingDim":1536,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"ImageClassification":0.7888967577237266,"ImageClustering":0.8144271459329184,"VisualSTS(eng)":0.15027495816217334,"VisualSTS(multi)":0.176674},"scoresByTask":{"CIFAR10Clustering":0.768401076933891,"STL10":0.978425,"PatchCamelyon":0.74169921875,"TinyImageNetClustering":0.701280301746393,"SUN397":0.7231356321839081,"METI2IRetrieval":0.43455,"STS13VisualSTS":0.011756144977994269,"ROxfordMediumI2IRetrieval":0.25913,"NIGHTSI2IRetrieval":0.25065,"UCF101":0.8918728898399648,"OxfordPets":0.9493049877350778,"Food101Classification":0.8809584158415842,"RParisEasyI2IRetrieval":0.11823,"Imagenet1k":0.78698,"VisualSTS-b-Eng":0.094832,"VOC2007":0.8543642990448005,"ROxfordEasyI2IRetrieval":0.4091,"RP2kI2IRetrieval":0.66818,"ROxfordHardI2IRetrieval":0.30242,"DTD":0.7608510638297872,"Country211":0.13533649289099525,"CIFAR10":0.9853400000000001,"CIFAR100":0.8962800000000002,"VisualSTS-b-Multilingual":0.181176,"VisualSTS17Eng":0.370662,"RESISC45":0.863015873015873,"GTSRB":0.5513855898653999,"FORBI2IRetrieval":0.43487,"MNIST":0.8581799999999999,"CIFAR100Clustering":0.7577140167049838,"RParisHardI2IRetrieval":0.05347,"SOPI2IRetrieval":0.49891,"ImageNet10Clustering":0.918444467895486,"StanfordCarsI2IRetrieval":0.68375,"FER2013":0.4106157704095848,"OxfordFlowersClassification":0.9970588235294118,"STS12VisualSTS":0.24337165341284864,"STS14VisualSTS":0.010553479111310389,"GLDv2I2IRetrieval":0.39726,"STS15VisualSTS":0.13645465160974304,"SketchyI2IRetrieval":0.72198,"Caltech101":0.8913872452333991,"VisualSTS17Multilingual":0.172172,"EuroSAT":0.8452592592592593,"Birdsnap":0.8188006482982171,"StanfordCars":0.8317870911578161,"STS16VisualSTS":0.184294778023317,"RParisMediumI2IRetrieval":0.02753,"ImageNetDog15Clustering":0.9262958663838387,"FGVCAircraft":0.7036903690369037,"CUB200I2IRetrieval":0.86244},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":76,"model":{"name":"facebook/dinov2-large","url":"https://huggingface.co/facebook/dinov2-large","zeroShotPct":100,"activeParamsB":0.304,"totalParamsB":0.304,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2023-07-18","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image"],"languages":["English"],"citation":"@article{oquab2023dinov2,\n  archiveprefix = {arXiv},\n  author = {Maxime Oquab and Timoth{\\'e}e Darcet and Th{\\'e}o Moutakanni and Huy V. Vo and Marc Szafraniec and Vasil Khalidov and Pierre Fernandez and Daniel HAZIZA and Francisco Massa and Alaaeldin El-Nouby and Mido Assran and Nicolas Ballas and Wojciech Galuba and Russell Howes and Po-Yao Huang and Shang-Wen Li and Ishan Misra and Michael Rabbat and Vasu Sharma and Gabriel Synnaeve and Hu Xu and Herve Jegou and Julien Mairal and Patrick Labatut and Armand Joulin and Piotr Bojanowski},\n  eprint = {2304.07193},\n  issn = {2835-8856},\n  journal = {Transactions on Machine Learning Research},\n  note = {Featured Certification},\n  title = {{DINO}v2: Learning Robust Visual Features without Supervision},\n  url = {https://openreview.net/forum?id=a68SUt6zFt},\n  year = {2024},\n}","memoryUsageMb":1161.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/facebookresearch/dinov2","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.304,"totalParamsB":0.304,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"ImageClassification":0.7798834329909838,"ImageClustering":0.8309526820420434,"VisualSTS(eng)":0.13816661061424648,"VisualSTS(multi)":0.1994775},"scoresByTask":{"CIFAR10Clustering":0.7990120240213314,"STL10":0.979275,"PatchCamelyon":0.74373779296875,"TinyImageNetClustering":0.7721811405936876,"SUN397":0.7275862068965517,"METI2IRetrieval":0.42852,"STS13VisualSTS":0.00047973495756825456,"ROxfordMediumI2IRetrieval":0.26639,"NIGHTSI2IRetrieval":0.24806,"UCF101":0.8807934918863719,"OxfordPets":0.9489779231398201,"Food101Classification":0.8778534653465346,"RParisEasyI2IRetrieval":0.11627,"Imagenet1k":0.786188,"VisualSTS-b-Eng":0.063094,"VOC2007":0.8551135559113904,"ROxfordEasyI2IRetrieval":0.42485,"RP2kI2IRetrieval":0.64291,"ROxfordHardI2IRetrieval":0.307,"DTD":0.764468085106383,"Country211":0.12727962085308056,"CIFAR10":0.97948,"CIFAR100":0.8882999999999999,"VisualSTS-b-Multilingual":0.170869,"VisualSTS17Eng":0.436739,"RESISC45":0.8621269841269841,"GTSRB":0.5322406967537608,"FORBI2IRetrieval":0.38423,"MNIST":0.82402,"CIFAR100Clustering":0.7992699292791358,"RParisHardI2IRetrieval":0.05252,"SOPI2IRetrieval":0.49259,"ImageNet10Clustering":0.922294933497504,"StanfordCarsI2IRetrieval":0.67827,"FER2013":0.41398718305934806,"OxfordFlowersClassification":0.996078431372549,"STS12VisualSTS":0.17447337999822643,"STS14VisualSTS":-0.023934185421068874,"GLDv2I2IRetrieval":0.39566,"STS15VisualSTS":0.12278281418592447,"SketchyI2IRetrieval":0.66153,"Caltech101":0.8958251150558845,"VisualSTS17Multilingual":0.228086,"EuroSAT":0.8022592592592593,"Birdsnap":0.8104808211777417,"StanfordCars":0.8099987563735853,"STS16VisualSTS":0.1935315305790751,"RParisMediumI2IRetrieval":0.02733,"ImageNetDog15Clustering":0.8620053828185583,"FGVCAircraft":0.6513651365136515,"CUB200I2IRetrieval":0.85899},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":77,"model":{"name":"facebook/dinov2-small","url":"https://huggingface.co/facebook/dinov2-small","zeroShotPct":100,"activeParamsB":0.022,"totalParamsB":0.022,"embeddingDim":384,"maxTokens":null,"releaseDate":"2023-07-18","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image"],"languages":["English"],"citation":"@article{oquab2023dinov2,\n  archiveprefix = {arXiv},\n  author = {Maxime Oquab and Timoth{\\'e}e Darcet and Th{\\'e}o Moutakanni and Huy V. Vo and Marc Szafraniec and Vasil Khalidov and Pierre Fernandez and Daniel HAZIZA and Francisco Massa and Alaaeldin El-Nouby and Mido Assran and Nicolas Ballas and Wojciech Galuba and Russell Howes and Po-Yao Huang and Shang-Wen Li and Ishan Misra and Michael Rabbat and Vasu Sharma and Gabriel Synnaeve and Hu Xu and Herve Jegou and Julien Mairal and Patrick Labatut and Armand Joulin and Piotr Bojanowski},\n  eprint = {2304.07193},\n  issn = {2835-8856},\n  journal = {Transactions on Machine Learning Research},\n  note = {Featured Certification},\n  title = {{DINO}v2: Learning Robust Visual Features without Supervision},\n  url = {https://openreview.net/forum?id=a68SUt6zFt},\n  year = {2024},\n}","memoryUsageMb":84.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/facebookresearch/dinov2","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.022,"totalParamsB":0.022,"embeddingDim":384,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"ImageClassification":0.7356461867074647,"ImageClustering":0.8020325235613154,"VisualSTS(eng)":0.12979017595333833,"VisualSTS(multi)":0.175937},"scoresByTask":{"CIFAR10Clustering":0.7924590716393438,"STL10":0.973575,"PatchCamelyon":0.745526123046875,"TinyImageNetClustering":0.7064735294077449,"SUN397":0.6664827586206897,"METI2IRetrieval":0.4239,"STS13VisualSTS":0.023881946051784057,"ROxfordMediumI2IRetrieval":0.25464,"NIGHTSI2IRetrieval":0.2599,"UCF101":0.805332304488384,"OxfordPets":0.9201417279912782,"Food101Classification":0.7768,"RParisEasyI2IRetrieval":0.11607,"Imagenet1k":0.693976,"VisualSTS-b-Eng":0.053715,"VOC2007":0.8694370359087007,"ROxfordEasyI2IRetrieval":0.39783,"RP2kI2IRetrieval":0.65299,"ROxfordHardI2IRetrieval":0.2625,"DTD":0.7242553191489361,"Country211":0.08098578199052134,"CIFAR10":0.9295399999999999,"CIFAR100":0.7701800000000001,"VisualSTS-b-Multilingual":0.111817,"VisualSTS17Eng":0.43104,"RESISC45":0.7685714285714285,"GTSRB":0.5221694378463975,"FORBI2IRetrieval":0.25645,"MNIST":0.8658199999999999,"CIFAR100Clustering":0.726175948994106,"RParisHardI2IRetrieval":0.05084,"SOPI2IRetrieval":0.4823,"ImageNet10Clustering":0.9123090941953348,"StanfordCarsI2IRetrieval":0.4931,"FER2013":0.37266648091390364,"OxfordFlowersClassification":0.9950980392156863,"STS12VisualSTS":0.13393947680008342,"STS14VisualSTS":-0.019013017914226483,"GLDv2I2IRetrieval":0.28942,"STS15VisualSTS":0.12023176458023736,"SketchyI2IRetrieval":0.40567,"Caltech101":0.885765943458251,"VisualSTS17Multilingual":0.240057,"EuroSAT":0.8185925925925925,"Birdsnap":0.7136682874122096,"StanfordCars":0.6988931724909837,"STS16VisualSTS":0.16473606215548986,"RParisMediumI2IRetrieval":0.02746,"ImageNetDog15Clustering":0.8727449735700475,"FGVCAircraft":0.5867386738673867,"CUB200I2IRetrieval":0.83535},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":78,"model":{"name":"intfloat/multilingual-e5-small","url":"https://huggingface.co/intfloat/multilingual-e5-small","zeroShotPct":100,"activeParamsB":0.022,"totalParamsB":0.118,"embeddingDim":384,"maxTokens":512.0,"releaseDate":"2024-02-08","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["text"],"languages":["Afrikaans","Albanian","Amharic (Latin)","Arabic (Latin)","Armenian (Latin)","Assamese (Latin)","Azerbaijani","Basque","Belarusian (Latin)","Bengali","Bengali (Latin)","Bosnian","Breton","Bulgarian (Latin)","Burmese (Latin)","Catalan","Chinese","Chinese (Traditional)","Croatian","Czech","Danish","Dutch","English","Esperanto","Estonian","Finnish","French","Galician","Georgian (Latin)","German","Gujarati (Latin)","Hausa","Hebrew (Latin)","Hindi","Hindi (Latin)","Hungarian","Icelandic","Indonesian","Irish","Italian","Japanese (Latin)","Javanese","Kannada (Latin)","Kazakh (Latin)","Khmer (Latin)","Kirghiz (Latin)","Korean (Latin)","Kurdish (Latin)","Lao (Latin)","Latin","Latvian","Lithuanian","Macedonian (Latin)","Malagasy (Latin)","Malay (macrolanguage)","Malayalam (Latin)","Marathi (Latin)","Modern Greek (1453-) (Latin)","Mongolian (Latin)","Nepali (macrolanguage) (Latin)","Norwegian Bokmål","Oriya (macrolanguage) (Latin)","Oromo (Latin)","Panjabi (Latin)","Persian (Latin)","Polish","Portuguese","Pushto (Latin)","Romanian","Russian (Latin)","Sanskrit (Latin)","Scottish Gaelic","Serbian (Latin)","Sindhi (Latin)","Sinhala (Latin)","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tamil","Tamil (Latin)","Telugu","Telugu (Latin)","Thai (Latin)","Turkish","Uighur (Latin)","Ukrainian (Latin)","Urdu","Urdu (Latin)","Uzbek","Vietnamese","Welsh","Western Frisian","Xhosa","Yiddish (Latin)"],"citation":"\n@article{wang2024multilingual,\n  title={Multilingual E5 Text Embeddings: A Technical Report},\n  author={Wang, Liang and Yang, Nan and Huang, Xiaolong and Yang, Linjun and Majumder, Rangan and Wei, Furu},\n  journal={arXiv preprint arXiv:2402.05672},\n  year={2024}\n}\n","memoryUsageMb":449.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"microsoft/Multilingual-MiniLM-L12-H384","supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["FEVER","FEVER-NL","FEVER-PL","FEVERHardNegatives","HotpotQA","HotpotQA-NL","HotpotQA-PL","HotpotQAHardNegatives","MIRACLReranking","MIRACLRetrieval","MIRACLRetrievalHardNegatives","MSMARCO","MSMARCO-PL","MSMARCOHardNegatives","MrTidyRetrieval","NQ","NQ-NL","NQ-PL","NQHardNegatives","NanoMSMARCORetrieval","NanoNQRetrieval","XQuADRetrieval","mMARCO-NL"]},"zeroShotPct":100,"activeParamsB":0.022,"totalParamsB":0.118,"embeddingDim":384,"maxTokens":512,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{},"scoresByTask":{"WebQAT2TRetrieval":0.66049},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":79,"model":{"name":"Haon-Chen/e5-omni-3B","url":"https://huggingface.co/Haon-Chen/e5-omni-3B","zeroShotPct":100,"activeParamsB":4.392,"totalParamsB":4.703,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2026-01-06","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@inproceedings{chen2026e5omni,\n  address = {San Diego, California, United States},\n  author = {Chen, Haonan and Gao, Sicheng and Timofte, Radu and Sakai, Tetsuya and Dou, Zhicheng},\n  booktitle = {Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026},\n  doi = {10.18653/v1/2026.findings-acl.970},\n  editor = {Liakata, Maria and Moreira, Viviane P. and Zhang, Jiajun and Jurgens, David},\n  isbn = {979-8-89176-395-1},\n  month = {jul},\n  pages = {19430--19443},\n  publisher = {Association for Computational Linguistics},\n  title = {e5-omni: Explicit Cross-modal Alignment for Omni-modal Embeddings},\n  url = {https://aclanthology.org/2026.findings-acl.970/},\n  year = {2026},\n}\n","memoryUsageMb":8971.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen2.5-Omni-3B","supersededBy":null,"extraRequirementsGroups":["multimodal-sbert"],"trainingDatasets":["AudioCapsT2ARetrieval","MSRVTTT2V","MSRVTTV2T"]},"zeroShotPct":100,"activeParamsB":4.392,"totalParamsB":4.703,"embeddingDim":2048,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.7348699999999999},"scoresByTask":{"CVBenchDistance":0.51833,"CVBenchRelation":0.58923,"OxfordPetsZeroShot":0.492232,"VidoreTabfquadRetrieval":0.90303,"VidoreDocVQARetrieval":0.49211,"VisualNewsI2TRetrieval":0.03705,"PatchCamelyon":0.627771,"TinyImageNetClustering":0.641787,"WebQAT2ITRetrieval":0.63288,"SUN397":0.696579,"STS13VisualSTS":0.715633,"CVBenchCount":0.62817,"NIGHTSI2IRetrieval":0.24114,"VidoreTatdqaRetrieval":0.62625,"ImageCoDe":0.123805,"Fashion200kI2TRetrieval":0.03638,"FGVCAircraftZeroShot":0.124212,"XM3600T2IRetrieval":0.53082,"OxfordPets":0.804797,"FER2013ZeroShot":0.342575,"OVENIT2TRetrieval":0.17976,"CVBenchDepth":0.575,"VidoreShiftProjectRetrieval":0.80226,"WITT2IRetrieval":0.48831454545454545,"RP2kI2IRetrieval":0.58388,"DTD":0.685957,"AROVisualAttribution":0.478607,"Country211":0.073754,"AROVisualRelation":0.55926,"InfoSeekIT2TRetrieval":0.29711,"VidoreInfoVQARetrieval":0.90509,"BLINKIT2IMultiChoice":0.72637,"VisualSTS-b-Multilingual":0.711861,"RESISC45":0.865048,"GTSRB":0.669533,"Country211ZeroShot":0.092038,"CIFAR100ZeroShot":0.6048,"VidoreSyntheticDocQAAIRetrieval":0.97047,"StanfordCarsZeroShot":0.539983,"AROCocoOrder":0.216194,"VQA2IT2TRetrieval":0.23068,"STS15VisualSTS":0.799923,"VisualSTS17Multilingual":0.757879,"EuroSAT":0.558,"CIRRIT2IRetrieval":0.21657,"Food101ZeroShot":0.65996,"Winoground":0.0625,"HatefulMemesI2TRetrieval":0.41552,"AROFlickrOrder":0.2262,"ImageNetDog15Clustering":0.718047,"CUB200I2IRetrieval":0.762},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.5144578342245989}},"trainedOnTasks":[],"experiments":null},{"rank":80,"model":{"name":"Haon-Chen/e5-omni-7B","url":"https://huggingface.co/Haon-Chen/e5-omni-7B","zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768.0,"releaseDate":"2026-01-06","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@inproceedings{chen2026e5omni,\n  address = {San Diego, California, United States},\n  author = {Chen, Haonan and Gao, Sicheng and Timofte, Radu and Sakai, Tetsuya and Dou, Zhicheng},\n  booktitle = {Findings of the {A}ssociation for {C}omputational {L}inguistics: {ACL} 2026},\n  doi = {10.18653/v1/2026.findings-acl.970},\n  editor = {Liakata, Maria and Moreira, Viviane P. and Zhang, Jiajun and Jurgens, David},\n  isbn = {979-8-89176-395-1},\n  month = {jul},\n  pages = {19430--19443},\n  publisher = {Association for Computational Linguistics},\n  title = {e5-omni: Explicit Cross-modal Alignment for Omni-modal Embeddings},\n  url = {https://aclanthology.org/2026.findings-acl.970/},\n  year = {2026},\n}\n","memoryUsageMb":17036.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen2.5-Omni-7B","supersededBy":null,"extraRequirementsGroups":["multimodal-sbert"],"trainingDatasets":["AudioCapsT2ARetrieval","MSRVTTT2V","MSRVTTV2T"]},"zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.7733485},"scoresByTask":{"CVBenchDistance":0.62833,"CVBenchRelation":0.83385,"OxfordPetsZeroShot":0.802398,"VidoreTabfquadRetrieval":0.93357,"VidoreDocVQARetrieval":0.57503,"VisualNewsI2TRetrieval":0.30339,"PatchCamelyon":0.641235,"TinyImageNetClustering":0.670528,"WebQAT2ITRetrieval":0.65218,"SUN397":0.749738,"STS13VisualSTS":0.763669,"CVBenchCount":0.63325,"NIGHTSI2IRetrieval":0.25158,"VidoreTatdqaRetrieval":0.70953,"ImageCoDe":0.124674,"Fashion200kI2TRetrieval":0.06847,"FGVCAircraftZeroShot":0.258926,"XM3600T2IRetrieval":0.6866272222222222,"OxfordPets":0.915072,"FER2013ZeroShot":0.3455,"OVENIT2TRetrieval":0.19889,"CVBenchDepth":0.605,"VidoreShiftProjectRetrieval":0.85601,"WITT2IRetrieval":0.6358218181818182,"RP2kI2IRetrieval":0.63586,"DTD":0.739149,"AROVisualAttribution":0.741512,"Country211":0.114199,"AROVisualRelation":0.577224,"InfoSeekIT2TRetrieval":0.34124,"VidoreInfoVQARetrieval":0.9248,"BLINKIT2IMultiChoice":0.76368,"VisualSTS-b-Multilingual":0.757094,"RESISC45":0.891048,"GTSRB":0.719303,"Country211ZeroShot":0.201043,"CIFAR100ZeroShot":0.7073,"VidoreSyntheticDocQAAIRetrieval":0.98893,"StanfordCarsZeroShot":0.767193,"AROCocoOrder":0.42375,"VQA2IT2TRetrieval":0.19045,"STS15VisualSTS":0.845704,"VisualSTS17Multilingual":0.789603,"EuroSAT":0.69737,"CIRRIT2IRetrieval":0.22294,"Food101ZeroShot":0.810257,"Winoground":0.1,"HatefulMemesI2TRetrieval":0.56448,"AROFlickrOrder":0.4594,"ImageNetDog15Clustering":0.79558,"CUB200I2IRetrieval":0.75992},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.5955352556941969}},"trainedOnTasks":[],"experiments":null},{"rank":81,"model":{"name":"eagerworks/eager-embed-v1","url":"https://huggingface.co/eagerworks/eager-embed-v1","zeroShotPct":100,"activeParamsB":4.049,"totalParamsB":4.438,"embeddingDim":2560,"maxTokens":262144.0,"releaseDate":"2025-11-20","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":false,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Spanish"],"citation":null,"memoryUsageMb":16929.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/eagerworks/eager-embed","publicTrainingData":"https://github.com/eagerworks/eager-embed/blob/main/dataset_config.yaml","adaptedFrom":"https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct","supersededBy":null,"extraRequirementsGroups":["eager_embed"],"trainingDatasets":["bge-ir","colpali","pixmo-docs","wiki-ss"]},"zeroShotPct":100,"activeParamsB":4.049,"totalParamsB":4.438,"embeddingDim":2560,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8486290000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.92312,"VidoreDocVQARetrieval":0.57055,"VidoreArxivQARetrieval":0.8602,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.96786,"VidoreTatdqaRetrieval":0.62069,"VidoreShiftProjectRetrieval":0.77822,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.94309,"VidoreInfoVQARetrieval":0.88397,"VidoreSyntheticDocQAEnergyRetrieval":0.96036,"VidoreSyntheticDocQAAIRetrieval":0.97823},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":82,"model":{"name":"encord-team/ebind-full","url":"https://huggingface.co/encord-team/ebind-full","zeroShotPct":98,"activeParamsB":1.739,"totalParamsB":1.79,"embeddingDim":1024,"maxTokens":32.0,"releaseDate":"2025-11-19","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@misc{broadbent2025ebindpracticalapproachspace,\n      title={{EBind}: a practical approach to space binding},\n      author={Jim Broadbent and Felix Cohen and Frederik Hvilshøj and Eric Landau and Eren Sasoglu},\n      year={2025},\n      eprint={2511.14229},\n      archivePrefix={arXiv},\n      primaryClass={cs.LG},\n      url={https://arxiv.org/abs/2511.14229},\n}\n","memoryUsageMb":6828.0,"license":"cc-by-nc-sa-4.0","publicTrainingCode":"https://github.com/encord-team/ebind","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["audio","video"],"trainingDatasets":["AudioCapsA2TRetrieval","AudioCapsAVA2VRetrieval","AudioCapsAVAT2VRetrieval","AudioCapsAVT2VARetrieval","AudioCapsAVT2VRetrieval","AudioCapsAVV2ARetrieval","AudioCapsAVV2TRetrieval","AudioCapsAVVA2TRetrieval","AudioCapsAVVT2ARetrieval","AudioCapsT2ARetrieval","Imagenet1k","Imagenet1kZeroShot","VALOR32KA2VRetrieval","VALOR32KAT2VRetrieval","VALOR32KT2VARetrieval","VALOR32KT2VRetrieval","VALOR32KV2ARetrieval","VALOR32KV2TRetrieval","VALOR32KVA2TRetrieval","VALOR32KVT2ARetrieval","VGGSoundAVA2VRetrieval","VGGSoundAVAT2VRetrieval","VGGSoundAVT2VARetrieval","VGGSoundAVT2VRetrieval","VGGSoundAVV2ARetrieval","VGGSoundAVV2TRetrieval","VGGSoundAVVA2TRetrieval","VGGSoundAVVT2ARetrieval","VGGSoundV","VGGSoundVA","VGGSoundVideoAudioZeroshot","VGGSoundVideoZeroshot"]},"zeroShotPct":98,"activeParamsB":1.739,"totalParamsB":1.79,"embeddingDim":1024,"maxTokens":32,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.4277225},"scoresByTask":{"CVBenchDistance":0.505,"CVBenchRelation":0.50769,"OxfordPetsZeroShot":0.961842,"VidoreTabfquadRetrieval":0.39834,"VidoreDocVQARetrieval":0.32923,"VisualNewsI2TRetrieval":0.41051,"PatchCamelyon":0.795465,"TinyImageNetClustering":0.824596,"WebQAT2ITRetrieval":0.48388,"SUN397":0.75292,"STS13VisualSTS":0.719333,"CVBenchCount":0.30584,"NIGHTSI2IRetrieval":0.25933,"VidoreTatdqaRetrieval":0.34562,"ImageCoDe":0.140313,"Fashion200kI2TRetrieval":0.14973,"FGVCAircraftZeroShot":0.671467,"XM3600T2IRetrieval":0.1754863888888889,"OxfordPets":0.926847,"FER2013ZeroShot":0.586514,"OVENIT2TRetrieval":0.01203,"CVBenchDepth":0.51,"VidoreShiftProjectRetrieval":0.32774,"WITT2IRetrieval":0.34496272727272725,"RP2kI2IRetrieval":0.70437,"DTD":0.716702,"AROVisualAttribution":0.701962,"Country211":0.258104,"AROVisualRelation":0.522873,"InfoSeekIT2TRetrieval":0.0151,"VidoreInfoVQARetrieval":0.69425,"BLINKIT2IMultiChoice":0.74876,"VisualSTS-b-Multilingual":0.465063,"RESISC45":0.880984,"GTSRB":0.763642,"Country211ZeroShot":0.427536,"CIFAR100ZeroShot":0.8846,"VidoreSyntheticDocQAAIRetrieval":0.77381,"StanfordCarsZeroShot":0.932844,"AROCocoOrder":0.468972,"VQA2IT2TRetrieval":0.0107,"STS15VisualSTS":0.790653,"VisualSTS17Multilingual":0.390382,"EuroSAT":0.882037,"CIRRIT2IRetrieval":0.14977,"Food101ZeroShot":0.958733,"Winoground":0.1025,"HatefulMemesI2TRetrieval":0.59561,"AROFlickrOrder":0.5252,"ImageNetDog15Clustering":0.821025,"CUB200I2IRetrieval":0.83673},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.5385803552188553}},"trainedOnTasks":[],"experiments":null},{"rank":83,"model":{"name":"tencent/EVIE-4.5B","url":"https://huggingface.co/tencent/EVIE-4.5B","zeroShotPct":93,"activeParamsB":3.909,"totalParamsB":4.545,"embeddingDim":2048,"maxTokens":262144.0,"releaseDate":"2026-09-04","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":9089.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/Tencent/EVIE","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["evie"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data"]},"zeroShotPct":93,"activeParamsB":3.909,"totalParamsB":4.545,"embeddingDim":2048,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9285779999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.97345,"VidoreDocVQARetrieval":0.67781,"VidoreArxivQARetrieval":0.9279,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":1.0,"VidoreTatdqaRetrieval":0.8397,"VidoreShiftProjectRetrieval":0.95065,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98524,"VidoreInfoVQARetrieval":0.94672,"VidoreSyntheticDocQAEnergyRetrieval":0.98431,"VidoreSyntheticDocQAAIRetrieval":1.0},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":84,"model":{"name":"tencent/EVIE-8B","url":"https://huggingface.co/tencent/EVIE-8B","zeroShotPct":93,"activeParamsB":7.392,"totalParamsB":8.409,"embeddingDim":4096,"maxTokens":262144.0,"releaseDate":"2026-09-04","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":16819.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/Tencent/EVIE","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["evie"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data"]},"zeroShotPct":93,"activeParamsB":7.392,"totalParamsB":8.409,"embeddingDim":4096,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9289569999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.97492,"VidoreDocVQARetrieval":0.68242,"VidoreArxivQARetrieval":0.92654,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.8414,"VidoreShiftProjectRetrieval":0.95417,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.99262,"VidoreInfoVQARetrieval":0.94795,"VidoreSyntheticDocQAEnergyRetrieval":0.97693,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":85,"model":{"name":"google/embeddinggemma-2","url":"https://huggingface.co/google/embeddinggemma-2","zeroShotPct":100,"activeParamsB":0.137,"totalParamsB":0.271,"embeddingDim":768,"maxTokens":8192.0,"releaseDate":"2026-10-06","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["Bengali","Chinese","Chinese (Traditional)","English","Finnish","French","German","Hindi","Indonesian","Iranian Persian","Japanese","Korean","Russian","Spanish","Standard Arabic","Swahili (individual language)","Telugu","Thai","Yoruba"],"citation":"\n@misc{vera2026embeddinggemma2unifiedefficient,\n      title={EmbeddingGemma 2: Unified and Efficient Multimodal Representations},\n      author={Henrique Schechter Vera and Sahil Dua and Jovan Andonov and Sara Smoot and Sebastian Russo and Ivan Llanos and Max Matveev and Jae Hyeon Yoo and Kat Black and Shih-Cheng Huang and Josiah Aklilu and Maarten Grootendorst and Omar Sanseviero and Chandramouli Amarnath and Ian Ballantyne and Zhe Li and Daniel Salz and Ibrahim Alabdulmohsin and Ilya Philippov and Kristen Quan and Glenn Cameron and Edouard Yvinec and Luciano Martins and Doug Reid and Sindhu Raghuram Panyam and Weiyi Wang and Hengxuan Ying and Madhuri Shanbhogue and Tanmaya Dabral and Shanfeng Zhang and Juyeong Ji and Renjie Wu and Joe Zou and Xu Chen and Yunan Zhang and Yu-Hui Chen and Matthew Soulanille and Vinod Mamillapalli and Yi-Chun Kuo and Wai Hon Law and Matt Kreileder and Alice Zheng and Suril Shah and Mogan Shieh and Pooja Consul and Alberto Montes and Andreas Hess and Andres Sevilla and Babak Samari and Brendan McCloskey and Brian Potetz and Daniel Cer and Georgios Kathareios and Gustavo Hernández Ábrego and Hesen Zhang and Ji Ma and Jiaxing Wu and Keegan Mosley and Paul Suganthan and Rui Meng and Shen Gao and Shengcao Cao and Yi-Ting Chen and Yichang Chen and Zhongli Ding and Kendra Byrne and Cormac Brick and Olivier Lacombe and Qin Yin and Tris Warkentin and Armand Joulin and Tom Duerig and Mojtaba Seyedhosseini},\n      year={2026},\n      eprint={2610.37364},\n      archivePrefix={arXiv},\n      primaryClass={cs.LG},\n      url={https://arxiv.org/abs/2610.37364},\n}\n","memoryUsageMb":1034.0,"license":"gemma","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["embeddinggemma"],"trainingDatasets":["FEVERHardNegatives","HotpotQAHardNegatives","MIRACLRetrievalHardNegatives","NQHardNegatives"]},"zeroShotPct":100,"activeParamsB":0.137,"totalParamsB":0.271,"embeddingDim":768,"maxTokens":8192,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.767703},"scoresByTask":{"CVBenchDistance":0.53667,"CVBenchRelation":0.53846,"OxfordPetsZeroShot":0.876806,"VidoreTabfquadRetrieval":0.91458,"VidoreDocVQARetrieval":0.39582,"VisualNewsI2TRetrieval":0.16911,"PatchCamelyon":0.693402,"TinyImageNetClustering":0.688473,"WebQAT2ITRetrieval":0.63998,"SUN397":0.730253,"STS13VisualSTS":0.773191,"CVBenchCount":0.36929,"NIGHTSI2IRetrieval":0.26247,"VidoreTatdqaRetrieval":0.42042,"ImageCoDe":0.122068,"Fashion200kI2TRetrieval":0.15249,"FGVCAircraftZeroShot":0.39664,"XM3600T2IRetrieval":0.6499105555555555,"OxfordPets":0.927828,"FER2013ZeroShot":0.619532,"OVENIT2TRetrieval":0.15997,"CVBenchDepth":0.45333,"VidoreShiftProjectRetrieval":0.64609,"WITT2IRetrieval":0.6421072727272727,"RP2kI2IRetrieval":0.73744,"DTD":0.755957,"AROVisualAttribution":0.780054,"Country211":0.208474,"AROVisualRelation":0.569119,"InfoSeekIT2TRetrieval":0.13511,"VidoreInfoVQARetrieval":0.77327,"BLINKIT2IMultiChoice":0.70896,"VisualSTS-b-Multilingual":0.755934,"RESISC45":0.88273,"GTSRB":0.797846,"Country211ZeroShot":0.193744,"CIFAR100ZeroShot":0.7553,"VidoreSyntheticDocQAAIRetrieval":0.87758,"StanfordCarsZeroShot":0.883223,"AROCocoOrder":0.301359,"VQA2IT2TRetrieval":0.16531,"STS15VisualSTS":0.856675,"VisualSTS17Multilingual":0.779472,"EuroSAT":0.83637,"CIRRIT2IRetrieval":0.22032,"Food101ZeroShot":0.793822,"Winoground":0.12,"HatefulMemesI2TRetrieval":0.78553,"AROFlickrOrder":0.4238,"ImageNetDog15Clustering":0.886017,"CUB200I2IRetrieval":0.78737},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.5794054280055456}},"trainedOnTasks":[],"experiments":null},{"rank":86,"model":{"name":"ibm-granite/granite-vision-3.3-2b-embedding","url":"https://huggingface.co/ibm-granite/granite-vision-3.3-2b-embedding","zeroShotPct":-1,"activeParamsB":2.879,"totalParamsB":2.98,"embeddingDim":128,"maxTokens":128000.0,"releaseDate":"2025-06-11","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@article{karlinsky2025granitevision,\n  title={Granite Vision: a lightweight, open-source multimodal model for enterprise Intelligence},\n  author={Granite Vision Team and Karlinsky, Leonid and Arbelle, Assaf and Daniels, Abraham and Nassar, Ahmed and Alfassi, Amit and Wu, Bo and Schwartz, Eli and Joshi, Dhiraj and Kondic, Jovana and Shabtay, Nimrod and Li, Pengyuan and Herzig, Roei and Abedin, Shafiq and Perek, Shaked and Harary, Sivan and Barzelay, Udi and Raz Goldfarb, Adi and Oliva, Aude and Wieles, Ben and Bhattacharjee, Bishwaranjan and Huang, Brandon and Auer, Christoph and Gutfreund, Dan and Beymer, David and Wood, David and Kuehne, Hilde and Hansen, Jacob and Shtok, Joseph and Wong, Ken and Bathen, Luis Angel and Mishra, Mayank and Lysak, Maksym and Dolfi, Michele and Yurochkin, Mikhail and Livathinos, Nikolaos and Harel, Nimrod and Azulai, Ophir and Naparstek, Oshri and de Lima, Rafael Teixeira and Panda, Rameswar and Doveh, Sivan and Gupta, Shubham and Das, Subhro and Zawad, Syed and Kim, Yusik and He, Zexue and Brooks, Alexander and Goodhart, Gabe and Govindjee, Anita and Leist, Derek and Ibrahim, Ibrahim and Soffer, Aya and Cox, David and Soule, Kate and Lastras, Luis and Desai, Nirmit and Ofek-koifman, Shila and Raghavan, Sriram and Syeda-Mahmood, Tanveer and Staar, Peter and Drory, Tal and Feris, Rogerio},\n  journal={arXiv preprint arXiv:2502.09927},\n  year={2025}\n}","memoryUsageMb":11351.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":2.879,"totalParamsB":2.98,"embeddingDim":128,"maxTokens":128000,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.86033},"scoresByTask":{"VidoreTabfquadRetrieval":0.88649,"VidoreDocVQARetrieval":0.54601,"VidoreArxivQARetrieval":0.84222,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98893,"VidoreTatdqaRetrieval":0.67935,"VidoreShiftProjectRetrieval":0.83921,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97286,"VidoreInfoVQARetrieval":0.89668,"VidoreSyntheticDocQAEnergyRetrieval":0.96262,"VidoreSyntheticDocQAAIRetrieval":0.98893},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":87,"model":{"name":"jinaai/jina-embeddings-v4","url":"https://huggingface.co/jinaai/jina-embeddings-v4","zeroShotPct":93,"activeParamsB":3.624,"totalParamsB":3.935,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2025-06-24","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":true,"modalities":["image","text"],"languages":["Afrikaans","Albanian","Amharic (Latin)","Arabic (Latin)","Armenian (Latin)","Assamese (Latin)","Azerbaijani","Basque","Belarusian (Latin)","Bengali","Bengali (Latin)","Bosnian","Breton","Bulgarian (Latin)","Burmese (Latin)","Catalan","Chinese","Chinese (Traditional)","Croatian","Czech","Danish","Dutch","English","Esperanto","Estonian","Finnish","French","Galician","Georgian (Latin)","German","Gujarati (Latin)","Hausa","Hebrew (Latin)","Hindi","Hindi (Latin)","Hungarian","Icelandic","Indonesian","Irish","Italian","Japanese (Latin)","Javanese","Kannada (Latin)","Kazakh (Latin)","Khmer (Latin)","Kirghiz (Latin)","Korean (Latin)","Kurdish (Latin)","Lao (Latin)","Latin","Latvian","Lithuanian","Macedonian (Latin)","Malagasy (Latin)","Malay (macrolanguage)","Malayalam (Latin)","Marathi (Latin)","Modern Greek (1453-) (Latin)","Mongolian (Latin)","Nepali (macrolanguage) (Latin)","Norwegian Bokmål","Oriya (macrolanguage) (Latin)","Oromo (Latin)","Panjabi (Latin)","Persian (Latin)","Polish","Portuguese","Pushto (Latin)","Romanian","Russian (Latin)","Sanskrit (Latin)","Scottish Gaelic","Serbian (Latin)","Sindhi (Latin)","Sinhala (Latin)","Slovak","Slovenian","Somali","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tagalog","Tamil","Tamil (Latin)","Telugu","Telugu (Latin)","Thai (Latin)","Turkish","Uighur (Latin)","Ukrainian (Latin)","Urdu","Urdu (Latin)","Uzbek","Vietnamese","Welsh","Western Frisian","Xhosa","Yiddish (Latin)"],"citation":"@inproceedings{günther2025jinaembeddingsv4universalembeddingsmultimodal,\n  address = {Suzhuo, China},\n  author = {G{\\\"u}nther, Michael and Sturua, Saba and Akram, Mohammad Kalim and Mohr, Isabelle and Ungureanu, Andrei and Wang, Bo and Eslami, Sedigheh and Martens, Scott and Werk, Maximilian and Wang, Nan and Xiao, Han},\n  booktitle = {Proceedings of the 5th Workshop on Multilingual Representation Learning ({MRL} 2025)},\n  doi = {10.18653/v1/2025.mrl-main.36},\n  editor = {Adelani, David Ifeoluwa and Arnett, Catherine and Ataman, Duygu and Chang, Tyler A. and Gonen, Hila and Raja, Rahul and Schmidt, Fabian and Stap, David and Wang, Jiayi},\n  isbn = {979-8-89176-345-6},\n  month = {nov},\n  pages = {531--550},\n  publisher = {Association for Computational Linguistics},\n  title = {jina-embeddings-v4: Universal Embeddings for Multimodal Multilingual Retrieval},\n  url = {https://aclanthology.org/2025.mrl-main.36/},\n  year = {2025},\n}","memoryUsageMb":7500.0,"license":"cc-by-nc-4.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen2.5-VL-3B-Instruct","supersededBy":null,"extraRequirementsGroups":["jina-v4"],"trainingDatasets":["AppsRetrieval","CodeFeedbackMT","CodeFeedbackST","CodeSearchNetRetrieval","CornStack","MSMARCO","MSMARCOHardNegatives","NQ","NQ-NL","NQ-PL","NQHardNegatives","NanoMSMARCORetrieval","NanoNQRetrieval","SICK-R","STS12","StackOverflowQA","VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","mMARCO-NL"]},"zeroShotPct":93,"activeParamsB":3.624,"totalParamsB":3.935,"embeddingDim":2048,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.903514},"scoresByTask":{"VidoreTabfquadRetrieval":0.95598,"VidoreDocVQARetrieval":0.60137,"VidoreArxivQARetrieval":0.88464,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99131,"VidoreTatdqaRetrieval":0.80348,"VidoreShiftProjectRetrieval":0.92934,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96585,"VidoreInfoVQARetrieval":0.93793,"VidoreSyntheticDocQAEnergyRetrieval":0.97262,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":88,"model":{"name":"LanguageBind/LanguageBind_Image","url":"https://github.com/PKU-YuanGroup/LanguageBind","zeroShotPct":100,"activeParamsB":0.39,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77.0,"releaseDate":"2023-10-03","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@inproceedings{zhu2024languagebind,\n  archiveprefix = {arXiv},\n  author = {Bin Zhu and Bin Lin and Munan Ning and Yang Yan and Jiaxi Cui and WANG HongFa and Yatian Pang and Wenhao Jiang and Junwu Zhang and Zongwei Li and Cai Wan Zhang and Zhifeng Li and Wei Liu and Li Yuan},\n  booktitle = {The Twelfth International Conference on Learning Representations},\n  eprint = {2310.01852},\n  title = {LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment},\n  url = {https://openreview.net/forum?id=QmZKc7UZCy},\n  year = {2024},\n}\n","memoryUsageMb":1631.0,"license":"mit","publicTrainingCode":"https://github.com/PKU-YuanGroup/LanguageBind","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["languagebind"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.39,"totalParamsB":0.428,"embeddingDim":768,"maxTokens":77,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{},"scoresByTask":{"CIFAR10":0.98436},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":89,"model":{"name":"LanguageBind/LanguageBind_Omni","url":"https://github.com/PKU-YuanGroup/LanguageBind","zeroShotPct":100,"activeParamsB":1.162,"totalParamsB":1.2,"embeddingDim":768,"maxTokens":77.0,"releaseDate":"2023-10-03","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text","audio","video"],"languages":["English"],"citation":"\n@inproceedings{zhu2024languagebind,\n  archiveprefix = {arXiv},\n  author = {Bin Zhu and Bin Lin and Munan Ning and Yang Yan and Jiaxi Cui and WANG HongFa and Yatian Pang and Wenhao Jiang and Junwu Zhang and Zongwei Li and Cai Wan Zhang and Zhifeng Li and Wei Liu and Li Yuan},\n  booktitle = {The Twelfth International Conference on Learning Representations},\n  eprint = {2310.01852},\n  title = {LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment},\n  url = {https://openreview.net/forum?id=QmZKc7UZCy},\n  year = {2024},\n}\n","memoryUsageMb":4578.0,"license":"mit","publicTrainingCode":"https://github.com/PKU-YuanGroup/LanguageBind","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["languagebind"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":1.162,"totalParamsB":1.2,"embeddingDim":768,"maxTokens":77,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisionCentricQA":0.52964,"VisualSTS(eng)":0.6958047142857142,"VisualSTS(multi)":0.3741},"scoresByTask":{"GTSRBZeroShot":0.570863,"CVBenchDistance":0.46333,"CVBenchRelation":0.55692,"VidoreTabfquadRetrieval":0.30972,"CIFAR10Clustering":0.896082,"VisualNewsI2TRetrieval":0.34635,"VidoreArxivQARetrieval":0.34713,"STL10":0.9798,"CLEVRZeroShot":0.2544,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.60058,"SUN397":0.738133,"STS13VisualSTS":0.666818,"CVBenchCount":0.4632,"Flickr30kI2TRetrieval":0.83114,"BLINKIT2TRetrieval":0.29508,"VidoreTatdqaRetrieval":0.15275,"Fashion200kI2TRetrieval":0.11056,"FGVCAircraftZeroShot":0.472247,"XM3600T2IRetrieval":0.13365583333333333,"MSCOCOI2TRetrieval":0.55599,"SugarCrepe":0.836373,"CVBenchDepth":0.56333,"VisualSTS-b-Eng":0.63327,"VOC2007":0.645642,"VidoreShiftProjectRetrieval":0.16052,"WITT2IRetrieval":0.3180627272727273,"DTD":0.73766,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.49728,"AROVisualAttribution":0.593537,"AROVisualRelation":0.480511,"BLINKIT2IRetrieval":0.36816,"CIFAR10":0.98436,"VidoreInfoVQARetrieval":0.56611,"BLINKIT2IMultiChoice":0.73383,"VidoreSyntheticDocQAEnergyRetrieval":0.56383,"CIFAR100":0.87822,"VisualSTS-b-Multilingual":0.377858,"EuroSATZeroShot":0.71,"VisualSTS17Eng":0.803548,"CIFAR10ZeroShot":0.9797,"STL10ZeroShot":0.9915,"GTSRB":0.813207,"CIFAR100ZeroShot":0.8518,"GLDv2I2TRetrieval":0.75744,"CIFAR100Clustering":0.832997,"VidoreSyntheticDocQAAIRetrieval":0.47756,"STS12VisualSTS":0.623501,"STS14VisualSTS":0.639602,"AROCocoOrder":0.289044,"CLEVRCountZeroShot":0.352333,"RenderedSST2":0.549149,"STS15VisualSTS":0.77198,"SciMMIRT2IRetrieval":0.18022,"WebQAT2TRetrieval":0.32532,"VizWizIT2TRetrieval":0.07815,"VisualSTS17Multilingual":0.370342,"BLINKIT2TMultiChoice":0.39723,"EuroSAT":0.885074,"CIRRIT2IRetrieval":0.13679,"DTDZeroShot":0.642553,"Food101ZeroShot":0.942535,"Flickr30kT2IRetrieval":0.84211,"STS16VisualSTS":0.731914,"HatefulMemesI2TRetrieval":0.57906,"SciMMIRI2TRetrieval":0.19334,"AROFlickrOrder":0.3482,"ImageNetDog15Clustering":0.870883,"FashionIQIT2IRetrieval":0.05068,"SUN397ZeroShot":0.739218,"FGVCAircraft":0.5952},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":90,"model":{"name":"LCO-Embedding/LCO-Embedding-Omni-3B","url":"https://huggingface.co/LCO-Embedding/LCO-Embedding-Omni-3B","zeroShotPct":100,"activeParamsB":4.392,"totalParamsB":4.703,"embeddingDim":2048,"maxTokens":32768.0,"releaseDate":"2025-10-23","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio","image","text","video"],"languages":["English"],"citation":"\n@inproceedings{xiao2025scalinglanguagecentricomnimodalrepresentation,\n  archiveprefix = {arXiv},\n  author = {Chenghao Xiao and Hou Pong Chan and Hao Zhang and Weiwen Xu and Mahani Aljunied and Yu Rong},\n  booktitle = {The Thirty-ninth Annual Conference on Neural Information Processing Systems},\n  eprint = {2510.11693},\n  title = {Scaling Language-centric Omnimodal Representation Learning},\n  url = {https://openreview.net/forum?id=eA3Aum0rpA},\n  year = {2025},\n}","memoryUsageMb":8978.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":4.392,"totalParamsB":4.703,"embeddingDim":2048,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.795494},"scoresByTask":{"CVBenchDistance":0.565,"CVBenchRelation":0.78769,"OxfordPetsZeroShot":0.901063,"VidoreTabfquadRetrieval":0.91226,"VidoreDocVQARetrieval":0.50271,"VisualNewsI2TRetrieval":0.11898,"PatchCamelyon":0.649518,"TinyImageNetClustering":0.670485,"WebQAT2ITRetrieval":0.54069,"SUN397":0.781103,"STS13VisualSTS":0.842814,"CVBenchCount":0.61802,"NIGHTSI2IRetrieval":0.24861,"VidoreTatdqaRetrieval":0.56522,"ImageCoDe":0.129887,"Fashion200kI2TRetrieval":0.06089,"FGVCAircraftZeroShot":0.632763,"XM3600T2IRetrieval":0.5227466666666667,"OxfordPets":0.937476,"FER2013ZeroShot":0.405405,"OVENIT2TRetrieval":0.18143,"CVBenchDepth":0.74833,"VidoreShiftProjectRetrieval":0.67252,"WITT2IRetrieval":0.5700845454545455,"RP2kI2IRetrieval":0.52276,"DTD":0.78117,"AROVisualAttribution":0.705301,"Country211":0.263611,"AROVisualRelation":0.571918,"InfoSeekIT2TRetrieval":0.17574,"VidoreInfoVQARetrieval":0.8324,"BLINKIT2IMultiChoice":0.73134,"VisualSTS-b-Multilingual":0.803046,"RESISC45":0.890603,"GTSRB":0.807728,"Country211ZeroShot":0.279526,"CIFAR100ZeroShot":0.727,"VidoreSyntheticDocQAAIRetrieval":0.91303,"StanfordCarsZeroShot":0.820172,"AROCocoOrder":0.712475,"VQA2IT2TRetrieval":0.20855,"STS15VisualSTS":0.871348,"VisualSTS17Multilingual":0.787942,"EuroSAT":0.711148,"CIRRIT2IRetrieval":0.27457,"Food101ZeroShot":0.868634,"Winoground":0.1225,"HatefulMemesI2TRetrieval":0.829,"AROFlickrOrder":0.6114,"ImageNetDog15Clustering":0.90315,"CUB200I2IRetrieval":0.83863},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.6103997492572787}},"trainedOnTasks":[],"experiments":null},{"rank":91,"model":{"name":"LCO-Embedding/LCO-Embedding-Omni-7B","url":"https://huggingface.co/LCO-Embedding/LCO-Embedding-Omni-7B","zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768.0,"releaseDate":"2025-10-15","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["audio","image","text","video"],"languages":["English"],"citation":"\n@inproceedings{xiao2025scalinglanguagecentricomnimodalrepresentation,\n  archiveprefix = {arXiv},\n  author = {Chenghao Xiao and Hou Pong Chan and Hao Zhang and Weiwen Xu and Mahani Aljunied and Yu Rong},\n  booktitle = {The Thirty-ninth Annual Conference on Neural Information Processing Systems},\n  eprint = {2510.11693},\n  title = {Scaling Language-centric Omnimodal Representation Learning},\n  url = {https://openreview.net/forum?id=eA3Aum0rpA},\n  year = {2025},\n}","memoryUsageMb":17043.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.8239765},"scoresByTask":{"CVBenchDistance":0.65,"CVBenchRelation":0.86154,"OxfordPetsZeroShot":0.917689,"VidoreTabfquadRetrieval":0.93088,"VidoreDocVQARetrieval":0.51524,"VisualNewsI2TRetrieval":0.13455,"PatchCamelyon":0.676599,"TinyImageNetClustering":0.67935,"WebQAT2ITRetrieval":0.59016,"SUN397":0.786005,"STS13VisualSTS":0.842387,"CVBenchCount":0.69797,"NIGHTSI2IRetrieval":0.24893,"VidoreTatdqaRetrieval":0.58844,"ImageCoDe":0.128149,"Fashion200kI2TRetrieval":0.07088,"FGVCAircraftZeroShot":0.692469,"XM3600T2IRetrieval":0.6125225,"OxfordPets":0.937803,"FER2013ZeroShot":0.3944,"OVENIT2TRetrieval":0.20747,"CVBenchDepth":0.70333,"VidoreShiftProjectRetrieval":0.68919,"WITT2IRetrieval":0.6343172727272728,"RP2kI2IRetrieval":0.53593,"DTD":0.761809,"AROVisualAttribution":0.764505,"Country211":0.269924,"AROVisualRelation":0.562811,"InfoSeekIT2TRetrieval":0.27153,"VidoreInfoVQARetrieval":0.85585,"BLINKIT2IMultiChoice":0.71891,"VisualSTS-b-Multilingual":0.814728,"RESISC45":0.888571,"GTSRB":0.763626,"Country211ZeroShot":0.316303,"CIFAR100ZeroShot":0.7302,"VidoreSyntheticDocQAAIRetrieval":0.93734,"StanfordCarsZeroShot":0.84977,"AROCocoOrder":0.74962,"VQA2IT2TRetrieval":0.14844,"STS15VisualSTS":0.881581,"VisualSTS17Multilingual":0.833225,"EuroSAT":0.718926,"CIRRIT2IRetrieval":0.2896,"Food101ZeroShot":0.890178,"Winoground":0.15,"HatefulMemesI2TRetrieval":0.79634,"AROFlickrOrder":0.6462,"ImageNetDog15Clustering":0.923138,"CUB200I2IRetrieval":0.84415},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.6294799171122994}},"trainedOnTasks":[],"experiments":null},{"rank":92,"model":{"name":"nyu-visionx/moco-v3-vit-b","url":"https://github.com/facebookresearch/moco-v3","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.087,"embeddingDim":768,"maxTokens":null,"releaseDate":"2024-06-03","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image"],"languages":["English"],"citation":"@inproceedings{chen2021mocov3,\n  author = {Chen, Xinlei and Xie, Saining and He, Kaiming},\n  booktitle = {2021 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv48922.2021.00950},\n  month = {Oct},\n  pages = {9620–9629},\n  publisher = {IEEE},\n  title = {An Empirical Study of Training Self-Supervised Vision Transformers},\n  url = {https://doi.org/10.1109/iccv48922.2021.00950},\n  year = {2021},\n}","memoryUsageMb":330.0,"license":"cc-by-nc-4.0","publicTrainingCode":"https://github.com/facebookresearch/moco-v3","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["timm"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.087,"embeddingDim":768,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"ImageClassification":0.6131174482136397,"ImageClustering":0.738550588179038,"VisualSTS(eng)":0.14889973076896323,"VisualSTS(multi)":0.1772605},"scoresByTask":{"CIFAR10Clustering":0.7469370718746896,"STL10":0.951375,"PatchCamelyon":0.7298828125,"TinyImageNetClustering":0.5952743489020317,"SUN397":0.5517977011494253,"METI2IRetrieval":0.42791,"STS13VisualSTS":0.021895488870740505,"ROxfordMediumI2IRetrieval":0.10335,"NIGHTSI2IRetrieval":0.22758,"UCF101":0.7010934824202335,"OxfordPets":0.8264922322158628,"Food101Classification":0.5054257425742574,"RParisEasyI2IRetrieval":0.1007,"Imagenet1k":0.6238720000000001,"VisualSTS-b-Eng":0.065583,"VOC2007":0.7101696368503744,"ROxfordEasyI2IRetrieval":0.20835,"RP2kI2IRetrieval":0.72768,"ROxfordHardI2IRetrieval":0.03148,"DTD":0.659468085106383,"Country211":0.06562085308056873,"CIFAR10":0.8935599999999999,"CIFAR100":0.7018,"VisualSTS-b-Multilingual":0.126758,"VisualSTS17Eng":0.395439,"RESISC45":0.7255238095238095,"GTSRB":0.4592557403008709,"FORBI2IRetrieval":0.21343,"MNIST":0.7673800000000001,"CIFAR100Clustering":0.6398580084543934,"RParisHardI2IRetrieval":0.02679,"SOPI2IRetrieval":0.56194,"ImageNet10Clustering":0.9029562882575083,"StanfordCarsI2IRetrieval":0.22522,"FER2013":0.32699916411256613,"OxfordFlowersClassification":0.8880392156862745,"STS12VisualSTS":0.24030745758768068,"STS14VisualSTS":0.006300856092500297,"GLDv2I2IRetrieval":0.08728,"STS15VisualSTS":0.11479279580720028,"SketchyI2IRetrieval":0.04882,"Caltech101":0.854404996712689,"VisualSTS17Multilingual":0.227763,"EuroSAT":0.8864814814814815,"Birdsnap":0.26871961102106967,"StanfordCars":0.20226340007461757,"STS16VisualSTS":0.19797951702462077,"RParisMediumI2IRetrieval":0.02619,"ImageNetDog15Clustering":0.8077272234065666,"FGVCAircraft":0.18895889588958897,"CUB200I2IRetrieval":0.38851},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":93,"model":{"name":"nyu-visionx/moco-v3-vit-l","url":"https://github.com/facebookresearch/moco-v3","zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.304,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2024-06-03","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image"],"languages":["English"],"citation":"@inproceedings{chen2021mocov3,\n  author = {Chen, Xinlei and Xie, Saining and He, Kaiming},\n  booktitle = {2021 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv48922.2021.00950},\n  month = {Oct},\n  pages = {9620–9629},\n  publisher = {IEEE},\n  title = {An Empirical Study of Training Self-Supervised Vision Transformers},\n  url = {https://doi.org/10.1109/iccv48922.2021.00950},\n  year = {2021},\n}","memoryUsageMb":1161.0,"license":"cc-by-nc-4.0","publicTrainingCode":"https://github.com/facebookresearch/moco-v3","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["timm"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":null,"totalParamsB":0.304,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"ImageClassification":0.6272722179510408,"ImageClustering":0.7169929312656019,"VisualSTS(eng)":0.14730809235262218,"VisualSTS(multi)":0.183664},"scoresByTask":{"CIFAR10Clustering":0.7164566910071223,"STL10":0.9539249999999999,"PatchCamelyon":0.73082275390625,"TinyImageNetClustering":0.5914166565983523,"SUN397":0.5715034482758621,"METI2IRetrieval":0.42604,"STS13VisualSTS":0.03380862097547285,"ROxfordMediumI2IRetrieval":0.10665,"NIGHTSI2IRetrieval":0.22094,"UCF101":0.7096921783879453,"OxfordPets":0.8485145816298718,"Food101Classification":0.5438970297029704,"RParisEasyI2IRetrieval":0.10219,"Imagenet1k":0.642724,"VisualSTS-b-Eng":0.047026,"VOC2007":0.7280332331685531,"ROxfordEasyI2IRetrieval":0.22311,"RP2kI2IRetrieval":0.72432,"ROxfordHardI2IRetrieval":0.03,"DTD":0.6704255319148935,"Country211":0.07028436018957346,"CIFAR10":0.9013,"CIFAR100":0.6995199999999999,"VisualSTS-b-Multilingual":0.132909,"VisualSTS17Eng":0.402703,"RESISC45":0.7375555555555555,"GTSRB":0.4980047505938242,"FORBI2IRetrieval":0.19313,"MNIST":0.7894,"CIFAR100Clustering":0.6060297526255436,"RParisHardI2IRetrieval":0.02651,"SOPI2IRetrieval":0.55223,"ImageNet10Clustering":0.8640595011746921,"StanfordCarsI2IRetrieval":0.2302,"FER2013":0.3478127612148231,"OxfordFlowersClassification":0.8992156862745098,"STS12VisualSTS":0.20902195975168564,"STS14VisualSTS":-0.011289194167665904,"GLDv2I2IRetrieval":0.0951,"STS15VisualSTS":0.129859146088944,"Caltech101":0.86620644312952,"VisualSTS17Multilingual":0.234419,"EuroSAT":0.8950000000000001,"Birdsnap":0.28654781199351703,"StanfordCars":0.22340504912324338,"STS16VisualSTS":0.22002711381991868,"RParisMediumI2IRetrieval":0.02601,"ImageNetDog15Clustering":0.8070020549222996,"FGVCAircraft":0.1861986198619862,"CUB200I2IRetrieval":0.36106},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":94,"model":{"name":"minishlab/potion-multilingual-128M","url":"https://huggingface.co/minishlab/potion-multilingual-128M","zeroShotPct":100,"activeParamsB":0.0,"totalParamsB":0.128,"embeddingDim":256,"maxTokens":null,"releaseDate":"2025-05-23","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":false,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":true,"modalities":["text"],"languages":["Afrikaans","Albanian","Amharic","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Corsican (Latin)","Czech","Danish","Dutch","English","Esperanto","Estonian","Filipino","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hausa","Hawaiian (Latin)","Hebrew","Hindi","Hmong (Latin)","Hungarian","Icelandic","Igbo","Indonesian","Irish","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Kirghiz","Korean","Kurdish (Latin)","Lao","Latin","Latvian","Lithuanian","Luxembourgish","Macedonian","Malagasy (Latin)","Malay (macrolanguage)","Malayalam","Maltese","Maori","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian Bokmål","Nyanja","Panjabi","Persian","Polish","Portuguese","Pushto","Romanian","Russian","Samoan","Scottish Gaelic","Serbian","Shona","Sindhi","Sinhala","Slovak","Slovenian","Somali","Southern Sotho","Spanish","Sundanese","Swahili (macrolanguage)","Swedish","Tajik (Latin)","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Uzbek","Vietnamese","Welsh","Western Frisian","Xhosa","Yiddish","Yoruba","Zulu"],"citation":null,"memoryUsageMb":489.0,"license":"mit","publicTrainingCode":"https://github.com/MinishLab/model2vec","publicTrainingData":"https://huggingface.co/datasets/minishlab/tokenlearn-c4-multilingual-bge-m3","adaptedFrom":"BAAI/bge-m3","supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["AmazonReviewsClassification","DuRetrieval","MLQARetrieval","NQ","NQ-NL","NQHardNegatives"]},"zeroShotPct":100,"activeParamsB":0.0,"totalParamsB":0.128,"embeddingDim":256,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{},"scoresByTask":{"WebQAT2TRetrieval":0.34304},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":95,"model":{"name":"nanovdr/NanoVDR-S-Multi","url":"https://huggingface.co/nanovdr/NanoVDR-S-Multi","zeroShotPct":93,"activeParamsB":0.067,"totalParamsB":0.069,"embeddingDim":2048,"maxTokens":512.0,"releaseDate":"2026-02-26","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":true,"modalities":["text","image"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"@inproceedings{nanovdr2026,\n  author = {Liu, Zhuchenyang and Zhang, Yao and Xiao, Yu},\n  booktitle = {Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing},\n  doi = {10.18653/2026.emnlp-main.71},\n  pages = {1631–1648},\n  publisher = {Association for Computational Linguistics},\n  title = {NanoVDR: Distilling a 2B Vision-Language Retriever into a 70M Text-Only Encoder for Visual Document Retrieval},\n  url = {https://doi.org/10.18653/2026.emnlp-main.71},\n  year = {2026},\n}","memoryUsageMb":282.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/datasets/nanovdr/NanoVDR-Train","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["qwen-vl"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":0.067,"totalParamsB":0.069,"embeddingDim":2048,"maxTokens":512,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8235510000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.95589,"VidoreDocVQARetrieval":0.46344,"VidoreArxivQARetrieval":0.7829,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.94978,"VidoreTatdqaRetrieval":0.58365,"VidoreShiftProjectRetrieval":0.8198,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.94478,"VidoreInfoVQARetrieval":0.85888,"VidoreSyntheticDocQAEnergyRetrieval":0.90728,"VidoreSyntheticDocQAAIRetrieval":0.96911},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":96,"model":{"name":"nvidia/llama-nemoretriever-colembed-1b-v1","url":"https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1","zeroShotPct":93,"activeParamsB":2.155,"totalParamsB":2.418,"embeddingDim":2048,"maxTokens":8192.0,"releaseDate":"2025-06-27","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@misc{xu2025llamanemoretrievercolembedtopperforming,\n      title={Llama Nemoretriever Colembed: Top-Performing Text-Image Retrieval Model},\n      author={Mengyao Xu and Gabriel Moreira and Ronay Ak and Radek Osmulski and Yauhen Babakhin and Zhiding Yu and Benedikt Schifferer and Even Oldridge},\n      year={2025},\n      eprint={2507.05513},\n      archivePrefix={arXiv},\n      primaryClass={cs.CV},\n      url={https://arxiv.org/abs/2507.05513}\n}","memoryUsageMb":4610.0,"license":"https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1/blob/main/LICENSE","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1#training-dataset","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["llama-nemotron-colembed-vl"],"trainingDatasets":["HotpotQA","MIRACLRetrieval","NQ","SQuAD","StackExchangeClustering","VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","WebInstructSub","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":2.155,"totalParamsB":2.418,"embeddingDim":2048,"maxTokens":8192,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.904992},"scoresByTask":{"VidoreTabfquadRetrieval":0.94298,"VidoreDocVQARetrieval":0.6448,"VidoreArxivQARetrieval":0.87608,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.79756,"VidoreShiftProjectRetrieval":0.92327,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96655,"VidoreInfoVQARetrieval":0.93606,"VidoreSyntheticDocQAEnergyRetrieval":0.96631,"VidoreSyntheticDocQAAIRetrieval":1.0},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":97,"model":{"name":"nvidia/llama-nemoretriever-colembed-3b-v1","url":"https://huggingface.co/nvidia/llama-nemoretriever-colembed-3b-v1","zeroShotPct":93,"activeParamsB":4.013,"totalParamsB":4.407,"embeddingDim":3072,"maxTokens":8192.0,"releaseDate":"2025-06-27","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@misc{xu2025llamanemoretrievercolembedtopperforming,\n      title={Llama Nemoretriever Colembed: Top-Performing Text-Image Retrieval Model},\n      author={Mengyao Xu and Gabriel Moreira and Ronay Ak and Radek Osmulski and Yauhen Babakhin and Zhiding Yu and Benedikt Schifferer and Even Oldridge},\n      year={2025},\n      eprint={2507.05513},\n      archivePrefix={arXiv},\n      primaryClass={cs.CV},\n      url={https://arxiv.org/abs/2507.05513}\n}","memoryUsageMb":8403.0,"license":"https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1/blob/main/LICENSE","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/nvidia/llama-nemoretriever-colembed-1b-v1#training-dataset","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["llama-nemotron-colembed-vl"],"trainingDatasets":["HotpotQA","MIRACLRetrieval","NQ","SQuAD","StackExchangeClustering","VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","WebInstructSub","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":4.013,"totalParamsB":4.407,"embeddingDim":3072,"maxTokens":8192,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9100360000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.95935,"VidoreDocVQARetrieval":0.66213,"VidoreArxivQARetrieval":0.88351,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.80567,"VidoreShiftProjectRetrieval":0.907,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97823,"VidoreInfoVQARetrieval":0.94923,"VidoreSyntheticDocQAEnergyRetrieval":0.96631,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":98,"model":{"name":"nvidia/llama-nemotron-colembed-vl-3b-v2","url":"https://huggingface.co/nvidia/llama-nemotron-colembed-vl-3b-v2","zeroShotPct":93,"activeParamsB":4.013,"totalParamsB":4.407,"embeddingDim":3072,"maxTokens":8192.0,"releaseDate":"2026-01-21","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@misc{moreira2026nemotroncolembedv2topperforming,\n    title={Nemotron ColEmbed V2: Top-Performing Late Interaction embedding models for Visual Document Retrieval},\n    author={Gabriel de Souza P. Moreira and Ronay Ak and Mengyao Xu and Oliver Holworthy and Benedikt Schifferer and Zhiding Yu and Yauhen Babakhin and Radek Osmulski and Jiarui Cai and Ryan Chesler and Bo Liu and Even Oldridge},\n    year={2026},\n    eprint={2602.03992},\n    archivePrefix={arXiv},\n    primaryClass={cs.IR},\n    url={https://arxiv.org/abs/2602.03992},\n}","memoryUsageMb":8403.0,"license":"https://huggingface.co/nvidia/llama-nemotron-colembed-vl-3b-v2/blob/main/LICENSE","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/nvidia/llama-nemotron-colembed-vl-3b-v2#training-dataset","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["llama-nemotron-colembed-vl"],"trainingDatasets":["HotpotQA","MIRACLRetrieval","NQ","SQuAD","StackExchangeClustering","VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","WebInstructSub","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":4.013,"totalParamsB":4.407,"embeddingDim":3072,"maxTokens":8192,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.917406},"scoresByTask":{"VidoreTabfquadRetrieval":0.97251,"VidoreDocVQARetrieval":0.67173,"VidoreArxivQARetrieval":0.90401,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98893,"VidoreTatdqaRetrieval":0.81037,"VidoreShiftProjectRetrieval":0.92002,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97954,"VidoreInfoVQARetrieval":0.94677,"VidoreSyntheticDocQAEnergyRetrieval":0.98018,"VidoreSyntheticDocQAAIRetrieval":1.0},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":99,"model":{"name":"nvidia/nemotron-colembed-vl-4b-v2","url":"https://huggingface.co/nvidia/nemotron-colembed-vl-4b-v2","zeroShotPct":93,"activeParamsB":4.411,"totalParamsB":4.8,"embeddingDim":2560,"maxTokens":262144.0,"releaseDate":"2026-01-07","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@misc{moreira2026nemotroncolembedv2topperforming,\n    title={Nemotron ColEmbed V2: Top-Performing Late Interaction embedding models for Visual Document Retrieval},\n    author={Gabriel de Souza P. Moreira and Ronay Ak and Mengyao Xu and Oliver Holworthy and Benedikt Schifferer and Zhiding Yu and Yauhen Babakhin and Radek Osmulski and Jiarui Cai and Ryan Chesler and Bo Liu and Even Oldridge},\n    year={2026},\n    eprint={2602.03992},\n    archivePrefix={arXiv},\n    primaryClass={cs.IR},\n    url={https://arxiv.org/abs/2602.03992},\n}","memoryUsageMb":9206.0,"license":"https://huggingface.co/nvidia/nemotron-colembed-vl-4b-v2/blob/main/LICENSE","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/nvidia/nemotron-colembed-vl-4b-v2#training-dataset","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["nemotron-colembed-vl-v2"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":4.411,"totalParamsB":4.8,"embeddingDim":2560,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9162269999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.98053,"VidoreDocVQARetrieval":0.67387,"VidoreArxivQARetrieval":0.92025,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98524,"VidoreTatdqaRetrieval":0.81192,"VidoreShiftProjectRetrieval":0.92258,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98024,"VidoreInfoVQARetrieval":0.93309,"VidoreSyntheticDocQAEnergyRetrieval":0.96193,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":100,"model":{"name":"nvidia/nemotron-colembed-vl-8b-v2","url":"https://huggingface.co/nvidia/nemotron-colembed-vl-8b-v2","zeroShotPct":93,"activeParamsB":8.078,"totalParamsB":8.7,"embeddingDim":4096,"maxTokens":262144.0,"releaseDate":"2026-01-07","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":true,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"\n@misc{moreira2026nemotroncolembedv2topperforming,\n    title={Nemotron ColEmbed V2: Top-Performing Late Interaction embedding models for Visual Document Retrieval},\n    author={Gabriel de Souza P. Moreira and Ronay Ak and Mengyao Xu and Oliver Holworthy and Benedikt Schifferer and Zhiding Yu and Yauhen Babakhin and Radek Osmulski and Jiarui Cai and Ryan Chesler and Bo Liu and Even Oldridge},\n    year={2026},\n    eprint={2602.03992},\n    archivePrefix={arXiv},\n    primaryClass={cs.IR},\n    url={https://arxiv.org/abs/2602.03992},\n}","memoryUsageMb":16722.0,"license":"https://huggingface.co/nvidia/nemotron-colembed-vl-8b-v2/blob/main/LICENSE","publicTrainingCode":null,"publicTrainingData":"https://huggingface.co/nvidia/nemotron-colembed-vl-8b-v2#training-dataset","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["nemotron-colembed-vl-v2"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":8.078,"totalParamsB":8.7,"embeddingDim":4096,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9265229999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.97742,"VidoreDocVQARetrieval":0.68052,"VidoreArxivQARetrieval":0.9308,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.83371,"VidoreShiftProjectRetrieval":0.93302,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98893,"VidoreInfoVQARetrieval":0.94559,"VidoreSyntheticDocQAEnergyRetrieval":0.97893,"VidoreSyntheticDocQAAIRetrieval":1.0},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":101,"model":{"name":"OpenSearch-AI/Ops-Colqwen3-4B","url":"https://huggingface.co/OpenSearch-AI/Ops-Colqwen3-4B","zeroShotPct":93,"activeParamsB":4.411,"totalParamsB":4.8,"embeddingDim":2560,"maxTokens":32768.0,"releaseDate":"2026-01-24","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":false,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["Afrikaans","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Latvian","Lithuanian","Macedonian","Malay (macrolanguage)","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Panjabi","Persian","Polish","Portuguese","Quechua (Latin)","Romanian","Russian","Sinhala","Slovak","Slovenian","Spanish","Swahili (macrolanguage)","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Yoruba"],"citation":null,"memoryUsageMb":9206.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/illuin-tech/colpali","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["FEVER","HotpotQA","MIRACLRetrieval","MrTyDi","NQ","VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","WebInstructSub","docmatix-ir"]},"zeroShotPct":93,"activeParamsB":4.411,"totalParamsB":4.8,"embeddingDim":2560,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.913575},"scoresByTask":{"VidoreTabfquadRetrieval":0.93554,"VidoreDocVQARetrieval":0.66452,"VidoreArxivQARetrieval":0.91781,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.82379,"VidoreShiftProjectRetrieval":0.90839,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98024,"VidoreInfoVQARetrieval":0.94022,"VidoreSyntheticDocQAEnergyRetrieval":0.97262,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":102,"model":{"name":"perplexity-ai/pplx-embed-v2-late-0.6b","url":"https://huggingface.co/perplexity-ai/pplx-embed-v2-late-0.6b","zeroShotPct":-1,"activeParamsB":0.34,"totalParamsB":0.594,"embeddingDim":128,"maxTokens":4096.0,"releaseDate":"2026-10-05","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["image","text"],"languages":["Afrikaans","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Latvian","Lithuanian","Macedonian","Malay (macrolanguage)","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Panjabi","Persian","Polish","Portuguese","Quechua (Latin)","Romanian","Russian","Sinhala","Slovak","Slovenian","Spanish","Swahili (macrolanguage)","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Yoruba"],"citation":"\n@misc{krimmel2026pplxembedv2latetechnicalreport,\n      title={pplx-embed-v2-late Technical Report},\n      author={Markus Krimmel and Louis Milliken and Ekaterina Glazkova and Maximilian Schall and Antoine Chaffin and Sedigheh Eslami and Jonas Golde and Denis Bykov and Claire Beamer and Jeremy Yang and Bo Wang},\n      year={2026},\n      eprint={2610.29523},\n      archivePrefix={arXiv},\n      primaryClass={cs.IR},\n      url={https://arxiv.org/abs/2610.29523},\n}\n","memoryUsageMb":2267.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3.5-0.8B","supersededBy":null,"extraRequirementsGroups":["pplx-embed-v2","image"],"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":0.34,"totalParamsB":0.594,"embeddingDim":128,"maxTokens":4096,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.897036},"scoresByTask":{"VidoreTabfquadRetrieval":0.97718,"VidoreDocVQARetrieval":0.60944,"VidoreArxivQARetrieval":0.83233,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99631,"VidoreTatdqaRetrieval":0.78067,"VidoreShiftProjectRetrieval":0.92833,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98024,"VidoreInfoVQARetrieval":0.90076,"VidoreSyntheticDocQAEnergyRetrieval":0.97248,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":103,"model":{"name":"perplexity-ai/pplx-embed-v2-late-9b","url":"https://huggingface.co/perplexity-ai/pplx-embed-v2-late-9b","zeroShotPct":-1,"activeParamsB":7.376,"totalParamsB":8.393,"embeddingDim":128,"maxTokens":4096.0,"releaseDate":"2026-10-05","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":true,"modalities":["image","text"],"languages":["Afrikaans","Arabic","Armenian","Azerbaijani","Basque","Belarusian","Bengali","Bulgarian","Burmese","Catalan","Cebuano","Chinese","Croatian","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Haitian","Hebrew","Hindi","Hungarian","Icelandic","Indonesian","Italian","Japanese","Javanese","Kannada","Kazakh","Khmer","Kirghiz","Korean","Lao","Latvian","Lithuanian","Macedonian","Malay (macrolanguage)","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Nepali (macrolanguage)","Norwegian","Norwegian Bokmål","Norwegian Nynorsk","Panjabi","Persian","Polish","Portuguese","Quechua (Latin)","Romanian","Russian","Sinhala","Slovak","Slovenian","Spanish","Swahili (macrolanguage)","Tagalog","Tamil","Telugu","Thai","Turkish","Ukrainian","Urdu","Vietnamese","Welsh","Yoruba"],"citation":"\n@misc{krimmel2026pplxembedv2latetechnicalreport,\n      title={pplx-embed-v2-late Technical Report},\n      author={Markus Krimmel and Louis Milliken and Ekaterina Glazkova and Maximilian Schall and Antoine Chaffin and Sedigheh Eslami and Jonas Golde and Denis Bykov and Claire Beamer and Jeremy Yang and Bo Wang},\n      year={2026},\n      eprint={2610.29523},\n      archivePrefix={arXiv},\n      primaryClass={cs.IR},\n      url={https://arxiv.org/abs/2610.29523},\n}\n","memoryUsageMb":32016.0,"license":"mit","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3.5-9B","supersededBy":null,"extraRequirementsGroups":["pplx-embed-v2","image"],"trainingDatasets":null},"zeroShotPct":-1,"activeParamsB":7.376,"totalParamsB":8.393,"embeddingDim":128,"maxTokens":4096,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9066780000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.97034,"VidoreDocVQARetrieval":0.62498,"VidoreArxivQARetrieval":0.87099,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.995,"VidoreTatdqaRetrieval":0.78867,"VidoreShiftProjectRetrieval":0.94678,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98893,"VidoreInfoVQARetrieval":0.91617,"VidoreSyntheticDocQAEnergyRetrieval":0.96492,"VidoreSyntheticDocQAAIRetrieval":1.0},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":104,"model":{"name":"sentence-transformers/static-similarity-mrl-multilingual-v1","url":"https://huggingface.co/sentence-transformers/static-similarity-mrl-multilingual-v1","zeroShotPct":100,"activeParamsB":0.0,"totalParamsB":0.108,"embeddingDim":1024,"maxTokens":null,"releaseDate":"2025-01-15","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":true,"modalities":["text"],"languages":["Albanian","Arabic","Armenian","Bulgarian","Burmese","Catalan","Chinese","Czech","Danish","Dutch","English","Estonian","Finnish","French","Galician","Georgian","German","Gujarati","Hebrew","Hindi","Hungarian","Indonesian","Italian","Japanese","Korean","Kurdish (Latin)","Latvian","Lithuanian","Macedonian","Malayalam","Marathi","Modern Greek (1453-)","Mongolian","Norwegian Bokmål","Persian","Polish","Portuguese","Romanian","Russian","Serbian","Slovak","Slovenian","Spanish","Swedish","Thai","Turkish","Ukrainian","Urdu","Vietnamese"],"citation":"@inproceedings{reimers-2019-sentence-bert,\n    title = \"Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks\",\n    author = \"Reimers, Nils and Gurevych, Iryna\",\n    booktitle = \"Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing\",\n    month = \"11\",\n    year = \"2019\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://arxiv.org/abs/1908.10084\",\n}","memoryUsageMb":413.0,"license":"apache-2.0","publicTrainingCode":"https://huggingface.co/blog/static-embeddings","publicTrainingData":"https://huggingface.co/collections/sentence-transformers/embedding-model-datasets-6644d7a3673a511914aa7552","adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["NanoQuoraRetrieval","Quora-NL","QuoraRetrieval","StackExchangeClustering.v2","StackExchangeClusteringP2P.v2","TatoebaBitextMining"]},"zeroShotPct":100,"activeParamsB":0.0,"totalParamsB":0.108,"embeddingDim":1024,"maxTokens":null,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{},"scoresByTask":{"WebQAT2TRetrieval":0.44096},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":105,"model":{"name":"google/siglip-so400m-patch14-224","url":"https://huggingface.co/google/siglip-so400m-patch14-224","zeroShotPct":100,"activeParamsB":0.84,"totalParamsB":0.877,"embeddingDim":1152,"maxTokens":16.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":3347.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.84,"totalParamsB":0.877,"embeddingDim":1152,"maxTokens":16,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.3821605},"scoresByTask":{"VidoreTabfquadRetrieval":0.30045,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.59869,"ROxfordMediumI2IRetrieval":0.17614,"Flickr30kI2TRetrieval":0.83785,"BLINKIT2TRetrieval":0.13745,"VidoreTatdqaRetrieval":0.11117,"RParisEasyI2IRetrieval":0.10241,"VisualSTS-b-Eng":0.568687,"ROxfordEasyI2IRetrieval":0.31161,"ROxfordHardI2IRetrieval":0.11335,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.56244,"BLINKIT2IRetrieval":0.30846,"BLINKIT2IMultiChoice":0.69652,"VidoreSyntheticDocQAEnergyRetrieval":0.57202,"VisualSTS-b-Multilingual":0.390173,"VisualSTS17Eng":0.756737,"RParisHardI2IRetrieval":0.04504,"VidoreSyntheticDocQAAIRetrieval":0.40562,"VisualSTS17Multilingual":0.374148,"BLINKIT2TMultiChoice":0.37831,"Flickr30kT2IRetrieval":0.85811,"RParisMediumI2IRetrieval":0.02672},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":106,"model":{"name":"google/siglip-so400m-patch16-256-i18n","url":"https://huggingface.co/google/siglip-so400m-patch16-256-i18n","zeroShotPct":100,"activeParamsB":0.841,"totalParamsB":1.129,"embeddingDim":1152,"maxTokens":64.0,"releaseDate":"2024-01-08","modelType":"dense","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English"],"citation":"@inproceedings{zhai2023sigmoid,\n  author = {Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},\n  booktitle = {2023 IEEE/CVF International Conference on Computer Vision (ICCV)},\n  doi = {10.1109/iccv51070.2023.01100},\n  month = {Oct},\n  pages = {11941–11952},\n  publisher = {IEEE},\n  title = {Sigmoid Loss for Language Image Pre-Training},\n  url = {https://doi.org/10.1109/iccv51070.2023.01100},\n  year = {2023},\n}","memoryUsageMb":4306.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/google-research/big_vision/blob/main/big_vision/trainers/proj/image_text/siglip.py","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":["siglip","image"],"trainingDatasets":null},"zeroShotPct":100,"activeParamsB":0.841,"totalParamsB":1.129,"embeddingDim":1152,"maxTokens":64,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.4402285},"scoresByTask":{"VidoreTabfquadRetrieval":0.56338,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.51405,"ROxfordMediumI2IRetrieval":0.17371,"Flickr30kI2TRetrieval":0.86593,"BLINKIT2TRetrieval":0.18537,"VidoreTatdqaRetrieval":0.08399,"RParisEasyI2IRetrieval":0.11149,"VisualSTS-b-Eng":0.572427,"ROxfordEasyI2IRetrieval":0.29784,"ROxfordHardI2IRetrieval":0.11633,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.42512,"BLINKIT2IRetrieval":0.3408,"BLINKIT2IMultiChoice":0.73134,"VidoreSyntheticDocQAEnergyRetrieval":0.44836,"VisualSTS-b-Multilingual":0.459485,"VisualSTS17Eng":0.798294,"RParisHardI2IRetrieval":0.04423,"VidoreSyntheticDocQAAIRetrieval":0.33943,"VisualSTS17Multilingual":0.420972,"BLINKIT2TMultiChoice":0.38209,"Flickr30kT2IRetrieval":0.87146,"RParisMediumI2IRetrieval":0.02733},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":[],"experiments":null},{"rank":107,"model":{"name":"VAGOsolutions/SauerkrautLM-ColLFM2-450M-v0.1","url":"https://huggingface.co/VAGOsolutions/SauerkrautLM-ColLFM2-450M-v0.1","zeroShotPct":93,"activeParamsB":null,"totalParamsB":0.451,"embeddingDim":128,"maxTokens":32768.0,"releaseDate":"2025-12-20","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":3,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":860.0,"license":"https://huggingface.co/LiquidAI/LFM2-VL-450M/blob/main/LICENSE","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"LiquidAI/LFM2-VL-450M","supersededBy":null,"extraRequirementsGroups":["sauerkrautlm-colpali"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":null,"totalParamsB":0.451,"embeddingDim":128,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.8355969999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.80914,"VidoreDocVQARetrieval":0.59114,"VidoreArxivQARetrieval":0.76111,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.97323,"VidoreTatdqaRetrieval":0.72875,"VidoreShiftProjectRetrieval":0.73137,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.94609,"VidoreInfoVQARetrieval":0.88359,"VidoreSyntheticDocQAEnergyRetrieval":0.94393,"VidoreSyntheticDocQAAIRetrieval":0.98762},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":108,"model":{"name":"VAGOsolutions/SauerkrautLM-ColMinistral3-3b-v0.1","url":"https://huggingface.co/VAGOsolutions/SauerkrautLM-ColMinistral3-3b-v0.1","zeroShotPct":93,"activeParamsB":3.849,"totalParamsB":4.252,"embeddingDim":128,"maxTokens":262144.0,"releaseDate":"2025-12-20","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":8110.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"mistralai/Ministral-3B-Instruct-2410","supersededBy":null,"extraRequirementsGroups":["sauerkrautlm-colpali"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":3.849,"totalParamsB":4.252,"embeddingDim":128,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.819809},"scoresByTask":{"VidoreTabfquadRetrieval":0.82336,"VidoreDocVQARetrieval":0.56233,"VidoreArxivQARetrieval":0.89371,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.94786,"VidoreTatdqaRetrieval":0.65676,"VidoreShiftProjectRetrieval":0.7001,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.92978,"VidoreInfoVQARetrieval":0.7601,"VidoreSyntheticDocQAEnergyRetrieval":0.94016,"VidoreSyntheticDocQAAIRetrieval":0.98393},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":109,"model":{"name":"VAGOsolutions/SauerkrautLM-ColQwen3-1.7b-Turbo-v0.1","url":"https://huggingface.co/VAGOsolutions/SauerkrautLM-ColQwen3-1.7b-Turbo-v0.1","zeroShotPct":93,"activeParamsB":1.445,"totalParamsB":1.757,"embeddingDim":128,"maxTokens":262144.0,"releaseDate":"2025-12-20","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":3350.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3-VL-2B-Instruct","supersededBy":null,"extraRequirementsGroups":["sauerkrautlm-colpali"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":1.445,"totalParamsB":1.757,"embeddingDim":128,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.88892},"scoresByTask":{"VidoreTabfquadRetrieval":0.87028,"VidoreDocVQARetrieval":0.60985,"VidoreArxivQARetrieval":0.91086,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98262,"VidoreTatdqaRetrieval":0.81189,"VidoreShiftProjectRetrieval":0.87758,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.94885,"VidoreInfoVQARetrieval":0.9208,"VidoreSyntheticDocQAEnergyRetrieval":0.97492,"VidoreSyntheticDocQAAIRetrieval":0.98155},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":110,"model":{"name":"VAGOsolutions/SauerkrautLM-ColQwen3-2b-v0.1","url":"https://huggingface.co/VAGOsolutions/SauerkrautLM-ColQwen3-2b-v0.1","zeroShotPct":93,"activeParamsB":1.817,"totalParamsB":2.128,"embeddingDim":128,"maxTokens":262144.0,"releaseDate":"2025-12-20","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":4058.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3-VL-2B-Instruct","supersededBy":null,"extraRequirementsGroups":["sauerkrautlm-colpali"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":1.817,"totalParamsB":2.128,"embeddingDim":128,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9024369999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.90317,"VidoreDocVQARetrieval":0.65062,"VidoreArxivQARetrieval":0.91238,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.82055,"VidoreShiftProjectRetrieval":0.88738,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96085,"VidoreInfoVQARetrieval":0.93138,"VidoreSyntheticDocQAEnergyRetrieval":0.96911,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":111,"model":{"name":"VAGOsolutions/SauerkrautLM-ColQwen3-4b-v0.1","url":"https://huggingface.co/VAGOsolutions/SauerkrautLM-ColQwen3-4b-v0.1","zeroShotPct":93,"activeParamsB":4.049,"totalParamsB":4.438,"embeddingDim":128,"maxTokens":262144.0,"releaseDate":"2025-12-20","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":8465.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3-VL-4B-Instruct","supersededBy":null,"extraRequirementsGroups":["sauerkrautlm-colpali"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":4.049,"totalParamsB":4.438,"embeddingDim":128,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.908025},"scoresByTask":{"VidoreTabfquadRetrieval":0.89485,"VidoreDocVQARetrieval":0.66956,"VidoreArxivQARetrieval":0.91831,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":1.0,"VidoreTatdqaRetrieval":0.82661,"VidoreShiftProjectRetrieval":0.90547,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96155,"VidoreInfoVQARetrieval":0.94235,"VidoreSyntheticDocQAEnergyRetrieval":0.96524,"VidoreSyntheticDocQAAIRetrieval":0.99631},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":112,"model":{"name":"VAGOsolutions/SauerkrautLM-ColQwen3-8b-v0.1","url":"https://huggingface.co/VAGOsolutions/SauerkrautLM-ColQwen3-8b-v0.1","zeroShotPct":93,"activeParamsB":7.523,"totalParamsB":8.145,"embeddingDim":128,"maxTokens":262144.0,"releaseDate":"2025-12-20","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":false,"open training data":false,"paper":true,"model card":true},"opennessScore":4,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":"\n@inproceedings{faysse2024colpali,\n  archiveprefix = {arXiv},\n  author = {Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and CELINE HUDELOT and Pierre Colombo},\n  booktitle = {The Thirteenth International Conference on Learning Representations},\n  eprint = {2407.01449},\n  title = {ColPali: Efficient Document Retrieval with Vision Language Models},\n  url = {https://openreview.net/forum?id=ogjBpZ8uSi},\n  year = {2025},\n}","memoryUsageMb":15536.0,"license":"apache-2.0","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3-VL-8B-Instruct","supersededBy":null,"extraRequirementsGroups":["sauerkrautlm-colpali"],"trainingDatasets":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":7.523,"totalParamsB":8.145,"embeddingDim":128,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9108370000000001},"scoresByTask":{"VidoreTabfquadRetrieval":0.92176,"VidoreDocVQARetrieval":0.64688,"VidoreArxivQARetrieval":0.93797,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.84038,"VidoreShiftProjectRetrieval":0.90409,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96786,"VidoreInfoVQARetrieval":0.94508,"VidoreSyntheticDocQAEnergyRetrieval":0.96524,"VidoreSyntheticDocQAAIRetrieval":0.98649},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":113,"model":{"name":"Tevatron/OmniEmbed-v0.1","url":"https://huggingface.co/Tevatron/OmniEmbed-v0.1","zeroShotPct":93,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768.0,"releaseDate":"2025-04-12","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":true,"paper":true,"model card":true},"opennessScore":6,"sentenceTransformersCompatible":true,"modalities":["text","image","audio","video"],"languages":["English"],"citation":"\n@inproceedings{zhuang2025tevatron,\n  author = {Ma, Xueguang and Gao, Luyu and Zhuang, Shengyao and Zhan, Jiaqi Samantha and Callan, Jamie and Lin, Jimmy},\n  booktitle = {Proceedings of the 48th International ACM SIGIR Conference on Research and Development in Information Retrieval},\n  collection = {SIGIR ’25},\n  doi = {10.1145/3726302.3730135},\n  month = {July},\n  pages = {4061–4065},\n  publisher = {ACM},\n  series = {SIGIR ’25},\n  title = {Tevatron 2.0: Unified Document Retrieval Toolkit across Scale, Language, and Modality},\n  url = {https://doi.org/10.1145/3726302.3730135},\n  year = {2025},\n}\n","memoryUsageMb":17036.0,"license":"mit","publicTrainingCode":"https://github.com/texttron/tevatron/tree/qwenomni","publicTrainingData":"https://huggingface.co/Tevatron/OmniEmbed-v0.1#training-data","adaptedFrom":"Qwen/Qwen2.5-Omni-7B","supersededBy":null,"extraRequirementsGroups":["multimodal-sbert","peft"],"trainingDatasets":["AudioCapsT2ARetrieval","FEVER","MSMARCO","MSRVTTT2V","MSRVTTV2T","NQ","SCIDOCS","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval"]},"zeroShotPct":93,"activeParamsB":8.387,"totalParamsB":8.932,"embeddingDim":3584,"maxTokens":32768,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"VisualSTS(multi)":0.70021},"scoresByTask":{"CVBenchDistance":0.41167,"CVBenchRelation":0.58,"OxfordPetsZeroShot":0.506405,"VidoreTabfquadRetrieval":0.91961,"VidoreDocVQARetrieval":0.53561,"VisualNewsI2TRetrieval":0.04266,"PatchCamelyon":0.652924,"TinyImageNetClustering":0.64827,"WebQAT2ITRetrieval":0.56234,"SUN397":0.753637,"STS13VisualSTS":0.6962,"CVBenchCount":0.29442,"NIGHTSI2IRetrieval":0.24572,"VidoreTatdqaRetrieval":0.72648,"ImageCoDe":0.129887,"Fashion200kI2TRetrieval":0.0274,"FGVCAircraftZeroShot":0.176718,"XM3600T2IRetrieval":0.6149,"OxfordPets":0.919815,"FER2013ZeroShot":0.349958,"OVENIT2TRetrieval":0.16019,"CVBenchDepth":0.49333,"VidoreShiftProjectRetrieval":0.82716,"WITT2IRetrieval":0.5748690909090909,"RP2kI2IRetrieval":0.55518,"DTD":0.75766,"AROVisualAttribution":0.716293,"Country211":0.101005,"AROVisualRelation":0.591177,"InfoSeekIT2TRetrieval":0.18709,"VidoreInfoVQARetrieval":0.9141,"BLINKIT2IMultiChoice":0.73383,"VisualSTS-b-Multilingual":0.662109,"RESISC45":0.881714,"GTSRB":0.731227,"Country211ZeroShot":0.115972,"CIFAR100ZeroShot":0.6503,"VidoreSyntheticDocQAAIRetrieval":0.96542,"StanfordCarsZeroShot":0.619575,"AROCocoOrder":0.372331,"VQA2IT2TRetrieval":0.00548,"STS15VisualSTS":0.775076,"VisualSTS17Multilingual":0.738311,"EuroSAT":0.702074,"CIRRIT2IRetrieval":0.15916,"Food101ZeroShot":0.54705,"Winoground":0.06,"HatefulMemesI2TRetrieval":0.45871,"AROFlickrOrder":0.3972,"ImageNetDog15Clustering":0.763326,"CUB200I2IRetrieval":0.79116},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{"Task Mean":0.5255431978609626}},"trainedOnTasks":["VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":114,"model":{"name":"webAI-Official/webAI-ColVec1.1-4b","url":"https://huggingface.co/webAI-Official/webAI-ColVec1.1-4b","zeroShotPct":93,"activeParamsB":4.539,"totalParamsB":4.541,"embeddingDim":640,"maxTokens":262144.0,"releaseDate":"2026-07-22","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":2,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":8661.0,"license":"https://huggingface.co/webAI-Official/webAI-ColVec1.1-4b/blob/main/LICENSE.md","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3.5-4B","supersededBy":null,"extraRequirementsGroups":["colvec1_1"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":4.539,"totalParamsB":4.541,"embeddingDim":640,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.909071},"scoresByTask":{"VidoreTabfquadRetrieval":0.96032,"VidoreDocVQARetrieval":0.6318,"VidoreArxivQARetrieval":0.89892,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.79203,"VidoreShiftProjectRetrieval":0.93917,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.97585,"VidoreInfoVQARetrieval":0.94107,"VidoreSyntheticDocQAEnergyRetrieval":0.96631,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":115,"model":{"name":"webAI-Official/webAI-ColVec1.1-8b","url":"https://huggingface.co/webAI-Official/webAI-ColVec1.1-8b","zeroShotPct":93,"activeParamsB":8.393,"totalParamsB":8.395,"embeddingDim":640,"maxTokens":262144.0,"releaseDate":"2026-07-22","modelType":"late-interaction","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":2,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French","German","Italian","Portuguese","Spanish"],"citation":null,"memoryUsageMb":16013.0,"license":"https://huggingface.co/webAI-Official/webAI-ColVec1.1-8b/blob/main/LICENSE.md","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3.5-9B","supersededBy":null,"extraRequirementsGroups":["colvec1_1"],"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTatdqaRetrieval","VisRAG-Ret-Train-In-domain-data","VisRAG-Ret-Train-Synthetic-data","docmatix-ir","wiki-ss-nq"]},"zeroShotPct":93,"activeParamsB":8.393,"totalParamsB":8.395,"embeddingDim":640,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.915277},"scoresByTask":{"VidoreTabfquadRetrieval":0.97352,"VidoreDocVQARetrieval":0.64497,"VidoreArxivQARetrieval":0.91477,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98762,"VidoreTatdqaRetrieval":0.80785,"VidoreShiftProjectRetrieval":0.9344,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.96972,"VidoreInfoVQARetrieval":0.94968,"VidoreSyntheticDocQAEnergyRetrieval":0.97762,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":116,"model":{"name":"webAI-Official/webAI-ColVec1-4b","url":"https://huggingface.co/webAI-Official/webAI-ColVec1-4b","zeroShotPct":93,"activeParamsB":4.539,"totalParamsB":4.541,"embeddingDim":640,"maxTokens":262144.0,"releaseDate":"2026-04-05","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":2,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French"],"citation":null,"memoryUsageMb":8661.0,"license":"multiple","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3.5-4B","supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VisRAGRetArxivQA","VisRAGRetChartQA","VisRAGRetInfoVQA","VisRAGRetMPDocVQA","VisRAGRetPlotQA","VisRAGRetSlideVQA"]},"zeroShotPct":93,"activeParamsB":4.539,"totalParamsB":4.541,"embeddingDim":640,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.904946},"scoresByTask":{"VidoreTabfquadRetrieval":0.94013,"VidoreDocVQARetrieval":0.66987,"VidoreArxivQARetrieval":0.92536,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.99262,"VidoreTatdqaRetrieval":0.79005,"VidoreShiftProjectRetrieval":0.87194,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.95865,"VidoreInfoVQARetrieval":0.94173,"VidoreSyntheticDocQAEnergyRetrieval":0.95911,"VidoreSyntheticDocQAAIRetrieval":1.0},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval"],"experiments":null},{"rank":117,"model":{"name":"webAI-Official/webAI-ColVec1-9b","url":"https://huggingface.co/webAI-Official/webAI-ColVec1-9b","zeroShotPct":93,"activeParamsB":9.41,"totalParamsB":9.42,"embeddingDim":2560,"maxTokens":262144.0,"releaseDate":"2026-04-05","modelType":"late-interaction","instructionTuned":false,"openWeights":true,"openness":{"open weights":true,"open license":false,"open training code":false,"open training data":false,"paper":false,"model card":true},"opennessScore":2,"sentenceTransformersCompatible":false,"modalities":["image","text"],"languages":["English","French"],"citation":null,"memoryUsageMb":17968.0,"license":"multiple","publicTrainingCode":null,"publicTrainingData":null,"adaptedFrom":"Qwen/Qwen3.5-9B","supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["VDRMultilingualRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VisRAGRetArxivQA","VisRAGRetChartQA","VisRAGRetInfoVQA","VisRAGRetMPDocVQA","VisRAGRetPlotQA","VisRAGRetSlideVQA"]},"zeroShotPct":93,"activeParamsB":9.41,"totalParamsB":9.42,"embeddingDim":2560,"maxTokens":262144,"meanTask":null,"meanTaskType":null,"meanPublic":null,"meanPrivate":null,"scoresByTaskType":{"DocumentUnderstanding":0.9129679999999999},"scoresByTask":{"VidoreTabfquadRetrieval":0.95006,"VidoreDocVQARetrieval":0.6868,"VidoreArxivQARetrieval":0.93639,"VidoreSyntheticDocQAHealthcareIndustryRetrieval":0.98893,"VidoreTatdqaRetrieval":0.78838,"VidoreShiftProjectRetrieval":0.8855,"VidoreSyntheticDocQAGovernmentReportsRetrieval":0.98024,"VidoreInfoVQARetrieval":0.95183,"VidoreSyntheticDocQAEnergyRetrieval":0.96893,"VidoreSyntheticDocQAAIRetrieval":0.99262},"scoresByCustomGroup":{"MIEB(eng) Task Mean":{},"MIEB(lite) Task Mean":{}},"trainedOnTasks":["VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval"],"experiments":null}],"aggregations":["mean_task_type","task_types","custom_groups"],"customGroupings":[{"name":"MIEB(eng) Task Mean","groups":[{"label":"Task Mean","description":"Unweighted mean across MIEB(eng)'s task set (a strict subset of MIEB(Multilingual)'s) — not the same statistic as MIEB(eng)'s own Mean(TaskType).","tasks":["Birdsnap","Caltech101","CIFAR10","CIFAR100","Country211","DTD","EuroSAT","FER2013","FGVCAircraft","Food101Classification","GTSRB","Imagenet1k","MNIST","OxfordFlowersClassification","OxfordPets","PatchCamelyon","RESISC45","StanfordCars","STL10","SUN397","UCF101","VOC2007","CIFAR10Clustering","CIFAR100Clustering","ImageNetDog15Clustering","ImageNet10Clustering","TinyImageNetClustering","BirdsnapZeroShot","Caltech101ZeroShot","CIFAR10ZeroShot","CIFAR100ZeroShot","CLEVRZeroShot","CLEVRCountZeroShot","Country211ZeroShot","DTDZeroShot","EuroSATZeroShot","FER2013ZeroShot","FGVCAircraftZeroShot","Food101ZeroShot","GTSRBZeroShot","Imagenet1kZeroShot","MNISTZeroShot","OxfordPetsZeroShot","PatchCamelyonZeroShot","RenderedSST2","RESISC45ZeroShot","StanfordCarsZeroShot","STL10ZeroShot","SUN397ZeroShot","UCF101ZeroShot","BLINKIT2IMultiChoice","BLINKIT2TMultiChoice","CVBenchCount","CVBenchRelation","CVBenchDepth","CVBenchDistance","AROCocoOrder","AROFlickrOrder","AROVisualAttribution","AROVisualRelation","SugarCrepe","Winoground","ImageCoDe","STS12VisualSTS","STS13VisualSTS","STS14VisualSTS","STS15VisualSTS","STS16VisualSTS","BLINKIT2IRetrieval","BLINKIT2TRetrieval","CIRRIT2IRetrieval","CUB200I2IRetrieval","EDIST2ITRetrieval","Fashion200kI2TRetrieval","Fashion200kT2IRetrieval","FashionIQIT2IRetrieval","Flickr30kI2TRetrieval","Flickr30kT2IRetrieval","FORBI2IRetrieval","GLDv2I2IRetrieval","GLDv2I2TRetrieval","HatefulMemesI2TRetrieval","HatefulMemesT2IRetrieval","ImageCoDeT2IRetrieval","InfoSeekIT2ITRetrieval","InfoSeekIT2TRetrieval","MemotionI2TRetrieval","MemotionT2IRetrieval","METI2IRetrieval","MSCOCOI2TRetrieval","MSCOCOT2IRetrieval","NIGHTSI2IRetrieval","OVENIT2ITRetrieval","OVENIT2TRetrieval","ROxfordEasyI2IRetrieval","ROxfordMediumI2IRetrieval","ROxfordHardI2IRetrieval","RP2kI2IRetrieval","RParisEasyI2IRetrieval","RParisMediumI2IRetrieval","RParisHardI2IRetrieval","SciMMIRI2TRetrieval","SciMMIRT2IRetrieval","SketchyI2IRetrieval","SOPI2IRetrieval","StanfordCarsI2IRetrieval","TUBerlinT2IRetrieval","VidoreArxivQARetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VidoreShiftProjectRetrieval","VidoreSyntheticDocQAAIRetrieval","VidoreSyntheticDocQAEnergyRetrieval","VidoreSyntheticDocQAGovernmentReportsRetrieval","VidoreSyntheticDocQAHealthcareIndustryRetrieval","VisualNewsI2TRetrieval","VisualNewsT2IRetrieval","VizWizIT2TRetrieval","VQA2IT2TRetrieval","WebQAT2ITRetrieval","WebQAT2TRetrieval","VisualSTS17Eng","VisualSTS-b-Eng"],"tasksComplete":true}]},{"name":"MIEB(lite) Task Mean","groups":[{"label":"Task Mean","description":"Unweighted mean across MIEB(lite)'s task set (a strict subset of MIEB(Multilingual)'s) — not the same statistic as MIEB(lite)'s own Mean(TaskType).","tasks":["Country211","DTD","EuroSAT","GTSRB","OxfordPets","PatchCamelyon","RESISC45","SUN397","ImageNetDog15Clustering","TinyImageNetClustering","CIFAR100ZeroShot","Country211ZeroShot","FER2013ZeroShot","FGVCAircraftZeroShot","Food101ZeroShot","OxfordPetsZeroShot","StanfordCarsZeroShot","BLINKIT2IMultiChoice","CVBenchCount","CVBenchRelation","CVBenchDepth","CVBenchDistance","AROCocoOrder","AROFlickrOrder","AROVisualAttribution","AROVisualRelation","Winoground","ImageCoDe","STS13VisualSTS","STS15VisualSTS","VisualSTS17Multilingual","VisualSTS-b-Multilingual","CIRRIT2IRetrieval","CUB200I2IRetrieval","Fashion200kI2TRetrieval","HatefulMemesI2TRetrieval","InfoSeekIT2TRetrieval","NIGHTSI2IRetrieval","OVENIT2TRetrieval","RP2kI2IRetrieval","VidoreDocVQARetrieval","VidoreInfoVQARetrieval","VidoreTabfquadRetrieval","VidoreTatdqaRetrieval","VidoreShiftProjectRetrieval","VidoreSyntheticDocQAAIRetrieval","VisualNewsI2TRetrieval","VQA2IT2TRetrieval","WebQAT2ITRetrieval","WITT2IRetrieval","XM3600T2IRetrieval"],"tasksComplete":true}]}],"showZeroShot":true}