{"model":{"name":"GritLM/GritLM-7B","url":"https://huggingface.co/GritLM/GritLM-7B","zeroShotPct":73,"activeParamsB":7.109,"totalParamsB":7.24,"embeddingDim":4096,"maxTokens":32768.0,"releaseDate":"2024-02-15","modelType":"dense","instructionTuned":true,"openWeights":true,"openness":{"open weights":true,"open license":true,"open training code":true,"open training data":false,"paper":true,"model card":true},"opennessScore":5,"sentenceTransformersCompatible":false,"modalities":["text"],"languages":["English","French","German","Italian","Spanish"],"citation":"\n@misc{muennighoff2024generative,\n      title={Generative Representational Instruction Tuning},\n      author={Niklas Muennighoff and Hongjin Su and Liang Wang and Nan Yang and Furu Wei and Tao Yu and Amanpreet Singh and Douwe Kiela},\n      year={2024},\n      eprint={2402.09906},\n      archivePrefix={arXiv},\n      primaryClass={cs.CL}\n}\n","memoryUsageMb":13813.0,"license":"apache-2.0","publicTrainingCode":"https://github.com/ContextualAI/gritlm","publicTrainingData":null,"adaptedFrom":null,"supersededBy":null,"extraRequirementsGroups":null,"trainingDatasets":["FEVER","FEVER-NL","FEVER-PL","FEVERHardNegatives","HotpotQA","HotpotQA-NL","HotpotQA-PL","HotpotQAHardNegatives","MIRACLReranking","MIRACLRetrieval","MIRACLRetrievalHardNegatives","MSMARCO","MSMARCO-PL","MSMARCOHardNegatives","MrTidyRetrieval","NQ","NQ-NL","NQ-PL","NQHardNegatives","NanoMSMARCORetrieval","NanoNQRetrieval","XQuADRetrieval","mMARCO-NL"]},"rows":[{"benchmarkName":"IFIR","benchmarkDisplayName":"IFIR","rank":1,"totalModels":7,"meanTask":0.30967428571428574,"meanTaskType":0.30967428571428574,"zeroShotPct":100,"taskTypes":["InstructionRetrieval"],"scoresByTaskType":{"InstructionRetrieval":0.30967428571428574}},{"benchmarkName":"BRIGHT","benchmarkDisplayName":"Reasoning Retrieval","rank":4,"totalModels":29,"meanTask":0.2062775,"meanTaskType":0.2062775,"zeroShotPct":100,"taskTypes":["Retrieval"],"scoresByTaskType":{"Retrieval":0.2062775}},{"benchmarkName":"R2MED","benchmarkDisplayName":"Reasoning-driven medical retrieval","rank":4,"totalModels":27,"meanTask":0.28026,"meanTaskType":0.28026,"zeroShotPct":100,"taskTypes":["Retrieval"],"scoresByTaskType":{"Retrieval":0.28026}},{"benchmarkName":"RAR-b","benchmarkDisplayName":"Reasoning as retrieval","rank":4,"totalModels":209,"meanTask":0.36163529411764705,"meanTaskType":0.36163529411764705,"zeroShotPct":100,"taskTypes":["Retrieval"],"scoresByTaskType":{"Retrieval":0.36163529411764705}},{"benchmarkName":"BuiltBench(eng)","benchmarkDisplayName":"BuiltBench(eng)","rank":6,"totalModels":54,"meanTask":0.66851775,"meanTaskType":0.6867726666666666,"zeroShotPct":100,"taskTypes":["Retrieval","Clustering","Reranking"],"scoresByTaskType":{"Retrieval":0.73872,"Clustering":0.613753,"Reranking":0.707845}},{"benchmarkName":"MTEB(rus, v1)","benchmarkDisplayName":"Russian","rank":9,"totalModels":255,"meanTask":0.6766590001449984,"meanTaskType":0.6507546452946785,"zeroShotPct":91,"taskTypes":["Classification","Clustering","STS","Retrieval","Reranking","MultilabelClassification","PairClassification"],"scoresByTaskType":{"Classification":0.6992400966579709,"Clustering":0.643037821152686,"STS":0.7463051786172544,"Retrieval":0.7579066666666666,"Reranking":0.699869372497077,"MultilabelClassification":0.41958500763815093,"PairClassification":0.5893383738329443}},{"benchmarkName":"MTEB(Scandinavian, v1)","benchmarkDisplayName":"Scandinavian","rank":10,"totalModels":246,"meanTask":0.640838394367255,"meanTaskType":0.6534739720453733,"zeroShotPct":100,"taskTypes":["Classification","Retrieval","Clustering","BitextMining"],"scoresByTaskType":{"Classification":0.6674939356359276,"Retrieval":0.66491,"Clustering":0.5121749934812383,"BitextMining":0.7693169590643274}},{"benchmarkName":"MTEB(Africa, v1)","benchmarkDisplayName":"African","rank":12,"totalModels":324,"meanTask":null,"meanTaskType":null,"zeroShotPct":88,"taskTypes":["BitextMining","Retrieval","Any2AnyMultilingualRetrieval","Classification","Reranking","Clustering","PairClassification","STS","MultilabelClassification"],"scoresByTaskType":{"BitextMining":0.4453548686516087,"Reranking":0.6476799999999999,"Clustering":0.460526329552675,"STS":0.620219456516204}},{"benchmarkName":"MTEB(jpn, v1)","benchmarkDisplayName":"Japanese","rank":13,"totalModels":248,"meanTask":null,"meanTaskType":null,"zeroShotPct":93,"taskTypes":["Classification","Clustering","Retrieval","STS","PairClassification","Reranking"],"scoresByTaskType":{"Classification":0.7233589598909136,"Clustering":0.5382795812109982,"STS":0.8405777424886041,"PairClassification":0.5342474317812214}},{"benchmarkName":"MINERSBitextMining","benchmarkDisplayName":"MINERSBitextMining","rank":13,"totalModels":201,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["BitextMining"],"scoresByTaskType":{}},{"benchmarkName":"MTEB(rus, v1.1)","benchmarkDisplayName":"Russian","rank":14,"totalModels":255,"meanTask":0.6502176957971723,"meanTaskType":0.6217951214851549,"zeroShotPct":91,"taskTypes":["Classification","STS","Retrieval","Clustering","Reranking","PairClassification","MultilabelClassification"],"scoresByTaskType":{"Classification":0.6992400966579709,"STS":0.7463051786172546,"Retrieval":0.5551900000000001,"Clustering":0.643037821152686,"Reranking":0.699869372497077,"PairClassification":0.5893383738329443,"MultilabelClassification":0.41958500763815093}},{"benchmarkName":"MTEB(Europe, v1)","benchmarkDisplayName":"European","rank":15,"totalModels":415,"meanTask":0.6297081861512202,"meanTaskType":0.5608971943348854,"zeroShotPct":100,"taskTypes":["Classification","PairClassification","Retrieval","Reranking","BitextMining","STS","Clustering","InstructionReranking","MultilabelClassification"],"scoresByTaskType":{"Classification":0.6473689600808005,"PairClassification":0.8993943952500313,"Retrieval":0.5710526336654212,"Reranking":0.6026964376195737,"BitextMining":0.9042070495128554,"STS":0.7605024661506241,"Clustering":0.4528110464598032,"InstructionReranking":0.034529763909089885,"MultilabelClassification":0.1755119963657701}},{"benchmarkName":"MTEB(Africa, v1, lite)","benchmarkDisplayName":"African Lite","rank":18,"totalModels":212,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["Classification","BitextMining","Clustering","PairClassification","Retrieval","MultilabelClassification"],"scoresByTaskType":{"BitextMining":0.45348130933485975,"Clustering":0.2251586314268726,"Retrieval":0.5906277777777778}},{"benchmarkName":"RTEB(Health, beta)","benchmarkDisplayName":"RTEB Healthcare","rank":20,"totalModels":236,"meanTask":0.6414945454545455,"meanTaskType":null,"zeroShotPct":100,"taskTypes":[],"scoresByTaskType":{}},{"benchmarkName":"MTEB(Medical, v1)","benchmarkDisplayName":"Medical","rank":21,"totalModels":389,"meanTask":0.6150722325387004,"meanTaskType":0.6026167381057502,"zeroShotPct":100,"taskTypes":["Retrieval","Clustering","Reranking"],"scoresByTaskType":{"Retrieval":0.6507978156565657,"Clustering":0.36663405089462797,"Reranking":0.7904183477660569}},{"benchmarkName":"FollowIR","benchmarkDisplayName":"Instruction Following","rank":22,"totalModels":204,"meanTask":0.034529763909089885,"meanTaskType":0.034529763909089885,"zeroShotPct":100,"taskTypes":["InstructionReranking"],"scoresByTaskType":{"InstructionReranking":0.034529763909089885}},{"benchmarkName":"RTEB(fra, beta)","benchmarkDisplayName":"RTEB French","rank":22,"totalModels":233,"meanTask":0.5711172727272728,"meanTaskType":null,"zeroShotPct":100,"taskTypes":[],"scoresByTaskType":{}},{"benchmarkName":"MTEB(slk, v1)","benchmarkDisplayName":"Slovak","rank":24,"totalModels":215,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["Classification","Clustering","BitextMining","Reranking","STS","PairClassification","Retrieval"],"scoresByTaskType":{}},{"benchmarkName":"BEIR-PL","benchmarkDisplayName":"BEIR-PL","rank":26,"totalModels":179,"meanTask":null,"meanTaskType":null,"zeroShotPct":76,"taskTypes":["Retrieval"],"scoresByTaskType":{}},{"benchmarkName":"MTEB(Law, v1)","benchmarkDisplayName":"Legal","rank":26,"totalModels":264,"meanTask":0.5672525,"meanTaskType":0.5672525,"zeroShotPct":100,"taskTypes":["Retrieval"],"scoresByTaskType":{"Retrieval":0.5672525}},{"benchmarkName":"Encodechka","benchmarkDisplayName":"Encodechka","rank":30,"totalModels":203,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["PairClassification","STS","Classification"],"scoresByTaskType":{"PairClassification":0.7809895233572838}},{"benchmarkName":"RTEB(eng, beta)","benchmarkDisplayName":"RTEB English","rank":32,"totalModels":294,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":[],"scoresByTaskType":{}},{"benchmarkName":"MTEB(Multilingual, v1)","benchmarkDisplayName":"Multilingual","rank":33,"totalModels":487,"meanTask":0.6093085533977807,"meanTaskType":0.5382545298276504,"zeroShotPct":99,"taskTypes":["Classification","STS","BitextMining","Clustering","Reranking","PairClassification","Retrieval","MultilabelClassification","InstructionReranking"],"scoresByTaskType":{"Classification":0.6183024740334713,"STS":0.7332796007418407,"BitextMining":0.7053172550560896,"Clustering":0.5048252047423882,"Reranking":0.6377875398491918,"PairClassification":0.7994441093062641,"Retrieval":0.5830671092003586,"MultilabelClassification":0.22773771161015946,"InstructionReranking":0.034529763909089885}},{"benchmarkName":"MTEB(Multilingual, v2)","benchmarkDisplayName":"Multilingual","rank":34,"totalModels":487,"meanTask":0.6092069877415862,"meanTaskType":0.5374365544830888,"zeroShotPct":99,"taskTypes":["Classification","STS","Reranking","Clustering","BitextMining","Retrieval","PairClassification","InstructionReranking","MultilabelClassification"],"scoresByTaskType":{"Classification":0.6183024740334713,"STS":0.7332796007418407,"Reranking":0.6377875398491918,"Clustering":0.4974634266413336,"BitextMining":0.7053172550560897,"Retrieval":0.5830671092003586,"PairClassification":0.7994441093062641,"InstructionReranking":0.034529763909089885,"MultilabelClassification":0.2277377116101595}},{"benchmarkName":"RTEB(fin, beta)","benchmarkDisplayName":"RTEB Finance","rank":34,"totalModels":110,"meanTask":0.7131666666666666,"meanTaskType":null,"zeroShotPct":100,"taskTypes":[],"scoresByTaskType":{}},{"benchmarkName":"JMTEB-lite(v1)","benchmarkDisplayName":"Japanese","rank":36,"totalModels":251,"meanTask":null,"meanTaskType":null,"zeroShotPct":89,"taskTypes":["Reranking","Classification","STS","Clustering","Retrieval"],"scoresByTaskType":{"STS":0.8405777424886041,"Clustering":0.5451800856197485}},{"benchmarkName":"MTEB(Indic, v1)","benchmarkDisplayName":"Indic","rank":37,"totalModels":224,"meanTask":0.6015634585352274,"meanTaskType":0.5794584088814619,"zeroShotPct":95,"taskTypes":["Classification","PairClassification","Reranking","Retrieval","BitextMining","Clustering","STS"],"scoresByTaskType":{"Classification":0.5996424109260997,"PairClassification":0.678381697108265,"Reranking":0.846951455026455,"Retrieval":0.7949682692307692,"BitextMining":0.5840255191164492,"Clustering":0.2797833137051044,"STS":0.27245619705709045}},{"benchmarkName":"MTEB(kor, v1)","benchmarkDisplayName":"Korean","rank":39,"totalModels":184,"meanTask":null,"meanTaskType":null,"zeroShotPct":66,"taskTypes":["Retrieval","Reranking","STS","Classification"],"scoresByTaskType":{"Reranking":0.55579,"STS":0.8304837553325621,"Classification":0.666796875}},{"benchmarkName":"MTEB(por, v1)","benchmarkDisplayName":"Portuguese","rank":39,"totalModels":247,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["Classification","Retrieval","Reranking","STS","MultilabelClassification"],"scoresByTaskType":{"Classification":0.6919387667002543,"MultilabelClassification":0.1884765625}},{"benchmarkName":"RTEB(beta)","benchmarkDisplayName":"RTEB Multilingual","rank":40,"totalModels":294,"meanTask":null,"meanTaskType":null,"zeroShotPct":94,"taskTypes":[],"scoresByTaskType":{}},{"benchmarkName":"JMTEB(v2)","benchmarkDisplayName":"Japanese","rank":41,"totalModels":251,"meanTask":null,"meanTaskType":null,"zeroShotPct":89,"taskTypes":["Clustering","Retrieval","Classification","Reranking","STS"],"scoresByTaskType":{"Clustering":0.5451800856197485,"STS":0.8405777424886041}},{"benchmarkName":"MTEB(Code, v1)","benchmarkDisplayName":"Code","rank":45,"totalModels":269,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["Retrieval"],"scoresByTaskType":{}},{"benchmarkName":"MTEB(tha, v1)","benchmarkDisplayName":"Thai","rank":47,"totalModels":230,"meanTask":null,"meanTaskType":null,"zeroShotPct":73,"taskTypes":["Retrieval","Classification","PairClassification","Clustering","Reranking"],"scoresByTaskType":{"PairClassification":0.6995784168672687,"Clustering":0.5344859540301232}},{"benchmarkName":"MTEB(kor, v2)","benchmarkDisplayName":"Korean","rank":48,"totalModels":266,"meanTask":null,"meanTaskType":null,"zeroShotPct":85,"taskTypes":["STS","Retrieval","PairClassification","Classification","Reranking","Clustering"],"scoresByTaskType":{"STS":0.8161228490470317,"Classification":0.666796875,"Reranking":0.55579}},{"benchmarkName":"MTEB(eng, v2)","benchmarkDisplayName":"English","rank":49,"totalModels":351,"meanTask":0.670707780399812,"meanTaskType":0.6322378334323554,"zeroShotPct":95,"taskTypes":["Clustering","Classification","PairClassification","Reranking","STS","Retrieval","Summarization"],"scoresByTaskType":{"Clustering":0.508190392698282,"Classification":0.8124520809951246,"PairClassification":0.8728504659734665,"Reranking":0.49590824795166166,"STS":0.8302657083266705,"Retrieval":0.5494579999999999,"Summarization":0.35653993808128315}},{"benchmarkName":"LongEmbed","benchmarkDisplayName":"Long-context Retrieval","rank":50,"totalModels":206,"meanTask":0.4709383333333334,"meanTaskType":0.4709383333333334,"zeroShotPct":100,"taskTypes":["Retrieval"],"scoresByTaskType":{"Retrieval":0.4709383333333334}},{"benchmarkName":"MTEB(fas, v2)","benchmarkDisplayName":"Persian","rank":51,"totalModels":292,"meanTask":null,"meanTaskType":null,"zeroShotPct":88,"taskTypes":["Retrieval","Clustering","BitextMining","Reranking","Classification","PairClassification","STS"],"scoresByTaskType":{"Reranking":0.7132988624338623}},{"benchmarkName":"RTEB(Code, beta)","benchmarkDisplayName":"RTEB Code","rank":52,"totalModels":251,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":[],"scoresByTaskType":{}},{"benchmarkName":"CoIR","benchmarkDisplayName":"Code Information Retrieval","rank":55,"totalModels":268,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["Retrieval"],"scoresByTaskType":{}},{"benchmarkName":"MTEB(fas, v1)","benchmarkDisplayName":"Persian","rank":55,"totalModels":257,"meanTask":null,"meanTaskType":null,"zeroShotPct":91,"taskTypes":["Retrieval","Clustering","STS","Classification","Reranking","PairClassification","BitextMining"],"scoresByTaskType":{"Reranking":0.7132988624338623}},{"benchmarkName":"MTEB(pol, v1)","benchmarkDisplayName":"Polish","rank":55,"totalModels":234,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["PairClassification","Classification","Clustering","STS"],"scoresByTaskType":{"PairClassification":0.8649716520201025,"Classification":0.7121731912225121}},{"benchmarkName":"RTEB(Law, beta)","benchmarkDisplayName":"RTEB Legal","rank":55,"totalModels":264,"meanTask":0.47979499999999997,"meanTaskType":null,"zeroShotPct":100,"taskTypes":[],"scoresByTaskType":{}},{"benchmarkName":"MTEB(deu, v1)","benchmarkDisplayName":"German","rank":60,"totalModels":265,"meanTask":null,"meanTaskType":null,"zeroShotPct":94,"taskTypes":["Classification","Clustering","Retrieval","PairClassification","Reranking","STS"],"scoresByTaskType":{"Classification":0.7665463390150918,"Retrieval":0.5286136282516636,"PairClassification":0.5472298406748641,"Reranking":0.54023}},{"benchmarkName":"MTEB(fra, v1)","benchmarkDisplayName":"French","rank":68,"totalModels":250,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["Clustering","PairClassification","Summarization","Retrieval","Classification","STS","Reranking"],"scoresByTaskType":{"PairClassification":0.6160253458678753,"Retrieval":0.6594513333333334,"Classification":0.7627336135579895,"Reranking":0.8527141351242308}},{"benchmarkName":"MTEB(nld, v1)","benchmarkDisplayName":"Dutch","rank":71,"totalModels":256,"meanTask":0.38373140692263075,"meanTaskType":0.5082895538567385,"zeroShotPct":100,"taskTypes":["Clustering","PairClassification","Classification","Retrieval","STS","MultilabelClassification","Reranking"],"scoresByTaskType":{"Clustering":0.17678660237072333,"PairClassification":0.730688,"Classification":0.6016567035670679,"Retrieval":0.18145750000000002,"STS":0.6852622774085835,"MultilabelClassification":0.26175833333333337,"Reranking":0.9204174603174603}},{"benchmarkName":"MTEB(spa, v1)","benchmarkDisplayName":"Spanish","rank":80,"totalModels":228,"meanTask":null,"meanTaskType":null,"zeroShotPct":87,"taskTypes":["STS","Retrieval","PairClassification","Clustering","Classification","Reranking"],"scoresByTaskType":{"PairClassification":0.6938281850411716,"Reranking":0.61474}},{"benchmarkName":"MTEB(cmn, v1)","benchmarkDisplayName":"Chinese","rank":83,"totalModels":264,"meanTask":null,"meanTaskType":null,"zeroShotPct":100,"taskTypes":["Reranking","Retrieval","STS","Classification","Clustering","PairClassification"],"scoresByTaskType":{"Reranking":0.6136783005698356,"Retrieval":0.6508,"Classification":0.7487144512249264}},{"benchmarkName":"RTEB(deu, beta)","benchmarkDisplayName":"RTEB German","rank":95,"totalModels":231,"meanTask":0.44181,"meanTaskType":null,"zeroShotPct":100,"taskTypes":[],"scoresByTaskType":{}},{"benchmarkName":"MTEB(eng, v1)","benchmarkDisplayName":"English","rank":135,"totalModels":353,"meanTask":null,"meanTaskType":null,"zeroShotPct":92,"taskTypes":["Retrieval","Classification","Clustering","PairClassification","STS","Reranking","Summarization"],"scoresByTaskType":{"Classification":0.7864604666069489,"PairClassification":0.8728504659734665,"STS":0.8335139824591742,"Reranking":0.6048580489010122,"Summarization":0.30385319320589765}},{"benchmarkName":"BEIR","benchmarkDisplayName":"BEIR","rank":172,"totalModels":321,"meanTask":null,"meanTaskType":null,"zeroShotPct":73,"taskTypes":["Retrieval"],"scoresByTaskType":{}}]}