README.md

---
tags:
- mteb
model-index:
- name: Zhihui_LLM_Embedding
  results:
  - task:
      type: Retrieval
    dataset:
      type: C-MTEB/CmedqaRetrieval
      name: MTEB CmedqaRetrieval
      config: default
      split: dev
      revision: None
    metrics:
    - type: map_at_1
      value: 29.012
    - type: map_at_10
      value: 41.998000000000005
    - type: map_at_100
      value: 43.821
    - type: map_at_1000
      value: 43.924
    - type: map_at_3
      value: 37.804
    - type: map_at_5
      value: 40.025
    - type: mrr_at_1
      value: 43.536
    - type: mrr_at_10
      value: 51.413
    - type: mrr_at_100
      value: 52.329
    - type: mrr_at_1000
      value: 52.366
    - type: mrr_at_3
      value: 49.058
    - type: mrr_at_5
      value: 50.291
    - type: ndcg_at_1
      value: 43.536
    - type: ndcg_at_10
      value: 48.693
    - type: ndcg_at_100
      value: 55.644000000000005
    - type: ndcg_at_1000
      value: 57.354000000000006
    - type: ndcg_at_3
      value: 43.627
    - type: ndcg_at_5
      value: 45.462
    - type: precision_at_1
      value: 43.536
    - type: precision_at_10
      value: 10.552999999999999
    - type: precision_at_100
      value: 1.624
    - type: precision_at_1000
      value: 0.184
    - type: precision_at_3
      value: 24.314
    - type: precision_at_5
      value: 17.299
    - type: recall_at_1
      value: 29.012
    - type: recall_at_10
      value: 59.123000000000005
    - type: recall_at_100
      value: 87.783
    - type: recall_at_1000
      value: 99.078
    - type: recall_at_3
      value: 43.474000000000004
    - type: recall_at_5
      value: 49.557
  - task:
      type: Retrieval
    dataset:
      type: C-MTEB/CovidRetrieval
      name: MTEB CovidRetrieval
      config: default
      split: dev
      revision: None
    metrics:
    - type: map_at_1
      value: 73.235
    - type: map_at_10
      value: 80.87100000000001
    - type: map_at_100
      value: 81.10300000000001
    - type: map_at_1000
      value: 81.105
    - type: map_at_3
      value: 79.171
    - type: map_at_5
      value: 80.163
    - type: mrr_at_1
      value: 73.235
    - type: mrr_at_10
      value: 80.80000000000001
    - type: mrr_at_100
      value: 81.024
    - type: mrr_at_1000
      value: 81.026
    - type: mrr_at_3
      value: 79.15299999999999
    - type: mrr_at_5
      value: 80.133
    - type: ndcg_at_1
      value: 73.34
    - type: ndcg_at_10
      value: 84.387
    - type: ndcg_at_100
      value: 85.348
    - type: ndcg_at_1000
      value: 85.411
    - type: ndcg_at_3
      value: 80.97
    - type: ndcg_at_5
      value: 82.757
    - type: precision_at_1
      value: 73.34
    - type: precision_at_10
      value: 9.631
    - type: precision_at_100
      value: 1.005
    - type: precision_at_1000
      value: 0.101
    - type: precision_at_3
      value: 28.837000000000003
    - type: precision_at_5
      value: 18.209
    - type: recall_at_1
      value: 73.235
    - type: recall_at_10
      value: 95.311
    - type: recall_at_100
      value: 99.473
    - type: recall_at_1000
      value: 100
    - type: recall_at_3
      value: 86.091
    - type: recall_at_5
      value: 90.411
  - task:
      type: Retrieval
    dataset:
      type: C-MTEB/DuRetrieval
      name: MTEB DuRetrieval
      config: default
      split: dev
      revision: None
    metrics:
    - type: map_at_1
      value: 28.059
    - type: map_at_10
      value: 85.309
    - type: map_at_100
      value: 87.926
    - type: map_at_1000
      value: 87.945
    - type: map_at_3
      value: 59.862
    - type: map_at_5
      value: 75.345
    - type: mrr_at_1
      value: 93.30000000000001
    - type: mrr_at_10
      value: 95.624
    - type: mrr_at_100
      value: 95.647
    - type: mrr_at_1000
      value: 95.649
    - type: mrr_at_3
      value: 95.42500000000001
    - type: mrr_at_5
      value: 95.572
    - type: ndcg_at_1
      value: 93.30000000000001
    - type: ndcg_at_10
      value: 91.338
    - type: ndcg_at_100
      value: 93.38
    - type: ndcg_at_1000
      value: 93.57
    - type: ndcg_at_3
      value: 90.512
    - type: ndcg_at_5
      value: 89.617
    - type: precision_at_1
      value: 93.30000000000001
    - type: precision_at_10
      value: 43.169999999999995
    - type: precision_at_100
      value: 4.868
    - type: precision_at_1000
      value: 0.49100000000000005
    - type: precision_at_3
      value: 80.7
    - type: precision_at_5
      value: 68.12
    - type: recall_at_1
      value: 28.059
    - type: recall_at_10
      value: 91.949
    - type: recall_at_100
      value: 98.777
    - type: recall_at_1000
      value: 99.816
    - type: recall_at_3
      value: 61.699000000000005
    - type: recall_at_5
      value: 79.134
  - task:
      type: Retrieval
    dataset:
      type: C-MTEB/EcomRetrieval
      name: MTEB EcomRetrieval
      config: default
      split: dev
      revision: None
    metrics:
    - type: map_at_1
      value: 57.599999999999994
    - type: map_at_10
      value: 67.169
    - type: map_at_100
      value: 67.654
    - type: map_at_1000
      value: 67.663
    - type: map_at_3
      value: 64.833
    - type: map_at_5
      value: 66.298
    - type: mrr_at_1
      value: 57.599999999999994
    - type: mrr_at_10
      value: 67.169
    - type: mrr_at_100
      value: 67.654
    - type: mrr_at_1000
      value: 67.663
    - type: mrr_at_3
      value: 64.833
    - type: mrr_at_5
      value: 66.298
    - type: ndcg_at_1
      value: 57.599999999999994
    - type: ndcg_at_10
      value: 71.95899999999999
    - type: ndcg_at_100
      value: 74.092
    - type: ndcg_at_1000
      value: 74.323
    - type: ndcg_at_3
      value: 67.212
    - type: ndcg_at_5
      value: 69.892
    - type: precision_at_1
      value: 57.599999999999994
    - type: precision_at_10
      value: 8.7
    - type: precision_at_100
      value: 0.9650000000000001
    - type: precision_at_1000
      value: 0.098
    - type: precision_at_3
      value: 24.7
    - type: precision_at_5
      value: 16.14
    - type: recall_at_1
      value: 57.599999999999994
    - type: recall_at_10
      value: 87
    - type: recall_at_100
      value: 96.5
    - type: recall_at_1000
      value: 98.3
    - type: recall_at_3
      value: 74.1
    - type: recall_at_5
      value: 80.7
  - task:
      type: Retrieval
    dataset:
      type: C-MTEB/MMarcoRetrieval
      name: MTEB MMarcoRetrieval
      config: default
      split: dev
      revision: None
    metrics:
    - type: map_at_1
      value: 72.858
    - type: map_at_10
      value: 81.559
    - type: map_at_100
      value: 81.808
    - type: map_at_1000
      value: 81.813
    - type: map_at_3
      value: 80.018
    - type: map_at_5
      value: 81.04299999999999
    - type: mrr_at_1
      value: 75.27199999999999
    - type: mrr_at_10
      value: 81.989
    - type: mrr_at_100
      value: 82.202
    - type: mrr_at_1000
      value: 82.206
    - type: mrr_at_3
      value: 80.647
    - type: mrr_at_5
      value: 81.53399999999999
    - type: ndcg_at_1
      value: 75.27199999999999
    - type: ndcg_at_10
      value: 84.772
    - type: ndcg_at_100
      value: 85.79599999999999
    - type: ndcg_at_1000
      value: 85.925
    - type: ndcg_at_3
      value: 81.884
    - type: ndcg_at_5
      value: 83.60300000000001
    - type: precision_at_1
      value: 75.27199999999999
    - type: precision_at_10
      value: 10.017
    - type: precision_at_100
      value: 1.051
    - type: precision_at_1000
      value: 0.106
    - type: precision_at_3
      value: 30.578
    - type: precision_at_5
      value: 19.261
    - type: recall_at_1
      value: 72.858
    - type: recall_at_10
      value: 94.197
    - type: recall_at_100
      value: 98.634
    - type: recall_at_1000
      value: 99.63499999999999
    - type: recall_at_3
      value: 86.6
    - type: recall_at_5
      value: 90.692
  - task:
      type: Retrieval
    dataset:
      type: C-MTEB/MedicalRetrieval
      name: MTEB MedicalRetrieval
      config: default
      split: dev
      revision: None
    metrics:
    - type: map_at_1
      value: 56.2
    - type: map_at_10
      value: 62.198
    - type: map_at_100
      value: 62.794000000000004
    - type: map_at_1000
      value: 62.829
    - type: map_at_3
      value: 60.699999999999996
    - type: map_at_5
      value: 61.660000000000004
    - type: mrr_at_1
      value: 56.49999999999999
    - type: mrr_at_10
      value: 62.348000000000006
    - type: mrr_at_100
      value: 62.944
    - type: mrr_at_1000
      value: 62.979
    - type: mrr_at_3
      value: 60.85
    - type: mrr_at_5
      value: 61.809999999999995
    - type: ndcg_at_1
      value: 56.2
    - type: ndcg_at_10
      value: 65.19200000000001
    - type: ndcg_at_100
      value: 68.341
    - type: ndcg_at_1000
      value: 69.392
    - type: ndcg_at_3
      value: 62.163999999999994
    - type: ndcg_at_5
      value: 63.894
    - type: precision_at_1
      value: 56.2
    - type: precision_at_10
      value: 7.46
    - type: precision_at_100
      value: 0.899
    - type: precision_at_1000
      value: 0.098
    - type: precision_at_3
      value: 22.133
    - type: precision_at_5
      value: 14.12
    - type: recall_at_1
      value: 56.2
    - type: recall_at_10
      value: 74.6
    - type: recall_at_100
      value: 89.9
    - type: recall_at_1000
      value: 98.4
    - type: recall_at_3
      value: 66.4
    - type: recall_at_5
      value: 70.6
  - task:
      type: Retrieval
    dataset:
      type: C-MTEB/T2Retrieval
      name: MTEB T2Retrieval
      config: default
      split: dev
      revision: None
    metrics:
    - type: map_at_1
      value: 28.79
    - type: map_at_10
      value: 81.687
    - type: map_at_100
      value: 85.19200000000001
    - type: map_at_1000
      value: 85.232
    - type: map_at_3
      value: 57.145999999999994
    - type: map_at_5
      value: 70.491
    - type: mrr_at_1
      value: 92.21000000000001
    - type: mrr_at_10
      value: 94.303
    - type: mrr_at_100
      value: 94.368
    - type: mrr_at_1000
      value: 94.37
    - type: mrr_at_3
      value: 93.94500000000001
    - type: mrr_at_5
      value: 94.175
    - type: ndcg_at_1
      value: 92.21000000000001
    - type: ndcg_at_10
      value: 88.29599999999999
    - type: ndcg_at_100
      value: 91.268
    - type: ndcg_at_1000
      value: 91.645
    - type: ndcg_at_3
      value: 89.031
    - type: ndcg_at_5
      value: 88.075
    - type: precision_at_1
      value: 92.21000000000001
    - type: precision_at_10
      value: 43.775
    - type: precision_at_100
      value: 5.097
    - type: precision_at_1000
      value: 0.518
    - type: precision_at_3
      value: 77.708
    - type: precision_at_5
      value: 65.473
    - type: recall_at_1
      value: 28.79
    - type: recall_at_10
      value: 87.457
    - type: recall_at_100
      value: 97.21499999999999
    - type: recall_at_1000
      value: 99.14
    - type: recall_at_3
      value: 58.606
    - type: recall_at_5
      value: 73.52300000000001
  - task:
      type: Retrieval
    dataset:
      type: C-MTEB/VideoRetrieval
      name: MTEB VideoRetrieval
      config: default
      split: dev
      revision: None
    metrics:
    - type: map_at_1
      value: 67
    - type: map_at_10
      value: 75.44999999999999
    - type: map_at_100
      value: 75.771
    - type: map_at_1000
      value: 75.776
    - type: map_at_3
      value: 73.867
    - type: map_at_5
      value: 74.837
    - type: mrr_at_1
      value: 67
    - type: mrr_at_10
      value: 75.44999999999999
    - type: mrr_at_100
      value: 75.771
    - type: mrr_at_1000
      value: 75.776
    - type: mrr_at_3
      value: 73.867
    - type: mrr_at_5
      value: 74.837
    - type: ndcg_at_1
      value: 67
    - type: ndcg_at_10
      value: 79.313
    - type: ndcg_at_100
      value: 80.894
    - type: ndcg_at_1000
      value: 80.989
    - type: ndcg_at_3
      value: 76.08500000000001
    - type: ndcg_at_5
      value: 77.845
    - type: precision_at_1
      value: 67
    - type: precision_at_10
      value: 9.13
    - type: precision_at_100
      value: 0.987
    - type: precision_at_1000
      value: 0.099
    - type: precision_at_3
      value: 27.500000000000004
    - type: precision_at_5
      value: 17.36
    - type: recall_at_1
      value: 67
    - type: recall_at_10
      value: 91.3
    - type: recall_at_100
      value: 98.7
    - type: recall_at_1000
      value: 99.4
    - type: recall_at_3
      value: 82.5
    - type: recall_at_5
      value: 86.8
license: cc-by-nc-4.0
language:
- zh
library_name: transformers
---



## Zhihui_LLM_Embedding

### Model Introduction

**Zhihui_LLM_Embedding** is an embedding model specifically designed to enhance Chinese text retrieval capabilities. It is built on a 7B LLM and enhanced bidirectional attention mechanism to improved contextual understanding. The model is trained on an extensive corpus from various fields within an extremely large batch. **Zhihui_LLM_Embedding** excels in retrieval tasks, ranking **1st position** on the C-MTEB leaderboard with a leading performance score of **76.74** as of June 25, 2024.

### Optimization points
* Data source enhancement: Leverages the knowledge of LLMs through three types of distillation methods.(GPT3.5 & GPT4)
    * Data Refinement: LLM scores candidate positive passages to select the most relevant examples.
    * Query Rewriting: LLM generates queries that can be answered by positive documents but are unrelated to negatives, thus enhancing the query's quality and diversity.
    * Query Expansion: Queries are expanded based on multiple topics for long documents.
* Negative example mining: Use multiple methods and different ranges of negative selection to mine hard negative examples. 
* Improved Contrastive Loss: Design a novel InfoNCE loss assigns higher weights to the harder negative examples to improve the fine-grained feature representation of the model.
* Bidirectional-attention: Remove the causal attention of LLMs during contrastive training of decoder-only LLM to produce rich contextualized representations.
* Training efficiency: Using Gradient Cache to scale contrastive learning batches beyond GPU memory constraints allows the model to learn from more challenging negative examples.
* Others： Dataset-Homogenous Batching、cross-batch negative sampling

### Model Details
* Base Decoder-only LLM: [gte-Qwen2-7B-instruct](https://huggingface.co./Alibaba-NLP/gte-Qwen2-7B-instruct) 
* Pooling Methods: Last token
* Embedding Dimension: 3584

### Usage
##### Requirements
```
transformers>=4.40.2
flash_attn>=2.5.8
sentence-transformers>=2.7.0
```
##### How to use
Here is an example of how to encode queries and passages using Huggingface-transformer and Sentence-transformer.
##### Usage (HuggingFace Transformers)
```python
import torch
import torch.nn.functional as F

from torch import Tensor
from transformers import AutoTokenizer, AutoModel


def last_token_pool(last_hidden_states: Tensor, attention_mask: Tensor) -> Tensor:
    left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0])
    if left_padding:
        return last_hidden_states[:, -1]
    else:
        sequence_lengths = attention_mask.sum(dim=1) - 1
        batch_size = last_hidden_states.shape[0]
        return last_hidden_states[torch.arange(batch_size, device=last_hidden_states.device), sequence_lengths]


def get_detailed_instruct(task_description: str, query: str) -> str:
    return f'Instruct: {task_description}\nQuery: {query}'



task = 'Given a web search query, retrieve relevant passages that answer the query'
queries = [
    get_detailed_instruct(task, "国家法定节假日共多少天"),
    get_detailed_instruct(task, "如何查看好友申请")
]

documents = [
    "一年国家法定节假日为11天。根据公布的国家法定节假日调整方案，调整的主要内容包括：元旦放假1天不变；春节放假3天，放假时间为农历正月初一、初二、初三；“五一”国际劳动节1天不变；“十一”国庆节放假3天；清明节、端午节、中秋节增设为国家法定节假日，各放假1天(农历节日如遇闰月，以第一个月为休假日)。3、允许周末上移下错，与法定节假日形成连休。",
    "这个直接去我的QQ中心不就好了么那里可以查到 我的好友单向好友好友恢复、 以及好友申请 啊可以是你加别人的 或 别人加你的都可以查得到QQ空间里 这个没注意 要有的话也会在你进空间的时候会提示你的QQ 空间里 上面消息 就可以看见了！望采纳！谢谢这个直接去我的QQ中心不就好了么那里可以查到 我的好友单向好友好友恢复、 以及好友申请 啊可以是你加别人的 或 别人加你的都可以查得到",
]
input_texts = queries + documents

tokenizer = AutoTokenizer.from_pretrained('Lenovo-Zhihui/Zhihui_LLM_Embedding', trust_remote_code=True)
model = AutoModel.from_pretrained('Lenovo-Zhihui/Zhihui_LLM_Embedding', trust_remote_code=True)

max_length = 512

# Tokenize the input texts
batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt')
outputs = model(**batch_dict)
embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask'])

# normalize embeddings
embeddings = F.normalize(embeddings, p=2, dim=1)
scores = (embeddings[:2] @ embeddings[2:].T)
print(scores.tolist())

```
##### Usage (Sentence-Transformers)
```python
from sentence_transformers import SentenceTransformer
model = SentenceTransformer("Lenovo-Zhihui/Zhihui_LLM_Embedding", trust_remote_code=True)
model.max_seq_length = 512
# 数据来源DuRetrieval https://huggingface.co./datasets/C-MTEB/DuRetrieval
queries = [
    "国家法定节假日共多少天",
    "如何查看好友申请",
]
documents = [
    "一年国家法定节假日为11天。根据公布的国家法定节假日调整方案，调整的主要内容包括：元旦放假1天不变；春节放假3天，放假时间为农历正月初一、初二、初三；“五一”国际劳动节1天不变；“十一”国庆节放假3天；清明节、端午节、中秋节增设为国家法定节假日，各放假1天(农历节日如遇闰月，以第一个月为休假日)。3、允许周末上移下错，与法定节假日形成连休。",
    "这个直接去我的QQ中心不就好了么那里可以查到 我的好友单向好友好友恢复、 以及好友申请 啊可以是你加别人的 或 别人加你的都可以查得到QQ空间里 这个没注意 要有的话也会在你进空间的时候会提示你的QQ 空间里 上面消息 就可以看见了！望采纳！谢谢这个直接去我的QQ中心不就好了么那里可以查到 我的好友单向好友好友恢复、 以及好友申请 啊可以是你加别人的 或 别人加你的都可以查得到",
]

query_embeddings = model.encode(queries, prompt_name="query", normalize_embeddings=True)
document_embeddings = model.encode(documents, normalize_embeddings=True)

scores = (query_embeddings @ document_embeddings.T)
print(scores.tolist())
```
### Reproduce our results（C-MTEB）:
Check out scripts/eval_mteb.py to reproduce evaluation results on C-MTEB benchmark.

| Model | T2Retrieval | MMarcoRetrieval | DuRetrieval | CovidRetrieval | CmedqaRetrieval | EcomRetrieval | MedicalRetrieval | VideoRetrieval | Avg |  
|:-------------------------------|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|  
|**Zhihui_LLM_Embedding** | 88.30 | 84.77 | 91.34 | 84.39 | 48.69 | 71.96 | 65.19 | 79.31 | **76.74** |
|zpoint_large_embedding_zh | 83.81 | 82.38 | 89.23 | 89.14 | 47.16 | 70.74 | 68.14 | 80.26 | 76.36 |
|gte-Qwen2-7B-instruct | 87.73 | 85.16 | 87.44 | 83.65 | 48.69 | 71.15 | 65.59 | 78.84 | 76.03 |
|360Zhinao-search | 87.12 | 83.32 | 87.57 | 85.02 | 46.73 | 68.9 | 63.69 | 78.09 | 75.06 |
|AGE_Hybrid | 86.88 | 80.65 | 89.28 | 83.66 | 47.26 | 69.28 | 65.94 | 76.79 | 74.97 |