Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Video-LLaVA-7B-7B-F16.gguf filter=lfs diff=lfs merge=lfs -text
+Video-LLaVA-7B-7B-F32.gguf filter=lfs diff=lfs merge=lfs -text
+llava.projector filter=lfs diff=lfs merge=lfs -text
+vvl_vit/mmproj-model-f16.gguf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,238 @@
----
-license: apache-2.0
----
+---
+license: apache-2.0
+---
+
+
+<p align="center">
+    <img src="https://z1.ax1x.com/2023/11/07/pil4sqH.png" width="150" style="margin-bottom: 0.2;"/>
+<p>
+<h2 align="center"> <a href="https://arxiv.org/abs/2311.10122">Video-LLaVA: Learning United Visual Representation by Alignment Before Projection</a></h2>
+<h5 align="center"> If you like our project, please give us a star ⭐ on GitHub for latest update.  </h2>
+
+
+
+
+## 📰 News
+* **[2024.01.27]**  👀👀👀 Our [MoE-LLaVA](https://github.com/PKU-YuanGroup/MoE-LLaVA) is released! A sparse model with 3B parameters outperformed the dense model with 7B parameters.
+* **[2024.01.17]**  🔥🔥🔥 Our [LanguageBind](https://github.com/PKU-YuanGroup/LanguageBind) has been accepted at ICLR 2024!
+* **[2024.01.16]**  🔥🔥🔥 We reorganize the code and support LoRA fine-tuning, checking [finetune_lora.sh](scripts/v1_5/finetune_lora.sh).
+* **[2023.11.30]**  🤝 Thanks to the generous contributions of the community, the [OpenXLab's demo](https://openxlab.org.cn/apps/detail/houshaowei/Video-LLaVA) is now accessible.
+* **[2023.11.23]**  We are training a new and powerful model.
+* **[2023.11.21]**  🤝 Check out the [replicate demo](https://replicate.com/nateraw/video-llava), created by [@nateraw](https://github.com/nateraw), who has generously supported our research!
+* **[2023.11.20]**  🤗 [Hugging Face demo](https://huggingface.co/spaces/LanguageBind/Video-LLaVA) and **all codes & datasets** are available now! Welcome to **watch** 👀 this repository for the latest updates.
+
+## 😮 Highlights
+
+Video-LLaVA exhibits remarkable interactive capabilities between images and videos, despite the absence of image-video pairs in the dataset.
+
+### 💡 Simple baseline, learning united visual representation by alignment before projection
+- With **the binding of unified visual representations to the language feature space**, we enable an LLM to perform visual reasoning capabilities on both images and videos simultaneously.
+
+### 🔥 High performance, complementary learning with video and image
+- Extensive experiments demonstrate **the complementarity of modalities**, showcasing significant superiority when compared to models specifically designed for either images or videos. 
+
+
+## 🤗 Demo
+
+### Gradio Web UI
+
+Highly recommend trying out our web demo by the following command, which incorporates all features currently supported by Video-LLaVA. We also provide [online demo](https://huggingface.co/spaces/LanguageBind/Video-LLaVA) in Huggingface Spaces.
+```bash
+python -m  videollava.serve.gradio_web_server
+```
+
+
+
+### CLI Inference
+
+```bash
+python -m videollava.serve.cli --model-path "LanguageBind/Video-LLaVA-7B" --file "path/to/your/video.mp4" --load-4bit
+```
+
+```bash
+python -m videollava.serve.cli --model-path "LanguageBind/Video-LLaVA-7B" --file "path/to/your/image.jpg" --load-4bit
+```
+
+
+
+## 🛠️ Requirements and Installation
+* Python >= 3.10
+* Pytorch == 2.0.1
+* CUDA Version >= 11.7
+* Install required packages:
+```bash
+git clone https://github.com/PKU-YuanGroup/Video-LLaVA
+cd Video-LLaVA
+conda create -n videollava python=3.10 -y
+conda activate videollava
+pip install --upgrade pip  # enable PEP 660 support
+pip install -e .
+pip install -e ".[train]"
+pip install flash-attn --no-build-isolation
+pip install decord opencv-python git+https://github.com/facebookresearch/pytorchvideo.git@28fe037d212663c6a24f373b94cc5d478c8c1a1d
+```
+
+## 🤖 API
+**We open source all codes.** If you want to load the model (e.g. ```LanguageBind/Video-LLaVA-7B```) on local, you can use the following code snippets.
+
+### Inference for image
+```python
+import torch
+from videollava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
+from videollava.conversation import conv_templates, SeparatorStyle
+from videollava.model.builder import load_pretrained_model
+from videollava.utils import disable_torch_init
+from videollava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+
+def main():
+    disable_torch_init()
+    image = 'videollava/serve/examples/extreme_ironing.jpg'
+    inp = 'What is unusual about this image?'
+    model_path = 'LanguageBind/Video-LLaVA-7B'
+    cache_dir = 'cache_dir'
+    device = 'cuda'
+    load_4bit, load_8bit = True, False
+    model_name = get_model_name_from_path(model_path)
+    tokenizer, model, processor, _ = load_pretrained_model(model_path, None, model_name, load_8bit, load_4bit, device=device, cache_dir=cache_dir)
+    image_processor = processor['image']
+    conv_mode = "llava_v1"
+    conv = conv_templates[conv_mode].copy()
+    roles = conv.roles
+
+    image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values']
+    if type(image_tensor) is list:
+        tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor]
+    else:
+        tensor = image_tensor.to(model.device, dtype=torch.float16)
+
+    print(f"{roles[1]}: {inp}")
+    inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
+    conv.append_message(conv.roles[0], inp)
+    conv.append_message(conv.roles[1], None)
+    prompt = conv.get_prompt()
+    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+    keywords = [stop_str]
+    stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+    with torch.inference_mode():
+        output_ids = model.generate(
+            input_ids,
+            images=tensor,
+            do_sample=True,
+            temperature=0.2,
+            max_new_tokens=1024,
+            use_cache=True,
+            stopping_criteria=[stopping_criteria])
+
+    outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip()
+    print(outputs)
+
+if __name__ == '__main__':
+    main()
+```
+
+### Inference for video
+```python
+import torch
+from videollava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
+from videollava.conversation import conv_templates, SeparatorStyle
+from videollava.model.builder import load_pretrained_model
+from videollava.utils import disable_torch_init
+from videollava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+
+def main():
+    disable_torch_init()
+    video = 'videollava/serve/examples/sample_demo_1.mp4'
+    inp = 'Why is this video funny?'
+    model_path = 'LanguageBind/Video-LLaVA-7B'
+    cache_dir = 'cache_dir'
+    device = 'cuda'
+    load_4bit, load_8bit = True, False
+    model_name = get_model_name_from_path(model_path)
+    tokenizer, model, processor, _ = load_pretrained_model(model_path, None, model_name, load_8bit, load_4bit, device=device, cache_dir=cache_dir)
+    video_processor = processor['video']
+    conv_mode = "llava_v1"
+    conv = conv_templates[conv_mode].copy()
+    roles = conv.roles
+
+    video_tensor = video_processor(video, return_tensors='pt')['pixel_values']
+    if type(video_tensor) is list:
+        tensor = [video.to(model.device, dtype=torch.float16) for video in video_tensor]
+    else:
+        tensor = video_tensor.to(model.device, dtype=torch.float16)
+
+    print(f"{roles[1]}: {inp}")
+    inp = ' '.join([DEFAULT_IMAGE_TOKEN] * model.get_video_tower().config.num_frames) + '\n' + inp
+    conv.append_message(conv.roles[0], inp)
+    conv.append_message(conv.roles[1], None)
+    prompt = conv.get_prompt()
+    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+    keywords = [stop_str]
+    stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+    with torch.inference_mode():
+        output_ids = model.generate(
+            input_ids,
+            images=tensor,
+            do_sample=True,
+            temperature=0.1,
+            max_new_tokens=1024,
+            use_cache=True,
+            stopping_criteria=[stopping_criteria])
+
+    outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip()
+    print(outputs)
+
+if __name__ == '__main__':
+    main()
+```
+
+## 🗝️ Training & Validating
+The training & validating instruction is in [TRAIN_AND_VALIDATE.md](TRAIN_AND_VALIDATE.md).
+
+## 👍 Acknowledgement
+* [LLaVA](https://github.com/haotian-liu/LLaVA) The codebase we built upon and it is an efficient large language and vision assistant.
+* [Video-ChatGPT](https://github.com/mbzuai-oryx/Video-ChatGPT) Great job contributing the evaluation code and dataset.
+
+## 🙌 Related Projects
+* [LanguageBind](https://github.com/PKU-YuanGroup/LanguageBind) An open source five modalities language-based retrieval framework.
+* [Chat-UniVi](https://github.com/PKU-YuanGroup/Chat-UniVi) This framework empowers the model to efficiently utilize a limited number of visual tokens.
+
+## 🔒 License
+* The majority of this project is released under the Apache 2.0 license as found in the [LICENSE](https://github.com/PKU-YuanGroup/Video-LLaVA/blob/main/LICENSE) file.
+* The service is a research preview intended for non-commercial use only, subject to the model [License](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md) of LLaMA, [Terms of Use](https://openai.com/policies/terms-of-use) of the data generated by OpenAI, and [Privacy Practices](https://chrome.google.com/webstore/detail/sharegpt-share-your-chatg/daiacboceoaocpibfodeljbdfacokfjb) of ShareGPT. Please contact us if you find any potential violation.
+
+## ✏️ Citation
+If you find our paper and code useful in your research, please consider giving a star :star: and citation :pencil:.
+
+```BibTeX
+@article{lin2023video,
+  title={Video-LLaVA: Learning United Visual Representation by Alignment Before Projection},
+  author={Lin, Bin and Zhu, Bin and Ye, Yang and Ning, Munan and Jin, Peng and Yuan, Li},
+  journal={arXiv preprint arXiv:2311.10122},
+  year={2023}
+}
+```
+
+```BibTeX
+@article{zhu2023languagebind,
+  title={LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment},
+  author={Zhu, Bin and Lin, Bin and Ning, Munan and Yan, Yang and Cui, Jiaxi and Wang, HongFa and Pang, Yatian and Jiang, Wenhao and Zhang, Junwu and Li, Zongwei and others},
+  journal={arXiv preprint arXiv:2310.01852},
+  year={2023}
+}
+```
+
+<!---->
+## ✨ Star History
+[![Star History](https://api.star-history.com/svg?repos=PKU-YuanGroup/Video-LLaVA&type=Date)](https://star-history.com/#PKU-YuanGroup/Video-LLaVA&Date)
+
+## 🤝 Contributors
+
+<a href="https://github.com/PKU-YuanGroup/Video-LLaVA/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=PKU-YuanGroup/Video-LLaVA" />
+</a>
+
+

Video-LLaVA-7B-7B-F16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd45874e9ea19cd22f55d45b09406a472fd51822e0887b8f99596b5e5707fcd3
+size 13478104832

Video-LLaVA-7B-7B-F32.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5208fa3b05cf91f098aa6220c2c5686355133da7d71d670409f8b942a175049e
+size 26954403584

config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "lmsys/vicuna-7b-v1.5",
+  "architectures": [
+    "LlavaLlamaForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "freeze_mm_mlp_adapter": false,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_aspect_ratio": "pad",
+  "image_grid_pinpoints": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mm_hidden_size": 1024,
+  "mm_image_tower": "LanguageBind/LanguageBind_Image",
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_video_tower": "LanguageBind/LanguageBind_Video_merge",
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "model_type": "llava",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.31.0",
+  "tune_mm_mlp_adapter": false,
+  "use_cache": true,
+  "use_mm_proj": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": 0.9,
+  "top_p": 0.6,
+  "transformers_version": "4.31.0"
+}

llava.projector

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37370de2db3f3a501e1aecff9712df83894017956d4870825849b193be7f0ed1
+size 83920640

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e342ae2c6d40d6ebba3328904310a00a2b82e57c728e0e4f5059fe6f7774a03
+size 9976576392

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb5c47457569d61be0d2b2d5739f1e30985acc5a61160406880fee64d2c8e111
+size 4957139888

pytorch_model-00001-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a29e845e426d3b6c1c40fee81a234873c36b9e7b1aebd374055a5ac3e9071a11
+size 9976631486

pytorch_model-00002-of-00002.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a5a315d504411e4619ff548fe54c721356e838d4b57581ed06d409b4306b96a
+size 4957395839

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vvl_vit/config.json

@@ -0,0 +1,62 @@
+{
+  "_name_or_path": "vit-large336-custom",
+  "architectures": [
+    "CLIPVisionModel"
+  ],
+  "attention_dropout": 0.0,
+  "dropout": 0.0,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 1024,
+  "image_size": 336,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "model_type": "clip_vision_model",
+  "num_attention_heads": 16,
+  "num_channels": 3,
+  "num_hidden_layers": 24,
+  "patch_size": 14,
+  "projection_dim": 768,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.31.0",
+  "image_aspect_ratio": "anyres",
+  "image_crop_resolution": 224,
+  "image_grid_pinpoints": [
+    [
+      336,
+      672
+    ],
+    [
+      672,
+      336
+    ],
+    [
+      672,
+      672
+    ],
+    [
+      1008,
+      336
+    ],
+    [
+      336,
+      1008
+    ]
+  ],
+  "image_split_resolution": 224,
+  "initializer_range": 0.02,
+  "max_length": 4096,
+  "max_position_embeddings": 4096,
+  "mm_hidden_size": 1024,
+  "mm_patch_merge_type": "spatial_unpad",
+  "mm_projector_lr": null,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_resampler_type": null,
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower__": "openai/clip-vit-large-patch14-336",
+  "mm_vision_tower_lr": 2e-06
+}

vvl_vit/llava.projector

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37370de2db3f3a501e1aecff9712df83894017956d4870825849b193be7f0ed1
+size 83920640

vvl_vit/mmproj-model-f16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab553f9a6ecf2c73d9073f56b3b45fcc30442cabbd3795f8aec0c9c1a3a24d4d
+size 624434720

vvl_vit/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d99f87cc57e547eb3a0287b753ca6500aa4e1505c3ead22066f48360bfd485c6
+size 607116558