Python 3.12
apt install git-lfs
git lfs install
git clone https://huggingface.co./unsloth/Meta-Llama-3.1-8B-Instruct
git clone https://github.com/mit-han-lab/deepcompressor
cd deepcompressor
poetry shell
poetry install
cd examples/llm
rm -rf runs ~/Meta-Llama-3.1-8B-Instruct-QServe-g128
python -m deepcompressor.app.llm.ptq configs/qoq-g128.yaml --model-name llama-3.1-8b-instruct --model-path ~/Meta-Llama-3.1-8B-Instruct --save-model ~/Meta-Llama-3.1-8B-Instruct-QServe-g128
cd ~/Meta-Llama-3.1-8B-Instruct-QServe-g128/
huggingface-cli upload rungalileo/Llama-3.1-8B-QServe-g128 model.pt --commit-message "Update model.pt"
huggingface-cli upload rungalileo/Llama-3.1-8B-QServe-g128 scale.pt --commit-message "Update scale.pt"