from auto_round import AutoRoundConfig ##must import for autoround format from transformers import AutoModelForCausalLM, AutoTokenizer import torch quantized_model_dir = "OPEA/DeepSeek-V3-int4-sym-gptq-inc" quantization_config = AutoRoundConfig( backend="cpu" ) model = AutoModelForCausalLM.from_pretrained( quantized_model_dir, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="cpu", revision="8fe0735",##use autoround format, the only difference is config.json quantization_config = quantization_config, ##cpu only machine don't need to set this value ) tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir,trust_remote_code=True) prompt = "There is a girl who likes adventure," messages = [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": prompt} ] text = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) model_inputs = tokenizer([text], return_tensors="pt").to(model.device) generated_ids = model.generate( model_inputs.input_ids, max_new_tokens=200, ##change this to align with the official usage do_sample=False ##change this to align with the official usage ) generated_ids = [ output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) ] response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] print(response) prompt = "9.11和9.8哪个数字大" ##INT4 """要比较 **9.11** 和 **9.8** 的大小,可以按照以下步骤进行: 1. **比较整数部分**: - 两个数的整数部分都是 **9**,所以整数部分相同。 2. **比较小数部分**: - **9.11** 的小数部分是 **0.11** - **9.8** 的小数部分是 **0.8**(即 **0.80**) 3. **分析小数部分**: - **0.80** 大于 **0.11** 因此,**9.8** 大于 **9.11**。 最终答案:\boxed{9.8} """ prompt = "strawberry中有几个r?" ##INT4 """ ### 第一步:理解问题 首先,我需要明确问题的含义。问题是:“strawberry中有几个r?”。这里的“strawberry”是一个英文单词,意思是“草莓”。问题问的是这个单 词中有多少个字母“r”。 ### 第二步:分解单词 为了找出“strawberry”中有多少个“r”,我需要将这个单词分解成单个字母。让我们逐个字母来看: - s - t - r - a - w - b - e - r - r - y ### 第三步:识别字母“r” 现在,我需要找出这些字母中哪些是“r”。让我们逐一检查: 1. s - 不是r 2. t - 不是r 3. r - 是r 4. a - 不是r 5. w - 不是r 6. b - 不是r 7. e - 不是r 8. r - 是r """ prompt = "How many r in strawberry." ##INT4 """The word "strawberry" contains **3 "r"s. """ prompt = "There is a girl who likes adventure," ##INT4: """That's wonderful! A girl who loves adventure is likely curious, brave, and eager to explore the world around her. Here are some ideas to fuel her adventurous spirit: ### **Outdoor Adventures** - **Hiking:** Explore local trails, national parks, or mountains. - **Camping:** Spend a night under the stars and connect with nature. - **Rock Climbing:** Challenge herself with bouldering or climbing walls. - **Kayaking/Canoeing:** Paddle through rivers, lakes, or even the ocean. - **Zip-lining:** Soar through the treetops for an adrenaline rush. ### **Travel Adventures** - **Road Trips:** Plan a journey to new cities or scenic destinations. - **Backpacking:** Travel light and explore different cultures and landscapes. - **Volunteer Abroad:** Combine adventure with helping others in a new country. ### **Creative Adventures** - **Photography:** Capture the beauty """ prompt = "Please give a brief introduction of DeepSeek company." ##INT4: """DeepSeek Artificial Intelligence Co., Ltd. (referred to as "DeepSeek" or "深度求索") , founded in 2023, is a Chinese company dedicated to making AGI a reality"""