update

config.json

@@ -21,5 +21,15 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.42.0.dev0",
   "use_cache": true,
-  "vocab_size": 32768
-}
+  "vocab_size": 32768,
+
+  "quantization_config": {
+    "dtype": "int4",                   // Data type for 4-bit quantization
+    "approach": "dynamic",             // Dynamic quantization approach
+    "bits": 4,                         // Set to 4 for 4-bit quantization
+    "layer_norm": false,               // Disable layer normalization during quantization
+    "activation_quantization": true,   // Quantize activations
+    "weight_quantization": true,       // Quantize weights
+    "weight_scaling": "learned"        // Use learned scaling for weights
+  }
+}