Text Generation
Transformers
Safetensors
llama
text-generation-inference
Inference Endpoints
jaeb commited on
Commit
9ad497c
·
verified ·
1 Parent(s): a7e67f6

Allow passing string to `save_tokenizer_config`

Browse files

See https://huggingface.co./openGPT-X/Teuken-7B-instruct-research-v0.4/discussions/7.
Need to implement this "upstream" in the tokenizer repo as well.

Files changed (1) hide show
  1. gptx_tokenizer.py +3 -1
gptx_tokenizer.py CHANGED
@@ -395,7 +395,9 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
395
  old_tokenizer_file_path.replace(new_tokenizer_file_path)
396
  return str(new_tokenizer_file_path)
397
 
398
- def save_tokenizer_config(self, save_dir: Path) -> None:
 
 
399
  # convert Path to str
400
  for k in self.tokenizer_config:
401
  if isinstance(self.tokenizer_config[k], Path):
 
395
  old_tokenizer_file_path.replace(new_tokenizer_file_path)
396
  return str(new_tokenizer_file_path)
397
 
398
+ def save_tokenizer_config(self, save_dir: Union[str, Path]) -> None:
399
+ save_dir = Path(save_dir)
400
+
401
  # convert Path to str
402
  for k in self.tokenizer_config:
403
  if isinstance(self.tokenizer_config[k], Path):