Allow passing string to `save_tokenizer_config`
Browse filesSee https://huggingface.co./openGPT-X/Teuken-7B-instruct-research-v0.4/discussions/7.
Need to implement this "upstream" in the tokenizer repo as well.
- gptx_tokenizer.py +3 -1
gptx_tokenizer.py
CHANGED
@@ -395,7 +395,9 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
|
|
395 |
old_tokenizer_file_path.replace(new_tokenizer_file_path)
|
396 |
return str(new_tokenizer_file_path)
|
397 |
|
398 |
-
def save_tokenizer_config(self, save_dir: Path) -> None:
|
|
|
|
|
399 |
# convert Path to str
|
400 |
for k in self.tokenizer_config:
|
401 |
if isinstance(self.tokenizer_config[k], Path):
|
|
|
395 |
old_tokenizer_file_path.replace(new_tokenizer_file_path)
|
396 |
return str(new_tokenizer_file_path)
|
397 |
|
398 |
+
def save_tokenizer_config(self, save_dir: Union[str, Path]) -> None:
|
399 |
+
save_dir = Path(save_dir)
|
400 |
+
|
401 |
# convert Path to str
|
402 |
for k in self.tokenizer_config:
|
403 |
if isinstance(self.tokenizer_config[k], Path):
|