Update gptx_tokenizer.py
Browse files- gptx_tokenizer.py +2 -1
gptx_tokenizer.py
CHANGED
@@ -62,7 +62,7 @@ class HFGPTXTokenizer(PreTrainedTokenizer):
|
|
62 |
f"<placeholder_tok_{i}>" for i in range(256)
|
63 |
]
|
64 |
|
65 |
-
def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optional[Path]:
|
66 |
if not os.path.isfile(config_path):
|
67 |
config_path = try_to_load_from_cache(repo_id=repo_id, filename=Path(config_path).name)
|
68 |
if not config_path:
|
@@ -141,6 +141,7 @@ def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optio
|
|
141 |
return tokenizer_config_file_or_name
|
142 |
except Exception as e:
|
143 |
raise OSError(f"Failed to download tokenizer model: {str(e)}")
|
|
|
144 |
def __init__(
|
145 |
self,
|
146 |
model_path: Optional[str] = None,
|
|
|
62 |
f"<placeholder_tok_{i}>" for i in range(256)
|
63 |
]
|
64 |
|
65 |
+
def find_tokenizer_config(self, config_path: Path, repo_id: str = None) -> Optional[Path]:
|
66 |
if not os.path.isfile(config_path):
|
67 |
config_path = try_to_load_from_cache(repo_id=repo_id, filename=Path(config_path).name)
|
68 |
if not config_path:
|
|
|
141 |
return tokenizer_config_file_or_name
|
142 |
except Exception as e:
|
143 |
raise OSError(f"Failed to download tokenizer model: {str(e)}")
|
144 |
+
|
145 |
def __init__(
|
146 |
self,
|
147 |
model_path: Optional[str] = None,
|