- 在apps/common/config/tokenizer_manage_config里面的使用本地的模型为运行联网下载,从
- class TokenizerManage:
tokenizer = None
# TODO 路径不知道要怎么改变,现在可以运行通,2025-10-20
@staticmethod
def get_tokenizer():
from transformers import BertTokenizer
if TokenizerManage.tokenizer is None:
TokenizerManage.tokenizer = BertTokenizer.from_pretrained(
'bert-base-cased',
cache_dir="/opt/maxkb-app/model/tokenizer",
local_files_only=False,
resume_download=True,
force_download=False)
return TokenizerManage.tokenizer修改为
class TokenizerManage:
tokenizer = None
# TODO 路径不知道要怎么改变,现在可以运行通,2025-10-20
@staticmethod
def get_tokenizer():
from transformers import BertTokenizer
if TokenizerManage.tokenizer is None:
TokenizerManage.tokenizer = BertTokenizer.from_pretrained(
'bert-base-cased',
cache_dir="/opt/maxkb-app/model/tokenizer",
# local_files_only=False,
resume_download=False,
force_download=False)
return TokenizerManage.tokenizer
