Update tokenizer.py

tokenizer.py

@@ -1,17 +1,17 @@
 import spacy
 
-spacy.cli.download("en_core_web_sm")
+#spacy.cli.download("en_core_web_sm")
 
-from spacy.tokens import Doc
+#from spacy.tokens import Doc
 
 # 加载英文模型
-nlp = spacy.load('en_core_web_sm')
+#nlp = spacy.load('en_core_web_sm')
 
 import nltk
 
-nltk.download('punkt')
+#nltk.download('punkt')
 
-from nltk.tokenize import word_tokenize
+#from nltk.tokenize import word_tokenize
 
 import jieba
 
@@ -34,18 +34,20 @@ with codecs.open('model2_data/bpecode.en', 'r', 'utf-8') as f:
 
 def spacy_tokenize(line):
     # 使用spaCy处理文本
-    doc = nlp(line)
+    #doc = nlp(line)
     # 获取单词列表
-    words = [token.text for token in doc]
+    #words = [token.text for token in doc]
     # 将单词连接成一个字符串，单词间用一个空格间隔
-    return ' '.join(words)
+    #return ' '.join(words)
+    return ""
 
 
 def nltk_tokenize(line):
     # 使用NLTK的word_tokenize进行分词
-    tokens = word_tokenize(line)
+    #tokens = word_tokenize(line)
     #print(tokens)
-    return tokens
+    #return tokens
+    return []
 
 
 def jieba_tokenize(line):