Merge pull request #13 from eltociear/patch-1

This commit is contained in:
jxxghp 2023-07-24 19:15:50 +08:00 committed by GitHub
commit 3b5863befa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -12,8 +12,8 @@ class Tokens:
self.load_text(text) self.load_text(text)
def load_text(self, text): def load_text(self, text):
splited_text = re.split(r"\.|\s+|\(|\)|\[|]|-|\+|【|】|/||;|&|\||#|_|「|」|~", text) splitted_text = re.split(r"\.|\s+|\(|\)|\[|]|-|\+|【|】|/||;|&|\||#|_|「|」|~", text)
for sub_text in splited_text: for sub_text in splitted_text:
if sub_text: if sub_text:
self._tokens.append(sub_text) self._tokens.append(sub_text)