Fix typo in tokens.py

splited_text -> splitted_text
This commit is contained in:
Ikko Eltociear Ashimine 2023-07-24 03:01:50 +09:00 committed by GitHub
parent d56eb149f7
commit 6c86e75872
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -12,8 +12,8 @@ class Tokens:
self.load_text(text)
def load_text(self, text):
splited_text = re.split(r"\.|\s+|\(|\)|\[|]|-|\+|【|】|/||;|&|\||#|_|「|」|~", text)
for sub_text in splited_text:
splitted_text = re.split(r"\.|\s+|\(|\)|\[|]|-|\+|【|】|/||;|&|\||#|_|「|」|~", text)
for sub_text in splitted_text:
if sub_text:
self._tokens.append(sub_text)