Merge pull request #13 from eltociear/patch-1
This commit is contained in:
commit
3b5863befa
@ -12,8 +12,8 @@ class Tokens:
|
|||||||
self.load_text(text)
|
self.load_text(text)
|
||||||
|
|
||||||
def load_text(self, text):
|
def load_text(self, text):
|
||||||
splited_text = re.split(r"\.|\s+|\(|\)|\[|]|-|\+|【|】|/|~|;|&|\||#|_|「|」|~", text)
|
splitted_text = re.split(r"\.|\s+|\(|\)|\[|]|-|\+|【|】|/|~|;|&|\||#|_|「|」|~", text)
|
||||||
for sub_text in splited_text:
|
for sub_text in splitted_text:
|
||||||
if sub_text:
|
if sub_text:
|
||||||
self._tokens.append(sub_text)
|
self._tokens.append(sub_text)
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user