illtellyoulater commited on
Commit
3b8d8cb
1 Parent(s): 542378c

fix(textnormalizer): consider utf8 on windows (#148)

Browse files
Files changed (1) hide show
  1. src/dalle_mini/model/text.py +2 -1
src/dalle_mini/model/text.py CHANGED
@@ -25,7 +25,8 @@ class HashtagProcessor:
25
  "dalle-mini/dalle-mini", filename="enwiki-words-frequency.txt"
26
  )
27
  self._word_cost = (
28
- l.split()[0] for l in Path(wiki_word_frequency).read_text().splitlines()
 
29
  )
30
  self._word_cost = {
31
  str(k): math.log(float(i + 1)) for i, k in enumerate(self._word_cost)
 
25
  "dalle-mini/dalle-mini", filename="enwiki-words-frequency.txt"
26
  )
27
  self._word_cost = (
28
+ l.split()[0]
29
+ for l in Path(wiki_word_frequency).read_text(encoding="utf8").splitlines()
30
  )
31
  self._word_cost = {
32
  str(k): math.log(float(i + 1)) for i, k in enumerate(self._word_cost)