Spaces:
Running
Running
illtellyoulater
commited on
Commit
•
3b8d8cb
1
Parent(s):
542378c
fix(textnormalizer): consider utf8 on windows (#148)
Browse files
src/dalle_mini/model/text.py
CHANGED
@@ -25,7 +25,8 @@ class HashtagProcessor:
|
|
25 |
"dalle-mini/dalle-mini", filename="enwiki-words-frequency.txt"
|
26 |
)
|
27 |
self._word_cost = (
|
28 |
-
l.split()[0]
|
|
|
29 |
)
|
30 |
self._word_cost = {
|
31 |
str(k): math.log(float(i + 1)) for i, k in enumerate(self._word_cost)
|
|
|
25 |
"dalle-mini/dalle-mini", filename="enwiki-words-frequency.txt"
|
26 |
)
|
27 |
self._word_cost = (
|
28 |
+
l.split()[0]
|
29 |
+
for l in Path(wiki_word_frequency).read_text(encoding="utf8").splitlines()
|
30 |
)
|
31 |
self._word_cost = {
|
32 |
str(k): math.log(float(i + 1)) for i, k in enumerate(self._word_cost)
|