Training in progress, step 1500
Browse files
.ipynb_checkpoints/eval-checkpoint.py
CHANGED
@@ -85,7 +85,7 @@ def create_vocabulary_from_data(
|
|
85 |
)
|
86 |
|
87 |
|
88 |
-
vocab_dict = {v: k for k, v in enumerate(sorted(
|
89 |
|
90 |
# replace white space with delimiter token
|
91 |
if word_delimiter_token is not None:
|
@@ -150,9 +150,9 @@ def main(args):
|
|
150 |
pad_token=pad_token,
|
151 |
)
|
152 |
print(vocab_dict)
|
153 |
-
print(set(vocab_dict) - set(tokenizer.get_vocab()))
|
154 |
-
|
155 |
|
|
|
156 |
if __name__ == "__main__":
|
157 |
parser = argparse.ArgumentParser()
|
158 |
|
|
|
85 |
)
|
86 |
|
87 |
|
88 |
+
vocab_dict = {v: k for k, v in enumerate(sorted(vocabs["test"]["vocab"][0]))}
|
89 |
|
90 |
# replace white space with delimiter token
|
91 |
if word_delimiter_token is not None:
|
|
|
150 |
pad_token=pad_token,
|
151 |
)
|
152 |
print(vocab_dict)
|
153 |
+
print("OOV chars:", set(vocab_dict) - set(tokenizer.get_vocab()))
|
|
|
154 |
|
155 |
+
|
156 |
if __name__ == "__main__":
|
157 |
parser = argparse.ArgumentParser()
|
158 |
|
eval.py
CHANGED
@@ -85,7 +85,7 @@ def create_vocabulary_from_data(
|
|
85 |
)
|
86 |
|
87 |
|
88 |
-
vocab_dict = {v: k for k, v in enumerate(sorted(
|
89 |
|
90 |
# replace white space with delimiter token
|
91 |
if word_delimiter_token is not None:
|
@@ -150,9 +150,9 @@ def main(args):
|
|
150 |
pad_token=pad_token,
|
151 |
)
|
152 |
print(vocab_dict)
|
153 |
-
print(set(vocab_dict) - set(tokenizer.get_vocab()))
|
154 |
-
|
155 |
|
|
|
156 |
if __name__ == "__main__":
|
157 |
parser = argparse.ArgumentParser()
|
158 |
|
|
|
85 |
)
|
86 |
|
87 |
|
88 |
+
vocab_dict = {v: k for k, v in enumerate(sorted(vocabs["test"]["vocab"][0]))}
|
89 |
|
90 |
# replace white space with delimiter token
|
91 |
if word_delimiter_token is not None:
|
|
|
150 |
pad_token=pad_token,
|
151 |
)
|
152 |
print(vocab_dict)
|
153 |
+
print("OOV chars:", set(vocab_dict) - set(tokenizer.get_vocab()))
|
|
|
154 |
|
155 |
+
|
156 |
if __name__ == "__main__":
|
157 |
parser = argparse.ArgumentParser()
|
158 |
|
pytorch_model.bin
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 1262054897
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:424c9eb48f2b29370f0999f3880f59229a4354c0e426a6210488c3332d18a561
|
3 |
size 1262054897
|
runs/Feb02_06-54-25_job-699ba53c-fea9-4eb2-81af-a97f440eaa45/events.out.tfevents.1643785646.job-699ba53c-fea9-4eb2-81af-a97f440eaa45.33872.0
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b4f9ad2746642360fd16b28a7e680fa5af325c16f6c7f11b572ffb75b09c8480
|
3 |
+
size 8058
|