lucio commited on
Commit
0cc9598
1 Parent(s): f99cbd1

Training in progress, step 1500

Browse files
.ipynb_checkpoints/eval-checkpoint.py CHANGED
@@ -85,7 +85,7 @@ def create_vocabulary_from_data(
85
  )
86
 
87
 
88
- vocab_dict = {v: k for k, v in enumerate(sorted(list(vocabs["test"]["vocab"])))}
89
 
90
  # replace white space with delimiter token
91
  if word_delimiter_token is not None:
@@ -150,9 +150,9 @@ def main(args):
150
  pad_token=pad_token,
151
  )
152
  print(vocab_dict)
153
- print(set(vocab_dict) - set(tokenizer.get_vocab()))
154
-
155
 
 
156
  if __name__ == "__main__":
157
  parser = argparse.ArgumentParser()
158
 
 
85
  )
86
 
87
 
88
+ vocab_dict = {v: k for k, v in enumerate(sorted(vocabs["test"]["vocab"][0]))}
89
 
90
  # replace white space with delimiter token
91
  if word_delimiter_token is not None:
 
150
  pad_token=pad_token,
151
  )
152
  print(vocab_dict)
153
+ print("OOV chars:", set(vocab_dict) - set(tokenizer.get_vocab()))
 
154
 
155
+
156
  if __name__ == "__main__":
157
  parser = argparse.ArgumentParser()
158
 
eval.py CHANGED
@@ -85,7 +85,7 @@ def create_vocabulary_from_data(
85
  )
86
 
87
 
88
- vocab_dict = {v: k for k, v in enumerate(sorted(list(vocabs["test"]["vocab"])))}
89
 
90
  # replace white space with delimiter token
91
  if word_delimiter_token is not None:
@@ -150,9 +150,9 @@ def main(args):
150
  pad_token=pad_token,
151
  )
152
  print(vocab_dict)
153
- print(set(vocab_dict) - set(tokenizer.get_vocab()))
154
-
155
 
 
156
  if __name__ == "__main__":
157
  parser = argparse.ArgumentParser()
158
 
 
85
  )
86
 
87
 
88
+ vocab_dict = {v: k for k, v in enumerate(sorted(vocabs["test"]["vocab"][0]))}
89
 
90
  # replace white space with delimiter token
91
  if word_delimiter_token is not None:
 
150
  pad_token=pad_token,
151
  )
152
  print(vocab_dict)
153
+ print("OOV chars:", set(vocab_dict) - set(tokenizer.get_vocab()))
 
154
 
155
+
156
  if __name__ == "__main__":
157
  parser = argparse.ArgumentParser()
158
 
pytorch_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0f8b5632f9e07f55fba70e10da1295b474bf5e556941e6a29a922295b5be21de
3
  size 1262054897
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:424c9eb48f2b29370f0999f3880f59229a4354c0e426a6210488c3332d18a561
3
  size 1262054897
runs/Feb02_06-54-25_job-699ba53c-fea9-4eb2-81af-a97f440eaa45/events.out.tfevents.1643785646.job-699ba53c-fea9-4eb2-81af-a97f440eaa45.33872.0 CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f4b9f85ce3736a50be3ac6b8b235dbf084d34a5ecbdea0c0d946a7ab0368daf4
3
- size 6955
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b4f9ad2746642360fd16b28a7e680fa5af325c16f6c7f11b572ffb75b09c8480
3
+ size 8058