<jupyter_start><jupyter_text>VISTEC-TP-TH-2021<jupyter_code>from bs4 import BeautifulSoup
from tqdm import tqdm

sent = []

def ignore(m, c):
  if c.endswith(" ๆ"):
    return True

  if m.replace(" ", "")==c.replace(" ", ""):
    return True

  if ("ฯ" in c) and (m not in ["ๆลๆ", "พณฯท่าน", "ฯล"]):
    return True

  if ("." in m) or ("." in c):
    return True
  
  return False

def process_dom(dom):
  tokens = []
  cntMisp, cntIgnored = 0, 0
  if dom.name is None:
    words = str(dom).split("|")
    for w in words:
      w = w.strip()
      if len(w)==0:
        continue
      tokens.append((w, w))
  elif dom.name=="ne":
    words = dom.text.strip().split("|")
    for w in words:
      w = w.strip()
      if len(w)==0:
        continue
      tokens.append((w, w))

  elif dom.name=="msp":
    m = dom.text.replace("|", "").strip()
    c = dom["value"].strip()
    # assert("|" not in m)

    cntMisp += 1
    if ignore(m, c):
      tokens.append((m, m))
      cntIgnored += 1
    else:
      tokens.append((m, c))
      mispTokens.add((m, c))
  elif dom.name=="compound":
    for child in dom.children:
      tkn, cms, cig = process_dom(child)
      tokens += tkn
      cntMisp += cms
      cntIgnored += cig
  else:
    print(dom)
    raise(f"Unknown Tag: {dom.name}")

  return tokens, cntMisp, cntIgnored


mispTokens = set()
cntIgnored = 0
cntMisp = 0
cntToken = 0
with open(f"{DIR}/VISTEC-TP-TH-2021/train/VISTEC-TP-TH-2021_train_proprocessed.txt", encoding="utf-8") as fin:
  for line in tqdm(fin, total=40000):
    line = line.strip()
    s = BeautifulSoup("<div id='text'>"+line+"</div>")
    tokens = []
    for dom in s.find("div", {"id": "text"}).children:
      tkn, cms, cig = process_dom(dom)
      tokens += tkn
      cntMisp += cms
      cntIgnored += cig

      cntToken += len(tkn)
        
    sent.append(tokens)
    # misp.append(" ".join([t[0] for t in tokens]))
    # corr.append(" ".join([t[1] for t in tokens]))

print()
print("#Tokens:", cntToken)
print("#Misspelling Tokens:", cntMisp)
print("#Misspelling Tokens[Skip]:", cntIgnored, cntIgnored*100/cntMisp)

with open(f"{DIR}/misp_VISTEC-TP-TH-2021_fasttext_training.txt", "w", encoding="utf-8") as fout:
    for tokens in sent:
      s = " ".join([t[0] for t in tokens])
      fout.write(s+"\n")

with open(f"{DIR}/corr_VISTEC-TP-TH-2021_fasttext_training.txt", "w", encoding="utf-8") as fout:
    for tokens in sent:
      s = " ".join([t[1] for t in tokens])
      fout.write(s+"\n")
with open(f"{DIR}/MST_VISTEC-TP-TH-2021_fasttext_training.txt", "w", encoding="utf-8") as fout:
    cnt = 0
    for tokens in sent:
      s = []
      for t in tokens:
        if t[1]!=t[0]:
          s.append(t[0])
          
          w = norm_word(t[0])
          if "<lol>" in w:
            s.append("<lol>")
          elif "<rep>" in w:
            s.append("<rep>")
          else:
            if w in MD:
              corr, mint = MD[w]
              if mint:
                s.append("<int>")
              else:
                s.append("<msp>")
            else:
              s.append("<msp>")
        else:
          s.append(t[0])
      # break
      s = " ".join(s)
      
      fout.write(s+"\n")<jupyter_output><empty_output><jupyter_text>Wisesight Train<jupyter_code>wisesight = load_jsonl(f"{DIR}/WisesightSentiment/tokenized_train.jsonl")
with open(f"{DIR}/wisesight_train_fasttext_training.txt", "w", encoding="utf-8") as fout:
    for sent in wisesight:
      s = " ".join(sent["tokenized"])
      fout.write(s+"\n")

cntToken = 0
cntMisp = 0
with open(f"{DIR}/MST_wisesight_train_fasttext_training.txt", "w", encoding="utf-8") as fout:
    for sent in wisesight:
      s = []
      for t in sent["tokenized"]:
        s.append(t)
        w = norm_word(t)
        cntToken += 1
        
        if "<lol>" in w:
          s.append("<lol>")
          cntMisp += 1 
        elif "<rep>" in w:
          s.append("<rep>")
          cntMisp += 1
        else:
          if w in MD:
            corr, mint = MD[w]
            if mint:
              s.append("<int>")
            else:
              s.append("<msp>")
            
            cntMisp += 1

      s = " ".join(s)
      # if "<lol>" in s:
      #   print(s)
      #   break
      fout.write(s+"\n")

print()
print("#Tokens:", cntToken)
print("#Misspelling Tokens:", cntMisp, cntMisp*100/cntToken)<jupyter_output>#Tokens: 608957
#Misspelling Tokens: 32419 5.323692805895983<jupyter_text>Train FastText<jupyter_code>ls Datasets/*_fasttext_training.txt
# unsupervised_default = {
#     'model': "skipgram",
#     'lr': 0.05,
#     'dim': 100,
#     'ws': 5,
#     'epoch': 5,
#     'minCount': 5,
#     'minCountLabel': 0,
#     'minn': 3,
#     'maxn': 6,
#     'neg': 5,
#     'wordNgrams': 1,
#     'loss': "ns",
#     'bucket': 2000000,
#     'thread': multiprocessing.cpu_count() - 1,
#     'lrUpdateRate': 100,
#     't': 1e-4,
#     'label': "__label__",
#     'verbose': 2,
#     'pretrainedVectors': "",
#     'seed': 0,
#     'autotuneValidationFile': "",
#     'autotuneMetric': "f1",
#     'autotunePredictions': 1,
#     'autotuneDuration': 60 * 5,  # 5 minutes
#     'autotuneModelSize': ""
# }
trainingDataFile = f"{DIR}/misp_VISTEC-TP-TH-2021_fasttext_training.txt"
model = fasttext.train_unsupervised(trainingDataFile, "cbow", neg=10)
for vec in model.get_nearest_neighbors("แมว"):
  print(vec)
model.save_model(f"{DIR}/../Models/fasttext_misp_VISTEC-TP-TH-2021.bin")
trainingDataFile = f"{DIR}/corr_VISTEC-TP-TH-2021_fasttext_training.txt"
model = fasttext.train_unsupervised(trainingDataFile, "cbow", neg=10)
for vec in model.get_nearest_neighbors("แมว"):
  print(vec)
model.save_model(f"{DIR}/../Models/fasttext_corr_VISTEC-TP-TH-2021.bin")
trainingDataFile = f"{DIR}/MST_VISTEC-TP-TH-2021_fasttext_training.txt"
model = fasttext.train_unsupervised(trainingDataFile, "cbow", neg=10)
for vec in model.get_nearest_neighbors("แมว"):
  print(vec)
model.save_model(f"{DIR}/../Models/fasttext_MST_VISTEC-TP-TH-2021.bin")
trainingDataFile = f"{DIR}/wisesight_train_fasttext_training.txt"
model = fasttext.train_unsupervised(trainingDataFile, "cbow", neg=10)
for vec in model.get_nearest_neighbors("แมว"):
  print(vec)
model.save_model(f"{DIR}/../Models/fasttext_wisesight_train.bin")
trainingDataFile = f"{DIR}/MST_wisesight_train_fasttext_training.txt"
model = fasttext.train_unsupervised(trainingDataFile, "cbow", neg=10)
for vec in model.get_nearest_neighbors("แมว"):
  print(vec)
model.save_model(f"{DIR}/../Models/fasttext_MST_wisesight_train.bin")
model["แมว"].size<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1718155531882514, 'fraction_numerical': 0.024805965606452596, 'mean_word_length': 2.759725400457666, 'pattern_counts': {'":': 4, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '70407', 'license': 'MIT', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2541, 'n_words': 490, 'path': 'notebooks/Train FastText.ipynb', 'repo_name': 'chameleonTK/misspelling-semantics', 'size': '6571'}