<jupyter_start><jupyter_text>**Preprocess**<jupyter_code>import nltk
nltk.download('punkt')
from nltk.tokenize import MWETokenizer, word_tokenize, RegexpTokenizer
import re
import nltk
import unicodedata

multiple_punctuation_pattern = re.compile(r"([\"\.\?\!\,\:\;\-])(?:[\"\.\?\!\,\:\;\-]){1,}")
word_tokenizer = MWETokenizer(separator='')
multiple_emoji_pattern = re.compile(u"(["
        u"\U0001F600-\U0001F64F"  # emoticons
        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
        u"\U0001F680-\U0001F6FF"  # transport & map symbols
        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
        u"\u00a9"
        u"\u00ae"
        u"\u2000-\u3300"
        "]){1,}", flags= re.UNICODE )

normalizer = {'òa': 'oà',
              'óa': 'oá',
              'ỏa': 'oả',
              'õa': 'oã',
              'ọa': 'oạ',
              'òe': 'oè',
              'óe': 'oé',
              'ỏe': 'oẻ',
              'õe': 'oẽ',
              'ọe': 'oẹ',
              'ùy': 'uỳ',
              'úy': 'uý',
              'ủy': 'uỷ',
              'ũy': 'uỹ',
              'ụy': 'uỵ',
              'Ủy': 'Uỷ'}
correct_mapping = {
      "m": "mình",
      "mik": "mình",
      "ko": "không",
      "k": " không ",
      "kh": "không",
      "khong": "không",
      "kg": "không",
      "khg": "không",
      "tl": "trả lời",
      "r": "rồi",
      "ok": "tốt",
      "dc": "được",
      "vs": "với",
      "đt": "điện thoại",
      "thjk": "thích",
      "thik": "thích",
      "qá": "quá",
      "trể": "trễ",
      "bgjo": "bao giờ",
      "''": '"',
      "``": '"'
}

def normalize_text(text):
  for absurd, normal in normalizer.items():
    text = text.replace(absurd, normal)

  # for l in vn_location:
  #   text = text.replace(l, ' location ')

  return text

def tokmap(tok):
  if tok.lower() in correct_mapping:
      return correct_mapping[tok.lower()]
  else:
      return tok

def preprocess(text):
  global i
  text = multiple_emoji_pattern.sub(r"\g<1> ", text) # \g<1>
  text = multiple_punctuation_pattern.sub(r" \g<1> ", text)
  text = unicodedata.normalize("NFC", text)
  text = re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%)*\b(\/)?', 'url', text)
  text = re.sub("\.", " . ", text)
  text = re.sub("'", "' ", text)
  text = re.sub('"', '" ', text)
  text = re.sub('/', ' / ', text)
  text = re.sub('-', ' - ', text)
  text = re.sub(',', ' , ', text)
  text = re.sub(r'\s{2,}', ' ', text)
  text = normalize_text(text)
  # text = re.sub(r'\#[^\s]+', ' hastag ', text)
  text = re.sub(r'(|\s)([\d]+k)(\s|$)', ' cureency_k ', text)
  text = re.sub(r'(([\d]{2,4}\s){2,}([\d]+)?|(09|01|[2|6|8|9]|03)+([0-9]{8})\b)', ' phone_number ', text)
  text = re.sub(r'\d', "_digit", text)
  tokens = word_tokenizer.tokenize(word_tokenize(text))
  tokens = list(map(tokmap, tokens))
  # return tokens
  return ' '.join(tokens)

print(preprocess('Địa chỉ : 103 d9 ngõ 63 , thái thịnh, đống đa, hà nội.'))<jupyter_output>[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
Địa chỉ : _digit_digit_digit d_digit ngõ _digit_digit , thái thịnh , đống đa , hà nội .<jupyter_text>**preprocess FastText tranining & testing data**<jupyter_code>with open('./sell_detection_test.v1.0.txt') as f:
    sell_detection_test = f.read().strip().split('\n')
    sell_detection_test = [ preprocess(e) for e in sell_detection_test]
print(sell_detection_test[:10])
print(len(sell_detection_test))

with open('sell_detection_fasttext_test.txt', 'w') as f:
  f.write('\n'.join(sell_detection_test))
with open('./sell_detection_train.v1.0.txt') as f:
  sell_detection_train = f.read().strip().split('\n')
  sell_detection_train = [ line.split(' ',1) for line in sell_detection_train]
  sell_detection_train = [ [lables, preprocess(descriptions)] for lables, descriptions in sell_detection_train]
  sell_detection_train = [ ' '.join(e) for e in sell_detection_train]
print(sell_detection_train[:2])
with open('sell_detection_fasttext_train.txt', 'w') as f:
  f.write('\n'.join(sell_detection_train))<jupyter_output>['__label__post_khong_ban Gsu khsi nek dep gai hk cho y kirn yk nhoaz khsi huong nhiu', '__label__post_khong_ban Haiz . chan wa . hoc - hoc - nua - hoc mau']<jupyter_text>**FastText Model**<jupyter_code>!pip install fasttext
import fasttext

#Train the model
model = fasttext.train_supervised(input="sell_detection_fasttext_train.txt", lr=0.22, wordNgrams=3) #  epoch=5)# dim=10)
# model.save_model("sell_dectection_60K_fasttext.bin")
prediction = []

for i in range(len(sell_detection_test)):
  prediction.append(model.predict(sell_detection_test[i])[0][0] )
print(len(prediction))

# def pre(pre):
#   if pre == '__label__post_khong_ban':
#     return 0
#   return 1

# prediction = list(map(pre, prediction))<jupyter_output>98893<jupyter_text>write solution<jupyter_code>with open('sell_detection_team03_solution3.result.txt', 'w') as f:
  f.write('\n'.join(prediction))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19798387096774195, 'fraction_numerical': 0.02379032258064516, 'mean_word_length': 3.2878133102852205, 'pattern_counts': {'":': 21, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '61690', 'license': 'Apache-2.0', 'n_tokens_mistral': 2133, 'n_tokens_neox': 2017, 'n_words': 367, 'path': 'Solution3_FastText_Classifier.ipynb', 'repo_name': 'nvlong198/Sell-Detection', 'size': '4960'}