<commit_before>#!/usr/bin/env python

import normalize_breton_lib as norm
from absl import app
from absl import flags
import re


FLAGS = flags.FLAGS

flags.DEFINE_string('string_to_normalize', None, 'the Breton string to normalize')

FILENAME = "./bre_wikipedia_2016_100K/bre_wikipedia_2016_100K-sentences.txt"

with open(FILENAME) as f:
  breton_sentences = f.readlines()
#print(breton_sentences[:5])

def main(argv):
  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')
  #TODO(azupon): Add the lib functions
  for line in breton_sentences[:20]:
    sentence_id, sentence_text = line.split("\t")[0], line.split("\t")[1]
  #  input_text = FLAGS.string_to_normalize
    remove_special_characters = re.sub(r"[^0-9a-zA-Z\s\-'’\u00C0-\u00FF]+", "", sentence_text)
    preprocessed_text = re.sub(r"[^\w\s'’-]","",sentence_text).strip()
#    normalized_text = norm.NormalizeBreton(preprocessed_text)
    print("SENTENCE:\t\t"+sentence_id)
    print("ORIGINAL TEXT:\t\t"+sentence_text.strip())
    print("PREPROCESSED TEXT:\t"+preprocessed_text)
    normalized_text = norm.NormalizeBreton(preprocessed_text)
    print("NORMALIZED TEXT:\t"+normalized_text)
   # print(norm.NormalizeBreton(FLAGS.string_to_normalize))
   # print(norm.NormalizeBretonSoftMutation(FLAGS.string_to_normalize))
if __name__ == '__main__':
  app.run(main)<commit_msg>removed preprocessing from binary, added functionality to write to file<commit_after>#!/usr/bin/env python

import normalize_breton_lib as norm
from absl import app
from absl import flags
import re
from tqdm import tqdm

FLAGS = flags.FLAGS

flags.DEFINE_string('string_to_normalize', None, 'the Breton string to normalize')

INFILE: str = "./bre_wikipedia_2016_100K/bre_wikipedia_2016_100K-sentences.txt"
OUTFILE: str = "./bre_normalized_sentences"

with open(INFILE) as f:
  breton_sentences = f.readlines()

def main(argv):
  """Normalize the Breton text by applying mutations and save a .tsv file of the changed sentences."""

  total_sentences: int = 0
  changed_sentences: int = 0

  output_file: file = open(OUTFILE, "w")
  output_file.write("SENTENCE_ID\tSENTENCE_TEXT\tNORMALIZED_TEXT\n")

  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')

  for line in tqdm(breton_sentences):
    total_sentences += 1
    sentence_id: str = line.split("\t")[0]
    sentence_text: str = line.split("\t")[1]
#    input_text: str = FLAGS.string_to_normalize
#    print(norm.NormalizeBreton(FLAGS.string_to_normalize))
#    print(norm.NormalizeBretonSoftMutation(FLAGS.string_to_normalize))
    normalized_text: str = norm.NormalizeBreton(sentence_text)

    if normalized_text != sentence_text.strip().lower():
      changed_sentences += 1
      newline = sentence_id+"\t"+sentence_text.strip().lower()+"\t"+normalized_text+"\n"
      output_file.write(newline)
#      print(sentence_id)
#      print(sentence_text.strip().lower())
#      print(normalized_text)

  print("Changed {} out of {} sentences!".format(changed_sentences, total_sentences))

if __name__ == '__main__':
  app.run(main)

{'fraction_non_alphanumeric': 0.12852969814995133, 'fraction_numerical': 0.01557935735150925, 'mean_word_length': 5.102970297029703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5268657', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1075, 'n_words': 207}