<jupyter_start><jupyter_text>Word2vec and vocabulary Import necessary libraries<jupyter_code>from tqdm.notebook import tqdm<jupyter_output><empty_output><jupyter_text>Mount Google Drive to load the dataset of lyrics and the word2vec pretrained embedding<jupyter_code># Mount drive
from google.colab import drive
drive.mount("/content/drive")

word2vec_path = "/content/drive/MyDrive/DM project - NLP lyrics generation/cc.en.300.vec"<jupyter_output>Mounted at /content/drive<jupyter_text>Check if words of interest are present in the Word2vec file<jupyter_code># OLD VERSION WHERE VOCABULARY WAS CREATED WITHOUT EXCLUSIVELY USING DATASET WORDS
'''# Create dictionary and word vector list for word embeddings
VOCABULARY_LEN = 390_000
VECTOR_SIZE    = 300

# Create dictionary and inverse dictionary of words and word vector list for word embeddings
words2indices = {}
indices2words = {}
word_vectors = []

# Append padding vector to the word_vectors (pad is at position 0)
word_vectors.append(torch.rand(VECTOR_SIZE))

# Append unknown vector to the word_vectors (unk is at position 1)
word_vectors.append(torch.rand(VECTOR_SIZE))

# Word2vec file contains words (including punctuation and contractions) with some permutations of lower and upper case letters,
# so don't worry about the case that a word may be present only in uppercase letters

# Take just a subset of the words that start with an apostrophe
contractions = { "'bout", "'cause", "'cos", "'coz", "'cuz", "'fore", "'em", "'nuff", "'round",  "'til", "'till" }

print("Creating vocabulary using Word2vec pre-trained embedding...\n")

with open(word2vec_path) as f:
    next(f) 
    #for i, l in tqdm(enumerate(f), total=2000000):
    for l in tqdm(f, total=2_000_000):
        # Check if the desired vocabulary size has been achieved
        if len(word_vectors) == VOCABULARY_LEN:
          print("Desired vocabulary size achieved, stop vectors iterations!")
          break

        # Get the word and the corresponding vector
        word, *vector = l.strip().split()

        # Check if a word is in the word2vec file
        #if word.lower() == 'i':
        #if word.lower() in punctuation:
        #if word.lower() in contractions:
        #if "'" in word:
        #if "’" in word:
        #if word.isdigit():
        #if re.match("^[a-z]+'*[a-z]*$", word):
        #if re.match("^[0-9]+$|^(19|20)[0-9]{1}0s$|^[0-9]0s$", word):
        #if re.match("^[0-9]{1}x$|^x[0-9]{1}$", word):
        #if word.lower() == "i've" or word.lower() == "ain't": # In Word2vec vectors, only I've is present while the lyrics include both i've and ain't
        #if word[len(word)-1] == "'":                          # In Word2vec vectors, no 'ing' ending contraction is present while the lyrics include them (e.g., crying -> cryin')
        #if word[len(word)-2:] == "'s":                        # In Word2vec vectors, no decade in the form XX's is present (e.g., 60's)
        #if re.match("^[0-9]+s$", word):                       # But are present in the form XXXXs or XXs, so we can use one of the two or both representations
        #  print("word: ", word)
        #else:
        #  continue

        # Skip words composed of at least one upper case or non-alpha letter (except contractions ,numbers and decades)
        if re.match("^[a-z]+'*[a-z]*$", word) or re.match("^[0-9]+$|^(19|20)[0-9]{1}0s|^[0-9]0s$$", word) or word in contractions:
          #print("Added word: ", word)

          # Convert to float the word's vector values
          vector = torch.tensor([float(c) for c in vector])
          
          # Calculate the word's index and add it and the word in the corresponding dictionaries
          word_idx = len(word_vectors)
          words2indices[word]     = word_idx
          indices2words[word_idx] = word

          # Append the word's vector to the list of vectors
          word_vectors.append(vector)

word_vectors = torch.stack(word_vectors)

# Try to move word_vectors on the GPU 
if torch.cuda.is_available():
  word_vectors.cuda()

print("Vocabulary created")'''








# Test if the following words are present in the word2vec file (useful for lyrics preprocessing)
test = { 
  "'s", "'d",
  "abouts", "abuse", "across", "ain't", "aiyo", "alabama", "alright", "ancient", "announced", "appeared", "arizona", "assquake", "awfully", \
  "backas", "banished", "bean", "beneath", "beyoncé", "bidness", "bigga", "billions", "bloodletting", "Brigadoon", "Brignac", "brilliant", "buzzy", \
  "calledst", "cadillac", "can't", "cah", "cating", "causе", "children", "chiraq", "cliche", "closa", "coolest", "Cobain", "copy", "crackhead", "creeping", "cuz", \
  "d'usse", "Dahrouge", "Defari", "delphinidin", "desiree", "dеtroit", "dirt", "dolemite", "don't", "dookie", "dope", "drooned", "dr.", \
  "'em", "embroiled", "except", \
  "fa", "feared", "finally", "frightening", "f**k", "fucking", \
  "gats", "gi", "gonna", "gorgeous", "gotta", "grabba", "groceries", "gster", \
  "halfpennies", "hallucinogen", "hast", "ho", \
  "irregular", \
  "jag", \
  "karat", "keep", "Khalil", "kilos", \
  "Lativia", "lightning", "lil", "lingering", "locked", "looed", "looters","loser", "logically", "lolo", "louisiana", \
  "mackerel", "madam", "memba", "mercedes", "mil", "mississippi", "monsta", "mothafucka", "mothafuckers", "motherfucker", "motherfuckers", "motherfucking", "mr", "mr.", "mrs.", "ms.", \
  "named", "naturally", "norsemen", "northwester", "north", "nourished", \
  "oclock", "orleannais", "Oshun", "ough", \
  "philistines", "pile", "pining", "played", "Portmore", "prof.", "profitable", "psychedelically", \
  "quicka", \
  "Rafah", "RnB", "restored", "ruled", \
  "scrambling", "screeching", "scuse", "shocking", "slippery", "smiled", "smouldering", "sovereign", "spinning", "strawberries", "strengthened", "suffering", "sumn", \
  "taken", "tho", "thou", "thrilla", "tobacco", "tow", "traveling", "travellin", "trembling", "trigga", "tryna", "twerk", \
  "unleash", "unutterable", \
  "Vedder", \
  "wanderer", "wandering", "wantcha", "wester", "wha", "whatevs", "whatna", "wondering", \
  "ya", "Yamazaki", "yo", \
  "...", "…", ",", ".", ";", ":", "?", "!", "$", "/", "(", ")", " ", "\n" }

found = set()

print("Scanning Word2vec pre-trained embedding looking for missing words...\n")

with open(word2vec_path) as f:
    next(f) 
    for l in tqdm(f, total=2_000_000):
        # Get the word and the corresponding vector
        word, *vector = l.strip().split()

        if word in test:
          found.add(word)
          #print("word: ", word)

missing_words = test.difference(found)
print("%d missing words" %len(missing_words))

for word in missing_words:
  if word == "\n":
    word = "newline"
  elif word == " ":
    word = "whitespace"
  print("missing word: ", word)
  
print("\nScanning Word2vec pre-trained embedding trying to find previous missing words ignoring capitalization...\n")

with open(word2vec_path) as f:
    next(f) 
    for l in tqdm(f, total=2_000_000):
        # Get the word and the corresponding vector
        word, *vector = l.strip().split()

        if word.lower() in missing_words:
          print("Required word is not present but ignoring capitalization there is: ", word)#'''<jupyter_output>Scanning Word2vec pre-trained embedding looking for missing words...<jupyter_text>Dataset preprocessing Enter the following code in the browser console to prevent Colab to suspend current session (may no longer work from March 2021 due to captcha)```javascriptfunction ConnectButton(){    console.log("Connect pushed");     document.querySelector("top-toolbar > colab-connect-button").shadowRoot.querySelector("connect").click();}let funId = setInterval(ConnectButton,60000);```\\Enter the following line to cancel continuous clicking```javascriptclearInterval(funId);```<jupyter_code># Check python version because from 3.10 we can use match ... case
!python3 -V<jupyter_output><empty_output><jupyter_text>Import necessary libraries<jupyter_code>from tqdm.notebook import tqdm

import torch
import re

import pandas as pd

# Set pandas option
pd.set_option("display.max_columns", None)
pd.set_option("display.max_rows", None)
pd.set_option("display.max_colwidth", None)<jupyter_output><empty_output><jupyter_text>Mount Google Drive to load the dataset of lyrics and the word2vec pretrained embedding<jupyter_code># Mount drive
from google.colab import drive
drive.mount("/content/drive")

dataset_path              = "/content/drive/MyDrive/DM project - NLP lyrics generation/genius_lyrics.csv"
preprocessed_dataset_path = "/content/drive/MyDrive/DM project - NLP lyrics generation/preprocessed_lyrics.csv"


# TEST
#input_dataset_path = "/content/drive/MyDrive/DM project - NLP lyrics generation/Dataset creation notebooks/genius ok/genius_lyrics_0-50.csv"
#dataset_path       = "/content/drive/MyDrive/DM project - NLP lyrics generation/preprocessed_lyrics.csv"<jupyter_output><empty_output><jupyter_text>Install googletrans to detect language of a lyrics<jupyter_code>#!pip install googletrans==4.0.0-rc1<jupyter_output><empty_output><jupyter_text>Define detecting function using googletrans<jupyter_code>'''from googletrans import Translator
import time

detector = Translator()
MAX_SPLIT = 10
SENTENCES_GOAL = 5

# Non-stable, sometimes throw errors
def is_english(text):
  if not text:
    return False

  sentences = text.split("\n", MAX_SPLIT)
  
  i = 0
  en_sentences = 0

  for s in sentences:
    if i == SENTENCES_GOAL:
      break

    #print("*"*40)
    #print("Sentence: ", s)

    if s:
      detected = detector.detect(s)
      #print(detected)

      if detected.lang == "en": # and detected.confidence > 0.6:
        en_sentences += 1

      i += 1
      time.sleep(0.5) # To avoid being blocked from Google

  confidence = en_sentences/i if i > 0 else 0.0
  #print("\nenglish sentences: %d/%d = %.2f" %(en_sentences, i, confidence))

  return True if confidence >= 0.8 else False


#lyrics = "Viva la vita\n¿Dónde está el hombre con fuego en la sangre?\nHe's been waiting around for the weekend\nOK it's alright with me\nSome things are just\nRight about the time I missed your call\nWhoa, whoa, whoa\nI'm going down to the\nI wanna hold em' like they do in texas please\nWhatsoever I feared has\nCome to life"
#print("Is english text?", is_english(lyrics))'''<jupyter_output><empty_output><jupyter_text>Install textblob to detect language of a lyrics<jupyter_code>!pip install textblob<jupyter_output>Requirement already satisfied: textblob in /usr/local/lib/python3.7/dist-packages (0.15.3)
Requirement already satisfied: nltk>=3.1 in /usr/local/lib/python3.7/dist-packages (from textblob) (3.2.5)
Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk>=3.1->textblob) (1.15.0)<jupyter_text>Define detecting function using textblob<jupyter_code>from textblob import TextBlob
import time
import random

MAX_SPLIT = 30 #10
SENTENCES_GOAL = 10 #5

TOLERANCE = int(SENTENCES_GOAL * 0.2)

# Return true if text is english with a certain high probability, false otherwise
def is_english(text):
  if not text:
    return False

  # Split the text in sentences
  sentences = text.split("\n", MAX_SPLIT)

  # Shuffle sentences to avoid taking only the beginning of text which can be misleading
  random.shuffle(sentences)
  
  i = 0
  '''en_sentences = 0

  for s in sentences:
    if i == SENTENCES_GOAL:
      break

    #print("*"*40)
    #print("Sentence: ", s)

    if s and len(s) > 2:
      detector = TextBlob(s)
      lang = detector.detect_language()
      print("Language: ", lang)

      if lang == "en":
        en_sentences += 1

      i += 1
      time.sleep(0.5) # To avoid being blocked from Google

  # Calculate confidence of the english language about the text
  confidence = en_sentences/i if i > 0 else 0.0
  #print("\nenglish sentences: %d/%d = %.2f" %(en_sentences, i, confidence))
  
  return True if confidence >= 0.8 else False'''



  # BETTER PERFORMANCES
  non_en_sentences = 0

  for s in sentences:
    # Stop iterations if number of sentences goal has been reached or
    # too many non english sentences have been detected or
    # enough english sentences have been detected
    if i == SENTENCES_GOAL or non_en_sentences > TOLERANCE or i-non_en_sentences == SENTENCES_GOAL-TOLERANCE:
      break

    #print("*"*40)
    #print("Sentence: ", s)

    if s and len(s) > 2:
      detector = TextBlob(s)
      lang = detector.detect_language()
      print("Language: ", lang)

      if lang != "en":
        non_en_sentences += 1

      i += 1
      time.sleep(0.5) # To avoid being blocked from Google

  # Tolerate at most TOLERANCE non english sentences over SENTENCES_GOAL
  return non_en_sentences <= TOLERANCE


#lyrics = "Viva la vita\n¿Dónde está el hombre con fuego en la sangre?\nHe's been waiting around for the weekend\nOK it's alright with me\nSome things are just\nRight about the time I missed your call\nWhoa, whoa, whoa\nI'm going down to the\nI wanna hold em' like they do in texas please\nWhatsoever I feared has\nCome to life"
#print("Is english text?", is_english(lyrics))'''

lyrics = "this a\nthis a\nthis ,  this\nthis a gyal yah waan\n( back it up )\nthis thi thi ,  this\nthis a gyal yah waan\n( back it up )\nthis a\nthis a gyal yah\nthis a gyal yah waan\n ( back it up )\nhey gyal !"
print("Is english text?", is_english(lyrics))

lyrics = "flying\nand you know i'm not coming down\nyou are trying\nbut you know you must soon go down\nall my colours turn to cloud\nall my colours turn to cloud\n zimbo ,  zimbo ,  zimbo ,  zimbo ,  zimbo\n zimbo ,  zimbo ,  zimbo ,  zimbo ,  zimbo\n zimbo ,  zimbo ,  zimbo ,  zimbo ,  zimbo\nzimbo ,  zimbo ,  zimbo ,  zimbo ,  zimbo\nwhat to say\nwhen your heart's in pieces ?\nhow to play"
print("Is english text?", is_english(lyrics))

lyrics = "intro :\nkid's voice :\nwise ... wise .... blessed ... blessed ...\nmore fire !  !  ... more fire !  !  .... red\ncapleton :\nmore fire !  !  !  red hot !  !  !  !  yuh see ah now well done ,  yo !  !\nlight up di fire from mi ready fi put it pon dem\nhey gimme di hey gimme di yo !  !  !  !\nchorus :\ndat one yah name cuyah cuyah cuyah when dem see mi wid di fire\ncuyah cuyah cuyah when mi bun di vampire\ncuyah cuyah cuyah when mi bun di obeah wuker\ntell dem cuyah !  !  !  ... cuyah !  !  !  ... cuyah !  !  !  ... again"
print("Is english text?", is_english(lyrics))

lyrics = "so sick of playing these games ,  it's\nimpossible for me to ever ever be happy ,  yeah\nand i'm so sick of doing the same shit\ni just wanna sleep a tad bit ,  more\nmore ,  more ,  more\n\ni don't want to die\ni'm in a elevator with snoop dogg and i just got a contact high\nso everything better except the cheddar except when\ni  start to close my eyes ,  or did i already close my eyes" 
print("Is english text?", is_english(lyrics))<jupyter_output>Language:  en
Language:  en
Language:  en
Language:  en
Language:  en
Language:  hi
Language:  en
Language:  en
Language:  en
Is english text? True
Language:  pt
Language:  pt
Language:  pt
Is english text? False
Language:  en
Language:  ms
Language:  en
Language:  it
Language:  en
Language:  en
Language:  en
Language:  en
Language:  en
Language:  ms
Is english text? False
Language:  en
Language:  en
Language:  en
Language:  en
Language:  en
Language:  en
Language:  en
Language:  en
Is english text? True<jupyter_text>Define lyrics preprocessing<jupyter_code>blacklisted_chars  = { '"', '“', '”', '<', '>', '@', '+', '|', '', '_', '·', '~', '#' }
apostrophes        = { "'", '`', '’', '‘', '´' }
dashes             = { '-', '‑', '–', '—', '−' }
punctuation_subset = { ',' , ';', ':', '!', '?', ')', '}', '[', ']', '$', '/', '…', '\n' }
string_ending_chars = blacklisted_chars.union(dashes).union(punctuation_subset).union({'.'})
dot_contractions = [".", "mr", "mrs", "ms", "dr", "st", "prof"]

# USE A WHITELIST INSTEAD OF A BLACKLIST
#whitelisted_chars = {  }

#print("string_ending_chars: ", string_ending_chars)


# Lyrics preprocessing.
# Convert text to lower case letters to reduce the word's space.
# Dataset does not include punctuation except contractions such as i'm, it's, isn't, ... and few uncleaned chars,
# so remove these and replace contractions not in the vocabulary with the corresponding complete forms
def lyrics_preprocessing(text):
  #print("*"*40)
  #print("lyrics: \n", text)
  #return text.lower() # TEST RAPIDO

  debug = False
  #if "weren't" in text.lower() or "mustn't" in text.lower() or "shouldn't" in text.lower():
  #if "'ve" in text.lower():
  #if "'twas" in text.lower():
  #if "pain'll" in text.lower() or "box'll" in text.lower():
  #if "'\"ve" in text:
  #if "forty-four lies" in text.lower() or "before i fall away" in text.lower():
  #if "votin''" in text:
  #  debug = True

  if not is_english(text):
    return None


  # Get the char at position i in the text if exists, otherwise return the empty char
  def get_char(text, i):
    try:
      char = text[i]
    except:
      char = ''

    return char

  # Return true if char is an allowed space (" " or "\n")
  def is_allowed_space(char):
    #print("is allowed space? ", char == ' ' or char == '\n')
    return char == ' ' or char == '\n'

  # Return true if char is a string ending (i.e. there is not a subsequent char or this is a whitespace or this is a double quotes)
  def is_string_ending(char):
    if debug:
      print("char: ", char)
      print("is char ending? ", not char or char.isspace() or char in string_ending_chars) # {'"', ',', '.', ';', ':', '!', '?', '-', '(', ')', '{', '}', '[', ']', '…'}) #, "'"})
    return not char or char.isspace() or char in string_ending_chars # {'"', ',', '.', ';', ':', '!', '?', '-', '(', ')', '{', '}', '[', ']', '…'}

  # Return true if the char at position i in the text is a string ending, otherwise false
  def __is_string_ending(text, i):
    next_char = get_char(text, i+1)
    if debug:
      print("Current char: ", text[i])
      print("Next char: ", next_char)
    return is_string_ending(next_char)

  def match_next_chars_regex(text, i, regex, regex_len):
    #print("Regex to match: ", regex)
    try:
      next_chars = text[i+1:i+1+regex_len]
      #print("Text next_chars: ", next_chars)
    except:
      return False

    return True if re.match(regex, next_chars) else False

  def match_prev_chars_regex(text, i, regex, regex_len):
    #print("Regex to match: ", regex)
    try:
      prev_chars = text[i-regex_len:i]
      #print("Text prev_chars: ", prev_chars)
    except:
      return False

    return True if re.match(regex, prev_chars) else False

  # Takes in input a list of strings or a string.
  # If string_ending parameter is false, return true if the next chars of the text match the word, otherwise false.
  # If string_ending parameter is true, return true if the next chars of the text match the word and such chars are a word ending in the text, otherwise false
  def match_next_chars(text, i, input, string_ending=False):
    def __match(word):
      if debug:
        #print("Next chars of the text: ", text[i:])
        print("word to match: ", word)
      try:
        next_chars = text[i+1:i+1+len(word)]
        if debug:
          print("text next_chars: ", next_chars)
      except:
        return False

      if debug:
        print("next_chars == word? ", next_chars.lower() == word)

      if string_ending and not __is_string_ending(text, i+len(word)):
        return False

      return next_chars.lower() == word
 

    # Check if input is a list of words
    if isinstance(input, list):
      # Iterate the words
      for word in input:
        if __match(word):
          return True
    elif isinstance(input, str):
      return __match(input)
    
    return False

    
    '''if debug:
      #print("Next chars of the text: ", text[i:])
      print("word to match: ", word)
    try:
      next_chars = text[i+1:i+1+len(word)]
      if debug:
        print("text next_chars: ", next_chars)
    except:
      return False

    if debug:
      print("next_chars == word? ", next_chars == word)

    if next_chars.lower() != word:
      return False

    if string_ending:
      return __is_string_ending(text, i+len(word))

    return True #next_chars.lower() == word'''


  # Takes in input a list of strings or a string.
  # If string_ending and string_beginning parameters are false, return true if the prev chars of the text match the word, otherwise false.
  # If string_ending parameter is true, return true if the prev chars of the text match the word and such chars are a word ending in the text, otherwise false.
  # If string_beginning parameter is true, return true if the prev chars of the text match the word and such chars are a word beginning in the text, otherwise false.
  def match_prev_chars(text, i, input, string_ending=False, string_beginning=False):
    def __match(word):
      if debug:
        #print("Last chars of the text: ", text[:i])
        print("word to match: ", word)

      if string_ending and not __is_string_ending(text, i):
        return False

      if string_beginning:
        char = get_char(text, i-len(word)-1)
        if not is_string_ending(char):
          return False

      try:
        prev_chars = text[i-len(word):i]
        if debug:
          print("text prev_chars: ", prev_chars)
      except:
        return False

      return prev_chars.lower() == word


    # Check if input is a list of words
    if isinstance(input, list):
      # Iterate the words
      for word in input:
        if __match(word):
          return True
    elif isinstance(input, str):
      return __match(input)
    
    return False


    '''if debug:
      #print("Last chars of the text: ", text[:i])
      print("word to match: ", word)
    try:
      prev_chars = text[i-len(word):i]
      if debug:
        print("text prev_chars: ", prev_chars)
    except:
      return False

    if prev_chars.lower() != word:
      return False

    if string_ending:
      return __is_string_ending(text, i)
    
    return True'''



  def check_prev_chars(text, i, words):
    for word in words:
      if debug:
        print("Check prev chars of word: ", word)
      match = True
      word_len = len(word)
      
      for j in range(word_len):
        prev_char_text = get_char(text, i-j-1)
        prev_char_word = get_char(word, word_len-j-1)
        if debug:
          print("Check if text char %c == word char %c" %(prev_char_text, prev_char_word))

        if prev_char_text != prev_char_word:
          match = False
          break

      if match and word_len > 0:
        return True

    return False


  def analyze_apostrophe(text, i, out_text):
    # Try to get the next char
    next_char = get_char(text, i+1).lower()
    if debug:
      print("="*20)
      print("char after apostrophe is: ", next_char)

    skip_char = 0
    is_contraction = True

    # Check if the apostrophe is a contraction of an 'ing' ending (e.g., cryin' -> crying)
    if match_prev_chars(text, i, "in", string_ending=True):
      # Replace the apostrophe with a 'g'
      out_text += "g"
    

    # Else check if the apostrophe is a contraction of a decade (e.g., 60', 60's, 1960's -> 60s)
    #elif i >= 2 and re.match("^[0-9]+$", text[i-2:i]):
    #  out_text += "s"

    #  # If the next char is a 's', skip it
    #  if next_char == 's':
    #    skip_char = 1

    # Else add the apostrophe (') to the output text
    #else:
    #  out_text += "'"


    # Else check if the apostrophe is followed by "ve"
    elif match_next_chars(text, i, "ve", string_ending=True):
      #print("'ve found")
      # Check if "'ve" follows a word in the list (s.t. the word + 've is not in the vocabulary)
      '''match = check_prev_chars(out_text, len(out_text), ["i", "should", "must", "might", "could", "would", "who", "that"])
      if match:
        if debug:
          print("Match found")
        # If a match is found, replace the contraction with the complete form (word + " have")
        skip_char = 2
        out_text += " have"
      else:
        out_text += "'"'''

      # Check if "'ve" follows a word in the list (s.t. the word + 've is not in the vocabulary)
      match = check_prev_chars(out_text, len(out_text), ["l"])
      if match:
        # Replace "l've" with "love"
        skip_char = 2
        out_text += "ove"
      else: 
        # Else replace the contraction with the complete form (word + " have")
        # COSÌ SOSTITUIAMO ANCHE ABBREVIAZIONI CON 're CHE SONO PRESENTI NEL FILE word2vec (POCO IMPORTA).
        skip_char = 2
        out_text += " have"

    # Else check if the apostrophe is followed by "re" or "r"         
    elif match_next_chars(text, i, "re", string_ending=True):
      '''#print("'re found")
      # Check if "'re" follows a word in the list (s.t. the word + 're is not in the vocabulary)
      match = check_prev_chars(out_text, len(out_text), ["why"])
      if match:
        if debug:
          print("Replace 're with are")
        # If a match is found, replace the contraction with the complete form (word + " are")
        skip_char = 2
        out_text += " are"
      else:
        out_text += "'"'''
      
      # COSÌ SOSTITUIAMO ANCHE ABBREVIAZIONI CON 're CHE SONO PRESENTI NEL FILE word2vec (POCO IMPORTA).
      # PER <NAME> VERIFICATO CHE I CARATTERI PRECEDENTI L'APOSTROFO NON SIANO PAROLE LE CUI ABBREVIAZIONI SONO GIÀ PRESENTI
      # If a match is found, replace the contraction with the complete form (word + " are")
      skip_char = 2
      out_text += " are"

    # Else check if the apostrophe is followed by "r"         
    elif match_next_chars(text, i, "r", string_ending=True):
      # If a match is found, replace the contraction with the complete form (word + " are")
      skip_char = 1
      out_text += " are"

    # Else check if the apostrophe is followed by "ll"
    elif match_next_chars(text, i, "ll", string_ending=True):
      #print("'ll found")
      # Check if "'ll" follows a word in the list (s.t. the word + 'll is not in the vocabulary)
      match = check_prev_chars(out_text, len(out_text), ["ya"])
      if match:
        if debug:
          print("Replace 'll with ou all")
        # If a match is found, replace the contraction with the complete form (word + " ou all")
        skip_char = 2
        out_text = out_text[:len(out_text)-1] + "ou all"
      else:
        # COSÌ SOSTITUIAMO ANCHE ABBREVIAZIONI CON 'll CHE SONO PRESENTI NEL FILE word2vec (POCO IMPORTA)
        if debug:
          print("Replace 'll with will")
        # Else replace the contraction with the complete form (word + " will")
        skip_char = 2
        out_text += " will"

    # Else add the apostrophe (') to the output text
    else:
      # Check if the apostrophe does not follow another one
      #if not check_prev_chars(out_text, len(out_text), ["'"]):
      #  out_text += "'"

      # Check if the apostrophe follows another one (e.g. votin'', i''ve)
      if check_prev_chars(out_text, len(out_text), ["'"]):
        # Perform the analysis of the apostrophe (removing from the output the apostrophe itself).

        # Try with contraction of subsequent chars (e.g. ve, re, ll)
        out_text, skip_char, is_contraction = analyze_apostrophe(text, i, out_text[:len(out_text)-1])
        if debug:
          print("is ' for subsequent chars a contraction? ", is_contraction)
        if is_contraction:
          return out_text, skip_char, is_contraction

        # Replace current apostrophe in the text to analyze previous chars (to recognize an eventual string ending)
        text = text[:i] + " " + text[i+1:]

        # Try with contraction of previous chars (e.g. "ing" ending)
        out_text, skip_char, is_contraction = analyze_apostrophe(text, i-1, out_text[:len(out_text)-1])
        if debug:
          print("is ' for previous chars a contraction? ", is_contraction)
      else:
        out_text += "'"
        is_contraction = False

    return out_text, skip_char, is_contraction

  
  skip_char = 0
  out_text = ""

  # Iterate char of the input lyrics
  for i in range(len(text)):
    # Check if the current char has to be skipped
    if skip_char > 0:
      skip_char -= 1
      continue

    # Get the current char
    char = text[i]

    if debug:
      print("="*20)
      print("current char is: ", char)

    # Remove chars in the blacklist
    if char not in blacklisted_chars:
      # Check if the char is a non-allowed space (i.e. \t, \v, \f, \r, etc.) or a dash
      if (char.isspace() and not is_allowed_space(char)) or char in dashes:
        if debug:
          print("Replace it with a whitespace")
        # Replace it with a whitespace
        out_text += ' '

      # Check if the char is an apostrophe (i.e. ',`,’, etc.)
      elif char in apostrophes:
        # Try to get the next char
        next_char = get_char(text, i+1).lower()
        if debug:
          print("="*20)
          print("char after apostrophe is: ", next_char)

        out_text, skip_char, _ = analyze_apostrophe(text, i, out_text)




        '''elif match_next_chars(text, i, "ll", True):
          #print("'ll found")
          # Check if "'ll" follows one word in the list (s.t. the word + 'll is not in the vocabulary)
          match = check_prev_chars(out_text, len(out_text), ["i", "he", "she", "it", "who", "there", "that", "what", "this"])
          if match:
            if debug:
              print("Match found")
            # If a match is found, replace the contraction with the complete form (word + " ou all")
            skip_char = 2
            out_text += " will"
          elif check_prev_chars(out_text, len(out_text), ["ya"]):
            # If a match is found, replace the contraction with the complete form (word + "ou all")
            skip_char = 2
            out_text = out_text[:len(out_text)-1] + "ou all"
          else:
            out_text += "'"

        elif match_next_chars(text, i, "all", True):
          #print("'all found")
          # Check if "'all" follows one word in the list (s.t. the word + 'all is not in the vocabulary)
          match = check_prev_chars(out_text, len(out_text), ["y"])
          if match:
            if debug:
              print("Match found")
            # If a match is found, replace the contraction with the complete form (word + " ou all")
            skip_char = 3
            out_text += "ou all"
          else:
            out_text += "'"

        elif match_next_chars(text, i, "d", True):
          #print("'d found")
          # Check if "'d" follows one word in the list (s.t. the word + 'd is not in the vocabulary)
          match = check_prev_chars(out_text, len(out_text), ["what", "where", "why"])
          if match:
            if debug:
              print("Match found")
            # If a match is found, replace the contraction with the complete form (word + " did")
            skip_char = 1
            out_text += " did"
          else:
            out_text += "'"

        elif match_next_chars(text, i, "s", True):
          #print("'s found")
          # Check if "'s" follows one word in the list (s.t. the word + 's is not in the vocabulary)
          match = check_prev_chars(out_text, len(out_text), ["everything", "everyone", "here", "now", "body", "number", "back", "time", "rain", "whatever"])
          if match:
            if debug:
              print("Match found")
            # If a match is found, replace the contraction with the complete form (word + " is")
            skip_char = 1
            out_text += " is"
          else:
            out_text += "'"

        elif match_next_chars(text, i, "t", True):
          # Try to get the previous char
          prev_char = get_char(out_text, len(out_text)-1)
          
          if debug:
            print("char before 't  is: ", prev_char)
            print("char before 't in out_text is: ", out_text[i-1])
            print("i:", i)
            print("len out_text: ", len(out_text))

          # Check if the char before the apostrophe is 'n' (for n't)
          if prev_char == 'n':
            if debug:
              print("Found n't in the text")
            # Check if n't follows one of the words s.t. the word + n't is not in the vocabulary
            match = check_prev_chars(out_text, len(out_text)-1, ["were", "had", "must", "should"])

            if match:
              if debug:
                print("Match found")
              # If a match is found, replace the contraction with the complete form (word + " not")
              skip_char = 1
              # Use len of out_text instead of i because after a replacement, i in the original text and in the out_text can refer to different chars
              out_text = out_text[:len(out_text)-1] + " not"
            else:
              out_text += "'"
          else:
            out_text += "'"


        elif is_string_ending(get_char(out_text, len(out_text)-1)):
          #("'twas", "it was")
          #("'cept", "except")
          #("'ceptin", "except")
          #("'cross", "across")
          #("'scuse", "scuse")
          #("'neath", "beneath")
          if match_next_chars(text, i, "twas", True):
            skip_char = 4
            out_text += "it was"
          elif match_next_chars(text, i, "cept", True):
            skip_char = 4
            out_text += "except"
          elif match_next_chars(text, i, "ceptin", True):
            skip_char = 6
            out_text += "except"
          elif match_next_chars(text, i, "cross", True):
            skip_char = 5
            out_text += "across"
          elif match_next_chars(text, i, "scuse", True):
            skip_char = 5
            out_text += "scuse"
          elif match_next_chars(text, i, "neath", True):
            skip_char = 5
            out_text += "beneath"


        # Else add the apostrophe (') to the output text
        else:
          out_text += "'"'''





        # Else check if the apostrophe is a contraction of 'have' (e.g., i've -> i have)
        '''elif next_char == 'v':
          # Try to get the char after the 'v'
          next_char = get_char(text, i+2)
          if debug:
            print("out_text so far:", out_text)
            print("char after 'v is: ", next_char)

          if next_char == 'e':
            # Try to get the char after the 'e'
            next_char = get_char(text, i+3)
            if debug:
              print("char after 've is: ", next_char)
            
            # Check if 'e' is the last character of the string (i.e. there is not a subsequent char or this is a whitespace)
            if not next_char or next_char == ' ':
              if debug:
                print("Found 've in the text")
              # Check if "'ve" follows one word in the list (s.t. the word + 've is not in the vocabulary)
              match = check_prev_chars(out_text, len(out_text), ["i", "should", "must", "might", "could", "would", "who", "that"])
              if match:
                if debug:
                  print("Match found")
                # If a match is found, replace the contraction with the complete form (word + " have")
                skip_char = 2
                out_text += " have"
              else:
                out_text += "'"
            else:
              out_text += "'"

        # Else check if the apostrophe is a contraction of 'are' (e.g., why're -> why are)
        elif next_char == 'r':
          # Try to get the char after the 'r'
          next_char = get_char(text, i+2)

          if next_char == 'e':
            # Try to get the char after the 'e'
            next_char = get_char(text, i+3)
            
            # Check if 'e' is the last character of the string (i.e. there is not a subsequent char or this is a whitespace)
            if not next_char or next_char == ' ':
              # Check if "'re" follows one word in the list (s.t. the word + 're is not in the vocabulary)
              match = check_prev_chars(out_text, len(out_text), ["why"])
              if match:
                # If a match is found, replace the contraction with the complete form (word + " are")
                skip_char = 2
                out_text += " are"
              else:
                out_text += "'"
            else:
              out_text += "'"

        # Else check if the apostrophe is a contraction of 'will' (e.g., i'll -> i will)
        elif next_char == 'l':
          # Try to get the char after the 'l'
          next_char = get_char(text, i+2)

          if next_char == 'l':
            # Try to get the char after the 'l'
            next_char = get_char(text, i+3)
            
            # Check if 'l' is the last character of the string (i.e. there is not a subsequent char or this is a whitespace)
            if not next_char or next_char == ' ':
              # Check if "'ll" follows one word in the list (s.t. the word + 'll is not in the vocabulary)
              match = check_prev_chars(out_text, len(out_text), ["i", "he", "she", "it", "who", "there", "that", "what", "this"])
              if match:
                # If a match is found, replace the contraction with the complete form (word + " will")
                skip_char = 2
                out_text += " will"
              elif check_prev_chars(out_text, len(out_text), ["ya"]):
                # If a match is found, replace the contraction with the complete form (word + "ou all")
                skip_char = 2
                out_text = out_text[:len(out_text)-1] + "ou all"
              else:
                out_text += "'"
            else:
              out_text += "'"
        
        # Check if the next char to the apostrophe is 't' (for n't)
        elif next_char == 't':
          # Try to get the next char to 't'
          next_char = get_char(text, i+2)

          if debug:
            print("out_text so far:", out_text)
            print("char after 't is: ", next_char)

          # If there is not a subsequent char or this is a whitespace
          if not next_char or next_char == ' ':
            # Try to get the previous char
            prev_char = get_char(out_text, len(out_text)-1)
            
            if debug:
              print("char before 't  is: ", prev_char)
              print("char before 't in out_text is: ", out_text[i-1])
              print("i:", i)
              print("len out_text: ", len(out_text))

            # Check if the char before the apostrophe is 'n' (for n't)
            if prev_char == 'n':
              if debug:
                print("Found n't in the text")
              # Check if n't follows one of the words s.t. the word + n't is not in the vocabulary
              match = check_prev_chars(out_text, len(out_text)-1, ["were", "had", "must", "should"])

              if match:
                if debug:
                  print("Match found")
                # If a match is found, replace the contraction with the complete form (word + " not")
                skip_char = 1
                # Use len of out_text instead of i because after a replacement, i in the original text and in the out_text can refer to different chars
                out_text = out_text[:len(out_text)-1] + " not"
              else:
                out_text += "'"
            else:
              out_text += "'"

        # Check 'd
        elif next_char == 'd':
          # Try to get the char after the 'd'
          next_char = get_char(text, i+2)

          # If there is not a subsequent char or this is a whitespace
          if not next_char or next_char == ' ':
            # Check if 'd follows one of the words s.t. the word + 'd is not in the vocabulary
            match = check_prev_chars(out_text, len(out_text), ["what", "where", "why"])
            if match:
              # If a match is found, replace the contraction with the complete form (word + " will")
              skip_char = 1
              out_text += " did"
            else:
              out_text += "'"
          else:
            out_text += "'"

        # Check 's
        elif next_char == 's':
          # Try to get the char after the 's'
          next_char = get_char(text, i+2)

          # If there is not a subsequent char or this is a whitespace
          if not next_char or next_char == ' ':
            # Check if 's follows one of the words s.t. the word + 's is not in the vocabulary
            match = check_prev_chars(out_text, len(out_text), ["everything", "everyone", "here", "now", "body", "number", "back", "time", "rain", "whatever"])
            if match:
              # If a match is found, replace the contraction with the complete form (word + " will")
              skip_char = 1
              out_text += " is"
            else:
              out_text += "'"
          else:
            out_text += "'"


        # Else add the apostrophe (') to the output text
        else:
          out_text += "'"'''




      # Else check if the char is a number
      elif re.match("^[0-9]$", char):
        #print("="*40)
        #print("Current char is a number: ", char)
        is_repetition = False
        is_decade     = False
        j = 0
        next_char = char

        # Iterate the string
        while not is_string_ending(next_char):
          # Try to get the next char
          next_char = get_char(text, i+j+1)
          #print("="*40)
          #print("Number next_char: ", next_char)

          # Check if the text is ended
          if not next_char:
            break

          # Advance the offset to the current char
          j += 1
      
          # Check if the number is followed by 'x' (repetition in the lyrics: e.g., '2x', '10x' -> skip it)
          if next_char.lower() == 'x' or next_char == '×':
            is_repetition = True

          # Else check if the number is followed by an apostrophe
          elif next_char == "'":
            # Check if this is a contraction of a decade (e.g., 60', 60's, 1960's)
            if j == 2 or j == 4:
              # Try to get the char next to the apostrophe.
              next_char = get_char(text, i+j+1)

              # Check if the char next to the apostrophe does not exist (i.e. the text is ended), is a 's' or a string ending
              if is_string_ending(char) or next_char.lower() == 's':
                is_decade = True

            # In any case break the loop and if the number is not a contraction of a decade, the apostrophe will be evaluated in the next iteration.
            # Don't skip the apostrophe (for the moment) to not include it in the out_text
            #j -= 1
            break

        # Set the chars to skip
        #skip_char = j
        skip_char = j-1

        # Check if the number is not a repetition
        if not is_repetition:
          # Try to include the subsequent whitespace if exists
          #try:
          #  substr = text[i:i+j+1]  
          #except:
          #  substr = text[i:i+j]

          substr = text[i:i+j]
          out_text += substr.lower()

          # Check if the number is a decade
          if is_decade: 
            # Try to get the char next to the apostrophe.
            # text[i+j+1] is the apostrophe because we performed j -= 1 to not include it in the out_text
            #next_char = get_char(text, i+j+2)                 

            # If the next char is a 's', skip the apostrophe and it
            if next_char == 's':
              skip_char += 2
            # Else skip only the apostrophe
            else:
              skip_char += 1

            out_text += "s"

      # Else check if the char is 'x' or '×'
      elif char.lower() == 'x' or char == '×':
        is_repetition = False
        j = 0
        next_char = char

        # Iterate the string
        while not is_string_ending(next_char):
          # Try to get the next char
          next_char = get_char(text, i+1+j)

          # Check if the text is ended
          if not next_char:
              break
  
          # Advance the offset to the current char
          j += 1

          # Check if the 'x' is followed by a number (i.e. repetition in the lyrics: e.g., 'x2', 'x10' -> skip it)
          if re.match("^[0-9]$", next_char):
            is_repetition = True
          else:
            break

        # Set the chars to skip
        skip_char = j-1 #j

        # Check if the number is not a repetition
        if not is_repetition:
          # Try to include the subsequent whitespace if exists
          #try:
          #  substr = text[i:i+j+1]  
          #except:
          #  substr = text[i:i+j]

          substr = text[i:i+j]

          out_text += substr.lower()

      # Check if the char is a punctuation one (subset) or newline
      elif char in punctuation_subset:
        # Surround it with whitespaces to include it in the subsequent splitting of words (surround for avoiding missing whitespaces)
        out_text += " " + char + " "

      # Check if the char is a dot
      elif char == '.':
        #print("\nCurrent char is a dot")
        # Check if "." belongs to an acronym (e.g. U.S.A., U.S.A)
        '''if match_next_chars_regex(text, i, "^[a-zA-Z]\.$", 2) or match_prev_chars_regex(text, i, "^\.[a-zA-Z]$", 2):
          #print("Acronym found")
          # Add the dot and the next char to the output
          out_text += text[i:i+2]
          skip_char = 1'''

        # Check if "." belongs to an acronym (e.g. U.S.A., U.S.A)
        if match_next_chars_regex(text, i, "^[a-zA-Z]\.$", 2):
          #print("Acronym found")
          # Add the dot and the next char to the output (replacing the previous character of the acronym with the original one, e.g. uppercase)
          out_text = out_text[:len(out_text)-1] + text[i-1:i+2]
          skip_char = 1

        elif match_prev_chars_regex(text, i, "^\.[a-zA-Z]$", 2):
          #print("Acronym found")
          if match_next_chars_regex(text, i, "^[a-zA-Z]$", 1):
            # Add the dot and the next char to the output
            out_text += text[i:i+2]
            skip_char = 1
          else:
            # Add the dot + whitespace
            out_text += '. '

        # The dot does not belong to an acronym
        else:
          #print("Acronym not found")
          # Check if "." follows a string in the list (i.e. multiple dots or contraction in which the dot must not be considered alone)
          #if not check_prev_chars(out_text, len(out_text), dot_contractions):
          if not match_prev_chars(out_text, len(out_text), dot_contractions, string_beginning=True):
            # No, so add a whitespace to include it alone in the splitting of words 
            out_text += " "

          # Add the dot in the output
          out_text += char

          # Check if "." precedes a whitespace (i.e. detect missing whitespace) or a dot (i.e. multiple dots)
          if not match_next_chars(text, i, " ", string_ending=False) and not match_next_chars(text, i, ".", string_ending=False):
            # No, so add a whitespace to include the next word in the splitting of words
            out_text += " "

      # Check if the char is "("
      elif char == '(':
        # Check if the text in parenthesis is a repetition with 1 digit (e.g. (2x), (x2))
        if match_next_chars_regex(text, i, "^([0-9]+[xX×]|[xX×][0-9]+)\)$", 3):
          skip_char = 3
        # Check if the text in parenthesis is a repetition with 2 digits (e.g. (10x), (x10))
        elif match_next_chars_regex(text, i, "^([0-9]+[xX×]|[xX×][0-9]+)\)$", 4):
          skip_char = 4
        else:
          # Surround it with whitespaces to include it in the subsequent splitting of words (surround for avoiding missing whitespaces)
          out_text += " " + char + " "

      # Check if the char is "("
      elif char == '{':
        # Check if the text in parenthesis is a repetition with 1 digit (e.g. {2x}, {x2})
        if match_next_chars_regex(text, i, "^([0-9]+[xX×]|[xX×][0-9]+)\}$", 3):
          skip_char = 3
        # Check if the text in parenthesis is a repetition with 2 digits (e.g. {10x}, {x10})
        elif match_next_chars_regex(text, i, "^([0-9]+[xX×]|[xX×][0-9]+)\}$", 4):
          skip_char = 4
        else:
          # Surround it with whitespaces to include it in the subsequent splitting of words (surround for avoiding missing whitespaces)
          out_text += " " + char + " "

      elif char == '*':
        # Check if "*" precedes or follows a parenthesis
        if match_prev_chars(text, i, ["(", "[", "{"], string_ending=False) or match_next_chars(text, i, [")", "]", "}"], string_ending=False):
          continue
        # Check if "*" follows a space or newline (probably is not really text, e.g. *drum solo*)
        elif match_prev_chars(text, i, [" ", "\n"]):
          # Replace with a parenthesis + whitespace
          out_text += '( '
        # Check if "*" precedes a string ending (probably is not really text, e.g. *drum solo*)
        elif is_string_ending(get_char(text, i+1)):
          # Replace with a whitespace + parenthesis
          out_text += ' )'
        else:
          # Add the * to the output text
          out_text += char

      # Else add the lower case version of the char
      else:
        out_text += char.lower()

    # Else this is a blacklisted char
    else:
      # Try to get the previous char
      prev_char = get_char(out_text, len(out_text)-1)

      if debug:
        print("Char before the blacklisted char is: ", prev_char)
        print("Text before the blacklisted char is: ", out_text)

      # Check if the blacklisted char follows an apostrophe (e.g. i'"ve, cryin'")
      if prev_char == "'":
        # Perform the analysis of the apostrophe (removing from the output the apostrophe itself)
        out_text, skip_char, _ = analyze_apostrophe(text, i, out_text[:len(out_text)-1])

        # We don't need to check previous chars because blacklisted chars are included in string_ending_chars set (i.e. cryin'" -> crying)
        

  # '90s '72 -> remove ' before number


  # Replace contractions not in the vocabulary with corresponding complete forms
  # HO INSERITO NEL CICLO SOPRA 've, 're, 'll, n't, 'd, 's  INVECE DI METTERLI QUI (6m E 6m, FORSE È PIÙ VELOCE replace)
  # UPDATE: NEL CICLO SOPRA 've, 're, 'll, IL RESTO QUI
  '''out_text = out_text.replace("i've", "i have")
  out_text = out_text.replace("should've", "should have")
  out_text = out_text.replace("must've", "must have")
  out_text = out_text.replace("might've", "might have")
  out_text = out_text.replace("could've", "could have")
  out_text = out_text.replace("would've", "would have")
  out_text = out_text.replace("who've", "who have")
  out_text = out_text.replace("that've", "that have")
  out_text = out_text.replace("when've", "when have")

  out_text = out_text.replace("why're", "why are")

  out_text = out_text.replace("i'll", "i will")
  out_text = out_text.replace("he'll", "he will")
  out_text = out_text.replace("she'll", "she will")
  out_text = out_text.replace("it'll", "it will")
  out_text = out_text.replace("who'll", "who will")
  out_text = out_text.replace("there'll", "there will")
  out_text = out_text.replace("that'll", "that will")
  out_text = out_text.replace("what'll", "what will")
  out_text = out_text.replace("this'll", "this will")'''

  out_text = out_text.replace("y'all", "you all")
  #out_text = out_text.replace("ya'll", "you all")

  out_text = out_text.replace("weren't", "were not")
  out_text = out_text.replace("hadn't", "had not")
  out_text = out_text.replace("mustn't", "must not")
  out_text = out_text.replace("shouldn't", "should not")
  out_text = out_text.replace("cain't", "can't")
  out_text = out_text.replace("needn't", "need not")

  out_text = out_text.replace("wouldn'ta", "would not have")
  out_text = out_text.replace("shouldn'a", "should not")
  out_text = out_text.replace("had'a", "had")
  out_text = out_text.replace("should'a", "should have")
  out_text = out_text.replace("ought'a", "ought to")
  out_text = out_text.replace("i'da", "i would have")
  out_text = out_text.replace("don'tcha", "don't you")
  out_text = out_text.replace("want'cha", "want you")

  out_text = out_text.replace("what'd", "what did")
  out_text = out_text.replace("where'd", "where did")
  out_text = out_text.replace("why'd", "why did")

  out_text = out_text.replace("i'ma", "i'm gonna")
  out_text = out_text.replace("i'mma", "i'm gonna")
  #out_text = out_text.replace("i'm-a", "i'm gonna") # ORA HO SOSTITUITO TUTTI I "-" CON " "
  
  out_text = out_text.replace("everything's", "everything is")
  out_text = out_text.replace("everyone's", "everyone is")
  out_text = out_text.replace("here's", "here is")
  out_text = out_text.replace("now's", "now is")
  out_text = out_text.replace("body's", "body is")
  out_text = out_text.replace("number's", "number is")
  out_text = out_text.replace("back's", "back is")
  out_text = out_text.replace("time's", "time is")
  out_text = out_text.replace("rain's", "rain is")
  out_text = out_text.replace("whatever's", "whatever is")

  #out_text = out_text.replace("call'st", "calledst") # NON PRESENTE NEL FILE WORD2VEC (NON TUTTI GLI 'st SONO edst)
  out_text = out_text.replace("call'st", "called")
  out_text = out_text.replace("gav'st", "gave")
  out_text = out_text.replace("thou'st", "thou hast")
  out_text = out_text.replace("would'st", "would")

  out_text = out_text.replace("'twas", "it was")
  out_text = out_text.replace("t'was", "it was")
  out_text = out_text.replace("'twere", "it were")
  out_text = out_text.replace("'cept", "except")
  out_text = out_text.replace("'ceptin", "except")
  out_text = out_text.replace("'cross", "across")
  out_text = out_text.replace("'scuse", "scuse")
  out_text = out_text.replace("'neath", "beneath")
  out_text = out_text.replace("'bouts", "abouts")#'''
  out_text = out_text.replace("'llac", "cadillac")
  out_text = out_text.replace("'lac", "cadillac")
  out_text = out_text.replace("'em", " 'em") # 'em È NEL FILE WORD2VEC MA NON TUTTE LE VARIE FORME let'em, kill'em
  out_text = out_text.replace("'stead", "instead")
  out_text = out_text.replace("'nough", "enough")
  out_text = out_text.replace("'doing", "doing")
  out_text = out_text.replace("'cause", "because")
  out_text = out_text.replace("'cah", "cah")
  out_text = out_text.replace("'ssassin", "assassin")
  out_text = out_text.replace("'member", "remember")
  out_text = out_text.replace("'memba", "memba") # remember
  out_text = out_text.replace("'gin", "begin")
  out_text = out_text.replace("'tween", "between")
  out_text = out_text.replace("'leven", "eleven")
  out_text = out_text.replace("'yoncé", "beyoncé")
  out_text = out_text.replace("yoncé", "beyoncé")
  out_text = out_text.replace("'gator", "alligator")
  out_text = out_text.replace("'while", "a while")
  out_text = out_text.replace("'gats", "gats")

  out_text = out_text.replace("tho'", "though")
  out_text = out_text.replace("cuz'", "cuz")
  out_text = out_text.replace("gotta'", "gotta")
  out_text = out_text.replace("tryna'", "tryna")
  out_text = out_text.replace("ya'", " ya")
  out_text = out_text.replace("gon'", "gonna")
  out_text = out_text.replace("ol'", "old")
  out_text = out_text.replace("jag'", "jag")
  out_text = out_text.replace("gi'", "gi")
  out_text = out_text.replace("more'", "more")
  out_text = out_text.replace("red'", "red")
  out_text = out_text.replace("girlfrien'", "girlfriend")
  out_text = out_text.replace("wan'", "want")
  out_text = out_text.replace("record'", "records")
  out_text = out_text.replace("politic'", "politics")
  out_text = out_text.replace("wit'", "with")
  #out_text = out_text.replace("an'", "and") # OCCHIO CHE SOSTITUISCE ANCHE can't CON candt, POSSO METTERE UNO SPAZIO PRIMA DI an'
  out_text = out_text.replace(" an'", " and")
  out_text = out_text.replace(" n'", " n") # and
  out_text = out_text.replace("'n'", " and ") # and
  out_text = out_text.replace("mam'", "mama")
  out_text = out_text.replace("sumn'", "sumn") # something
  out_text = out_text.replace("fa'", "fa")     # for
  out_text = out_text.replace("fo'", "fo")     # for
  out_text = out_text.replace("roun'", "round")
  out_text = out_text.replace("ho'", "ho") # hoes
  out_text = out_text.replace("mothafuckers'", "motherfuckers")
  out_text = out_text.replace("motherfuckers'", "motherfuckers")
  out_text = out_text.replace("mo'", "more")
  out_text = out_text.replace("yo'", "yo")
  out_text = out_text.replace("lil'", "lil")
  out_text = out_text.replace("billi'", "billions")
  out_text = out_text.replace("aroon'", "around")
  out_text = out_text.replace("frien'", "friend")
  out_text = out_text.replace("cliche'", "cliche")
  out_text = out_text.replace("oft'", "often")
  out_text = out_text.replace("looters'", "looters")
  out_text = out_text.replace("ta'", "to")
  out_text = out_text.replace("abusa'", "abuse")
  out_text = out_text.replace("losa'", "loser")
  out_text = out_text.replace("cus'", "cuz")
  out_text = out_text.replace("fam'", "family")
  out_text = out_text.replace("bis'", "business")
  out_text = out_text.replace("morn'", "morning")
  out_text = out_text.replace("quicka'", "quicker")
  out_text = out_text.replace("bigga'", "bigga") # bigger
  out_text = out_text.replace("trigga'", "trigga") # trigger
  out_text = out_text.replace("flo'", "flow") 
  out_text = out_text.replace("thrilla'", "thrilla")

  out_text = out_text.replace("o' ", "of ") # (SPAZIO ALLA FINE PER EVITARE DI SOSTITUIRE PAROLE COME o'shea)
  out_text = out_text.replace(" em'", " 'em'") # (SPAZIO DAVANTI PER EVITARE DI SOSTITUIRE PAROLE COME 'em')
  out_text = out_text.replace("do'", "door") # (NON SO SE SEMPRE PUÒ ESSERE SOSTITUITA COSÌ)
  out_text = out_text.replace("wha'", "wha")
  out_text = out_text.replace("mil'", "mil") # million
    
  out_text = out_text.replace("c'mon", "come on")
  out_text = out_text.replace("mem'ry", "memory")
  out_text = out_text.replace("mem'ries", "memories")
  out_text = out_text.replace("ev'ry", "every")
  out_text = out_text.replace("e'ry", "every")
  out_text = out_text.replace("ev'y", "every")
  #out_text = out_text.replace("e'ery", "every") # INCLUSO SOTTO
  out_text = out_text.replace("e'er", "ever")
  #out_text = out_text.replace("ne'er", "never")       # INCLUSO SOPRA
  #out_text = out_text.replace("whate'er", "whatever") # INCLUSO SOPRA
  #out_text = out_text.replace("fore'er", "")          # INCLUSO SOPRA
  out_text = out_text.replace("myst'ry", "mystery")
  out_text = out_text.replace("m'sippi", "mississippi")
  #out_text = out_text.replace("let'em", "let them") # INCLUSO SOPRA
  out_text = out_text.replace("whatev's", "whatevs")
  out_text = out_text.replace("o'clock", "oclock")
  out_text = out_text.replace("prob'ly", "probably")
  out_text = out_text.replace("ki's", "kilos")
  out_text = out_text.replace("meetin's", "meetings")
  out_text = out_text.replace("fam'ly", "family")
  out_text = out_text.replace("o'er", "over")
  out_text = out_text.replace("slip'n ", "slip'n'")
  out_text = out_text.replace("fa'sho", "for sure")
  out_text = out_text.replace("fo'sure", "for sure")
  out_text = out_text.replace("fa'sure", "for sure")
  out_text = out_text.replace("fin'lly", "finally")
  out_text = out_text.replace("awf'lly", "awfully")
  out_text = out_text.replace("logic'lly", "logically")
  out_text = out_text.replace("trav'llin", "travellin")
  out_text = out_text.replace("natur'lly", "naturally")
  out_text = out_text.replace("natu'lly", "naturally")
  out_text = out_text.replace("f'real", "for real")
  out_text = out_text.replace("fo'real", "for real")
  out_text = out_text.replace("fa'real", "for real")
  #out_text = out_text.replace("diff'rent", "different")   # INCLUSO SOTTO
  #out_text = out_text.replace("diff'rence", "difference") # INCLUSO SOTTO
  out_text = out_text.replace("diff'ren", "differen")
  #out_text = out_text.replace("pref'rence", "preference") # INCLUSO SOTTO
  out_text = out_text.replace("ref'rence", "reference")
  out_text = out_text.replace("chil'ren", "children")
  out_text = out_text.replace("chi'ren", "children")
  out_text = out_text.replace("int'rest", "interest")
  out_text = out_text.replace("mack'rel", "mackerel")
  out_text = out_text.replace("sov'reign", "sovereign")
  out_text = out_text.replace("des'ree", "desiree")
  out_text = out_text.replace("wand'rer", "wanderer")
  out_text = out_text.replace("wand'ring", "wandering")
  #out_text = out_text.replace("l've", "love") # INCLUSO NEL CICLO
  #out_text = out_text.replace("lo'es", "loves") # INCLUSO SOTTO
  out_text = out_text.replace("lo'e", "love")
  out_text = out_text.replace("lov'd", "loved")
  out_text = out_text.replace("b'day", "birthday")
  out_text = out_text.replace("full'a", "full of")
  out_text = out_text.replace("creep'n", "creeping")
  out_text = out_text.replace("loc'ed", "locked")
  out_text = out_text.replace("ye'se", "you shall")
  out_text = out_text.replace("ta'en", "taken")
  out_text = out_text.replace("e'en", "even")
  out_text = out_text.replace("de'il", "devil")
  out_text = out_text.replace("ma'am", "madam")
  out_text = out_text.replace("cap'n", "captain")
  out_text = out_text.replace("sev'ral", "several")
  out_text = out_text.replace("monsta'", "monsta")
  out_text = out_text.replace("closa'", "close")
  out_text = out_text.replace("s'posed", "supposed")
  out_text = out_text.replace("s'pposed", "supposed")
  out_text = out_text.replace("wonda'", "wonder")
  out_text = out_text.replace("kind'a", "kind of")
  out_text = out_text.replace("irregula'", "irregular")
  out_text = out_text.replace("gon'be", "gonna be")
  out_text = out_text.replace("fright'ning", "frightening")
  out_text = out_text.replace("sump'n", "something")
  out_text = out_text.replace("some'n", "something")
  out_text = out_text.replace("s'all", "it is all")
  out_text = out_text.replace("do'ya", "do you")
  out_text = out_text.replace("trav'lin", "traveling")
  #out_text = out_text.replace("trav'ling", "traveling") # INCLUSO SOPRA
  out_text = out_text.replace("wond'rin", "wondering")
  out_text = out_text.replace("ling'ring", "lingering")
  out_text = out_text.replace("strawb'ries", "strawberries")
  out_text = out_text.replace("smould'ring", "smouldering")
  out_text = out_text.replace("nam'd", "named")
  out_text = out_text.replace("appear'd", "appeared")
  out_text = out_text.replace("announc'd", "announced")
  out_text = out_text.replace("fulfill'd", "fulfilled")
  out_text = out_text.replace("rul'd", "ruled")
  out_text = out_text.replace("releas'd", "released")
  out_text = out_text.replace("banish'd", "banished")
  out_text = out_text.replace("restor'd", "restored")
  out_text = out_text.replace("nourish'd", "nourished")
  out_text = out_text.replace("fear'd", "feared")
  out_text = out_text.replace("pro'ly", "probably")
  out_text = out_text.replace("wit'chu", "with you")
  out_text = out_text.replace("b'ness", "business")
  out_text = out_text.replace("a'ight", "alright")
  out_text = out_text.replace("suff'ring", "suffering")
  out_text = out_text.replace("mem'rable", "memorable")
  out_text = out_text.replace("alabam'", "alabama")
  out_text = out_text.replace("arizon'", "arizona")
  out_text = out_text.replace("louisian'", "louisiana")
  out_text = out_text.replace("groc'ries", "groceries")
  out_text = out_text.replace("s'rambling", "scrambling")
  out_text = out_text.replace("t'night", "tonight")
  out_text = out_text.replace("anotha'", "another")
  out_text = out_text.replace("t'ward", "toward")
  out_text = out_text.replace("why'm", "why am")
  out_text = out_text.replace("hap'nin", "happening")
  out_text = out_text.replace("whad'ya", "what do you")
  out_text = out_text.replace("mothafucka'", "mothafucka")
  out_text = out_text.replace("muhh'fucka", "mothafucka")
  out_text = out_text.replace("'lectricity", "electricity")
  out_text = out_text.replace("didn'tcha", "didn't you")
  out_text = out_text.replace("choc'late", "chocolate")
  out_text = out_text.replace("bi'ness", "business")
  out_text = out_text.replace("b'iness", "business")
  out_text = out_text.replace("light'ning", "lightning")
  out_text = out_text.replace("big'un", "big one")
  out_text = out_text.replace("slipp'ry", "slippery")
  out_text = out_text.replace("joyf'ly", "joyfully")
  out_text = out_text.replace("nor'wester", "Northwester")
  out_text = out_text.replace("thund'ry", "thundery")
  out_text = out_text.replace("fi'th", "fifth")
  out_text = out_text.replace("y'know", "you know")
  out_text = out_text.replace("play'd", "played")
  out_text = out_text.replace("who'da", "who would have")
  out_text = out_text.replace("light'nin", "lightning")
  #out_text = out_text.replace("i'm'a", "") # i'm gonna O i'm a?
  out_text = out_text.replace("fuckin'day", "fucking day")
  out_text = out_text.replace("aw'right", "all right")
  out_text = out_text.replace("gi'me", "give me")
  out_text = out_text.replace("a'waitin", "awaiting")
  out_text = out_text.replace("n'ggas", "niggas")
  #out_text = out_text.replace("ass'quake", "assquake") # NON PRESENTE NEL FILE WORD2VEC
  out_text = out_text.replace("ass'quake", "twerk")
  out_text = out_text.replace("ha'pennies", "halfpennies")
  out_text = out_text.replace("wanderin'up", "wandering up")
  out_text = out_text.replace("p'liceman", "policeman")
  out_text = out_text.replace("oughn't", "ough not")
  out_text = out_text.replace("p'hapz", "perhaps")
  out_text = out_text.replace("what'sup", "what's up")
  out_text = out_text.replace("mo'nin", "morning")
  out_text = out_text.replace("bless'n", "blessing")
  out_text = out_text.replace("sittin'on", "sitting on")
  out_text = out_text.replace("challeng'd", "challenged")
  out_text = out_text.replace("terr'ble", "terrible")
  out_text = out_text.replace("tell'n", "telling")
  out_text = out_text.replace("gall'ry", "gallery")
  out_text = out_text.replace("try'n", "trying")
  out_text = out_text.replace("sick'ning", "sickening")
  out_text = out_text.replace("g'mornin", "good morning")
  out_text = out_text.replace("'liminate", "eliminate")
  out_text = out_text.replace("d'stroy", "destroy")
  out_text = out_text.replace("ass'state", "ass state")
  out_text = out_text.replace("d'you", "do you") # MA POTREBBE ESSERE ANCHE did you
  out_text = out_text.replace("x'tra", "extra")
  out_text = out_text.replace("n'don't", "and don't")
  out_text = out_text.replace("loo'd", "looed")
  #out_text = out_text.replace("droon'd", "drooned") # MANCANTE
  out_text = out_text.replace("droon'd", "drowned")
  out_text = out_text.replace("ladies' man", "seducer")
  out_text = out_text.replace("it'e", "it's")
  out_text = out_text.replace("i'e", "i've")
  out_text = out_text.replace("y'get", "you get")
  out_text = out_text.replace("y'say", "you say")
  out_text = out_text.replace("bid'ness", "bidness")
  out_text = out_text.replace("pu**y", "pussy")
  out_text = out_text.replace("emotion'ly", "emotionally")
  out_text = out_text.replace("fin'ly", "finally")
  
  # ORA HO SOSTITUITO TUTTI I "-" CON " "
  '''out_text = out_text.replace("bone-out", "bone out")
  out_text = out_text.replace("lo-lo", "lolo") # low rider car
  out_text = out_text.replace("g-ster", "gangster")
  out_text = out_text.replace("motherfu-", "motherfucker")
  out_text = out_text.replace("chi-raq", "Chiraq")'''

  # Capitalize missing words
  out_text = out_text.replace("chiraq", "Chiraq")
  out_text = out_text.replace("norsemen", "Norsemen")
  out_text = out_text.replace("dolemite", "Dolemite")
  out_text = out_text.replace("kahlil", "Khalil")
  out_text = out_text.replace("vedders", "Vedder")
  out_text = out_text.replace("cobains", "Cobain")
  #out_text = out_text.replace("dahrouge", "Dahrouge") MISSING
  out_text = out_text.replace("yamazaki", "Yamazaki")
  out_text = out_text.replace("oshun", "Oshun")
  out_text = out_text.replace("rafah", "Rafah")
  out_text = out_text.replace("brigadoon", "Brigadoon")
  out_text = out_text.replace("brignac", "Brignac")  
  out_text = out_text.replace("portmore", "Portmore")
  out_text = out_text.replace("defari", "Defari")
  out_text = out_text.replace("r&b", "RnB")

  out_text = out_text.replace("copycating", "copy cating")
  out_text = out_text.replace("grabba", "tobacco")
  out_text = out_text.replace("f**k", "fuck")
  out_text = out_text.replace("whatna", "what kind of")
  out_text = out_text.replace("twizza", "pussy")
  out_text = out_text.replace("hallucigent", "hallucinogen")
  out_text = out_text.replace("psychadelically", "psychedelically")
  out_text = out_text.replace("skanless", "shocking")
  out_text = out_text.replace("dooky", "dookie")
  out_text = out_text.replace("thugsta", "gangster")
  out_text = out_text.replace("thugster", "gangster")
  out_text = out_text.replace("mugster", "robber")
  out_text = out_text.replace("downest", "coolest")
  out_text = out_text.replace("mothafucking", "motherfucking")
  out_text = out_text.replace("muthaphuckkin", "motherfucking")
  out_text = out_text.replace("dopeman", "dope man")
  out_text = out_text.replace("cluckhead", "crackhead")
  out_text = out_text.replace("mothafucker", "mothafucka")
  out_text = out_text.replace("mutherfucka", "mothafucka")
  out_text = out_text.replace("tamika", "gorgeous")
  out_text = out_text.replace("hunnid", "hundred")
  out_text = out_text.replace("geah", "yeah")
  out_text = out_text.replace("ugk", "underground kings")
  out_text = out_text.replace("whoadie", "friend")
  out_text = out_text.replace(" lluminati", " illuminati")
  out_text = out_text.replace("motherfunk", "motherfuck")
  out_text = out_text.replace("spinfire", "spinning fire")
  out_text = out_text.replace("brilling", "brilliant")
  out_text = out_text.replace("embrangled", "embroiled")
  out_text = out_text.replace("modicum", "bean")
  out_text = out_text.replace("mecedes", "mercedes")
  out_text = out_text.replace("pineing", "pining")
  out_text = out_text.replace("karot", "karat")
  out_text = out_text.replace("smilled", "smiled")
  out_text = out_text.replace("trembing", "trembling")
  out_text = out_text.replace("towsack", "tow sack")
  out_text = out_text.replace("bloodlet", "bloodletting")
  out_text = out_text.replace("dirtpile", "dirt pile")
  out_text = out_text.replace("inutterable", "unutterable")
  out_text = out_text.replace("profitous", "profitable")
  out_text = out_text.replace("unlash", "unleash")
  out_text = out_text.replace("delphinidins", "delphinidin")
  out_text = out_text.replace("phillistines", "philistines")
  out_text = out_text.replace("shwip", "thanks")
  out_text = out_text.replace("hoah", "woah")
  out_text = out_text.replace("favirite", "favorite")
  out_text = out_text.replace("strenthened", "strengthened")
  out_text = out_text.replace("bolb", "bomb")
  out_text = out_text.replace("bebeyoncé", "beyoncé")
  out_text = out_text.replace("cowpoking", "lazy")
  out_text = out_text.replace("couchie", "vagina")
  out_text = out_text.replace("strictly dickly", "attracted only to men")
  out_text = out_text.replace("doublemint", "cocaine")
  out_text = out_text.replace("kimosabe", "friend")
  out_text = out_text.replace("permastoned", "permanently stoned")
  out_text = out_text.replace("whappen", "what's happening")
  out_text = out_text.replace("audiotune", "audio tune")
  out_text = out_text.replace("birdback", "bird back")
  out_text = out_text.replace("spirtualism", "spiritualism")
  out_text = out_text.replace("surviellance", "surveillance")
  out_text = out_text.replace("skrrt", "wheels screeching")
  out_text = out_text.replace("skrt", "wheels screeching")
  out_text = out_text.replace("im", "i'm")  # CONTIENE CARATTERI SPECIALI
  out_text = out_text.replace("its", "it's")  # CONTIENE CARATTERI SPECIALI
  out_text = out_text.replace("dont", "don't")  # CONTIENE CARATTERI SPECIALI
  out_text = out_text.replace("backaz", "backaS")
  out_text = out_text.replace("blaow", "boom")
  out_text = out_text.replace("wrecka", "wreck")
  out_text = out_text.replace("wwoah", "woah")
  out_text = out_text.replace("cuyah", "look here")
  out_text = out_text.replace("aiyyo", "aiyo")
  out_text = out_text.replace("bzzy", "buzzy")
  out_text = out_text.replace("gangsto", "gangster")
  out_text = out_text.replace("dkm", "don't know me") # IL TESTO CONTIENE DELLE ABBREVIAZIONI CON LEGENDA ALL'INIZIO DEL TESTO (IL TESTO REALE INIZIA PIÙ AVANTI, ANDREBBE TAGLIATA LA PRIMA PARTE)


  
  # Put a space before 's and 'd to recognize remaining words + 's or 'd (OCCHIO SOSTITUISCE ANCHE it's e altri)
  # ALTRA TECNICA COMBINANDO I VETTORI WORD2VEC
  #out_text = out_text.replace("'s", " 's") 
  #out_text = out_text.replace("'d", " 'd")

  #out_text = out_text.replace("", "")
  
  #out_text = out_text.replace("i'd", "i would") # but can also be 'i had'
  #out_text = out_text.replace("ain't", "am not") # but ain't can also be 'are not', 'is not', ...

  # NEI TESTI CI SONO ANCHE DEGLI ERRORI DI BATTITURA, QUINDI ALCUNE SEQUENZE SIAMO COSTRETTI A SALTARLE

  #if '"' in text:
  #if "in'" in text:
  #if "60'" in text:
  #if "3x" in text:
  #if "x3" in text.lower():
  #if "'ve" in text.lower():
  #if "why're" in text.lower():
  #if "'ll" in text.lower():
  #if "ya'll" in text.lower() or "y'all" in text.lower():
  #if "weren't" in text.lower() or "mustn't" in text.lower() or "shouldn't" in text.lower():
  #if "everything's" in text.lower() or "here's" in text.lower():
  #if "'twas" in text.lower() or "'ceptin'" in text.lower():
  #if "38s'll" in text.lower() or "5th'll" in text.lower() or "187'll" in text.lower():
  #if "i got a story a story to tell about a long road back from a sory of hell i been flying almost every night" in text.lower():
  #if "'\"" in text:
  
  #if "g'mornin" in text.lower():# or "" in text.lower() or "'st" in text.lower() or "" in text.lower():

  #if "forty-four lies" in text.lower() or "before i fall away" in text.lower():
  #if "*" in text:
  #if "you know they are not even interested in votin" in text.lower():
  #if "interested in votin" in text.lower():
  #if "alchemist." in text:
  
  #if "when i'm behind the wheel i'm in the fast lane got the the radio on to bob tench" in out_text or \
  #   "busted the safe stole the key to the city made out with your sister the one i think is pretty i think" in out_text or \
  #   "baby baby's looking better than fine she's got a brand new dress and she's ready to shine got her heart broke bad by some silly guy didn't know who the" in out_text or \ # Errore nel testo (anche online): noody invece di nobody
  #   "washington d c is pretty but it ain't my city i need a town here where i can get down i follow traces of faces of people baby and when they get it on the world turns brown in my " in out_text \ # Errore nel testo (anche online): washingto invece di washington
  #   "she got out of the car w a cocaine nose and a nosejob she wore her pants so tight wasn't too bright it doesn't matter much when you're" in out_text \ # Errore nel testo (anche online): allrigt invece di allright
  #   "melody griffins sing a song of liberty ga ga ga ga eclectic melody's popping out oh hairy lips oompah" in out_text \ #  Errore nel testo: see'est invece di c'est
  #   :
    #print("text before preprocessing: ", text)
    #print("text after preprocessing: ", out_text)
    #print("*"*40)

  #  print("*"*40)
  #  print("text before preprocessing:")
  #  print(text)
  #  print("="*40)
  #  print("text after preprocessing:")
  #  print(out_text)
  #  print("*"*40)

  #print("="*40)
  #print("lyrics after preprocessing:\n", out_text)

  '''print("*"*40)
  print("text before preprocessing:")
  print(text)
  print("="*40)
  print("text after preprocessing:")
  print(out_text)'''
  
  return out_text<jupyter_output><empty_output><jupyter_text>Read input data<jupyter_code># Read CSV file and get columns of interest
data = pd.read_csv(dataset_path)
#data = pd.read_csv(input_dataset_path) # PER TEST TRA TESTI PRIMA PREPROCESSING E DOPO

data = data[['genre', 'lyrics']][:5000].dropna()<jupyter_output><empty_output><jupyter_text>Display input data (optionally using an interactive table)<jupyter_code>#from google.colab.data_table import DataTable
#DataTable(data)

data[:20]<jupyter_output><empty_output><jupyter_text>Preprocess input data<jupyter_code># Preprocess lyrics
print("Preprocessing lyrics...")
data['lyrics'] = data['lyrics'].apply(lyrics_preprocessing)

#print("\data:")
#print(data[:10])

# Removing non-english lyrics
print("Removing non-english lyrics detected during preprocessing...")
data.dropna(inplace=True)

print("\data:")
print(data[:10])<jupyter_output><empty_output><jupyter_text>Display preprocessed data (optionally using an interactive table)<jupyter_code>#data = pd.read_csv(dataset_path) # PER TEST TRA TESTI PRIMA PREPROCESSING E DOPO

#from google.colab.data_table import DataTable
#DataTable(data)

data[:20]<jupyter_output><empty_output><jupyter_text>Store preprocessed lyrics into a CSV file<jupyter_code>data.to_csv(preprocessed_dataset_path, mode="w+", encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13986495260355797, 'fraction_numerical': 0.0056226464095572, 'mean_word_length': 3.0858977079796266, 'pattern_counts': {'":': 11, '<': 64, '<?xml version=': 0, '>': 88, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '580534', 'license': 'MIT', 'n_tokens_mistral': 26895, 'n_tokens_neox': 25797, 'n_words': 7518, 'path': 'Lyrics_preprocessing.ipynb', 'repo_name': 'NLP-Lyrics-Team/nlp-lyrics', 'size': '77021'}