<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd
import os
import pathlib
data_root = pathlib.Path('/content/drive/MyDrive/tobaco_OCR/')

print(data_root)
for item in data_root.iterdir():
  print(item)
def get_file_paths_and_labels(data_root):
     text_paths = [str(path) for path in data_root.glob('*/*.txt')]
     return text_paths

text_paths = get_file_paths_and_labels(data_root)
print(text_paths)
texts = []
for this_text in text_paths:
    with open(this_text) as f:
        lines = f.readlines()
        lines  = ' '.join(lines)
        texts.append(lines)

print(len(texts))
df = pd.DataFrame(list(zip(text_paths, texts)),
               columns =['Path', 'text'])
df.head()
import re
df['text'] = [re.sub(r'[^\w\s]','',s) for s in df['text']]
df['text']  = [s.replace('\n','') for s in df['text']]
df
## Tokenize, Lemmatize, stopwords removal
import spacy 
import nltk
nlp = spacy.load("en", disable=['parser', 'tagger', 'ner'])
from nltk.corpus import stopwords
nltk.download('stopwords')
stops = stopwords.words("english")

def normalize(comment, lowercase, remove_stopwords):
    if lowercase:
        comment = comment.lower()
    comment = nlp(comment)
    lemmatized = list()
    for word in comment:
        lemma = word.lemma_.strip()
        if lemma:
            if not remove_stopwords or (remove_stopwords and lemma not in stops):
                lemmatized.append(lemma)
    return " ".join(lemmatized)


df['text'] = df['text'].apply(normalize, lowercase=True, remove_stopwords=True)
df
#Tokenize
import nltk
nltk.download('punkt')
nltk.download('wordnet')
df['text'] = [nltk.word_tokenize(s) for s in df['text']]
df
from gensim.models.fasttext import FastText
word_tokens = df['text']

# Defining values for parameters
embedding_size = 300
window_size = 5
min_word = 5
down_sampling = 1e-2
 
# %%time 
fast_Text_model = FastText(word_tokens,
                      size=embedding_size,
                      window=window_size,
                      min_count=min_word,
                      sample=down_sampling,
                      workers = 4,
                      sg=1,
                      iter=100)
!mkdir -p saved_model
fast_Text_model.save('saved_model/my_model')
!zip -r '/content/saved_model.zip' '/content/saved_model'
from google.colab import files
files.download('/content/saved_model.zip')
# from gensim.models import Word2Vec
# # Save fastText gensim model
# fast_Text_model.save("model/ft_model_yelp")
# # Load saved gensim fastText model
# fast_Text_model = Word2Vec.load("model/ft_model_yelp")
!pip install fasttext
import fasttext
import fasttext.util
ft = fasttext.load_model('cc.en.300.bin')


	
# Check word embedding for a perticular word
l = fast_Text_model.wv['chicken']
print(len(l))
print(l)
def vectorize_str(token_list):
    embedd = []
    for word in token_list:
        embedd.append(fast_Text_model.wv['word'])

df['text'] = [fast_Text_model.wv['chicken'] for word df['text']]
biglist = []
for this_list in df['text']:
    for i in this_list:
        biglist.append(i)

print(len(biglist))
print(len(set(biglist)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13029627269831157, 'fraction_numerical': 0.00541573749601784, 'mean_word_length': 3.6041055718475072, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '235920', 'license': 'MIT', 'n_tokens_mistral': 1114, 'n_tokens_neox': 1077, 'n_words': 243, 'path': 'old_and_other_codes/tobaco3482_text.ipynb', 'repo_name': 'R4HUL-ROY/Multimodal_feature_extraction', 'size': '3139'}