<jupyter_start><jupyter_text>Text preprocessing<jupyter_code>#import some libraries

from datatable import by, dt, f
import datatable, os, pandas as pd, numpy as np

import matplotlib.pyplot as plt
import seaborn as sns

%cd ..

data_r = datatable.fread("./input/data_no_dupes_top50.csv", max_nrows=10)
data = data_r[:, f[:].remove([f["#tags"], f["count_duples"], f[ "Tags"]])]

data.head(2)
X = data[:, ["Title", "Body"]].to_pandas()
Y = data[:, data.names[2:]]

print("X --> ", X.columns)
Y.head(1)<jupyter_output>X -->  Index(['Title', 'Body'], dtype='object')<jupyter_text>Logic below<jupyter_code>row = X.iloc[0]
X.head(1)
row = X.iloc[0]
row<jupyter_output><empty_output><jupyter_text>Steps - * Sample* Remove code snippets* Remove special characters from remaining text* Remove stop words* remove HTML tags* convert to lowercase* Stem the words using snowball stemmer<jupyter_code>import re
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer

from nltk.tokenize import word_tokenize # to break sentences into words

def striphtml(data):
    # find the html tags and remove them
    patt = re.compile('<.*?>')
    cleantxt = re.sub(patt, ' ', str(data))
    return cleantxt

def process_row(row):
    stop_words = set(stopwords.words('english'))
    stemmer = SnowballStemmer("english")

    questions_with_code =  0
    len_before = 0
    len_after = 0

    is_code = 0
    title, question = row[0], row[1]

    if "<code>" in question:
        questions_with_code += 1
        is_code = 1

    x = len(question) + len(title)
    len_before += x

    # Find the code piece of question
    code = str(re.findall(r'<code>(.*?)</code>', question, flags=re.DOTALL))
    # remove the code
    question = re.sub('<code>(.*?)</code>', '', question, flags=re.MULTILINE|re.DOTALL)

    title=title.encode('utf-8')

    # combine question and title, remove special characters, convert to lowercase
    queti = str(title) + str(question)
    queti = re.sub(r'[^A-Za-z]+',' ',queti).lower()

    # get words
    words = word_tokenize(str(queti))

    #Removing all single letter and and stopwords from question except for the letter 'c'
    queti_stop = ' '.join(str(stemmer.stem(j)) for j in words if j not in stop_words and (len(j)!=1 or j=='c'))

    len_after += len(queti_stop)

    return queti_stop, len(queti_stop), is_code, code, x

import pandas as pd
import dask.dataframe as dd
from dask.distributed import Client
from dask import delayed
from dask.multiprocessing import get
import numpy as np

n = 1000000

data = X
ddata = dd.from_pandas(data, npartitions=12)
res = ddata.map_partitions(lambda df: df.apply(lambda row: process_row(row), axis = 1, result_type="expand")).compute(scheduler="threads")
res.columns = ['queti_stop', 'len_queti_stop', 'is_code', 'code', 'len_pre']
res<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13934713934713935, 'fraction_numerical': 0.01053001053001053, 'mean_word_length': 4.407969639468691, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '587076', 'license': 'MIT', 'n_tokens_mistral': 1039, 'n_tokens_neox': 1005, 'n_words': 269, 'path': 'notebooks/test.ipynb', 'repo_name': 'k-pulkit/Keywork-Extraction-Kaggle-Stackoverflow', 'size': '2849'}