<jupyter_start><jupyter_text>1. Data extractionExtract text contained in json files and save it in a dataframe for further pre-processing.<jupyter_code>import os
import json
import numpy as np
import pandas as pd

from tqdm.notebook import tqdm
tqdm.pandas()

#import spacy, en_core_web_sm
import nltk
from nltk.tokenize import sent_tokenize
nltk.download('punkt')
# Check the different keys in the json file
length=0
#for id_file in tqdm(range(1,14)):
for id_file in range(1,14):
    file_path = "/raid/antoloui/Master-thesis/Data/Original/" + str(id_file) + ".json"

    with open(file_path) as f:
        data = json.load(f)

        # Loop over each document
        keys = []
        for i, doc in enumerate(data):
            for key, value in doc.items():
                keys.append(key)

        myset = set(keys)
        print("File {} - Keys: {} - Documents: {}".format(id_file, myset, len(data), length/len(data)))
id_file = 1

rows = []
file_path = "/raid/antoloui/Master-thesis/Data/Original/" + str(id_file) + ".json"

with open(file_path) as f:
    data = json.load(f)  # data is a list of dict of the form: {'text':['...'], 'uri':['...']}

    print("Extracting text from {} documents in file '{}.json'...".format(len(data), id_file))
    for i, doc in enumerate(data):
        if doc.get('text') is not None:
            text = ' '.join(doc.get('text')) # Flatten list of strings
            row_dict = {'Text': text, 'Length': len(text)}
            rows.append(row_dict)

print("Creating dataframe...")
df = pd.DataFrame(rows)
print("Done !")
print("Max number of characters in a doc: {}".format(df.Length.max()))
print("Total number of docs: {}".format(len(df.index)))<jupyter_output>Max number of characters in a doc: 2621440
Total number of docs: 11692<jupyter_text>2. Data ProcessingIn order to use "create_pretraining_data.py" from BERT repository, the input must be a plain text file, with one sentence per line and one blank line between documents:  * One sentence per line. These should ideally be actual sentences, not entire paragraphs or arbitrary spans of text. (Because we use the sentence boundaries for the "next sentence prediction" task).  * Blank lines between documents. Document boundaries are needed so that the "next sentence prediction" task doesn't span between documents.  They advise to perform sentence segmentation with an off-the-shelf NLP toolkit such as spaCy.<jupyter_code>df = df.head()<jupyter_output><empty_output><jupyter_text>2.1. Corpus Cleaning<jupyter_code>print("Cleaning corpus of text...")
df.Text = df.Text.replace('\s+', ' ', regex=True)  # Remove duplicate spaces
df.Text = df.Text.str.encode('ascii', 'ignore').str.decode('utf-8')   # Encode in ascii to remove weird characters such as \uf0a7
#df.Text = df.Text.str.lower()  # Lower case all strings
print("Done !")<jupyter_output>Cleaning corpus of text...
Done !<jupyter_text>2.2. Sentence segmentation<jupyter_code>def spacy_segmentation(doc_text):
    """
    Given a string, segment it by sentences (performed by Spacy).
    """
    nlp = en_core_web_sm.load()
    nlp.max_length = 2621500  # because larger document has a size of 2621440 char
    doc = nlp(doc_text)
    sentences = list(doc.sents)
    return [sent.text for sent in sentences]


def nltk_segmentation(doc_text):
    """
    Given a string, segment it by sentences (performed by nltk).
    """
    return sent_tokenize(doc_text)<jupyter_output><empty_output><jupyter_text>2.3. Sentence cleaning<jupyter_code>def sent_cleaning(list_sent):
    """
    """
    # Remove sequences of special characters
    spec_char = set(',?;.:/=+%`¨*$€-_())°!§\'\"&@#~®†ºπ‡¬≈©◊~∞µ…÷≠<>^')
    list_sent = [' '.join([x for x in sent.split() if len(x)<=2 or not all(c in spec_char for c in x)]) for sent in list_sent]
    
    # If line begins with a number, remove the number   
    list_sent = [sent.split(maxsplit=1)[1] if (len(sent.split(maxsplit=1))>1 and sent.split(maxsplit=1)[0].isdigit()) else sent for sent in list_sent]
    
    # If line begins with a unique special char, remove that char
    list_sent = [sent.split(maxsplit=1)[1] if (len(sent.split(maxsplit=1))>1 and len(sent.split(maxsplit=1)[0])==1 and sent.split(maxsplit=1)[0] in spec_char) else sent for sent in list_sent]
    
    # Keep only sentences with more than 2 words and less than 200 words
    list_sent = [sent for sent in list_sent if (len(sent.split())>2 and len(sent.split())<200)]
    return list_sent<jupyter_output><empty_output><jupyter_text>2.4. Plain text conversion<jupyter_code>def sent_convert(list_sent):
    """
    Given a list of string sentences, return one unique string where
    sentences are separated by newlines.
    """
    return "\n".join(list_sent)<jupyter_output><empty_output><jupyter_text>2.5. Apply all cleaning functions<jupyter_code>print("Segmenting sentences...")
df['Text'] = df['Text'].progress_apply(nltk_segmentation)
print("Cleaning sentences...")
df['Text'] = df['Text'].progress_apply(sent_cleaning)
print("Concatenating all sentences...")
df['Text'] = df['Text'].progress_apply(sent_convert)<jupyter_output>Segmenting sentences...<jupyter_text>2.6. Concatenate all documents<jupyter_code>sentences = "\n\n".join(df["Text"])
print("Saving sentences to output file (total of {} words)...".format(len(sentences.split())))
output_file = "../../Data/Preprocessed/output.txt"
with open(output_file, "w+") as f:
    f.write(sentences)
print("Done !")<jupyter_output>Saving sentences to output file (total of 102836 words)...
Done !<jupyter_text>2.7. Check size of output files<jupyter_code>directory = '../../Data/Preprocessed/'
files = os.listdir(directory).sort()
for filename in sorted(os.listdir(directory)):
    if filename.endswith(".txt"):
        file_path = directory + filename
        print("Size of '{}': {:.4f} GB".format(filename, os.path.getsize(file_path)/(1024*1024*1024)))<jupyter_output>Size of 'all_text.txt': 16.1553 GB
Size of 'output.txt': 0.0006 GB
Size of 'text_1.txt': 1.8405 GB
Size of 'text_10.txt': 0.9190 GB
Size of 'text_11.txt': 2.1406 GB
Size of 'text_12.txt': 3.0498 GB
Size of 'text_13.txt': 0.0011 GB
Size of 'text_2.txt': 0.1328 GB
Size of 'text_3.txt': 0.3557 GB
Size of 'text_4.txt': 2.9462 GB
Size of 'text_5.txt': 1.1260 GB
Size of 'text_6.txt': 0.9585 GB
Size of 'text_7.txt': 0.8667 GB
Size of 'text_8.txt': 1.0881 GB
Size of 'text_9.txt': 0.7303 GB
{'fraction_non_alphanumeric': 0.14122257053291537, 'fraction_numerical': 0.02962382445141066, 'mean_word_length': 4.179383116883117, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '596745', 'license': 'MIT', 'n_tokens_mistral': 2290, 'n_tokens_neox': 2162, 'n_words': 629, 'path': 'scripts/data_cleaning/-/initial_cleaning/data_processing.ipynb', 'repo_name': 'antoilouis/netbert', 'size': '6380'}