<jupyter_start><jupyter_text>Pipeline to split sentences and store them in S3 bucket - Methods for sentence splitting **WARNING!!** before installing *fasttext* you need to install *pybind11* which does not install automatically as dependency.<jupyter_code>import re
import random
import nltk.data
import spacy 
import string
from collections import Counter
from collections import defaultdict
import nltk
import uuid
import json
import boto3
import csv
import s3fs
import codecs
import fasttext

path_to_pretrained_model = "/Users/dafirebanks/Projects/models/fasttext/lid.176.bin"
path_to_pretrained_model = "C:/Users/jordi/Documents/models/fasttext/lid.176.bin"
fmodel = fasttext.load_model(path_to_pretrained_model)

en_tokenizer = nltk.data.load("tokenizers/punkt/english.pickle")
es_tokenizer = nltk.data.load("tokenizers/punkt/spanish.pickle")
def remove_html_tags(text):
    """Remove html tags from a string"""
    return re.sub(re.compile('<.*?>'), '', text)

def replace_links(text):
    text = re.sub(r'http\S+', '[URL]', text)
    return re.sub(r'www\S+', '[URL]', text)

def remove_multiple_spaces(text):
    return re.sub('\s+', ' ', text)

def parse_emails(text):
    """ 
    Remove the periods from emails in text, except the last one
    """
    emails = [email if email[-1] != "." else email[:-1] for email in re.findall(r"\S*@\S*\s?", text)]
    
    for email in emails:
        new_email = email.replace(".", "")
        text = text.replace(email, new_email)
        
    return text

def parse_acronyms(text):
    """ 
    Remove the periods from acronyms in the text (i.e "U.S." becomes "US") 
    """

    acronyms = re.findall(r"\b(?:[a-zA-Z]\.){2,}", text)
         
    for acronym in acronyms:
        new_acronym = acronym.replace(".", "")
        text = text.replace(acronym, new_acronym)
        
    return text

def english_preprocessing(txt):
    """
    Steps in the preprocessing of text:
        1. Remove HTML tags
        2. Replace URLS by a tag [URL]
        3. Replace new lines and tabs by normal spaces - sometimes sentences have new lines in the middle
        4. Remove excessive spaces (more than 1 occurrence)
        5. Parse abreviations and acronyms
    """
    txt = replace_links(remove_html_tags(txt)).strip()#.replace("\n", " ").replace("\t", " ").strip()
    txt = remove_multiple_spaces(txt)
    txt = parse_emails(txt)
    txt = parse_acronyms(txt)
    
    new_txt = ""
    all_period_idx = set([indices.start() for indices in re.finditer("\.", txt)])
    
    for i, char in enumerate(txt):
        if i in all_period_idx:
            # Any char following a period that is NOT a space means that we should not add that period
            if i + 1 < len(txt) and txt[i + 1] != " ":
                continue
            
            # Any char that is a number following a period will not count. 
            # For enumerations, we're counting on docs being enumerated as "(a)" or "(ii)", and if not, they will be separated by the . after the number ("3. Something" will just be "Something" as a sentence)
            if i + 2 < len(txt) and txt[i + 2].isnumeric(): 
                continue
            
            # If we wanted to have all numbered lists together, uncomment this, and comment out the previous condition
#             if i + 2 < len(txt) and not txt[i + 2].isalpha(): 
#                 continue
            
        new_txt += char

    return new_txt

def english_postprocessing(sents, min_num_words=4):
    """
    Remove sentences that are made of less than a given number of words. Default is 4
    """
    
    return [sent for sent in sents if len(sent.split()) >= min_num_words]

def get_nltk_sents(txt, tokenizer, extra_abbreviations=None):
    if extra_abbreviations:
        tokenizer._params.abbrev_types.update(extra_abbreviations)
        
    sents = tokenizer.tokenize(txt)
    return sents

def aws_credentials_from_file(f_name):
    with open(f_name, "r") as f:
        creds = json.load(f)
    
    return creds["aws"]["id"], creds["aws"]["secret"]

def format_sents_for_output(sents, doc_id):
    formatted_sents = {}

    for i, sent in enumerate(sents):
        formatted_sents.update({f"{doc_id}_sent_{i}": {"text": sent, "label": []}})

    return formatted_sents


def output_sents(sents, f_name, f_uuid, country, bucket, output_dir):

    sents_json = {}
    fformat = f_name.split(".")[-1]
    sents_json[f_uuid] = {"metadata":
                              {"n_sentences": len(sents),
                               "file_name": f_name,
                               "file_format": fformat,
                               "country": country},
                          "sentences": format_sents_for_output(sents, f_uuid)}
    
    s3.Object(bucket, f"{output_dir}/{f_uuid}_sents.json").put(Body=(json.dumps(sents_json, indent=4)))
    
    
def filenames_for_country(country, s3, bucket):
    metadata_fname = f"metadata/{country}_metadata.csv"
    obj = s3.Object(bucket_name = bucket, key = metadata_fname)
    
    filenames = []
    i = 0
    for row in csv.reader(codecs.getreader("utf-8")(obj.get()['Body'])):
        # Add original file ID without the file format
        filenames.append(row[3][:-4])
    
    return filenames<jupyter_output><empty_output><jupyter_text>- Actual pipelineReads from the S3 Bucket english documents, outputs to test_sentencesFormat of credentials JSON file:```{    "aws": {        "id": "AWS ID",        "secret": "AWS SECRET"    }}```<jupyter_code># Connection to AWS Jordi's way
folder = 'C:/Users/jordi/Google Drive/Els_meus_documents/projectes/CompetitiveIntelligence/WRI/Notebooks/credentials/' # TODO: change to your local path
file_name = 'AWS_S3_keys_wri.json' # TODO: Change to your filename
file = folder + file_name

with open(file, 'r') as dict:
    credentials = json.load(dict)
                                      
aws_id = list(credentials)[0]
aws_secret = list(credentials.values())[0]
region = 'us-east-1'
bucket = "wri-nlp-policy"
countries = ['India', 'USA']
s3 = boto3.resource(
    service_name = 's3',
    region_name = region,
    aws_access_key_id = aws_id,
    aws_secret_access_key = aws_secret
)
credentials_file = '/Users/dafirebanks/Documents/credentials.json'
aws_id, aws_secret = aws_credentials_from_file(credentials_file)
region = 'us-east-1'

s3 = boto3.resource(
    service_name = 's3',
    region_name = region,
    aws_access_key_id = aws_id,
    aws_secret_access_key = aws_secret
)

bucket = 'wri-nlp-policy'
countries = ['India', 'USA']<jupyter_output><empty_output><jupyter_text>This pipeline is ready to be ran on the India documents only<jupyter_code># Define abbreviations per country - this can work for all english documents
usa_abrevs = {"no", "sec", "cong", "dist", "doc"}
india_abrevs = {"sub", "subs", "ins", "govt", "dy", "dept", "deptt", "ptg"} 

# Define country and output directory in S3 bucket
country = "India"
out_dir = "english_documents/test_sentences"
select_filenames = filenames_for_country(country, s3, bucket)

i = 0
for obj in s3.Bucket(bucket).objects.all().filter(Prefix="english_documents/text_files/"):
    
    # Don't get the directory itself
    if not obj.key.endswith("/"):
        file_id = obj.key.replace("english_documents/text_files/", "").replace(".txt", "")
        if file_id in select_filenames:
            text = obj.get()['Body'].read().decode('utf-8')
            preprocessed = english_preprocessing(text)
            # NOTE: Change abreviations in the future specific to country
            sents = get_nltk_sents(preprocessed, en_tokenizer, india_abrevs)
            post_processed_sents = english_postprocessing(sents, min_num_words=5)
            output_sents(post_processed_sents, obj.key, file_id, country, bucket, out_dir)

    i += 1
    if i == 2:
        break
    if i % 100 == 0:
        print(f"Processed {i} documents")<jupyter_output><empty_output><jupyter_text>This cell incorporates the language detection<jupyter_code>all_abrevs = {"no", "sec", "cong", "dist", "doc", "sub", "subs", "ins", "govt", "dy", "dept", "deptt", "ptg"} 

# Define country and output directory in S3 bucket
country = "India"
out_dir = "english_documents/sentences"
# select_filenames = filenames_for_country(country, s3, bucket)
select_filenames = ['732b7e29d73091c2d10b3f16146ca7ed70ca83de']

i = 0
for obj in s3.Bucket(bucket).objects.all().filter(Prefix = "english_documents/text_files/new/"):
    
    # Don't get the directory itself
    if not obj.key.endswith("/"):
        file_id = obj.key.replace("english_documents/text_files/new/", "").replace(".txt", "")
#         if file_id in select_filenames:
#             # Uncomment to do language detection before sentence splitting        
        text = ''
        for line in obj.get()['Body'].read().splitlines():
            line = line.decode('utf-8')
            try:
                result = fmodel.predict([line])

                if result[0][0][0][-2:] == 'en' and result[1][0][0] > 0.7:
                    text = text + line + "\n"
            except:
                print(f"*** In document{obj.key}, this line could not be processed by the language detection\n{line} ")
                continue
        preprocessed = english_preprocessing(text)
        # NOTE: Change abreviations in the future specific to country
        sents = get_nltk_sents(preprocessed, en_tokenizer, all_abrevs)
        post_processed_sents = english_postprocessing(sents, min_num_words=5)
        output_sents(post_processed_sents, obj.key, file_id, country, bucket, out_dir)

    i += 1
#     if i == 2:
#         break
    if i % 100 == 0:
        print(f"Processed {i} documents")
select_filenames<jupyter_output><empty_output><jupyter_text>Extra functions to read the abbreviations file from S3, and move files between S3 bucket folders<jupyter_code>abbreviations_fname = f"abbreviations/english_abbreviations.txt"
obj = s3.Object(bucket_name = bucket, key = abbreviations_fname)
obj.get()['Body'].read().decode('utf-8').split("\n")
def move_s3_object(obj_name, obj_old_folder, obj_new_folder, s3, bucket_name):
    """
    Moves an object from a given S3 folder to another by copying it to the new folder it and then deleting it from the old one
    """
    try:
        s3.Object(bucket_name, f"{obj_new_folder}/{obj_name}").copy_from(CopySource=f"{bucket_name}/{obj_old_folder}/{obj_name}")
        response = s3.Object(bucket_name, f"{obj_old_folder}/{obj_name}").delete()
    except Exception as e:
        print(f"Exception while moving {obj_name} from {obj_old_folder} to {obj_new_folder}:")
        print(e)
move_s3_object("english_abbreviations.txt", "english_documents", "abbreviations", s3, "wri-nlp-policy")<jupyter_output><empty_output><jupyter_text>Testing language detection on sample file<jupyter_code>with open("/Users/dafirebanks/Downloads/011e041eced3f90558b955e5f65c7682608e35cc.txt", "r") as f:
    text = f.read()<jupyter_output><empty_output><jupyter_text>1. Apply language detection before sentence splitting<jupyter_code>for line in text.splitlines():
    try:
        result = fmodel.predict([line])
        print("--------------")
        print("Line:", line, " || Language:", result[0][0][-2:])
        print("lang_detect:", detect_langs(line))
        print("Probability:", result[1][0])
        print("--------------")
        if result[0][0][0][-2:] == 'en' and result[1][0][0] > 0.7:
            text = text + line + "\n"
            print(">>>>>>>>>>> English sentence:", line)
    except:
        print("*** ", line)
        continue<jupyter_output>--------------
Line: 12456  || Language: ['__label__gu']
lang_detect:
***  12456
--------------
Line: THE GAZETTE OF INDIA, NOVEMBER 10, 2012 (KARTIKA 19, 1934)  || Language: ['__label__en']
lang_detect:
[en:0.999994980952714]
Probability: [0.3980843]
--------------
--------------
Line: [PART III-SEC. 4  || Language: ['__label__en']
lang_detect:
[id:0.9999939202976486]
Probability: [0.5178548]
--------------
--------------
Line: University Grants Commission  || Language: ['__label__en']
lang_detect:
[en:0.9999950418111176]
Probability: [0.7218409]
--------------
>>>>>>>>>>> English sentence: University Grants Commission
--------------
Line: No. F. 15-3/2012 (ARC)  || Language: ['__label__en']
lang_detect:
[pt:0.9999956692468923]
Probability: [0.62202793]
--------------
--------------
Line: 8 October, 2012  || Language: ['__label__en']
lang_detect:
[en:0.9999956111997939]
Probability: [0.83474034]
--------------
>>>>>>>>>>> English sentence: 8 October, 2012
--------------
Line: In exerc[...]<jupyter_text>2. Apply language detection after sentence splitting<jupyter_code>all_abrevs = {"no", "sec", "cong", "dist", "doc", "sub", "subs", "ins", "govt", "dy", "dept", "deptt", "ptg"} 
preprocessed = english_preprocessing(text)
# NOTE: Change abreviations in the future specific to country
sents = get_nltk_sents(preprocessed, en_tokenizer, all_abrevs)
post_processed_sents = english_postprocessing(sents, min_num_words=5)
for sent in post_processed_sents:
    try:
        result = fmodel.predict(sent)
        print("--------------")
        print("Sentence:", sent, " || Language:", result[0][0][-2:])
        print("lang_detect:", detect_langs(sent))
        print("Probability:", result[1][0])
        print("--------------")
        if result[0][0][0][-2:] == 'en' and result[1][0][0] > 0.7:
            print(">>>>> English sentence:", sent)
    except:
        print("*** ", sent)
        continue
sample_text = 'SHIAT ARd TIYT R did TE ford and of at for fineft 3ite de 27 34G GRI Fine GIT to of 52 of 34-8RT (5) to Artafid HISTH to of anfull GRI 7th to fited 3+1 stut to finet 3/rl and the of ETRI (2) TETT (6) A, 2/06 as of FETTH R, sther: 2106 to affered Ane of ETRI 53 to FETTH WR, thell th fore GIR, of 34165 FOLTH GRI FOUTER and VR, Affer are of ETRI 63 =, 2106 'Te 21 frest at des of at 27 \T ARTH to thent 34d°T of and of thind 3m5 thet - st fil ETRI 68 to FETTH 9R, Artified The of and of - (1) d/d this Hifing the to 577 of TR of 21/0 ofl n. ARE feel 317 of TS Hiffi, his Ì, Affer the of TETT 13 21/0 21 Deft to fare The of do AM, 34-8IRT (1) of dog () to CTEF to  ETH of finefi of à 3461 + of at H off TH PREAT of Hiff AFE 211 DRA T7 3461 as  H(ed Uch R TH at of ETRI 71 to TETTH 4R, Prifical fail "71. of feed and of = FROH of Urto 401 to for Thell for TE 3fect thing TF to for Hetch 35d to fre *t 548, Writh YER, Ad At, RT of Theld to 039 43 to - GRT to JeTT Fatter of TS Haled R Filler 5097/i'
result = fmodel.predict(sample_text)
from langdetect import DetectorFactory, detect, detect_langs
# detect(sample_text) #  'cs'
detect_langs(sample_text)  # [cs:0.7142840957132709, pl:0.14285810606233737, sk:0.14285779665739756]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.140484100616991, 'fraction_numerical': 0.03390060343074107, 'mean_word_length': 3.143258426966292, 'pattern_counts': {'":': 13, '<': 36, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '141808', 'license': 'RSA-MD', 'n_tokens_mistral': 5297, 'n_tokens_neox': 4908, 'n_words': 1339, 'path': 'tasks/text_preprocessing/notebooks/sent_splitting_lang_detection.ipynb', 'repo_name': 'thefirebanks/policy-data-analyzer', 'size': '14749'}