<jupyter_start><jupyter_text>SCRAPER JW.ORG  Load modules<jupyter_code>#pip install -U selenium 
# install geckodriver if necessary

from selenium import webdriver
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.keys import Keys

from selenium.webdriver.support.ui import WebDriverWait
import time
g =r'C:\\Users\\romeo\\OneDrive\\Documents\\ELU\\geckodriver\\geckodriver.exe'<jupyter_output><empty_output><jupyter_text>Scrap yearly publications links<jupyter_code>h =r'https://www.jw.org/fon/nus%C9%9Bxwet%C9%9Bn/xojlawema-l%C9%9B/'
dr = webdriver.Firefox(executable_path = g)

dr.get(h)
yee = dr.find_elements_by_xpath("//select[@id='yearFilter']")
year = yee[0].text.split('\n')[1:]
year
li = []
for y in year:
    select = Select(dr.find_element_by_xpath("//select[@id='yearFilter']"))
    select.select_by_value(y)
    
    # click on the ba button to refresh page
    button = dr.find_element_by_xpath("//input[@class='primaryButton inputSearchButton']")
    button.click()
    
    time.sleep(5)
    #collect links
    al_li = dr.find_elements_by_xpath("//div[@id='pubsViewResults']//a[starts-with(@href,'/fon/')]")
    
    for el in al_li:
        li.append(el.get_attribute("href"))
        #print(el.get_attribute("href"))
        
    #remove duplicates
    
    mypee = list(dict.fromkeys(li))
    #len(mypee)
len(mypee)<jupyter_output><empty_output><jupyter_text>Scrap monthly articles links<jupyter_code>drv = webdriver.Firefox(executable_path = g)
links = []
for l in mypee:
    drv.get(l)
    
    all_elements = drv.find_elements_by_xpath("//div[@class='toc clearfix']//a[starts-with(@href,'/fon/nus%C9%9Bxwet%C9%9Bn/')]")

    for elem in all_elements:
        links.append(elem.get_attribute("href"))
#f = r'https://www.jw.org/fon/nus%C9%9Bxwet%C9%9Bn/xojlawema-l%C9%9B/at%C9%94xw%C9%9B-aluunsun-2016-t%C9%94n-t%C9%94n/'
#links
#all_elements = drv.find_elements_by_xpath("//div[@class='toc clearfix']//a[starts-with(@href,'/fon/nus%C9%9Bxwet%C9%9Bn/')]")

#for elem in all_elements:
#    links.append(elem.get_attribute("href"))
#len(links)
#remove duplicates
mylist = links
mylist = list(dict.fromkeys(mylist))
len(mylist)<jupyter_output><empty_output><jupyter_text>Scrap page content<jupyter_code>#g =r'C:\\Users\\romeo\\OneDrive\\Documents\\ELU\\geckodriver\\geckodriver.exe'
driver = webdriver.Firefox(executable_path = g)
fon = []
fre = []

def artscrap(ur):
    driver.get(ur)
    #elefon = driver.find_element_by_xpath("//div[@class='pGroup']")
    elefon = driver.find_elements_by_class_name("section")
    
    for j in elefon:
        fon.append(j.text)
    
    #inputElement = driver.find_element_by_xpath("//input[@class='jsAutoComplete jsShowFeatured jsAutoCompleteInput autoCompleteLanguageChoices']")
    inputElement = driver.find_element_by_xpath("//input[contains(@class, 'jsAutoComplete jsShowFeatured')]")

    inputElement.send_keys('Flansegbe')
    time.sleep(10)
    inputElement.send_keys(Keys.ENTER)
    inputElement.send_keys(Keys.ENTER)
    
    time.sleep(10) 
    
    elefre = driver.find_elements_by_class_name("section")

    for k in elefre:
        fre.append(k.text)
i=0
for u in mylist:
    print(i)
    artscrap(u)
    i=i+1
#mylist[241]
i=241
for u in mylist[241:]:
    print(i)
    artscrap(u)
    i=i+1
i=526
for u in mylist[526:]:
    print(i)
    artscrap(u)
    i=i+1
print(len(fon))
print(len(fre))
#till 344 it's ok. From 345 a difference of 1 noted with + 1 for french, ie should remove 344 for FRENCH.
a=345
print(fon[a])
print(fre[a+1])
# till 996 ok. At 997, need to add 2 to fre. So should remove 997 to fre
i = 999
print(fon[i])
print(fre[i+2])
ffon = fon
ffre = fre
ffre.pop(344)
ffre.pop(996)
print(len(ffon))
print(len(ffre))
# Export to CSV
# importing pandas as pd   
import pandas as pd   
     
# dictionary of lists   
dict = {'fon': ffon, 'french': ffre}   
df = pd.DataFrame(dict)  
df = df.drop_duplicates()
df['fon'].replace('', np.nan, inplace=True)
df['french'].replace('', np.nan, inplace=True)
#
df = df.dropna()
df.head()
# saving the dataframe  
df.to_csv('JW1.csv') 
# Transformations
df = df.apply(lambda x: x.str.replace('.”','”'))
df = df.apply(lambda x: x.str.replace(' »','»'))
df = df.apply(lambda x: x.str.replace('« ','«'))
df = df.apply(lambda x: x.str.replace('.»','»'))
df = df.apply(lambda x: x.str.replace('. »','»'))
df['french'][3] = re.sub(r'"[^"]*(?:""[^"]*)*"', lambda m: m.group(0).replace("\n", ""), df['french'][3])
df['french'][3] = re.sub(r'«[^"]*(?:""[^"]*)*»', lambda m: m.group(0).replace("\n", ""), df['french'][3])
df['french'][3] = re.sub(r'"[^"]+"', lambda m: m.group(0).replace("\n", ""), df['french'][3])
df['french'][3]
t = re.sub(r'"[^"]*(?:""[^"]*)*"', lambda m: m.group(0).replace("\n", ""), df['french'][3])
t = re.sub(r'«[^"]*(?:""[^"]*)*»', lambda m: m.group(0).replace("\n", ""), t)
t = re.sub(r'"[^"]+"', lambda m: m.group(0).replace("\n", ""), t)
t = re.sub(r'“[^"]*(?:""[^"]*)*”', lambda m: m.group(0).replace("\n", ""), t)
t
v = re.sub(r'"[^"]*(?:""[^"]*)*"', lambda m: m.group(0).replace("\n", ""), df['fon'][3])
v = re.sub(r'«[^"]*(?:""[^"]*)*»', lambda m: m.group(0).replace("\n", ""), v)
v = re.sub(r'"[^"]+"', lambda m: m.group(0).replace("\n", ""), v)
v = re.sub(r'“[^"]*(?:""[^"]*)*”', lambda m: m.group(0).replace("\n", ""), v)
v
# Tokenization test
import nltk
fontest = nltk.tokenize.sent_tokenize(v.replace(' »','»').replace('« ','«'))
frenchtest = nltk.tokenize.sent_tokenize(t.replace(' »','»').replace('« ','«'))
print(len(frenchtest))
print(len(fontest))
from nltk.tokenize import RegexpTokenizer
tokenizer = RegexpTokenizer(r'[^.?!\n]+')
fontest2 = list(map(str.strip, tokenizer.tokenize(v)))
frenchtest2 = list(map(str.strip, tokenizer.tokenize(t)))
frenchtest2
fontest2
print(len(frenchtest2))
print(len(fontest2))
df['fon'][3]
t<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2202258726899384, 'fraction_numerical': 0.021218343600273786, 'mean_word_length': 4.868473895582329, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '388795', 'license': 'MIT', 'n_tokens_mistral': 2483, 'n_tokens_neox': 2399, 'n_words': 418, 'path': 'DynamicScraperFon.ipynb', 'repo_name': 'Meomeoowww/JW_Fon_French_Scraper', 'size': '5844'}