<jupyter_start><jupyter_text>US Presidents & find judges - data wrangling  - Imports<jupyter_code>import requests
import pandas as pd
US_Presidents = pd.read_csv('data/us_presidents.csv')
US_Presidents.drop(['Unnamed: 0'], axis=1, inplace=True)
US_Presidents.tail()
US_Presidents.info()
US_Presidents['start'].index
def cleandates(df):
    df['end'].loc[43] = df['start'].loc[44]
    df['end'].loc[44] = 'January 20, 2021'
    for i in range(0,len(df['start'])):
        # spilts each whole string into list of 3 strings
        df['start'].loc[i] = df['start'].loc[i].split()
        # clean comma
        US_Presidents['start'][i][1] = US_Presidents['start'][i][1][0:1]
        # change str month to int month
        US_Presidents['start'][i][1] = US_Presidents['start'][i][1][0:1]
        # spilts each whole string into list of 3 strings
        df['end'].loc[i] = df['end'].loc[i].split()
        # clean comma
        US_Presidents['end'][i][1] = US_Presidents['end'][i][1][0:1]
        # change str month to int month
        US_Presidents['end'][i][1] = US_Presidents['end'][i][1][0:1]
        # change data columns format to largest date type to smallest
        month = US_Presidents['end'][i][0]
        day = US_Presidents['end'][i][1]
        year = US_Presidents['end'][i][2]
        US_Presidents['end'][i][2] = int(day)
        US_Presidents['end'][i][1] = month
        US_Presidents['end'][i][0] = int(year)
        month = US_Presidents['start'][i][0]
        day = US_Presidents['start'][i][1]
        year = US_Presidents['start'][i][2]
        US_Presidents['start'][i][2] = int(day)
        US_Presidents['start'][i][1] = month
        US_Presidents['start'][i][0] = int(year)
        # Turning month string to int
        if US_Presidents['end'][i][1] == "January":
            US_Presidents['end'][i][1] = 1
        if US_Presidents['end'][i][1] == "March":
            US_Presidents['end'][i][1] = 3
        if US_Presidents['end'][i][1] == "April":
            US_Presidents['end'][i][1] = 4
        if US_Presidents['end'][i][1] == "July":
            US_Presidents['end'][i][1] = 7
        if US_Presidents['end'][i][1] == "August":
            US_Presidents['end'][i][1] = 8
        if US_Presidents['end'][i][1] == "September":
            US_Presidents['end'][i][1] = 9
        if US_Presidents['end'][i][1] == "November":
            US_Presidents['end'][i][1] = 11
        if US_Presidents['start'][i][1] == "January":
            US_Presidents['start'][i][1] = 1
        if US_Presidents['start'][i][1] == "March":
            US_Presidents['start'][i][1] = 3
        if US_Presidents['start'][i][1] == "April":
            US_Presidents['start'][i][1] = 4
        if US_Presidents['start'][i][1] == "July":
            US_Presidents['start'][i][1] = 7
        if US_Presidents['start'][i][1] == "August":
            US_Presidents['start'][i][1] = 8
        if US_Presidents['start'][i][1] == "September":
            US_Presidents['start'][i][1] = 9
        if US_Presidents['start'][i][1] == "November":
            US_Presidents['start'][i][1] = 11
    return df
US_Presidents = cleandates(US_Presidents)
#change # US_Presidents['end'][i][0] - i = row number
US_Presidents<jupyter_output><empty_output><jupyter_text>-  Board of Immigration appeals data mining/web scraping (BIA)  - Imports<jupyter_code>from bs4 import BeautifulSoup as Bs4
# dates
import re
# Government website with 30 appeal immagration judges
url = 'https://www.justice.gov/eoir/board-of-immigration-appeals-bios#WilliamACassidy'

# bs4 gets web html
response = requests.get(url)
soup = Bs4(response.text, 'html.parser')

# Current judges of the appeal's court
bodytext = soup.find('div', class_='bodytext')
#names = []
bio = []
# find all names
for content in bodytext.find_all('p'):
    bio.append(content.text)
# Clean list of information
for i in bio:
    if i == 'Return to top of the page':
        j = bio.index(i)
        bio.pop(j)
        j = None
bio = bio[4:]
# Separate bio and name into individual lists, also, clean data
judges_bio = []
judges_names = []
for i in bio:
    # find all bios
    if len(i) > 100:
        # adding bios to bio list
        judges_bio.append(i)
        # checking for names in bios
        if '\n' in i:
            j = i.index('\n')
            # adding the names to list
            judges_names.append(i[:j])
    else:
        # adding the names to list
        judges_names.append(i)
judges_names.pop(-2)
judges_names.pop(-15)
judges_names.pop(-8)
# check if we have a bio for each appeals court judge
len(judges_names) == len(judges_bio)
judges_names[0]
judges_names
x = judges_bio
x[1][207:219]<jupyter_output><empty_output><jupyter_text>- find out which president administration appointed the judge<jupyter_code># find dates of appointment
date = []
l = []
target = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
def find_dates(x, target, date, l):
    for j in range(0,len(x)):
        for i in target:
            index = x[j].find(i)
            if index == -1:
                index = 10000
            l.append(index)
            if i == "December":
                # end = index+13
                s = min(l)
                string = x[j][s:s+15]
                date.append(string)
                l = []
    return date
date = find_dates(x, target, date, l)
date
# cleaning dates and finding administration
def clean_createadminlist(list):
    p = '.'
    administration = []
    for i in list:
        j = date.index(i)
        index = i.find(p)
        date[j] = date[j][0:index]
    return date, administration 
date, administration = clean_createadminlist(date)
#print(date, administration)
def cleanstrmonth(date):
    administration = []
    department = []
    target = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
    for i in range(0,len(date)):
        month = ((target.index(date[i][:-5])) + 1)
        date[i] = date[i].split()
        date[i][0] = month
        department.append("The Board of Immigration Appeals (BIA)")
    for i in range(0,(len(date)-1)):
        for j in range(0, len(US_Presidents['end'])):
            if (int(date[i][1]) >= US_Presidents['start'][j][0]) and (int(date[i][1]) <= US_Presidents['end'][j][0]):
                if date[i][0] >= US_Presidents['start'][j][1] or date[i][0] < US_Presidents['end'][j][1]:
                     administration.append(US_Presidents['president'][j])
    return date, administration, department
date, administration, department = cleanstrmonth(date)
US_Presidents<jupyter_output><empty_output><jupyter_text>- create dataframe<jupyter_code># create dataframe for appeals judges
dict = {'name': judges_names, 'bio': judges_bio, 'appointed_date': date, 'administration': administration, "department": department}  
BIA_judges = pd.DataFrame(dict)
BIA_judges['name'][0] = BIA_judges['name'][0][:-33]
BIA_judges['name'][1] = BIA_judges['name'][1][:-41]
BIA_judges['name'][2] = BIA_judges['name'][2][:-41]
# finished dataframe
print(BIA_judges.shape)
BIA_judges<jupyter_output>(30, 5)<jupyter_text>-  The Office of the Chief Immigration Judge (OCIJ) data mining/web scraping  - Imports<jupyter_code># Government website with 43 Chief Immigration Judge (OCIJ)
url = 'https://www.justice.gov/eoir/office-of-the-chief-immigration-judge-bios#Top'

# bs4 gets web html
response = requests.get(url)
soup = Bs4(response.text, 'html.parser')

# Current judges of the appeal's court
bodytext = soup.find('div', class_='bodytext')
#names = []
bio = []
# find all names
for content in bodytext.find_all('p'):
    bio.append(content.text)
# Clean list of information
for i in bio:
    if i == 'Return to top of the page':
        j = bio.index(i)
        bio.pop(j)
        j = None
bio = bio[3:]
bio
# Separate bio and name into individual lists, also, clean data
judges_bio = []
judges_names = []
for i in bio:
    # find all bios
    if len(i) > 100:
        # adding bios to bio list
        judges_bio.append(i)
        # checking for names in bios
        if '\n' in i:
            j = i.index('\n')
            # adding the names to list
            judges_names.append(i[:j])
    else:
        # adding the names to list
        judges_names.append(i)
judges_names.pop(-12)
judges_names.pop(-14)
judges_names.pop(-15)
judges_names
# check if we have a bio for each appeals court judge
len(judges_names) == len(judges_bio)<jupyter_output><empty_output><jupyter_text>- find out which president administration appointed the judge<jupyter_code># find dates of appointment
x = judges_bio
date = []
l = []
target = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
def find_dates(x, target, date, l):
    for j in range(0,len(x)):
        for i in target:
            index = x[j].find(i)
            if index == -1:
                index = 10000
            l.append(index)
            if i == "December":
                # end = index+13
                s = min(l)
                string = x[j][s:s+15]
                date.append(string)
                l = []
    return date
date = find_dates(x, target, date, l)
date
date[29] = 'March 2020.'
# cleaning dates and finding administration
def clean_createadminlist(list):
    p = '.'
    c = ","
    administration = []
    for i in list:
        j = date.index(i)
        index = i.find(p)
        indexc = i.find(c)
        date[j] = date[j][0:index]
        if indexc != -1:
            date[j] = date[j][0:indexc]
    return date, administration 
date, administration = clean_createadminlist(date)
print(date, administration)
def cleanstrmonth(date):
    administration = []
    department = []
    target = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
    for i in range(0,len(date)):
        month = ((target.index(date[i][:-5])) + 1)
        date[i] = date[i].split()
        date[i][0] = month
        department.append("The Office of the Chief Immigration Judge (OCIJ)")
    for i in range(0,(len(date)-1)):
        for j in range(0, len(US_Presidents['end'])):
            if (int(date[i][1]) >= US_Presidents['start'][j][0]) and (int(date[i][1]) <= US_Presidents['end'][j][0]):
                if date[i][0] >= US_Presidents['start'][j][1] or date[i][0] < US_Presidents['end'][j][1]:
                     administration.append(US_Presidents['president'][j])
           # if (int(date[i][1]) <= US_Presidents['end'][j][0]) and (int(date[i][1]) >= US_Presidents['start'][j][0]):
               # if date[i][0] >= US_Presidents['start'][j][1] or date[i][0] < US_Presidents['end'][j][1]:
                     #administration.append(US_Presidents['president'][j])
    return date, administration, department
date, administration, department = cleanstrmonth(date)
date
US_Presidents<jupyter_output><empty_output><jupyter_text>- Create OCIJ dataframe<jupyter_code>date # 5 O 8 O 7 O 9 O 6 O 5 G T after
administration # 5 O 8 O 7 O 9 O 6 O 5 G T after
administration.pop(9)
administration.pop(-2)
administration.pop(25)
administration.pop(-7)
d = '<NAME>'
administration.append(d)
print(len(judges_names), len(judges_bio), len(date), len(administration),len(department))
# create dataframe for appeals judges
dict = {'name': judges_names, 'bio': judges_bio, 'appointed_date': date, 'administration': administration, "department": department}  
OCIJ_judges = pd.DataFrame(dict)
OCIJ_judges['name'][0] = OCIJ_judges['name'][0][:-24]
OCIJ_judges['name'][1] = OCIJ_judges['name'][1][:-49]
OCIJ_judges['name'][2] = OCIJ_judges['name'][2][:-39]
print(OCIJ_judges.shape)
OCIJ_judges
s = 'Assistant Chief Immigration Judge'
len(s)
# clean OCIJ dataframe names
for i in range(0,len(OCIJ_judges['name'])):
    if i >= 3 and i != 30:
        OCIJ_judges['name'][i] = OCIJ_judges['name'][i][:-33]
    if i ==30:
        OCIJ_judges['name'][i] = '<NAME> III'
OCIJ_judges['name']
# finished dataframe
print(OCIJ_judges.shape)
OCIJ_judges<jupyter_output>(43, 5)<jupyter_text>- Combine finished dataframes<jupyter_code>all_dfs = [OCIJ_judges, BIA_judges]

# Give all df's common column names
for df in all_dfs:
    df.columns = ['name', 'bio', 'appointed_date', 'administration', 'department']

judges= pd.concat(all_dfs).reset_index(drop=True)
# saving the DataFrame as a CSV file 
judges_appointed = judges.to_csv('judges_appointed.csv', index = True) 
print('\nCSV String:\n', judges_appointed)<jupyter_output>CSV String:
 None<jupyter_text>- EOIR Immigration Judge's data mining/web scraping in progress  - Imports<jupyter_code># imports as needed
import nltk
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
from nltk.corpus import stopwords
stop = stopwords.words('english')
# Government website with 460 immagration judges names
url = 'https://www.justice.gov/eoir/eoir-immigration-court-listing'

# bs4 gets web html
response = requests.get(url)
soup = Bs4(response.text, 'html.parser')

# Current judges of the EOIR court
bodytext = soup.find('div', class_="field__item even")
table = bodytext.find('div')
a = table.find('a')
j = []
u = []
# find all names
for content in table.find_all('p'):
    j.append(content)
for content in table.find_all('td'):
    u.append(content.text)
#bio = bio[4:]
print(table)
print(str(j[2]))
len(u), len(j)
def cleanfornames(u):
    for i in range(0,len(u)):
        
u
u[1]
u[2]
u[4]
u[8]
u[10]
u[14]
u[18]
u[22]
d = u[18]
d = test.join([i for i in d.split() if i not in stop])
s = nltk.sent_tokenize(document)
s = [nltk.word_tokenize(sent) for sent in s]
s = [nltk.pos_tag(sent) for sent in s]
s
sentences[0][-25:]
type(sentences[0][1])
print(j[0])
len(j[0])
u[0]
u[1]
# 53
u[1]
print(u[2])
u[2]
len(u[2])
u[2][89:-19]
print(len(u[3]))
print(len(u[7]))
len(u[11])
print(d)
u[11][d:]
u[11][y:]
for i in range(2,len(u),4):
    if u[i] == u[2]:
        i+4
    print(i)
u[394]
len("\n\t\t\t")
y = u[11].find("\t\t\t") + 3
print(y)
x = u[11][y:].find("\t\t\t")
print(x)
u[11][x:]
x
u[4]
len(u[4])
key = "\t\t\t"
key2 = '\n\n'
y = u[4].find(key) + 3
x = u[4][y:].find(key) + 3 + y
w = u[4][x:].find(key) + 3 + x
d = u[4][w:].find(key2) + 2 + w
a = u[4][d:].find(key2)+d
t = u[4][d:a]
c = count_seqence(t, key)
t, c
u[4][x:]
u[11][y:]
key = "\t\t\t"
key2 = '\n\n'
y = u[11].find(key) + 3
x = u[11][y:].find(key) + 3 + y
w = u[11][x:].find(key) + 3 + x
d = u[11][w:].find(key2) + 2 + w
a = u[11][d:].find(key2)+d
t = u[11][d:a]
c = count_seqence(t, key)
t, c
t
j = t.find(key)-1
f = t[j+4:]
m = f.find(key)-1
f[m+4:]
print(m)
def count_seqence(str, seq):
    return str.count(seq)
count_seqence("kjdsflsdnf lskmfldsknffsdlkfnsldkmf", "ds")
def scrapelotsofjudges(u):
    # create space with list
    name = [] 
    bio = []
    appointed_date = []
    administration = []
    department = []
    # loop though each section of text
    for i in range(0,len(u)):
        # catch larger text sections
        while 
        if 135 <= len(u[i]):
            key = "\t\t\t"
            key2 = '\n\n'
            key_end ='\n\n\n'
            y = u[i].find(key) + 3
            x = u[i][y:].find(key) + 3 + y
            w = u[i][x:].find(key) + 3 + x
            d = u[i][w:].find(key2) + 2 + w
            a = u[i][d:].find(key_end)+d
            t = u[i][d:a]
            j = t.find(key)-1
            # first judge add its the outliner in the html
            name.append(t[:j])
            # finding the count of the rest of names in text
            c = count_seqence(t, key)
            f = t[j+4:]
            while c >= 1:
                m = f.find(key)-1
                judge = f[:m]
                name.append(judge)
                # reformat for while loop
                f = f[m+4:]
    return name, bio, appointed_date, administration, department
name, bio, appointed_date, administration, department = scrapelotsofjudges(u)
print(name)
print(name)
s = '<NAME> (Acting)'
len(s)
cc = 'california'
len(cc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17213318424649024, 'fraction_numerical': 0.023046341160392596, 'mean_word_length': 3.186996098829649, 'pattern_counts': {'":': 18, '<': 44, '<?xml version=': 0, '>': 47, 'https://': 3, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '359628', 'license': 'MIT', 'n_tokens_mistral': 6061, 'n_tokens_neox': 5807, 'n_words': 1361, 'path': 'notebooks/Judge-appointed-by-data.ipynb', 'repo_name': 'navroz-lamba/human-rights-first-asylum-ds-a', 'size': '16132'}