<jupyter_start><jupyter_text>Importing Data - WIKI- Scraping names and committee names of Senate Committee Members- Scraping individual data from wikipedia from the previous list created- Creating dataframes of both committee memebers ranking member and chair - Further work will include voter data, state, assumed office, time in office<jupyter_code>import re
import pandas as pd
import numpy as np
import requests
import os
import json
import matplotlib.pyplot as plt
from bs4 import BeautifulSoup as bs
from splinter import Browser
import time
from IPython.core.display import HTML
from datetime import date, datetime
# Run this code to download/scrape the data off of wikipedia page
# url = "https://en.wikipedia.org/wiki/List_of_United_States_Senate_committees"
# response = requests.get(url)
# soup = bs(response.text, 'html.parser')
# #print(soup.prettify())
# names = pd.read_html(url)
# sliceNames = names[0]
# #Exportng data from Webscraping to CSV
# sliceNames.to_csv('names.csv', index = False)
#Importing Webscraping from CSV
importNames = pd.read_csv('names.csv')
committee = importNames['Committee']
chairNames = importNames['Chair']
rankingMembers = importNames['Ranking Member']
chairNames = list(chairNames)
chairNames = chairNames[1:]
rankingMembers = rankingMembers[1:]
#Pulling out list of names from scrape
#chairNames[1][:12]
def stripNames(stripList):
    stateParty = []
    fullName = []
    party = []
    state = []
    for i in stripList:
        counter = 0
        for j in i:
            counter += 1
            if j == '(':
                stateParty.append(i[counter-1:])
                fullName.append(i[:counter-2])
                party.append(i[-5:len(i)-4])
                state.append(i[-3:len(i)-1])
                #print(counter)
    return fullName, stateParty, party, state
chairNamesCols = stripNames(chairNames)
rankingMembersCols = stripNames(rankingMembers)
len(importNames)
importNames = importNames.drop(importNames.index[0])
#print(len(importNames))
#Renaming columns to normalize data
importNames['Chair_Names'] = chairNamesCols[0]
importNames['Party'] = chairNamesCols[1]
importNames['PartyAbbv'] = chairNamesCols[2]
importNames['State'] = chairNamesCols[3]
importNames['Ranking_Members'] = rankingMembersCols[0]
importNames['Party'] = rankingMembersCols[1]
importNames['PartyAbbv'] = rankingMembersCols[2]
importNames['State'] = rankingMembersCols[3]
importNames.head(2)
#Renaming and dropping columns to normalize data
importNames['Committee'] = importNames['Committee'].replace(np.NaN, 'SubCommittee')
importNames = importNames.drop(columns=['Refs', 'Ranking Member', 'Chair', 'Party'])
importNames = importNames.rename(columns={'Committee.1':'SubCommittee', 'PartyAbbv':'RM_Party', 'State':'State_Abbrv'})
importNames
#Creating a column Committee which has if member is in a subCommittee or main Committee
replaceCommittee = []
for i in importNames['Committee']:
    if i == 'SubCommittee':
        replaceCommittee.append(i)
        continue
    else:
        replaceCommittee.append('Committee')
importNames = importNames.drop(columns=['Committee'])
importNames['Committee'] = replaceCommittee
importNames.tail()
#Creating a column for full state name for visualizations as there are only abbreviations in the original data
stateName = pd.read_csv('stateAbbrv.csv')
stateDict = dict(zip(stateName['Code'], stateName['State']))
def longState(search_value):
    for key, val in stateDict.items():
        if key == search_value:
            return val
fullStateName = []
for i in importNames['State_Abbrv']:
    fullStateName.append(longState(i))
importNames['State'] = fullStateName
#Creating Columns for both Parties of Chair Members and Ranking Members
#The Chair Members are always the member of the majority party
rankingParty = []
chairParty = []
chairPartyAbbv = []
for i in importNames['RM_Party']:
    rankingParty.append('Democrat')
    chairParty.append('Republican')
    chairPartyAbbv.append('R')
importNames['R_Party_Name'] = rankingParty
importNames['C_Party_Name'] = chairParty
importNames['C_Party'] = chairPartyAbbv
importNames.head()
#Locating Names with a Middle Name in Chair Names for Renaming
# i = 1
# while i < len(importNames['Chair_Names']):
#     printHolder = importNames["Chair_Names"][i]
#     #print(printHolder)
#     if 'Shelley' in printHolder:
#         #print(f'{printHolder}, {i}')
#         i += 1
#     elif 'VACANT' in printHolder:
#         #print(f'{printHolder}, {i}')
#         i += 1
#     else:
#         i += 1
#Locating Names with a Middle Name in Ranking Members for Renaming
# i = 1
# while i < len(importNames['Ranking_Members']):
#     printHolder = importNames["Ranking_Members"][i]
#     #print(printHolder)
#     if 'Catherine' in printHolder:
#         print(f'{printHolder}, {i}')
#         i += 1
#     elif 'VACANT' in printHolder:
#         print(f'{printHolder}, {i}')
#     elif 'Van' in printHolder:
#         print(f'{printHolder}, {i}')
#         i += 1
#     else:
#         i += 1
#Renaming columns to better split names 
#Ideally there would be a third column with middle name or middle initial if that were available
#However, there would be alot of NaN's and the information would have to come from elsewhere
importNames["Chair_Names"][13] = "<NAME>"
importNames["Chair_Names"][51] = "<NAME>"
importNames["Chair_Names"][65] = "VACANT VACANT"
importNames['Ranking_Members'][29] = '<NAME>'
importNames['Ranking_Members'][46] = '<NAME>'
importNames['Ranking_Members'][33] = '<NAME>'
#importNames.tail(60)
def splitName(column):
    splitName = []
    for i in column:
        splitName.append(i.split())
    Namelist = []
    noMiddle =[]
    for i in column:
        Namelist.append(i.split(' '))
    len(Namelist)
    noMiddle = []
    counter = 1
    for i in Namelist:
        for j in i:
            if len(j) > 1:
                noMiddle.append(j)
                counter += 1
            elif len(j) <= 2:
                print(f'{j} <= len(j)')
                noMiddle.append(j)
                counter += 1
                #continue
    #             query = input('Keep this value, Y/N?')
    #             if query == 'Y':
    #                 noMiddle.append(j)
    #                 counter += 1
    #             else:
    #                 continue
    #                 counter += 1
            else:
                print(f'{j}')
                counter += 1
    #print(len(noMiddle))
    firstName = noMiddle[:len(noMiddle)+1:2]
    lastName = noMiddle[1:len(noMiddle)+1:2]
    #print(len(lastName))
    #print(len(firstName))
    #print(len(column))
    return firstName, lastName
chairNamesSplit = splitName(importNames["Chair_Names"])
rankingNamesSplit = splitName(importNames['Ranking_Members'])
importNames['C_First_Name'] = chairNamesSplit[0]
importNames['C_Last_Name'] = chairNamesSplit[1]
importNames['R_First_Name'] = rankingNamesSplit[0]
importNames['R_Last_Name'] = rankingNamesSplit[1]
#importNames.head(2)
importNames = importNames[['Committee','SubCommittee', 'C_First_Name', 'C_Last_Name', 'C_Party', 'C_Party_Name', 'State_Abbrv', 'R_First_Name', 'R_Last_Name', 'RM_Party', 'R_Party_Name', 'State', 'Chair_Names', 'Ranking_Members']]
importNames.head(2)
rankingMembers = importNames[['Committee', 'SubCommittee', 'R_First_Name', 'R_Last_Name', 'RM_Party', 'State_Abbrv', 'Ranking_Members', 'State', 'R_Party_Name']]
chairNames = importNames[['Committee','SubCommittee', 'C_First_Name', 'C_Last_Name', 'C_Party', 'State_Abbrv', 'Chair_Names', 'State', "C_Party_Name"]]
#chairNames.head()
importNames.to_csv('Chairs_RankingMembers.csv', index = False)
rankingMembers.to_csv('Ranking_Members.csv', index = False)
chairNames.to_csv('Chair_Members.csv', index = False)
chairNames[:50]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12938906752411575, 'fraction_numerical': 0.00887459807073955, 'mean_word_length': 3.9910141206675225, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '550149', 'license': 'MIT', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2417, 'n_words': 603, 'path': 'cleaningApps/02_committee_scrape/Chairs_RankingMembers.ipynb', 'repo_name': 'nguyenkevn/heran', 'size': '7840'}