<jupyter_start><jupyter_text>Executive Summary1. Extract Data Fields2. Build ML Model3. Export  Process  Extract Data<jupyter_code>def get_titles(urlsX):
    '''
    Input: List of urls
    Returns 3 lists: (1) Bill Title (2) Bill Sponsors (3) Party
    '''
    titlesX = sponsersX = partiesX = []
    i=0
    for urlX in tqdm(urlsX):
        #print(urlX)
        try:
            r = requests.get(urlX) # Use requests to get the contents
            html_content = r.text # Get the text of the contents
            soup = BeautifulSoup(html_content, 'lxml') # Convert the html content into a beautiful soup object

            titleX = str(soup.title.string)
            titlesX.append(str(titleX))

            '''soup = None
            soupX = BeautifulSoup(html_content, 'lxml')
            sponserX = soupX.findAll(['sponsors','fullname'])[1:]
            sponserX = [x.string for x in sponserX]
            sponsersX.append(sponserX)'''
            #print(sponsersX)
            #print(titlesX)

        except(Exception):
            titlesX.append("NA")
    return titlesX#,sponsersX

def get_sponsers(urlsX):
    '''
    Input: List of urls
    Returns 3 lists: (1) Bill Title (2) Bill Sponsors (3) Party
    '''
    titlesX = sponsersX = partiesX = []
    i=0
    for urlX in tqdm(urlsX):
        #print(urlX)
        try:
            r = requests.get(urlX) # Use requests to get the contents
            html_content = r.text # Get the text of the contents
            soup = BeautifulSoup(html_content, 'lxml') # Convert the html content into a beautiful soup object

            soupX = BeautifulSoup(html_content, 'lxml')
            sponserX = soupX.findAll(['sponsors','fullname'])[1:]
            sponserX = [str(x.string) for x in sponserX]
            sponsersX.append(sponserX)
            #print(sponsersX)
            #print(titlesX)

        except(Exception):
            sponsersX.append("NA")
    return sponsersX#,sponsersX
def get_texts(urlsX):
    '''
    Input: List of urls
    Returns 3 lists: (1) Bill Title (2) Bill Sponsors (3) Party
    '''
    titlesX = sponsersX = partiesX = textsX = []
    i=0
    for urlX in tqdm(urlsX):
        #print(urlX)
        try:
            r = requests.get(urlX) # Use requests to get the contents
            html_content = r.text # Get the text of the contents
            soup = BeautifulSoup(html_content, 'lxml') # Convert the html content into a beautiful soup object

            soupY = BeautifulSoup(html_content, 'lxml')
            textX = soup.p.string
            '''sponserX = soupX.findAll(['sponsors','fullname'])[1:]
            sponserX = [str(x.string) for x in sponserX]'''
            textsX.append(textX)
            #print(sponsersX)
            #print(titlesX)

        except(Exception):
            textsX.append("NA")
    return textsX#,sponsersX

urls =['https://www.gpo.gov/fdsys/bulkdata/BILLSTATUS/115/s/BILLSTATUS-115s1000.xml',
      'https://www.gpo.gov/fdsys/bulkdata/BILLSTATUS/115/s/BILLSTATUS-115s100.xml',
      'https://www.gpo.gov/fdsys/bulkdata/BILLSTATUS/115/s/BILLSTATUS-115s200.xml']
aaa = get_titles(urls)
bbb = get_sponsers(urls)
ccc = get_texts(urls)
print(aaa)
print(bbb)
print(ccc)
ccc
years = [115, 114, 113]
urls = []
bill_nums = list(range(2500))
url ='https://www.gpo.gov/fdsys/bulkdata/BILLSTATUS/115/s/BILLSTATUS-115s1000.xml'
#print(bill_nums)
print(len(urls))

for year in years:
    for bill_num in (bill_nums):
        #try:
        url = str(url[:46])+str(year)+'/s/BILLSTATUS-'+str(year)+'s'+str(bill_num)+'.xml'  #Needs to be optimized
        urls.append(url)
        #print(urls[0])
print(len(urls))

titles = get_titles(urls)
print(len(titles))
sponsers = get_sponsers(urls)
print(len(sponsers))
texts = get_texts(urls)
print(len(texts))<jupyter_output>0
7500<jupyter_text>Create Pandas DF<jupyter_code>print(len(titles), len(sponsers), len(texts))
df = pd.DataFrame()
df['title'] = titles
df['sponsers'] = sponsers
df['texts'] = texts
#df = pd.DataFrame({'title': titles, 'text':texts, 'sponsor': sponsors, 'party': parties_list})
df=df.dropna()
df.head(20)
#df.tail(20)
#data_dir = '/Users/whs/Documents/GitHub/gpo-ai.com/data/processed/1.0-billTitleSponsors.csv'
data_dir = 'C:\\Users\\wsolomon\\Desktop\\DataScience\\projects\\loc_challenge-master\\1.0-billTitleSponsors.csv'
df.to_csv(data_dir, encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.135067305498517, 'fraction_numerical': 0.020305726671229752, 'mean_word_length': 2.8388791593695273, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 11}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '464432', 'license': 'MIT', 'n_tokens_mistral': 1634, 'n_tokens_neox': 1547, 'n_words': 320, 'path': 'notebooks/1.0-whs-xmlExtract.ipynb', 'repo_name': 'whs2k/loc_challenge_ai', 'size': '4383'}