<jupyter_start><jupyter_text>Automate other columns as much as possible<jupyter_code>import pandas as pd
import numpy as np
import requests
import time
from bs4 import BeautifulSoup
from pandas.io.json import json_normalize
import re
allcontent = pd.read_pickle('lr_eposides_master.pkl')
allcontent.head(3)
sunsetlist=[]
epnumlist=[]
for ep in allcontent['episodenum']:
    t = allcontent.loc[allcontent['episodenum']==ep,'title'].tolist()[0]
    c = allcontent.loc[allcontent['episodenum']==ep,'content'].tolist()[0]
    dumt=re.findall('sunset',t.lower())
    dumc=re.findall('sunset',c.lower())
    if (len(dumt)>0) or (len(dumc)>0):
        sunsetlist.append(t)
        epnumlist.append(ep)
sunsetlist_df = pd.DataFrame({'title':sunsetlist}, index=epnumlist)
sunsetlist_df.head(3)
setrevlist=[]
setrevcontent=[]
epnumlist=[]
for ep in allcontent['episodenum']:
    t = allcontent.loc[allcontent['episodenum']==ep,'title'].tolist()[0]
    c = allcontent.loc[allcontent['episodenum']==ep,'content'].tolist()[0]
    dumts=re.findall('set',t.lower())
    dumtr=re.findall('review',t.lower())
    
    dumcs=re.findall('set',c.lower())
    dumcr=re.findall('review',c.lower())
    
    if (len(dumts)>0 and len(dumtr)>0): #or (len(dumcs)>0 and len(dumcr)>0):
        setrevlist.append(t)
        setrevcontent.append(c)
        epnumlist.append(ep)
setrevlist_df = pd.DataFrame({'title':setrevlist, 'content':setrevcontent}, index=epnumlist)
setrevlist_df.head(3)
leveluplist=[]
levelupcontent=[]
epnumlist=[]
for ep in allcontent['episodenum']:
    t = allcontent.loc[allcontent['episodenum']==ep,'title'].tolist()[0]
    c = allcontent.loc[allcontent['episodenum']==ep,'content'].tolist()[0]
    dumts=re.findall('level',t.lower())
    dumtr=re.findall('up',t.lower())
    
    dumcs=re.findall('level',c.lower())
    dumcr=re.findall('up',c.lower())
    
    if (len(dumts)>0 and len(dumtr)>0) or (len(dumcs)>0 and len(dumcr)>0):
        leveluplist.append(t)
        levelupcontent.append(c)
        epnumlist.append(ep)
leveluplist_df = pd.DataFrame({'title':leveluplist, 'content':levelupcontent}, index=epnumlist)
leveluplist_df.head(3)
guestlist=[]
guestcontent=[]
epnumlist=[]
for ep in allcontent['episodenum']:
    t = allcontent.loc[allcontent['episodenum']==ep,'title'].tolist()[0]
    c = allcontent.loc[allcontent['episodenum']==ep,'content'].tolist()[0]
    dumts=re.findall('guest',t.lower())
    dumtr=re.findall('welcome',t.lower())
    
    dumcs=re.findall('guest',c.lower())
    dumcr=re.findall('welcome',c.lower())
    
    if (len(dumts)>0 or len(dumtr)>0) or (len(dumcs)>0 or len(dumcr)>0):
        guestlist.append(t)
        guestcontent.append(c)
        epnumlist.append(ep)
guestlist_df = pd.DataFrame({'title':guestlist, 'content':guestcontent}, index=epnumlist)
sunsetlist_df.to_csv('sunsetlist.csv')
setrevlist_df.to_csv('setrevlist.csv')
leveluplist_df.to_csv('leveluplist.csv')
guestlist_df.to_csv('guestlist.csv')<jupyter_output><empty_output><jupyter_text>ADD LISTS TO MASTER  NB: Before running this, manually check the CSV files generated from above and FIX<jupyter_code>sunsetfinal = pd.read_csv('sunsetlistFIXED.csv')
setrevfinal = pd.read_csv('setrevlistFIXED.csv')
levelupfinal = pd.read_csv('leveluplistFIXED.csv')
guestfinal = pd.read_csv('guestlistFIXED.csv')
lrmaster = pd.read_csv('lr_eposides_master_05.csv')
lrmaster.head(3)
sunsetcol=[]
setrevcol=[]
levelupcol=[]
guestcol=[]
for ep in lrmaster['episodenum']:
    dumss=sunsetfinal.loc[sunsetfinal['epnum']==ep,'sunset'].tolist()
    dumsr=setrevfinal.loc[setrevfinal['epnum']==ep,'setreview'].tolist()
    dumlu=levelupfinal.loc[levelupfinal['epnum']==ep,'levelup'].tolist()
    dumgs=guestfinal.loc[guestfinal['epnum']==ep,'guest'].tolist()
    
    if len(dumss)>0:
        sunsetcol.append(dumss[0])
    else:
        sunsetcol.append(' ')
        
    if len(dumsr)>0:
        setrevcol.append(dumsr[0])
    else:
        setrevcol.append(' ')
    
    if len(dumlu)>0:
        levelupcol.append(dumlu[0])
    else:
        levelupcol.append(' ')
    
    if len(dumgs)>0:
        guestcol.append(dumgs[0])
    else:
        guestcol.append(' ')

lrmaster['sunsets']=sunsetcol
lrmaster['setreviews']=setrevcol
lrmaster['levelups']=levelupcol
lrmaster['guests']=guestcol
lrmaster.head(3)
lrmaster.to_csv('lr_master_FINAL.csv')<jupyter_output><empty_output><jupyter_text>VIEW LISTS<jupyter_code>guestlist_df['content'].tolist()
sunsetlist_df['title'].tolist()
leveluplist_df['content'].tolist()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17491166077738515, 'fraction_numerical': 0.008613074204946997, 'mean_word_length': 5.405940594059406, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '597156', 'license': 'MIT', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1779, 'n_words': 217, 'path': 'LRcolumns.ipynb', 'repo_name': 'M055/LRepisodes', 'size': '4528'}