<jupyter_start><jupyter_text>OverviewThis is an updated form that takes the place of the original NLSS notebook. However, it uses the more complete CSV files instead of the three smaller CSVs used in the original. In this Notebook, I organize data from a text document of dockets. The dockets include the date of the stream, who was on the show, and what games were played. I split up the data into categories and make lists out of the participants and games. I also take data on stream statistics from a CSV file. The data from the dockets and the stream stats are put together into one dataframe.<jupyter_code># Every returned Out[] is displayed, not just the last one. 
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
import nltk  
import pandas as pd
import numpy as np<jupyter_output><empty_output><jupyter_text>Cleaning up dataframe<jupyter_code>#Open up the docket and take a look
with open(r'data\NLSS_Dockets.txt') as f:
    file = f.read()
shows = file.split('\n\n') #split into every show
shows[:5]
#Take out unneeded text
index = 0
for s in shows:
    shows[index] = s.replace(' Nick View', '')
    index+=1
shows[-10:]
#Create a list of games
games = []
for s in shows:
    g = s.split('\n') #Text files has games on second line
    games.append(g[1])
games
#Create a list of crew members and show dates
date_crew = []
for s in shows:
    dc = s.split('\n')[0]
    date_crew.append(dc)
print(date_crew)
#Split into list for crew and list for dates
import re
date = []
crew = []
for entry in date_crew:
    foo = re.search(r'\((.*)\)', entry).group(1)
    d = foo.split(r')')[0]
    date.append(d)
    c = foo.split(r'(')[-1]
    crew.append(c)
date_df = pd.DataFrame(date, columns = ["Date"])
date_df.head()
games_df = pd.DataFrame(games, columns = ["Docket"])
games_df.head()
crew_df = pd.DataFrame(crew, columns = ["Crew"])
crew_df.head()
#Put these data frames together
nlss_df = pd.DataFrame()
nlss_df['Date'] = date_df['Date']
nlss_df['Crew'] = crew_df['Crew']
nlss_df['Docket'] = games_df['Docket']
nlss_df.head()
improved = []
#For each docket
for d in nlss_df['Docket']:
    #Split docket into list of games
    d = d.split(r',')
    #For each game
    for g in d:
        #If game matches string to remove
        if g == r" (continued)" or g == r" (Continued)":
            #Remove game
            d.remove(g)
    improved.append(d)
nlss_df['Docket'] = improved
improved = []
#For each cast of crew
for e in nlss_df['Crew']:
    #Split cast into list of members
    e = e.split(r',')
    #For each member
    for m in e:
        #If member contains a /w
        if r'w/' in m:
            both = m.split(r'w/')
            e.remove(m)
            e.extend(both)
    improved.append(e)
improved[:20]
#Clean up members list
fullstripped = []
for entry in improved:
    stripped = []
    for member in entry:
        member = member.strip(' ')
        stripped.append(member)
    fullstripped.append(stripped)
fullstripped[:10]
#A list of all unique members
names = []
for entry in fullstripped:
    for user in entry:
        if user not in names:
            names.append(user)
print(names)
#Clean up the list so that it follows a consistent format
foo = "Northernlion, RockLeeSmile, CobaltStreak, AlpacaPatrol, LastGreyWolf, HCJustin, BaerTaffy, JSmithOTI, Sinvicta, DanGheesling, MALF, FlackBlag, TotalBiscuit, LovelyMomo, Blueman, BaerTaffy, MathasGames, Crendor, BananasaurusRex, NOTREAL, AlpacaPatrol, DanGheesling, NOTREAL, BananasaurusRex, MALF, Arumba, BaerTaffy, CobaltStreak, MALF, Magresta, Northernlion, JSmithOTI, RockLeeSmile, LovelyMomo, MikeBithell, RedPandaGamer, OhmWrecker, PrescriptionPixel, Green9090"
translated = foo.split(", ")
translated
guests = []
for cast in fullstripped:
    guests.append([translated[names.index(user)] for user in cast])
#Replace first names with second names
guests[0]
nlss_df['Crew'] = guests
nlss_df.head()
nlss_df.tail()<jupyter_output><empty_output><jupyter_text>FULL CSVNow import the stream stats CSV<jupyter_code>import os
import glob
print(os.getcwd())

stream_df = pd.read_csv(r'data\FullCSV\Northernlion_Full.csv',index_col=None, header=0)
stream_df.drop('GamesPlayed',axis=1, inplace=True)
stream_df[150:160]
#Remove extra line we don't need
stream_df.drop(154, inplace=True)
stream_df[150:160]
datetime = [date.split(' ')for date in stream_df['StartTime']]
datetime
months = {'01':'January', '02':'February', '03':'March', '04':'April', '05':'May', '06':'June', '07':'July', '08':'August', '09':'September', '10':'October', '11':'November', '12':'December'}
#Make the dates in the CSV match the format of the docket
formatted = []
for stream in datetime:
    ymd = stream[0].split('-')
    entry = months.get(ymd[1])+" "+ymd[2]+", "+ymd[0]
    formatted.append(entry)
formatted
stream_df['Date'] = formatted
stream_df.head()
len(stream_df)
combined = pd.merge(nlss_df, stream_df)
combined.head()
combined.tail()
len(combined)<jupyter_output><empty_output><jupyter_text>Almost 200 lines that overlap. These are the videos we will get the comments for.<jupyter_code>combined.to_pickle('Pipeline\combined.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12702702702702703, 'fraction_numerical': 0.012934362934362934, 'mean_word_length': 4.436516264428121, 'pattern_counts': {'":': 1, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '583711', 'license': 'MIT', 'n_tokens_mistral': 1873, 'n_tokens_neox': 1775, 'n_words': 555, 'path': 'OldFiles/NLSS_V2.ipynb', 'repo_name': 'AndrewRyan95/Twitch_Sentiment_Analysis', 'size': '5180'}