<jupyter_start><jupyter_text>Writer's Correlation Detector--> <NAME> 2021<jupyter_code>#!pip3 install nltk ner requests
import tqdm
from tqdm import tqdm
import requests
import time
import tenacity
import openpyxl
from lxml import html
from requests import HTTPError
import pandas
from wikipedia.exceptions import *
session = requests.Session()

import wikipedia
import numpy
import requests
import pandas as pd
import tqdm
from tqdm import tqdm
from wikipedia.exceptions import *
import icecream
from icecream import *
#import NER
#from NER import extract_entities_labelstoo
#from NER import extract_entities
#from NER import cleanup_string # includes cleanup_string


def installpackages(array):
    for name in array:
        print("\nInstalling python package: ",name)
        !pip3 install name<jupyter_output><empty_output><jupyter_text>Now we want to pull the wikipedia pages for each character in our story. This will allow us to find correlated entities within the group. one idea is list comprehension : {key:value for key in iterable}we are trying a different approach here.<jupyter_code># create the writefile
write_file = open("/Users/grahamwaters/Documents/GitHub Main Repository/wikid/interactions.txt",'w+')
write_file.close()
df = {'name':'example name',
    'dates':[],
    'valence_names':[],
    'moments': pd.DataFrame()
}
# dictionary parameters
df_keys = ['name', 'dates', 'valence_names','moments']
df_moments_keys = ['interactee', 'contexts', 'context_dates','adjacent_names','adjacent_locations','weight_of_interaction']
# these keys will hold all our character information
df_moments = ['Person_Two',['Person_Two and Person_One were talking to Person_Three.'],[1778],['Person_Two','Person_Three'],[''],0.78]
ex_values = ['Person_One',[1778,1784],['Person_Three','Person_Two'],df_moments]
values = [1, 2, 3]
# create dictionary func.
def create_dictionary(keys, values):
    result = {} # empty dictionary
    for key, value in zip(keys, values):
        result[key] = value
    return result
# use create_dictionary func.
df_story = create_dictionary(keys=df_keys, values=ex_values)
print(df_story)<jupyter_output>{'name': 'Person_One', 'dates': [1778, 1784], 'valence_names': ['Person_Three', 'Person_Two'], 'moments': ['Person_Two', ['Person_Two and Person_One were talking to Person_Three.'], [1778], ['Person_Two', 'Person_Three'], [''], 0.78]}<jupyter_text>These are ways to query the dictionary in pythonMethod 	Descriptionclear()	Removes all the elements from the dictionarycopy()	Returns a copy of the dictionaryfromkeys()	Returns a dictionary with the specified keys and valueget()	Returns the value of the specified keyitems()	Returns a list containing a tuple for each key value pairkeys()	Returns a list containing the dictionary's keyspop()	Removes the element with the specified keypopitem()	Removes the last inserted key-value pairsetdefault()	Returns the value of the specified key. If the key does not exist: insert the key, with the specified valueupdate()	Updates the dictionary with the specified key-value pairsvalues()	Returns a list of all the values in the dictionary<jupyter_code>x = df_story.values()
y = df_story.keys()
print('\nThese are the keys: \n', y)
print("\nThese are the values in the dict: \n", x)
df_story.setdefault('name')
print(df_story)<jupyter_output>These are the keys: 
 dict_keys(['name', 'dates', 'valence_names', 'moments'])

These are the values in the dict: 
 dict_values(['Person_One', [1778, 1784], ['Person_Three', 'Person_Two'], ['Person_Two', ['Person_Two and Person_One were talking to Person_Three.'], [1778], ['Person_Two', 'Person_Three'], [''], 0.78]])
{'name': 'Person_One', 'dates': [1778, 1784], 'valence_names': ['Person_Three', 'Person_Two'], 'moments': ['Person_Two', ['Person_Two and Person_One were talking to Person_Three.'], [1778], ['Person_Two', 'Person_Three'], [''], 0.78]}<jupyter_text>These are the keys:  dict_keys(['name', 'dates', 'valence_names', 'moments'])These are the values in the dict:  dict_values(['<NAME>', [1778, 1784], ['<NAME>', '<NAME>'], ['<NAME>', ['<NAME> and <NAME> were talking to <NAME>.'], [1778], ['<NAME>', '<NAME>'], [''], 0.78]])<jupyter_code># Now we add in the people/concepts we want to scan for...
# todo --> people = ['<NAME>', 'Beast', 'Hyde', 'Unsolved Murders', '1918', 'Spanish Flu', '<NAME>', 'Mysticism', 'Antarctica', 'Builderbergs', 'Deep State', 'Dragon', 'Sea Serpant', 'Monster', 'Loch Ness', 'English Royalty', '<NAME>', '1800s', '1889', '1888', '1887', 'Paris France', 'Spiritualism', 'Conspiracy Theories', 'Isle of Wight', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', 'Napoleon', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME> Jr.', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', 'Professor <NAME>', '<NAME>', '<NAME>', 'President Kennedy', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', 'J.P. Morgan Jr. ', 'J.P. Morgan Sr. ', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', '<NAME>', 'Edison', 'Mina Edison', 'Time Travel', 'Time Machine', 'Dimensional', 'Dimensions', 'Ghost', 'Invisible man', 'invisibility', 'big foot', 'hiroshima', 'nagasaki', 'los alamos', 'october 1941', 'elisabe<NAME>', 'hitler', 'soviet union', 'USSR', 'J<NAME>']#dictionary functions for NumPy library

people = ['<NAME>','<NAME>']

#! the data frame below contains all dates and moments and data that will be stored about interactions between characters
df_associated_data = {'dates':[],
                    'valence_names':[],
                    'contexts':[],
                    'moments':{ # this is a nested dictionary containing the interaction data 
                        'second_party':'default_name', # second party is person interacted with
                        'context':['ex string one','ex string two'], # todo 'dates in context': datesinside[(df_story[key].get(df_associated_data)).get("moments")]
                        'adjacent_names':[''], #todo get this from the context array above
                        'adjacent_locations':['']
                        },
                    }

df_associated_data = {'dates':[],
                    'valence_names':[],
                    'contexts':[],
                    }
prepped_names = {}
for name in people:
    #time.sleep(1)
    entry = {name:df_associated_data} # associated data must be entered previously
    prepped_names.update(entry)
print("prepared ",len(people),' entities for analysis')
print(prepped_names)
for key in prepped_names:
    print(key)
    print(prepped_names[key].get("dates")) # gets dates for each key in dict... 
df_story = prepped_names
# example of pulling all dates in the Tesla File
print(prepped_names["<NAME>"].get("dates")) # gets dates for each key in dict... 
print(prepped_names["<NAME>"].get("moments").get("context")) # gets nested contexts for each key in dict...<jupyter_output>[]<jupyter_text>for key in data:  Iterates just through the keys, ignoring the valuesfor key, value in d.items():  Iterates through the pairsfor key in d.keys():  Iterates just through key, ignoring the valuesfor value in d.values():  Iterates just through value, ignoring the keys<jupyter_code>morepeople = []
this_var = "-"
print("To move on to the next data entry field simply hit RETURN without typing anything into the box.")

print("welcome writer... get ready for the power of analytics")
print("Make the following Selections")
#?ENT_CHOSEN = input("Which Entities should be mined? [Date,Noun,Proper Noun,")
min_date = 999 # earliest year 
max_date = 1980 # latest date 
#people = ['<NAME>', 'Beast', 'Hyde', 'Unsolved Murders', '1918', 'Spanish Flu', '<NAME>', 'Mysticism', 'Antarctica', 'Builderbergs', 'Deep State', 'Dragon', 'Sea Serpant', 'Monster', ...]
#xtra = input("waiting for your enter")
print("adding ",len(morepeople)," to people array...")
for person in tqdm(morepeople):
    people.append(person)
enter = '_' #init 
print(people)
this_var = "-"
#todo change to '-' for input to be allowed
this_var = ''
people_dates = []
people_adjacent_entities = [] # holds adjacent entities extracted with NER nltk function
people_adjacent_entities_wlabels = [] # holds same as above but with labels. 

# create the connections file
# /Users/grahamwaters/Documents/GitHub Main Repository/wikid/main.py
connections_file = open("/Users/grahamwaters/Documents/GitHub Main Repository/wikid/character_connections.txt",'w+')
connections_file.close()
import collections
def makehash():
    return collections.defaultdict(makehash)
df_story = makehash()
for perp in tqdm(people):
    # see if the person_page text contains the perp
    try:
        person_page = wikipedia.page(perp).content
    except Exception:
        try:
            person_page = wikipedia.page(str(wikipedia.search(perp)[0])).content
        except Exception:
            person_page = "n/a"
    # quick topic scanner
    for person in tqdm(people):
        for topic in topics:
                if topic in person_page:
                    print("found mention of this phrase:",topic," in the ",person," article on wikipedia")
    # deep scan of people and articles 
    current = []
    for person in people:
            
        if person in person_page and person != perp:
            
            contextfile = open("/Users/grahamwaters/Documents/GitHub Main Repository/wikid/contextfile.txt",'a+')
            # if the person is in person_page 
            headerstring = '\n'+'\n'+ " #### Wikipedia Article: [ " + perp + " mentions " + person + "] #### " + '\n' + '\n'
            contextfile.write(headerstring)
            
            print(perp," mentions ",person)

            index = person_page.find(person)
            for smalli in range(0,1000): # go through to find maximum text it can find on the subject
                for smallj in range(0,1000):
                    try: 
                        context = person_page[(index-(1000-smalli)):(index+(1000-smallj))]
                    except Exception:
                        break
            #!print("...",person_page[index-smalli:index+smallj],"...")


            current = df_story["Nikola Tesla"].get("contexts")# gets contexts for tesla
            if type(current) is None:
                current = []

            current.append(context) # add the new contextsf
            
            print(df_story)
            df_story["Nikola Tesla"].update({'contexts':current})
            
            print("df_story --> ",df_story['contexts'])









            contextfile.write(context)
            contextfile.write('\n')
contextfile.close()
#simple method of checking the entire page string for a substring 
#contextfile = open("/Users/grahamwaters/Documents/GitHub Main Repository/wikid/contextfile.txt",'w+')
#xtra = input("waiting for your enter")
topics = ''

for perp in tqdm(people):
    # see if the person_page text contains the perp
    try:
        person_page = wikipedia.page(perp).content
    except Exception:
        try:
            person_page = wikipedia.page(str(wikipedia.search(perp)[0])).content
        except Exception:
            person_page = "n/a"
    # quick topic scanner
    #for person in tqdm(people):
        #for topic in topics:
        #        if topic in person_page:
        #            print("found mention of this phrase:",topic," in the ",person," article on wikipedia")
    # deep scan of people and articles 
    for person in people:
        if person in person_page and person != perp:
            #contextfile = open("/Users/grahamwaters/Documents/GitHub Main Repository/wikid/contextfile.txt",'a+')
            # if the person is in person_page 
            #headerstring = '\n'+'\n'+ " #### Wikipedia Article: [ " + perp + " mentions " + person + "] #### " + '\n' + '\n'
            #contextfile.write(headerstring)
            
            print(perp," mentions ",person)

            index = person_page.find(person)
            for smalli in range(0,1000): # go through to find maximum text it can find on the subject
                for smallj in range(0,1000):
                    try: 
                        context = person_page[(index-(1000-smalli)):(index+(1000-smallj))]
                    except Exception:
                        break
            #!print("...",person_page[index-smalli:index+smallj],"...")
            current = prepped_names["<NAME>"].get("moments").get("context") # gets nested contexts for each key in dict... 
            
            current.append(context)
            
            df_story[name][contexts]=current
            
            print("df_story --> ",df_story[name][moments][context])
            
            #contextfile.write(context)
            #contextfile.write('\n')
#contextfile.close()



print("context file has been populated. Proceeding to phase 2")


print("proceeding to create numpy arrays")
import numpy as np

years = np.fromstring('1 2', dtype=int, sep=' ')

from BOLO import analysisengine

# stage 2 

print("beginning stage 2 - Analysis for Correlations")

analysisengine()<jupyter_output><empty_output><jupyter_text>Work on a 3 function approach1. Extract2. Transform3. Load<jupyter_code>def extract(page):
    # get custom user agents
    headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:89.0) Gecko/20100101 Firefox/89.0'}
    url = f'https://www.indeed.com/jobs?q=data+visualization&l=Pflugerville%2C+TX&start={page}'
    r = requests.get(url,headers)
    # add soup variable
    soup = BeautifulSoup(r.content,'html.parser')
    return soup<jupyter_output><empty_output><jupyter_text>Div Sections1. Job Title --> class = "heading4 color-text-primary singleLineTitle tapItem-gutter"    and then 'h2',class_ = 'jobTitle jobTitle-color-purple'2. Company Location3. OuterDiv > Inner Div: Salary    outer_class_ = "heading6 tapItem-gutter metadataContainer"    inner_class_ = "metadata salary-snippet-container"    span>class_ = "salary-snippet"4. Nothing5. class_ = 'urgentlyHiring'6. Job Snippet     outer_class = "heading6 tapItem-gutter result-footer"    inner_class = "job-snippet" 7. span class = "date" (i.e. active 5 days ago)(7 can be found once job snippet closes with /div)<jupyter_code>c = extract(0) # which was the first page
#length_c = len(transform(c))
page = transf(c)<jupyter_output><empty_output><jupyter_text>item -->                                      Data Scientist                             KronologicAustin, TX 78701 (Downtown area)•Temporarily Remote                               $90,000 - $200,000 a year                                                                     Easily apply                                    Urgently hiring                                                      Proficiency in statistical                   data                  analysis and                   visualization                  using R or Python statistical analysis tools.                         Compensation*: Kronologic is a premium brand in the…                                  Active 5 days ago                       1 > trying to retrieve jobtitle text...Data Scientist[-] WHERE I LEFT OFF...NEXT STEPS1. FIND SALARY TO PRINT2. RATING PRINTING3. EXTRA DATA AND THEN MULTIPAGE RETRIEVAL<jupyter_code>@tenacity.retry(retry=tenacity.retry_if_exception_type(HTTPError),
                wait=tenacity.wait_exponential(multiplier=1, min=2, max=30))
def get_character_page(url):
    #url = f'https://www.pro-football-reference.com/players/uniform.cgi?number={number}'
    url = str(url)
    response = session.get(url)
    try:
        response.raise_for_status()
    except HTTPError as e:
        print(e)
        raise

    tree = html.fromstring(response.text)
    return tree

def parse_player_data(number):
    tree = get_character_page(character_url)
    
    number_on_page = tree.xpath('//h1[contains(text(), "All Players To Wear")]')[0].text.split()[-1]
    if int(number_on_page) != number:
        print(f'Jersey number mismatch. On page {number_on_page}, expected {number}')
        return

    for row in tree.xpath('//th/a[contains(@href, "players/")]/ancestor::tr'):
        href = row.find('th/a')
        player_name = href.text
        player_url = 'https://www.pro-football-reference.com' + href.attrib['href']
        year_min = row.findtext('td[@data-stat="year_min"]')
        year_max = row.findtext('td[@data-stat="year_max"]')

        player = {
            'jersey': number,
            'name': player_name,
            'url': player_url,
            'year_min': year_min,
            'year_max': year_max,
        }
        yield player

#get more of the player's data
def parse_player_data_urls(player_url):
    tree = get_player_page(player_url)

    number_on_page = tree.xpath('//h1[contains(text(), "All Players To Wear")]')[0].text.split()[-1]

    #xpath for passing table --> //*[@id="passing"]
    # table id ="div_fantasy"
    #   table id = "fantasy"

    if int(number_on_page) != number:
        print(f'Jersey number mismatch. On page {number_on_page}, expected {number}')
        return

    for row in tree.xpath('//th/a[contains(@href, "players/")]/ancestor::tr'):
        href = row.find('th/a')
        player_name = href.text
        player_url = 'https://www.pro-football-reference.com' + href.attrib['href']
        year_min = row.findtext('td[@data-stat="year_min"]')
        year_max = row.findtext('td[@data-stat="year_max"]')

        player = {
            'jersey': number,
            'name': player_name,
            'url': player_url,
            'year_min': year_min,
            'year_max': year_max,
        }
        yield player

if __name__ == '__main__':
    skipdownload = False # init as false
    try:
        with open("player_data.xlsx") as f:
                print("File present")
                skipdownload = True
    except FileNotFoundError:
        print('File is not present')
        data = []
    
    if skipdownload == False:
        for number in tqdm(range(0, 100)):
            for player in parse_player_data(number):
                data.append(player)

        df = pandas.DataFrame(data)
        df.to_excel('/Users/grahamwaters/Documents/GitHub Main Repository/FANTASY_2021/player_data.xlsx', index=False)
    else:
        df = pandas.read_excel('/Users/grahamwaters/Documents/GitHub Main Repository/FANTASY_2021/player_data.xlsx')
        # file exists
        player_urls = df.url.values

        print(df) # df contains all the player data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13572618343990114, 'fraction_numerical': 0.013164257697060878, 'mean_word_length': 3.093248295579503, 'pattern_counts': {'":': 0, '<': 99, '<?xml version=': 0, '>': 109, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '520675', 'license': 'MIT', 'n_tokens_mistral': 5753, 'n_tokens_neox': 5475, 'n_words': 1577, 'path': 'wikiwriter_bground.ipynb', 'repo_name': 'icehelmetminer/my_butler_jeeves', 'size': '19196'}