<jupyter_start><jupyter_text>Jupyter notebook for testing and breaking things<jupyter_code>import os
import json
import natsort
from collections import OrderedDict
# hardcoding in path for testing
def get_people():
    filepath = '/home/osboxes/Code/forks/rse-skills-graph/people.json' 
    return get_file_contents(filepath)

# thanks to <NAME> for adding this code originally
def get_skills_list():
    skills_list = {}
    json_results = get_people()
    for supervisor, data in json_results.items():
        for section in data:
            for item in json_results[str(supervisor)][section]:
                if item not in skills_list:
                    skills_list[item] = 1;
                else:
                    skills_list[item] = skills_list[item] +1;

    skills_list_new = OrderedDict(natsort.natsorted(skills_list.items()))
    return skills_list_new


def get_file_contents(filename):
    data = None

    try:
        fp = open(filename, 'rb')
        try:
            contents = fp.read()
            data = json.loads(contents)
        finally:
            fp.close()
    except IOError:
        print('Could not open JSON file:' + filename, file=sys.stderr)
        sys.exit(1)

    return data
data = get_people()
for sup, data in data.items():
    
    print(sup)
    print(data)
    for section in data:
        print(section)
    break
def get_skills_list():
    skills_list = {}
    json_results = get_people()
    for supervisor, data in json_results.items():
        for section in data:
            if section != "email":
                for item in json_results[str(supervisor)][section]:
                    if item not in skills_list:
                        skills_list[item] = 1;
                    else:
                        skills_list[item] = skills_list[item] +1;
            else:
                pass

    skills_list_new = OrderedDict(natsort.natsorted(skills_list.items()))
    return skills_list_new
get_skills_list()
import sys

__file__ = 'rse-skills-graph'
os.environ['PATH'] = os.environ['PATH'] + ':/usr/local/bin'
os.environ['GV_FILE_PATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'static/images/')) + '/'

print('PATH: ' + os.environ['PATH'], file=sys.stderr)
print('GV_FILE_PATH: ' + os.environ['GV_FILE_PATH'], file=sys.stderr)
import urllib
def get_titles(topic):
    
    url = 'https://en.wikipedia.org/w/api.php'
    values = {
        'action' : 'query',
        'list' : 'search',
        'srwhat' : 'text',
        'srsearch' : topic.encode('utf8'),
        'format' : 'json',
        'srlimit' : '40'
    }

    data = urllib.parse.urlencode(values)
    print(data)
    data = data.encode('utf-8') # necessary for python 3
    print(data)
    
    # build wikipedia request from data values
    request = urllib.request.Request(url, data)
    # example full request 
    # https://en.wikipedia.org/w/api.php?action=query&list=search&srwhat=text&srsearch=Linux&format=json&srlimit=40
    
    # open API request and collect response
    response = urllib.request.urlopen(request)
    
    # read the response (open file)
    json_response = response.read()
    # load file as a json
    json_result = json.loads(json_response)
    
    # iterate over json returned and extract related titles in json
    # error if topic not found
    try:
        results = []
        for result in json_result['query']['search']:
            results.append(result['title'])
    except:
        results = []
        print("ERROR: no title found for topic: " + topic.encode('utf8'))
    
    # return full list of related titles
    return results
get_titles('Web Development')<jupyter_output>action=query&list=search&srwhat=text&srsearch=Web+Development&format=json&srlimit=40
b'action=query&list=search&srwhat=text&srsearch=Web+Development&format=json&srlimit=40'<jupyter_text>Graph vis stuff<jupyter_code>import pygraphviz as pgv
from flask import Flask, render_template, request, redirect, url_for

def get_graph_string(graph):
    #output = StringIO.StringIO()
    output = io.BytesIO()
    graph.draw(output, format = 'svg')
    svg = output.getvalue()
    output.close()

    svg_parser = etree.XMLParser()
    svg_obj = etree.fromstring(svg, svg_parser)

    svg_obj.attrib['width'] = '100%'
    del svg_obj.attrib['height']

    images = svg_obj.findall('.//{http://www.w3.org/2000/svg}image')

    # Convert all image href links to match server
    # For example, anonymous.png becomes static/images/anonymous.png
    # Have to do this because graphviz doesn't allow you to specify the
    # URL to an image, only the file path.
    for image in images:
        image_filename = image.attrib['{http://www.w3.org/1999/xlink}href']
        image_url = url_for('static', filename = 'images/' + image_filename)
        image.attrib['{http://www.w3.org/1999/xlink}href'] = image_url

    return etree.tostring(svg_obj, pretty_print = True).decode('utf-8')

def get_image_files():
    image_files = []
    image_dir = '/home/osboxes/Code/forks/rse-skills-graph/static/images/' #os.environ['GV_FILE_PATH']

    for root, sub_folders, files in os.walk(image_dir):
        for filename in files:
            actual_file_name = os.path.join(root, filename)
            if filename.endswith('.png'):
                image_files.append(filename)

    return image_files

def build_graph(name, results, topics):
    graph = pgv.AGraph(overlap = 'false', name = name)
    people = get_people()

    for person in results:
        forename, surname = person.lower().split()

        image_file = 'anonymous.png'
        image_files = get_image_files()

        for filename in image_files:
            #newstr = starturlsource[index+len(pattern):index+len(pattern)+17]
            if str.find(filename, surname) != -1 and str.find(filename, forename) != -1:
                image_file = '%s' % (filename)

        # check added for _ in name e.g. <NAME>; that is: convert _ to space
        myperson= person.replace(' ', '\n')
        graph.add_node(person, label = myperson.replace('_' , ' '), fontname = 'Helvetica', fixedsize = True, imagescale = True, width = '1.5', height = '1.5', fontcolor = 'white', shape = 'circle', style = 'filled', color = '#303030', URL = url_for('show_person', name = person), image = image_file)

        interests = people[person]['interests']
        for interest in interests:
            if interest in topics:
                color = '#A02020FF'
                shape = 'ellipse'
            else:
                color = '#105060EE'
                shape = 'ellipse'

            label = re.sub('\(.*\)', '', interest)

            graph.add_node(interest, label = label, style = 'filled', fontname = 'Helvetica', shape = shape, color = color, fontcolor = 'white', URL = url_for('show_topic', name = interest))
            graph.add_edge(person, interest, color = '#00000050')

        if 'technologies' in people[person]:
            for technology in people[person]['technologies']:
                if technology in topics:
                    color = '#B01050FF'
                    shape = 'ellipse'
                else:
                    color = '#701050EE'
                    shape = 'ellipse'

                label = re.sub('\(.*\)', '', technology)

                graph.add_node(technology, label = label, style = 'filled', fontname = 'Helvetica', shape = shape, color = color, fontcolor = 'white')
                graph.add_edge(person, technology, color = '#00000050')

    graph.layout(prog = 'neato')

    return graph
build_graph(name='Test', results=['Bob Up','Jane Down'], topics=['trains','planes'])
get_image_files()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12529335071707953, 'fraction_numerical': 0.011473272490221642, 'mean_word_length': 2.6355450236966824, 'pattern_counts': {'":': 1, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '354462', 'license': 'Apache-2.0', 'n_tokens_mistral': 2455, 'n_tokens_neox': 2341, 'n_words': 556, 'path': 'dev_notes/testing_and_breaking.ipynb', 'repo_name': 'ARCLeeds/rse-skills-graph', 'size': '7683'}