<jupyter_start><jupyter_text><jupyter_code>import requests;      #used for connecting to the API
import sys
from time import sleep
from math import log
import os
from urllib.request import urlopen
import re
from operator import itemgetter
if 'google.colab' in str(get_ipython()):
    environment = "Colab"
elif 'BINDER_SERVICE_HOST' in os.environ:
    environment = "Binder"
else:
    environment = "Unknown"
# For development purposes only when making changes in Github
import shutil
shutil.rmtree('ComputationalAccess')
if environment == "Colab":
    !git clone https://github.com/mark-bell-tna/ComputationalAccess.git
    sys.path.insert(0, 'ComputationalAccess')
    github_data = "ComputationalAccess/Data/"
    os.listdir(github_data)
    # Connect to gdrive
    from google.colab import drive
    drive.mount('/content/gdrive')
    data_folder = "/content/gdrive/My Drive/Data/"
else:
    github_data = "Data/"
    data_folder = "Data/"

from ukgwa_index import UKGWAIndex
from text_utils import SuffixTree, text_to_parts
from web_structure import UKGWAStructure
from ukgwa_query import QueryEngine
from cdx_indexer import TemporalIndexer
# Get entries from AtoZ index

refresh = False

print("Getting index...")
idx = UKGWAIndex()
if refresh:
    idx.indexfromweb()  # Read the A to Z index from the UKGWA website
else:
    idx.indexfromfile(data_folder + "atoz_index.txt")  # Read from a saved file

print("Loaded index...")
# Updated the entries with Discovery catalogue references

idx.discoveryfromfile(github_data + "discovery_ukgwa_links.txt")
# Test the last command worked - should see a catalogue reference in last position of list
# If it says 'N' then try a few other numbers.
idx.index['UKGWA.100']
# Only run this if you want to save the results of "indexfromweb" for next time
idx.indextofile(data_folder + "atoz_index.txt")
# Index the text of the index to make it searchable and to get common ngrams

st = SuffixTree(0, stopwords = set(['on','for','and','of','&','the']))

print("Loading suffixes...")
for key in idx:
    w_p = text_to_parts(idx.get_field(key, 'TEXT'))
    i = 0
    for p in w_p:
        if p[3] == "web":
            continue
        i += 1
        st.add_tokens(p[0].split(" "), key + "." + str(i))

print("Loaded")

# Print examples of top N common ngrams

print("Getting phrases...")
s = st.get_phrases(30,2,10)
s.sort(key=itemgetter(1), reverse=True)

# TODO: Need to incorporate nearest match code to get references
# Do inside suffix tree class
N = 10
topNindex = {}
for row in s[0:N]:
    this_list = [idx.lookup(st.get_ngram_reference(row[2])[:-2])]
    ngram_text = " ".join(row[0])
    print(ngram_text, row[1])
    print("\t", this_list[0])
    for r in row[4]:
        this_list.append(idx.lookup(st.get_ngram_reference(r)[:-2]))
    topNindex[ngram_text] = this_list

topNindex["National Health Service"][0]
WS = UKGWAStructure()

for identifier in idx:
    url = idx.get_field(identifier, 'URL')
    WS.add_entry(url, identifier)
Q = QueryEngine()
ngram_of_interest = "Advisory Committee"
domain_summary = {}
[Q.exclude(x[0]) for x in topNindex['Twitter Archive']]
[Q.exclude(x[0]) for x in topNindex['YouTube Archive']]
[Q.include(x[0]) for x in topNindex[ngram_of_interest]]

for identifier in Q:
    url_parts = WS.index[identifier]
    dt = WS.domaintotree(url_parts[WS.fields['NETLOC']], strip_www=True)
    dom_name = ".".join(reversed(dt[:3]))
    if dom_name in domain_summary:
        domain_summary[dom_name] += 1
    else:
        domain_summary[dom_name] = 1

sorted_domains = sorted([(k,v) for k,v in domain_summary.items()], key=itemgetter(1), reverse=True)
print(sorted_domains)
sum([v[1] for v in sorted_domains])
for identifier in Q:
    print(idx.lookup(identifier))
    break
series_summary = {}
for identifier in Q:
    entry = idx.lookup(identifier)
    series = entry[idx.fields['CATREF']].split(" ")[0]
    if series in series_summary:
        series_summary[series] += 1
    else:
        series_summary[series] = 1

sorted_series = sorted([(k,v) for k,v in series_summary.items()], key=itemgetter(1), reverse=True)
print(sorted_series)
T = TemporalIndexer()
T.fields
# This takes quite a while to run
# Get snapshot data from the CDX API
for identifier in Q:
    T.add_entry(idx.get_field(identifier, 'URL'), identifier)
# Add the 3 views to the Query Engine
Q.add_view("Temp", T)
Q.add_view("AtoZ", idx)
Q.add_view("Domains", WS)
# Get entries where first snapshot is greater than 1/1/2015
t_filt = Q.filter_view("Temp", 'MIN', '>', 20150101000000)
for t in t_filt:
    print(t, idx.lookup(t))<jupyter_output>UKGWA.4920 ['UKGWA.4920', 'Social Security Advisory Committee – GOV.UK Blog', '*', 'https://ssac.blog.gov.uk/', 'N']
UKGWA.61 ['UKGWA.61', 'Advisory Committee for Social Science (ACSS)', '*', 'https://acss.food.gov.uk/', 'N']
UKGWA.146 ['UKGWA.146', 'Air Travel Insolvency Protection Advisory Committee (ATIPAC) (http://www.atipac.org.uk/)', '*', 'http://www.atipac.org.uk/', 'DFT 91']
UKGWA.147 ['UKGWA.147', 'Air Travel Insolvency Protection Advisory Committee (ATIPAC) (http://www.atipac.org.uk/home/)', '*', 'http://www.atipac.org.uk/home/', 'DFT 91']
{'fraction_non_alphanumeric': 0.1330423230214462, 'fraction_numerical': 0.016511672044031124, 'mean_word_length': 3.8616236162361623, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 3, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '274612', 'license': 'MIT', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1807, 'n_words': 493, 'path': 'UKGWA_AtoZ.ipynb', 'repo_name': 'mark-bell-tna/ComputationalAccess', 'size': '5269'}