<jupyter_start><jupyter_text>Trove Newspaper HarvesterDownload large quantities of digitised newspaper articles from [Trove](https://trove.nla.gov.au/newspaper/) with this simplified, web-based interface to the [TroveHarvester tool](https://github.com/wragge/troveharvester).<jupyter_code>import time
import argparse
import os
import shutil
import datetime
import arrow
from tqdm import tqdm_notebook
import json
import ipywidgets as widgets
from IPython.display import display, HTML, FileLink, clear_output
from pprint import pprint
import re
import unicodecsv as csv
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
try:
    from urllib.parse import urlparse, parse_qsl
except ImportError:
    from urlparse import urlparse, parse_qsl
# Import everything from the troveharvester package
from troveharvester.__main__ import *

s = requests.Session()
retries = Retry(total=5, backoff_factor=1, status_forcelist=[ 500, 502, 503, 504 ])
s.mount('http://', HTTPAdapter(max_retries=retries))
s.mount('https://', HTTPAdapter(max_retries=retries))
# These basically replace functions in the TroveHarvester package
# Instead of getting parameters from the command line, they get them from the widgets.

def nb_save_meta(data_dir, harvest):
    '''
    Save the query metadata in a JSON file.
    Useful for documenting your harvest.
    '''
    meta = {}
    meta['query'] = query_url.value
    meta['key'] = api_key.value
    meta['max'] = None
    meta['pdf'] = pdf.value
    meta['text'] = text.value
    meta['image'] = image.value
    meta['harvest'] = harvest
    meta['date_started'] = datetime.datetime.now().isoformat()
    meta['start'] = '*'
    with open(os.path.join(data_dir, 'metadata.json'), 'w') as meta_file:
        json.dump(meta, meta_file, indent=4)

def nb_prepare_harvest(b):
    '''
    Create ouput directories, get parameters from widgets & run the harvest.
    Triggered by the Start button.
    '''
    out.clear_output()
    harvest = str(int(time.time()))  # Get rid of fractions
    data_dir = os.path.join(os.getcwd(), 'data', harvest)
    make_dir(data_dir)
    nb_save_meta(data_dir, harvest)
    if pdf.value == True:
        make_dir(os.path.join(data_dir, 'pdf'))
    if text.value == True:
        make_dir(os.path.join(data_dir, 'text'))
    if image.value == True:
        make_dir(os.path.join(data_dir, 'image'))
    with out:
        nb_start_harvest(data_dir=data_dir, key=api_key.value, query=query_url.value, pdf=pdf.value, text=text.value, image=image.value, start='*', max=None)
        # out.clear_output(wait=True)
        shutil.make_archive(data_dir, 'zip', data_dir)
        display(HTML('<b>Download results</b>'))
        display(FileLink('data/{}.zip'.format(harvest)))
    
def nb_start_harvest(data_dir, key, query, pdf, text, image, start, max):
    '''
    Start a harvest.
    '''
    # Turn the query url into a dictionary of parameters
    params = prepare_query(query, text, key)
    # Create the harvester
    harvester = nb_Harvester(query_params=params, data_dir=data_dir, pdf=pdf, text=text, image=image, start=start, max=max)
    # Go!
    harvester.harvest()

class nb_Harvester(Harvester):
    
        def harvest(self):
            '''
            Start the harvest and loop over the result set until finished.
            '''
            number = self.number
            params = self.query_params.copy()
            params['n'] = self.number
            with tqdm_notebook(total=self.maximum, unit='article') as pbar:
                while self.start and (self.harvested < self.maximum):
                    params['s'] = self.start
                    response = s.get(self.api_url, params=params, timeout=30)
                    # print(response.url)
                    try:
                        results = response.json()
                    except (AttributeError, ValueError):
                        # Log errors?
                        pass
                    else:
                        records = results['response']['zone'][0]['records']
                        self.process_results(records, pbar)
                        # pbar.update(len(records['article']))
# All the UI stuff

# Somewhere to put the results
out = widgets.Output(layout=widgets.Layout(padding='40px'))

api_key = widgets.Text(
    placeholder='Enter your Trove API key',
    description='API key:',
    disabled=False
)

query_url = widgets.Text(
    placeholder='Enter the url of your search',
    description='Query url:',
    disabled=False,
    layout=widgets.Layout(width='100%')
)

text = widgets.Checkbox(
    value=False,
    description='Save full text',
    disabled=False
)

pdf = widgets.Checkbox(
    value=False,
    description='Save PDFs (this can be slow)',
    disabled=False
)

image = widgets.Checkbox(
    value=False,
    description='Save articles as images',
    disabled=False
)

start_button = widgets.Button(
        description='Start harvest',
        disabled=False,
        button_style='primary', # 'success', 'info', 'warning', 'danger' or ''
        tooltip='Start harvest',
        icon=''
    )

start_button.on_click(nb_prepare_harvest)<jupyter_output><empty_output><jupyter_text>Enter your Trove API keyThe harvester gets its data from the Trove API. To use the API [you need a key](http://help.nla.gov.au/trove/building-with-trove/api) — the process is quick, painless, and free. Once you have a key, paste it in below.<jupyter_code>display(api_key)<jupyter_output><empty_output><jupyter_text>Enter your search queryUse the [Trove web interface](https://trove.nla.gov.au/newspaper/) to construct your search. Remember that the harvester will get **all** of the matched results, not just the first 2,000 you see in the web interface. Once you're happy with your search, just copy the url and paste it below.<jupyter_code>display(query_url)<jupyter_output><empty_output><jupyter_text>Set harvest optionsBy default the harvester only saves the metadata (date, page, title, newspaper etc) from the search results. If you want to save the full text content of each article, just check the `Text` box. You can also save PDF copies of every article by checking the `PDF` option, but be warned that this will slow down your harvest and generate large download files. If you want to save PDFs from large harvests, you're probably better off installing and running the harvester on your own computer.<jupyter_code>display(text)
display(pdf)
display(image)
display(start_button)
display(out)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10733861144945189, 'fraction_numerical': 0.004263093788063338, 'mean_word_length': 3.393979933110368, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 19, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '129076', 'license': 'MIT', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1937, 'n_words': 570, 'path': 'newspaper_harvester_app.ipynb', 'repo_name': 'WebSepp/trove-newspaper-harvester', 'size': '6568'}