<commit_before>"""
Author: <NAME>
Date: 12 December 2016
Purpose: This script will do the following things:
1. If there is no hash record of a PDF file in the `figures/` directory, then
   the script will create a hash and store it.
2. If there is a hash record, then the script will compare the current file
   hash against the existing file hash, and return whether the hash has
   changed or not.
"""

import logging
import os
import hashlib
import yaml

logging.basicConfig(level=logging.INFO)


def hashfile(f, hasher, blocksize=65536):
    buf = f.read(blocksize)
    while len(buf) > 0:
        hasher.update(buf)
        buf = f.read(blocksize)
    return hasher.hexdigest()


figures = [
    'networks',
    'recommender',
    'collaborative-filtering',
    'panama',
    'influenza',
    'convolutions',
    'hairballs',
    'rational-viz',
]

hashes = dict()
for fig in figures:
    with open('figures/{0}.pdf'.format(fig), 'rb') as f:
        fhash = hashfile(f, hashlib.sha256())
        hashes[fig] = fhash

with open('hash.log', 'w+') as f:
    yaml.dump(hashes, f, canonical=False, default_flow_style=False)
# logging.info(yaml.dump(hashes))

with open('hash.log', 'r+') as f:
    loaded_hashes = yaml.load(f)
    # logging.info(loaded_hashes)

# Check if hashes are identical or not.
for fig in figures:
    if hashes[fig] == loaded_hashes[fig]:
        logging.info('figure {0} unchanged.'.format(fig))
    else:
        logging.info('figure {0}.pdf has changed')
        os.system("convert -density 300 -resize 50% figures/{0}.pdf \
            figures/{0}.png".format(fig))

logging.info('converting slides to PDF...')
os.system("pandoc slides.md -t beamer -o slides.pdf --template=default.beamer")
logging.info('converting slides to HTML...')
os.system("pandoc slides.md -o index.html -H docs/css/styles.css \
    --template=default.html")

logging.info('checking status of the repository')
os.system('git status')<commit_msg>shifted build script into a python script<commit_after>"""
Author: <NAME>
Date: 12 December 2016
Purpose: This script will do the following things:
1. If there is no hash record of a PDF file in the `figures/` directory, then
   the script will create a hash and store it.
2. If there is a hash record, then the script will compare the current file
   hash against the existing file hash, and return whether the hash has
   changed or not.
"""

import logging
import os
import hashlib
import yaml

logging.basicConfig(level=logging.INFO)


def hashfile(f, hasher, blocksize=65536):
    buf = f.read(blocksize)
    while len(buf) > 0:
        hasher.update(buf)
        buf = f.read(blocksize)
    return hasher.hexdigest()


files = [
    'figures/networks.pdf',
    'figures/recommender.pdf',
    'figures/collaborative-filtering.pdf',
    'figures/panama.pdf',
    'figures/influenza.pdf',
    'figures/convolutions.pdf',
    'figures/hairballs.pdf',
    'figures/rational-viz.pdf',
    'slides.md',
]

hashes = dict()
for f in files:
    with open('{0}'.format(f), 'rb') as f:
        fhash = hashfile(f, hashlib.sha256())
        hashes[f] = fhash

with open('hash.log', 'r+') as f:
    prev_hashes = yaml.load(f)

# Check if hashes are identical or not.
for f in files:
    if hashes[f] == prev_hashes[f]:
        logging.info('file {0} unchanged.'.format(f))
    else:
        logging.info('file {0}.pdf has changed')
        if f[-4:] == '.pdf':
            fstr = f[:-4]
            os.system("convert -density 300 -resize 50% {0}.pdf \
                {1}.png".format(f, fstr))
        elif f == 'slides.md':
            logging.info('converting slides to PDF...')
            os.system("pandoc slides.md -t beamer -o slides.pdf \
                --template=default.beamer")

            logging.info('converting slides to HTML...')
            os.system("pandoc slides.md -o index.html -H docs/css/styles.css \
                --template=default.html")

# Write new hashes to disk.
with open('hash.log', 'w+') as f:
    yaml.dump(hashes, f, canonical=False, default_flow_style=False)
    logging.info('file hashes are:')
    logging.info(hashes)



logging.info('checking status of the repository')
os.system('git status')
{'fraction_non_alphanumeric': 0.1342426417803302, 'fraction_numerical': 0.01340033500837521, 'mean_word_length': 2.9923591212989495, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4948059', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1344, 'n_words': 410}