<commit_before>import abc
import base64
import datetime
import hashlib
import json
import os
import requests
import scraperwiki
import urllib.request
from collections import OrderedDict
from hashlib import sha1
from retry import retry
from urllib.error import HTTPError


def truncate(table):
    scraperwiki.sqlite.execute("DROP TABLE IF EXISTS %s;" % table)
    scraperwiki.sqlite.commit_transactions()


def summarise(table):
    count = scraperwiki.sqlite.execute(
        "SELECT COUNT(*) AS count FROM %s;" % table)
    print("%i %s in database" % (count['data'][0].count, table))


def save(unique_keys, data, table_name):
    scraperwiki.sqlite.save(
        unique_keys=unique_keys,
        data=data,
        table_name=table_name)
    scraperwiki.sqlite.commit_transactions()


def get_github_credentials():
    return GitHubCredentials(
        repo=os.environ['MORPH_GITHUB_REPO'],
        name=os.environ['MORPH_GITHUB_USERNAME'],
        email=os.environ['MORPH_GITHUB_EMAIL'],
        api_key=os.environ['MORPH_GITHUB_API_KEY']
    )


def dump_table_to_json(table_name, key):
    if isinstance(key, list):
        order_clause = ", ".join(key)
    else:
        order_clause = key
    records = scraperwiki.sqlite.select(
        " * FROM %s ORDER BY %s;" % (table_name, order_clause))
    return json.dumps(
        [OrderedDict(sorted(rec.items())) for rec in records],
        sort_keys=True, indent=4)


def sync_file_to_github(council_id, file_name, content):
    try:
        creds = get_github_credentials()
        g = GitHubClient(creds)
        path = "%s/%s.json" % (council_id, file_name)
        g.put_file(content, path)
    except KeyError:
        # if no credentials are defined in env vars
        # just ignore this step
        pass


def sync_db_to_github(council_id, table_name, key):
    content = dump_table_to_json(table_name, key)
    sync_file_to_github(council_id, table_name, content)<commit_msg>if we haven't specified an extension, assume .json<commit_after>import abc
import base64
import datetime
import hashlib
import json
import os
import requests
import scraperwiki
import urllib.request
from collections import OrderedDict
from hashlib import sha1
from retry import retry
from urllib.error import HTTPError


def truncate(table):
    scraperwiki.sqlite.execute("DROP TABLE IF EXISTS %s;" % table)
    scraperwiki.sqlite.commit_transactions()


def summarise(table):
    count = scraperwiki.sqlite.execute(
        "SELECT COUNT(*) AS count FROM %s;" % table)
    print("%i %s in database" % (count['data'][0].count, table))


def save(unique_keys, data, table_name):
    scraperwiki.sqlite.save(
        unique_keys=unique_keys,
        data=data,
        table_name=table_name)
    scraperwiki.sqlite.commit_transactions()


def get_github_credentials():
    return GitHubCredentials(
        repo=os.environ['MORPH_GITHUB_REPO'],
        name=os.environ['MORPH_GITHUB_USERNAME'],
        email=os.environ['MORPH_GITHUB_EMAIL'],
        api_key=os.environ['MORPH_GITHUB_API_KEY']
    )


def dump_table_to_json(table_name, key):
    if isinstance(key, list):
        order_clause = ", ".join(key)
    else:
        order_clause = key
    records = scraperwiki.sqlite.select(
        " * FROM %s ORDER BY %s;" % (table_name, order_clause))
    return json.dumps(
        [OrderedDict(sorted(rec.items())) for rec in records],
        sort_keys=True, indent=4)


def sync_file_to_github(council_id, file_name, content):
    try:
        creds = get_github_credentials()
        g = GitHubClient(creds)
        path = "%s/%s" % (council_id, file_name)

        # if we haven't specified an extension, assume .json
        if '.' not in path:
            path = "%s.json" % (path)

        g.put_file(content, path)
    except KeyError:
        # if no credentials are defined in env vars
        # just ignore this step
        pass


def sync_db_to_github(council_id, table_name, key):
    content = dump_table_to_json(table_name, key)
    sync_file_to_github(council_id, table_name, content)
{'fraction_non_alphanumeric': 0.10086526576019778, 'fraction_numerical': 0.002472187886279357, 'mean_word_length': 3.426695842450766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2456987', 'n_tokens_mistral': 1353, 'n_tokens_neox': 1339, 'n_words': 313}