<commit_before>#!/bin/python2
# -*- coding: utf-8 -*-
# Tool for downloading OADOI update files
# Default config file
"""
[Unpaywall]
changelist_url = http://api.unpaywall.org/feed/changefile
api_key = MY_API_KEY
local_update_file_dir = /tmp/oadoi
changelist_file_regex = changed_dois_with_versions_([\d-]+)(.*)([\d-]).*.jsonl.gz
"""

from subprocess import call
from operator import itemgetter
import gzip
import json
import os
import process_util
import re
import subprocess
import sys
import traceback
import urllib
import util


TEMPORARY_TEST_CHANGEFILE = "/tmp/changefiles.json"


def GetRemoteUpdateObjects(url, api_key):
    print "Get Changelists"
    response = open(TEMPORARY_TEST_CHANGEFILE)
    #response = urllib.urlopen(url + '?api_key=' + api_key)
    jdata = json.load(response)
    # Get only JSON update entries, no CSV
    json_update_objects = [item for item in jdata['list'] if item['filetype'] == 'jsonl']
    return json_update_objects


def GetRemoteUpdateFiles(json_update_objects):
    json_filenames = [item['filename'] for item in json_update_objects]
    return json_filenames


def GetLocalUpdateFiles(config, local_directory=None):
    def GetDirectoryContents():
        if local_directory is None:
            return os.listdir(".")
        else:<commit_msg>Remove test code<commit_after>#!/bin/python2
# -*- coding: utf-8 -*-
# Tool for downloading OADOI update files
# Default config file
"""
[Unpaywall]
changelist_url = http://api.unpaywall.org/feed/changefile
api_key = MY_API_KEY
local_update_file_dir = /tmp/oadoi
changelist_file_regex = changed_dois_with_versions_([\d-]+)(.*)([\d-]).*.jsonl.gz
"""

from subprocess import call
from operator import itemgetter
import gzip
import json
import os
import process_util
import re
import subprocess
import sys
import traceback
import urllib
import util


def GetRemoteUpdateObjects(url, api_key):
    print "Get Changelists"
    response = urllib.urlopen(url + '?api_key=' + api_key)
    jdata = json.load(response)
    # Get only JSON update entries, no CSV
    json_update_objects = [item for item in jdata['list'] if item['filetype'] == 'jsonl']
    return json_update_objects


def GetRemoteUpdateFiles(json_update_objects):
    json_filenames = [item['filename'] for item in json_update_objects]
    return json_filenames


def GetLocalUpdateFiles(config, local_directory=None):
    def GetDirectoryContents():
        if local_directory is None:
            return os.listdir(".")
        else:
{'fraction_non_alphanumeric': 0.10301810865191147, 'fraction_numerical': 0.0016096579476861167, 'mean_word_length': 4.5, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5654436', 'n_tokens_mistral': 805, 'n_tokens_neox': 787, 'n_words': 207}