<commit_before>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import dzclient
import glob
import json
import os

import stoneridge

class StoneRidgeReporter(object):
    def __init__(self):
        self.rootdir = stoneridge.get_config('server', 'uploads')
        self.pattern = os.path.join(self.rootdir, '*.json')
        self.host = stoneridge.get_config('report', 'host')
        self.project = stoneridge.get_config('report', 'project')
        self.key = stoneridge.get_config('report', 'key')
        self.secret = stoneridge.get_config('report', 'secret')

    def run(self):
        files = glob.glob(self.pattern)
        for fpath in files:
            with file(fpath, 'rb') as f:
                try:
                    dataset = json.load(f)
                except:
                    # This one is crap, trash it so we never try it again
                    os.unlink(fpath)
                    continue

            request = dzclient.DatazillaRequest(self.host, self.project,
                    self.key, self.secret)
            response = request.send(dataset)
            if response.status != 200:
                continue

            result = json.load(response)
            if result['status'] != 'ok':
                continue

            # If we get here, everything went ok, so we can delete the file
            os.unlink(fpath)

@stoneridge.main
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', dest='config', required=True)
    args = parser.parse_args()

    stoneridge._conffile = args.config

    reporter = StoneRidgeReporter()
    reporter.run()
<commit_msg>DatazillaRequest now takes the scheme, as well<commit_after>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import dzclient
import glob
import json
import os

import stoneridge

class StoneRidgeReporter(object):
    def __init__(self):
        self.rootdir = stoneridge.get_config('server', 'uploads')
        self.pattern = os.path.join(self.rootdir, '*.json')
        self.host = stoneridge.get_config('report', 'host')
        self.project = stoneridge.get_config('report', 'project')
        self.key = stoneridge.get_config('report', 'key')
        self.secret = stoneridge.get_config('report', 'secret')

    def run(self):
        files = glob.glob(self.pattern)
        for fpath in files:
            with file(fpath, 'rb') as f:
                try:
                    dataset = json.load(f)
                except:
                    # This one is crap, trash it so we never try it again
                    os.unlink(fpath)
                    continue

            request = dzclient.DatazillaRequest('https', self.host,
                    self.project, self.key, self.secret)
            response = request.send(dataset)
            if response.status != 200:
                continue

            result = json.load(response)
            if result['status'] != 'ok':
                continue

            # If we get here, everything went ok, so we can delete the file
            os.unlink(fpath)

@stoneridge.main
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', dest='config', required=True)
    args = parser.parse_args()

    stoneridge._conffile = args.config

    reporter = StoneRidgeReporter()
    reporter.run()

{'fraction_non_alphanumeric': 0.10503519220357337, 'fraction_numerical': 0.003789929615592853, 'mean_word_length': 2.1313559322033897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2322373', 'n_tokens_mistral': 1089, 'n_tokens_neox': 1027, 'n_words': 301}