<commit_before>#!/usr/bin/env python

import sys
import urllib2
import argparse

JOB_LOG_URL = "http://atlas-panda-jobsarchived.s3.amazonaws.com"


def download_log(date_string, save_raw=False):
    """
    Download job log files from Amazon EC2 machines

    parameters:
    start_date - beginning date to start downloading from
    end_date   - last date to dowload job data for
    work_directory  - directory to download files to
    """
    if save_raw:
        csv_file = "jobsarchived{0}-cleaned.csv".format(date_string)
    else:
        csv_file = "jobsarchived{0}.csv".format(date_string)
    csv_url = "{0}/{1}".format(JOB_LOG_URL, csv_file)
    try:
        request = urllib2.urlopen(csv_url)
        if request.getcode() != 200:
            sys.stderr.write("Can't download {0}".format(csv_url))
            return None
    except urllib2.HTTPError:
        sys.stderr.write("Can't download {0}".format(csv_url))
        return None
    output_file = open(csv_file, 'w')
    error_lines = 0
    for line in request:
        if len(line.split(',')) != 87:
            error_lines += 1
        else:
            output_file.write(line)
    sys.stderr.write("{0} lines skipped due to errors".format(error_lines))
    return None


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Download and process ATLAS'
                                                 'job records')
    parser.add_argument('--date', dest='date', default=None,
                        help='Date to download')
    parser.add_argument('--save-raw', dest='save_raw',
                        action='store_true',
                        help='Save raw log files instead of replacing in place')

    args = parser.parse_args(sys.argv[1:])
    if len(args.date) != 8:
        sys.stderr.write("Invalid date argument: {0}\n".format(args.date))
    try:
        int(args.date)
    except ValueError:
        sys.stderr.write("Invalid date argument: {0}\n".format(args.date))

    download_log(sys.argv[1], args.save_raw)<commit_msg>Pass date correctly<commit_after>#!/usr/bin/env python

import sys
import urllib2
import argparse

JOB_LOG_URL = "http://atlas-panda-jobsarchived.s3.amazonaws.com"


def download_log(date_string, save_raw=False):
    """
    Download job log files from Amazon EC2 machines

    parameters:
    start_date - beginning date to start downloading from
    end_date   - last date to dowload job data for
    work_directory  - directory to download files to
    """
    url_file = "jobsarchived{0}.csv".format(date_string)
    if save_raw:
        csv_file = "jobsarchived{0}-cleaned.csv".format(date_string)
    else:
        csv_file = "jobsarchived{0}.csv".format(date_string)
    csv_url = "{0}/{1}".format(JOB_LOG_URL, csv_file)
    try:
        request = urllib2.urlopen(csv_url)
        if request.getcode() != 200:
            sys.stderr.write("Can't download {0}".format(csv_url))
            return None
    except urllib2.HTTPError:
        sys.stderr.write("Can't download {0}".format(csv_url))
        return None
    output_file = open(csv_file, 'w')
    error_lines = 0
    for line in request:
        if len(line.split(',')) != 87:
            error_lines += 1
        else:
            output_file.write(line)
    sys.stderr.write("{0} lines skipped due to errors".format(error_lines))
    return None


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Download and process ATLAS'
                                                 'job records')
    parser.add_argument('--date', dest='date', default=None,
                        help='Date to download')
    parser.add_argument('--save-raw', dest='save_raw',
                        action='store_true',
                        help='Save raw log files instead of replacing in place')

    args = parser.parse_args(sys.argv[1:])
    if len(args.date) != 8:
        sys.stderr.write("Invalid date argument: {0}\n".format(args.date))
    try:
        int(args.date)
    except ValueError:
        sys.stderr.write("Invalid date argument: {0}\n".format(args.date))

    download_log(args.date, args.save_raw)
{'fraction_non_alphanumeric': 0.12030985233599613, 'fraction_numerical': 0.011619462599854757, 'mean_word_length': 2.4839797639123105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '497898', 'n_tokens_mistral': 1309, 'n_tokens_neox': 1256, 'n_words': 287}