<commit_before>#! /usr/env/python
"""Makes HTTP requests for datasets as JSON objects, 
then creates a CSV file containing the results.

Script usage
------------
user@host: python OnlineAnalyzer.py <fourby_list> <destination_file>

"""
import sys
import utilities
import logging
import re

logging.basicConfig(filename="online_analyzer.log", filemode="w", level=logging.DEBUG)


if __name__ == "__main__":
    docstring = """USAGE: python OnlineAnalyzer.py <fourby_list> <destination_file>
    """
    if len(sys.argv) < 3:
        sys.exit(docstring)
    datafile = sys.argv[1]
    outfile = sys.argv[2]

    with open(datafile) as infile:
        fours_file = infile.read()
    soc_ids = re.findall('[0-9a-z]{4}-[0-9a-z]{4}?', fours_file, re.DOTALL)
    for soc in soc_ids:
        print "soc:", soc
    Requester = utilities.ViewRequestHandler()
    Analyzer = utilities.DatasetAnalyzer()

    for fourby in soc_ids:
        dataset = Requester.get_view(fourby)
        print "Processing %s" % fourby
        if dataset == "null":
            continue
        Analyzer.add_dataset(dataset)

    Analyzer.make_csv(outfile)<commit_msg>Online analyzer changed<commit_after>#! /usr/env/python
"""Makes HTTP requests for datasets as JSON objects, 
then creates a CSV file containing the results.

Script usage
------------
user@host: python OnlineAnalyzer.py <destination_file>

"""
import sys
import utilities
import logging
import re

logging.basicConfig(filename="online_analyzer.log", filemode="w", level=logging.DEBUG)


if __name__ == "__main__":
    docstring = """USAGE: python OnlineAnalyzer.py  <destination_file>
    """
    if len(sys.argv) < 2:
        sys.exit(docstring)
    outfile = sys.argv[1]

    # with open(datafile) as infile:
    #     fours_file = infile.read()
    # soc_ids = re.findall('[0-9a-z]{4}-[0-9a-z]{4}?', fours_file, re.DOTALL)
    # for soc in soc_ids:
    #     print "soc:", soc

    IdGetter = utilities.SocIdGetter()
    soc_ids = IdGetter.soc_ids
    ViewRequester = utilities.ViewRequestHandler()
    Analyzer = utilities.DatasetAnalyzer()

    for fourby in soc_ids:
        dataset = ViewRequester.get_view(fourby)
        print "Processing %s" % fourby
        if dataset == "null":
            continue
        Analyzer.add_dataset(dataset)

    Analyzer.make_csv(outfile)
{'fraction_non_alphanumeric': 0.12721000431220353, 'fraction_numerical': 0.007330746011211729, 'mean_word_length': 3.2181818181818183, 'pattern_counts': {'":': 4, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5349251', 'n_tokens_mistral': 774, 'n_tokens_neox': 752, 'n_words': 181}