<commit_before>import boto
import os
from StringIO import StringIO
from time import sleep
import zlib
from lxml import etree
import re

class MissingTagException(Exception):
    pass


def tag_match(tagname, str, allow_none=False, concat=False):
    regex_str = "<{}>(.+?)</{}>".format(tagname, tagname)
    matches = re.findall(regex_str, str)

    if len(matches) == 0:
        if allow_none:
            return None
        else:
            print "something broke in this record:"
            print str
            raise MissingTagException

    if concat:
        return "|".join(matches)
    else:
        return matches[0]


def main():
    print "running main()"
    conn = boto.connect_s3(
        os.getenv("AWS_ACCESS_KEY_ID"),
        os.getenv("AWS_SECRET_ACCESS_KEY")
    )

    my_bucket = conn.get_bucket('base-initial')

    i = 0
    for key in my_bucket.list():
        if not key.name.startswith("base_dc_dump") or not key.name.endswith(".gz"):
            continue

        if i >= 2:
            break

        print "getting this key...", key.name



        print "done."

        # that second arg is important. see http://stackoverflow.com/a/18319515
        res = zlib.decompress(key.get_contents_as_string(), 16+zlib.MAX_WBITS)


        obj = {}
        records = re.findall("<record>.+?</record>", res, re.DOTALL)
        for record in records:

            obj["id"] = tag_match("identifier", record)
            obj["title"] = tag_match("dc:title", record)
            obj["oa"] = tag_match("base_dc:oa", record)
            obj["license"] = tag_match("base_dc:rights", record, allow_none=True)

            obj["creators"] = tag_match("dc:creator", record, concat=True)
            obj["identifiers"] = tag_match("dc:identifier", record, concat=True)

            obj["relations"] = tag_match("dc:relation", record, concat=True, allow_none=True)
            obj["hosts"] = tag_match("base_dc:collname", record, concat=True, allow_none=True)





            print obj







        i += 1






if __name__ == "__main__":<commit_msg>gets all we need from the xml records.<commit_after>import boto
import os
from time import sleep
from util import elapsed
import zlib
import re

class MissingTagException(Exception):
    pass


def tag_match(tagname, str, return_list=False):
    regex_str = "<{}>(.+?)</{}>".format(tagname, tagname)
    matches = re.findall(regex_str, str)

    if return_list:
        return matches  # will be empty list if we found naught
    else:
        try:
            return matches[0]
        except IndexError:  # no matches.
            return None


def is_complete(record):
    required_keys = [
        "id",
        "title",
        "urls"
    ]

    for k in required_keys:
        if not record[k]:  # empty list is falsey
            print u"Record is missing required key '{}'!".format(k)
            print record
            return False

    if record["oa"] == 0:
        print u"record {} is closed access. skipping.".format(record["id"])
        return False

    return True




def main():
    print "running main()"
    conn = boto.connect_s3(
        os.getenv("AWS_ACCESS_KEY_ID"),
        os.getenv("AWS_SECRET_ACCESS_KEY")
    )

    my_bucket = conn.get_bucket('base-initial')

    i = 0
    for key in my_bucket.list():
        if not key.name.startswith("base_dc_dump") or not key.name.endswith(".gz"):
            continue

        if i >= 100:
            break

        print "getting this key...", key.name
        print "done."

        # that second arg is important. see http://stackoverflow.com/a/18319515
        res = zlib.decompress(key.get_contents_as_string(), 16+zlib.MAX_WBITS)

        xml_records = re.findall("<record>.+?</record>", res, re.DOTALL)

        records_to_save = []
        for xml_record in xml_records:
            record = {}
            record["id"] = tag_match("identifier", xml_record)
            record["title"] = tag_match("dc:title", xml_record)
            record["license"] = tag_match("base_dc:rights", xml_record)

            try:
                record["oa"] = int(tag_match("base_dc:oa", xml_record))
            except TypeError:
                record["oa"] = 0

            record["urls"] = tag_match("dc:identifier", xml_record, return_list=True)
            record["authors"] = tag_match("dc:creator", xml_record, return_list=True)
            record["relations"] = tag_match("dc:relation", xml_record, return_list=True)
            record["sources"] = tag_match("base_dc:collname", xml_record, return_list=True)


            if is_complete(record):
                records_to_save.append(record)



        i += 1

        # save it!
        print u"saving a chunk of {} records.".format(len(records_to_save))






if __name__ == "__main__":
{'fraction_non_alphanumeric': 0.12484316185696362, 'fraction_numerical': 0.007319113341698034, 'mean_word_length': 2.3447552447552447, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6487401', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1447, 'n_words': 353}