<commit_before>"""High level, HTTP-agnostic record access and processing"""

import re, json

from pcapi import logtool
import time

log = logtool.getLogger("provider", "records")


class Record(object):
    """Class to store record bodies and metadata in memory for fast access"""

    def __init__(self, content, metadata):
        """Store content and metadata"""
        self.content = content  # as a parsed json (dict)
        self.metadata = metadata  # as a dbox_provider.Metadata object


def convertToGeoJSON(records):
    """
    Export all records to geojson and return result.
    """
    # python dicts should be automatically converted to geojson by bottle
    # self.response.headers['Content-Type'] = 'application/json'
    features = []
    for r in records:
        # log.debug(r.content)
        # get first -and only- value of dictionary because records are an array of
        # [ { <name> : <geojson feature> } ....]
        f = r.content.values()[0]
        features.append(f)

    geojson_dict = {"type": "FeatureCollection", "features": features}
    return geojson_dict


def create_records_cache(provider, path):
    """Creates an array of Records classed (s.a.) after parsing all records under `path'.
    Assumes a valid session
    """
    ret = []
    # If we are in `/' then get all records
    for d in provider.metadata(path).lsdirs():
        recpath = "{0}/record.json".format(d)
        log.debug("Parsing records -- requesting {0}".format(recpath))

        try:
            buf, meta = provider.get_file_and_metadata(recpath)
            rec = json.loads(buf.read())
            folder = re.split("/+", recpath)[2]
            record = Record({folder: rec}, meta)
            ret.append(record)
        except Exception as e:
            log.exception("Exception: " + str(e))
        buf.close()
    return ret<commit_msg>bugfix<commit_after>"""High level, HTTP-agnostic record access and processing"""

import re, json

from pcapi import logtool
import time

log = logtool.getLogger("provider", "records")


class Record(object):
    """Class to store record bodies and metadata in memory for fast access"""

    def __init__(self, content, metadata):
        """Store content and metadata"""
        self.content = content  # as a parsed json (dict)
        self.metadata = metadata  # as a dbox_provider.Metadata object


def convertToGeoJSON(records):
    """
    Export all records to geojson and return result.
    """
    # python dicts should be automatically converted to geojson by bottle
    # self.response.headers['Content-Type'] = 'application/json'
    features = []
    for r in records:
        # log.debug(r.content)
        # get first -and only- value of dictionary because records are an array of
        # [ { <name> : <geojson feature> } ....]
        f = r.content.values()[0]
        features.append(f)

    geojson_dict = {"type": "FeatureCollection", "features": features}
    return geojson_dict


def create_records_cache(provider, path):
    """Creates an array of Records classed (s.a.) after parsing all records under `path'.
    Assumes a valid session
    """
    ret = []
    # If we are in `/' then get all records
    for d in provider.metadata(path).lsdirs():
        recpath = "{0}/record.json".format(d)
        log.debug("Parsing records -- requesting {0}".format(recpath))

        try:
            buf, meta = provider.get_file_and_metadata(recpath)
            rec = json.loads(buf.read())
            folder = re.split("/+", recpath)[2]
            record = Record({folder: rec}, meta)
            ret.append(record)
            buf.close()
        except Exception as e:
            log.exception("Exception: " + str(e))
    return ret
{'fraction_non_alphanumeric': 0.11878527277613544, 'fraction_numerical': 0.002149959688255845, 'mean_word_length': 2.645445641527914, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6118039', 'n_tokens_mistral': 1056, 'n_tokens_neox': 1033, 'n_words': 339}