<commit_before>import json
import os
import shutil
import sys
import tempfile
import traceback
import copy
import re
import math

import requests

from girder.constants import AccessType
from girder.utility import config
from girder.utility.model_importer import ModelImporter
from girder.plugins.jobs.constants import JobStatus
from girder.plugins.minerva.utility.bsve.bsve_utility import BsveUtility
from girder.plugins.minerva.utility.minerva_utility import mM

import girder_client

# This job will accumulate and append metadata into
# the us states geojson properties.


#: geojson of state boundaries source url
states_source = (
    'https://gist.githubusercontent.com/jbeezley/82150e34cee8512815cf/'
    'raw/32549660c18149698a849b00cac6df8ddd2d1f1a/us-states.json'
)

#: cached geojson object
_states = None

# formatted like this:
"""
{
    "type": "FeatureCollection",
    "features": [
        {
            "type": "Feature",
            "properties": {
                "name": "New York",
                "abbr": "NY"
            },
            "geometry": {
                "type": "MultiPolygon",
                "coordinates": [
                    [ # polygon 1
                        [ # outer ring
                           [], [], ...
                        ],
                        [ # inner ring 1
                            [], [], ...
                        ],
                        ...
                    ],
                    ...
                ]
            }
        },
        ...
    ]
"""


#: A set of properties to ignore when accumulating
ignored_properties = set([
    'mmwr_week',
    'mmwr_year'
])


def accumulate(data):
    """Accumulate data by state returning a geojson object.

    The argument is expected to be structured as follows:
        [
            {
                "location_1": {
                    "city": "New York City, NY", # city, state pair
                },
                "key1": "value1",
                "key2": "value2",
                ...
            }
        ]

    The state is parsed from the `reporting_area` key.  If any error occurs,
    the record is skipped.  For the remaining keys, the
    value is coerced as a number and accumulated into the states' properties.
    Any error in the conversion causes the key to be skipped.  An additional
    key is added to each state, `num_records` indicating the number of records
    existed for that state.

    Finally, any state for which no record was encountered is filtered out of
    the returned value.

    :param list data: A list of dicts containing mappings of keys to counts
    :returns dict: A geojson object with ``properties`` containing accumulated
        values.  The top level ``FeatureCollection`` property contains a list
        of list of keys available in the individual features.
    """
    global _states
    if _states is None:
        _states = requests.get(states_source).json()
    geojson = copy.deepcopy(_states)

    # create a abbr -> properties mapping
    states = {
        feature.get('properties', {}).get('abbr'):
            feature.get('properties', {})
        for feature in geojson.get('features', [])
    }

    # city, state regex
    citystate = re.compile(r', ([A-Z][A-Z])$')

    # store all properties encountered here
    all_props = set()

    # loop over records
    for record in data:
        m = citystate.search(record.get('reporting_area', ''))
        if m and m.group(1) in states:  # it is a valid state

            props = states[m.group(1)]
            props['num_records'] = props.get('num_records', 0) + 1

            for key, value in record.iteritems():
                if key in ignored_properties:
                    continue

                try:
                    v = float(value)
                except Exception:
                    v = float('NaN')

                # check that it's a real value
                if math.isnan(v) or math.isinf(v):
                    continue

                if v < 0:  # maybe this is a valid case?
                    print('Encountered a negative value!')

                # accumulate the field as a property
                props[key] = props.get(key, 0) + v

                # add the property to the global list of properties found
                all_props.add(key)

    # filter out states that have no records
    # geojson['properties'] = filter(
    #     lambda f: states.get(
    #         f.get('properties', {}).get('abbr')
    #     ).get('num_records', 0) >= 0,
    #     geojson.get('properties', [])
    # )

    # store the global list of properties
    geojson['properties'] = {
        'values': list(all_props)
    }
    return geojson


def run(job):
    job_model = ModelImporter.model('job', 'jobs')
    job_model.updateJob(job, status=JobStatus.RUNNING)

    try:
        configFile = os.path.join(os.path.dirname(__file__), "bsve.json")
        if os.path.exists(configFile):
            bsveConfig = json.load(open(configFile))['bsve']
        else:
            bsveConfig = {}

        kwargs = job['kwargs']
        datasetId = str(kwargs['dataset']['_id'])

        # TODO better to create a job token rather than a user token?
        token = kwargs['token']

        params = kwargs['params']

        bsveUtility = BsveUtility(
            user=bsveConfig.get(
                'USER_NAME', os.environ.get('BSVE_USERNAME')),
            apikey=bsveConfig.get(
                'API_KEY', os.environ.get('BSVE_APIKEY')),<commit_msg>Remove commented-out code<commit_after>import json
import os
import shutil
import sys
import tempfile
import traceback
import copy
import re
import math

import requests

from girder.constants import AccessType
from girder.utility import config
from girder.utility.model_importer import ModelImporter
from girder.plugins.jobs.constants import JobStatus
from girder.plugins.minerva.utility.bsve.bsve_utility import BsveUtility
from girder.plugins.minerva.utility.minerva_utility import mM

import girder_client

# This job will accumulate and append metadata into
# the us states geojson properties.


#: geojson of state boundaries source url
states_source = (
    'https://gist.githubusercontent.com/jbeezley/82150e34cee8512815cf/'
    'raw/32549660c18149698a849b00cac6df8ddd2d1f1a/us-states.json'
)

#: cached geojson object
_states = None

# formatted like this:
"""
{
    "type": "FeatureCollection",
    "features": [
        {
            "type": "Feature",
            "properties": {
                "name": "New York",
                "abbr": "NY"
            },
            "geometry": {
                "type": "MultiPolygon",
                "coordinates": [
                    [ # polygon 1
                        [ # outer ring
                           [], [], ...
                        ],
                        [ # inner ring 1
                            [], [], ...
                        ],
                        ...
                    ],
                    ...
                ]
            }
        },
        ...
    ]
"""


#: A set of properties to ignore when accumulating
ignored_properties = set([
    'mmwr_week',
    'mmwr_year'
])


def accumulate(data):
    """Accumulate data by state returning a geojson object.

    The argument is expected to be structured as follows:
        [
            {
                "location_1": {
                    "city": "New York City, NY", # city, state pair
                },
                "key1": "value1",
                "key2": "value2",
                ...
            }
        ]

    The state is parsed from the `reporting_area` key.  If any error occurs,
    the record is skipped.  For the remaining keys, the
    value is coerced as a number and accumulated into the states' properties.
    Any error in the conversion causes the key to be skipped.  An additional
    key is added to each state, `num_records` indicating the number of records
    existed for that state.

    Finally, any state for which no record was encountered is filtered out of
    the returned value.

    :param list data: A list of dicts containing mappings of keys to counts
    :returns dict: A geojson object with ``properties`` containing accumulated
        values.  The top level ``FeatureCollection`` property contains a list
        of list of keys available in the individual features.
    """
    global _states
    if _states is None:
        _states = requests.get(states_source).json()
    geojson = copy.deepcopy(_states)

    # create a abbr -> properties mapping
    states = {
        feature.get('properties', {}).get('abbr'):
            feature.get('properties', {})
        for feature in geojson.get('features', [])
    }

    # city, state regex
    citystate = re.compile(r', ([A-Z][A-Z])$')

    # store all properties encountered here
    all_props = set()

    # loop over records
    for record in data:
        m = citystate.search(record.get('reporting_area', ''))
        if m and m.group(1) in states:  # it is a valid state

            props = states[m.group(1)]
            props['num_records'] = props.get('num_records', 0) + 1

            for key, value in record.iteritems():
                if key in ignored_properties:
                    continue

                try:
                    v = float(value)
                except Exception:
                    v = float('NaN')

                # check that it's a real value
                if math.isnan(v) or math.isinf(v):
                    continue

                if v < 0:  # maybe this is a valid case?
                    print('Encountered a negative value!')

                # accumulate the field as a property
                props[key] = props.get(key, 0) + v

                # add the property to the global list of properties found
                all_props.add(key)

    # store the global list of properties
    geojson['properties'] = {
        'values': list(all_props)
    }
    return geojson


def run(job):
    job_model = ModelImporter.model('job', 'jobs')
    job_model.updateJob(job, status=JobStatus.RUNNING)

    try:
        configFile = os.path.join(os.path.dirname(__file__), "bsve.json")
        if os.path.exists(configFile):
            bsveConfig = json.load(open(configFile))['bsve']
        else:
            bsveConfig = {}

        kwargs = job['kwargs']
        datasetId = str(kwargs['dataset']['_id'])

        # TODO better to create a job token rather than a user token?
        token = kwargs['token']

        params = kwargs['params']

        bsveUtility = BsveUtility(
            user=bsveConfig.get(
                'USER_NAME', os.environ.get('BSVE_USERNAME')),
            apikey=bsveConfig.get(
                'API_KEY', os.environ.get('BSVE_APIKEY')),
{'fraction_non_alphanumeric': 0.09883290107447203, 'fraction_numerical': 0.010003705075954057, 'mean_word_length': 1.7949779963758736, 'pattern_counts': {'":': 26, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7568509', 'n_tokens_mistral': 3114, 'n_tokens_neox': 2896, 'n_words': 899}