<commit_before>from collections import namedtuple
from cStringIO import StringIO
from shapely.geometry import MultiPolygon
from shapely import geometry
from shapely.wkb import loads
from tilequeue.tile import calc_meters_per_pixel_dim
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.transform import mercator_point_to_lnglat
from tilequeue.transform import transform_feature_layers_shape
from zope.dottedname.resolve import resolve


def make_transform_fn(transform_fns):
    if not transform_fns:
        return None

    def transform_fn(shape, properties, fid, zoom):
        for fn in transform_fns:
            shape, properties, fid = fn(shape, properties, fid, zoom)
        return shape, properties, fid
    return transform_fn


def resolve_transform_fns(fn_dotted_names):
    if not fn_dotted_names:
        return None
    return map(resolve, fn_dotted_names)


def _preprocess_data(feature_layers):
    preproc_feature_layers = []

    for feature_layer in feature_layers:
        layer_datum = feature_layer['layer_datum']
        geometry_types = layer_datum['geometry_types']
        padded_bounds = feature_layer['padded_bounds']
        shape_padded_bounds = geometry.box(*padded_bounds)

        features = []
        for row in feature_layer['features']:
            wkb = bytes(row.pop('__geometry__'))
            shape = loads(wkb)

            if shape.is_empty:
                continue

            if not shape.is_valid:
                continue

            if geometry_types is not None:
                if shape.type not in geometry_types:
                    continue

            # since a bounding box intersection is used, we
            # perform a more accurate check here to filter out
            # any extra features
            # the formatter specific transformations will take
            # care of any additional filtering
            if not shape_padded_bounds.intersects(shape):
                continue

            feature_id = row.pop('__id__')
            props = dict()
            for k, v in row.iteritems():
                if v is None:
                    continue
                if k == 'mz_properties':
                    for output_key, output_val in v.items():
                        if output_val is not None:
                            # all other tags are utf8 encoded, encode
                            # these the same way to be consistent
                            output_key = output_key.encode('utf-8')
                            if isinstance(output_val, unicode):
                                output_val = output_val.encode('utf-8')
                            props[output_key] = output_val
                else:
                    props[k] = v

            feature = shape, props, feature_id
            features.append(feature)

        preproc_feature_layer = dict(
            name=layer_datum['name'],
            layer_datum=layer_datum,
            features=features,
            padded_bounds=padded_bounds,
        )
        preproc_feature_layers.append(preproc_feature_layer)

    return preproc_feature_layers


# shared context for all the post-processor functions. this single object can
# be passed around rather than needing all the parameters to be explicit.<commit_msg>Guard json props key with isinstance check<commit_after>from collections import namedtuple
from cStringIO import StringIO
from shapely.geometry import MultiPolygon
from shapely import geometry
from shapely.wkb import loads
from tilequeue.tile import calc_meters_per_pixel_dim
from tilequeue.tile import coord_to_mercator_bounds
from tilequeue.transform import mercator_point_to_lnglat
from tilequeue.transform import transform_feature_layers_shape
from zope.dottedname.resolve import resolve


def make_transform_fn(transform_fns):
    if not transform_fns:
        return None

    def transform_fn(shape, properties, fid, zoom):
        for fn in transform_fns:
            shape, properties, fid = fn(shape, properties, fid, zoom)
        return shape, properties, fid
    return transform_fn


def resolve_transform_fns(fn_dotted_names):
    if not fn_dotted_names:
        return None
    return map(resolve, fn_dotted_names)


def _preprocess_data(feature_layers):
    preproc_feature_layers = []

    for feature_layer in feature_layers:
        layer_datum = feature_layer['layer_datum']
        geometry_types = layer_datum['geometry_types']
        padded_bounds = feature_layer['padded_bounds']
        shape_padded_bounds = geometry.box(*padded_bounds)

        features = []
        for row in feature_layer['features']:
            wkb = bytes(row.pop('__geometry__'))
            shape = loads(wkb)

            if shape.is_empty:
                continue

            if not shape.is_valid:
                continue

            if geometry_types is not None:
                if shape.type not in geometry_types:
                    continue

            # since a bounding box intersection is used, we
            # perform a more accurate check here to filter out
            # any extra features
            # the formatter specific transformations will take
            # care of any additional filtering
            if not shape_padded_bounds.intersects(shape):
                continue

            feature_id = row.pop('__id__')
            props = dict()
            for k, v in row.iteritems():
                if v is None:
                    continue
                if k == 'mz_properties':
                    for output_key, output_val in v.items():
                        if output_val is not None:
                            # all other tags are utf8 encoded, encode
                            # these the same way to be consistent
                            if isinstance(output_key, unicode):
                                output_key = output_key.encode('utf-8')
                            if isinstance(output_val, unicode):
                                output_val = output_val.encode('utf-8')
                            props[output_key] = output_val
                else:
                    props[k] = v

            feature = shape, props, feature_id
            features.append(feature)

        preproc_feature_layer = dict(
            name=layer_datum['name'],
            layer_datum=layer_datum,
            features=features,
            padded_bounds=padded_bounds,
        )
        preproc_feature_layers.append(preproc_feature_layer)

    return preproc_feature_layers


# shared context for all the post-processor functions. this single object can
# be passed around rather than needing all the parameters to be explicit.
{'fraction_non_alphanumeric': 0.056063558686853546, 'fraction_numerical': 0.000899415380002998, 'mean_word_length': 1.9865711727842434, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5070731', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1653, 'n_words': 500}