<commit_before>from osgeo import ogr
import re
import urllib
import json

uvmfeatures = []

def filterLayer(layer):
    if layer is None:
        return None
    print layer.GetName()
    return layer

def filterFeature(ogrfeature, fieldNames, reproject):
    if ogrfeature is None:
        return
    layer = ogrfeature.GetFieldAsString("Layer")
    if (layer == "VA-BLDG-UVM" or
        layer == "VA-BLDG-NON UVM"):
        return ogrfeature
    elif (layer == "VA-BLDG-ATTRIBUTES" and len(ogrfeature.GetFieldAsString("Text")) == 4):
        return ogrfeature
    else:
        return

def filterFeaturePost(feature, ogrfeature, ogrgeometry):
    if feature is None and ogrfeature is None and ogrgeometry is None:
        return
    global uvmfeatures
    uvmfeatures.append((feature, ogrfeature, ogrgeometry))

def filterTags(tags):
    if tags is None:
        return
    newtags = {}
    for (key, value) in tags.items():
        if (key == "Layer" or 
            (key == "Text" and value != "")):
            newtags[key] = value
        if (key == "Layer" and value != "VA-UVM-BLDG-CODE"):
            newtags["building"] = "yes"
    return newtags
        

def preOutputTransform(geometries, features):
    if geometries is None and features is None:
        return
    global uvmfeatures
    buildingcodes = [(feature, ogrfeature, ogrgeometry) for (feature, ogrfeature, ogrgeometry) in uvmfeatures if ogrfeature.GetFieldAsString("Layer") == "VA-BLDG-ATTRIBUTES"]<commit_msg>uvmtrans: Ignore soccer stands<commit_after>from osgeo import ogr
import re
import urllib
import json

uvmfeatures = []

def filterLayer(layer):
    if layer is None:
        return None
    print layer.GetName()
    return layer

def filterFeature(ogrfeature, fieldNames, reproject):
    if ogrfeature is None:
        return
    layer = ogrfeature.GetFieldAsString("Layer")
    if (layer == "VA-BLDG-UVM" or
        layer == "VA-BLDG-NON UVM"):
        return ogrfeature
    elif (layer == "VA-BLDG-ATTRIBUTES" and len(ogrfeature.GetFieldAsString("Text")) == 4):
        # Ignore CFC Soccer Stands - has no building outline
        if ogrfeature.GetFieldAsString("Text") == "0979":
            return
        return ogrfeature
    else:
        return

def filterFeaturePost(feature, ogrfeature, ogrgeometry):
    if feature is None and ogrfeature is None and ogrgeometry is None:
        return
    global uvmfeatures
    uvmfeatures.append((feature, ogrfeature, ogrgeometry))

def filterTags(tags):
    if tags is None:
        return
    newtags = {}
    for (key, value) in tags.items():
        if (key == "Layer" or 
            (key == "Text" and value != "")):
            newtags[key] = value
        if (key == "Layer" and value != "VA-UVM-BLDG-CODE"):
            newtags["building"] = "yes"
    return newtags
        

def preOutputTransform(geometries, features):
    if geometries is None and features is None:
        return
    global uvmfeatures
    buildingcodes = [(feature, ogrfeature, ogrgeometry) for (feature, ogrfeature, ogrgeometry) in uvmfeatures if ogrfeature.GetFieldAsString("Layer") == "VA-BLDG-ATTRIBUTES"]
{'fraction_non_alphanumeric': 0.10272, 'fraction_numerical': 0.00192, 'mean_word_length': 2.8450184501845017, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1041378', 'n_tokens_mistral': 968, 'n_tokens_neox': 942, 'n_words': 284}