<commit_before>#! /usr/bin/env python

import json
import geojson
import shapely.geometry
import csv
import sys
import optparse

colnames = {
    'bbox': 'bbox',
    'the_geom': 'geom',
    'geom': 'geom',
    'gemoetry': 'geom',
    'lat': 'lat',
    'latitude': 'lat',
    'lon': 'lon',
    'lng': 'lon',
    'longitude': 'lon'
    }

def get_geocols(info, prefix = []):
    geocols = {}
    for key, value in info.iteritems():
        if key in colnames:
            geocols[colnames[key]] = prefix + [key]
        if isinstance(value, dict):
            geocols.update(get_geocols(value, prefix = prefix + [key]))
    return geocols

def getvalue(row, col):
    for item in col:
        if not isinstance(row, dict) or item not in row:
            return None
        row = row[item]
    return row

def selvalue(row, col, value):
    for item in col[:-1]:
        if item not in row or not isinstance(row[item], dict):
            row[item] = {}
        row = row[item]
    row[col[-1]] = value

def delvalue(row, col):
    for item in col[:-1]:
        if not isinstance(row, dict) or item not in row:
            return
        row = row[item]
    if col[-1] in row:
        del row[col[-1]]

def to_csv_value(value):
    if isinstance(value, unicode):
        return value.encode("utf-8")
    elif isinstance(value, str):
        return value
    else:
        return json.dumps(value)

def from_csv_value(value):
    value = value.decode("utf-8")<commit_msg>Bugfix<commit_after>#! /usr/bin/env python

import json
import geojson
import shapely.geometry
import csv
import sys
import optparse

colnames = {
    'bbox': 'bbox',
    'the_geom': 'geom',
    'geom': 'geom',
    'gemoetry': 'geom',
    'lat': 'lat',
    'latitude': 'lat',
    'lon': 'lon',
    'lng': 'lon',
    'longitude': 'lon'
    }

def get_geocols(info, prefix = []):
    geocols = {}
    for key, value in info.iteritems():
        if key in colnames:
            geocols[colnames[key]] = prefix + [key]
        if isinstance(value, dict):
            geocols.update(get_geocols(value, prefix = prefix + [key]))
    return geocols

def getvalue(row, col):
    for item in col:
        if not isinstance(row, dict) or item not in row:
            return None
        row = row[item]
    return row

def setvalue(row, col, value):
    for item in col[:-1]:
        if item not in row or not isinstance(row[item], dict):
            row[item] = {}
        row = row[item]
    row[col[-1]] = value

def delvalue(row, col):
    for item in col[:-1]:
        if not isinstance(row, dict) or item not in row:
            return
        row = row[item]
    if col[-1] in row:
        del row[col[-1]]

def to_csv_value(value):
    if isinstance(value, unicode):
        return value.encode("utf-8")
    elif isinstance(value, str):
        return value
    else:
        return json.dumps(value)

def from_csv_value(value):
    value = value.decode("utf-8")
{'fraction_non_alphanumeric': 0.13734198838401093, 'fraction_numerical': 0.004783054321831227, 'mean_word_length': 2.3539518900343643, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2033213', 'n_tokens_mistral': 999, 'n_tokens_neox': 979, 'n_words': 293}