<commit_before>#!/usr/bin/env python

# converts a geojson file with polygons to a sqlite database with individuals features as rows
# run: python build.py file.geojson
# creates two sqlite databases, named [file].sqlite and [file]-difficult.sqlite
    # [file] is taken from the source file
import json
import sqlite3
from shapely.geometry import asShape, mapping
from argparse import ArgumentParser

# MAX_EDITABLE_AREA = 1/float(69.11*69.11)
    # square mile
MAX_EDITABLE_AREA = 1/float(138.22*138.22)
    # half sqaure mile
    # obviously not very accurate but it's good enough

parser = ArgumentParser()

parser.add_argument(
    'source',
    help='Source geojson file to parse')
parser.add_argument(
    '--simplify',
    help='Simplification tolerance.',
    type=float,
    default=0.0001)

args = vars(parser.parse_args())


def isEditable(geo):
    # items that are easily editable for leaflet
    # mirrors isEditable() in osmly.item.js
    if geo.geom_type == 'Polygon' and geo.interiors:
        return False
    elif geo.geom_type == 'MultiPolygon':
        return False
    if geo.area > MAX_EDITABLE_AREA:
        return False
    if geo.area == 0:
        return False
    return True

data = open(args['source'])
data = json.load(data)
dbName = args['source'].split('.')

editableDB = dbName[0] + '.sqlite'
edit_conn = sqlite3.connect(editableDB)
edit_c = edit_conn.cursor()
edit_c.execute("DROP TABLE IF EXISTS osmly")
edit_c.execute('''CREATE TABLE osmly (id INTEGER PRIMARY KEY, geo TEXT, remote TEXT, problem TEXT, done TEXT)''')
edit_conn.commit()

difficultDB = dbName[0] + '-difficult.sqlite'
diff_conn = sqlite3.connect(difficultDB)
diff_c = diff_conn.cursor()
diff_c.execute("DROP TABLE IF EXISTS osmly")
diff_c.execute('''CREATE TABLE osmly (id INTEGER PRIMARY KEY, geo TEXT, remote TEXT, problem TEXT, done TEXT)''')
diff_conn.commit()

count = 0
easy_count = 0
diff_count = 0

for feature in data['features']:
    geo = asShape(feature['geometry'])
    bounds = geo.bounds
    geoarea = geo.area
    editable = isEditable(geo)

    # we want to use simplify() with False because it's faster
    # but it occasionally deletes all nodes and that upsets mapping()
    try:
        simple = geo.simplify(0.0001, False)
        geo = mapping(simple)
    except:
        simple = geo.simplify(0.0001, True)
        geo = mapping(simple)

    feature['properties']['bounds'] = bounds
    feature['geometry']['coordinates'] = geo['coordinates']

    statement = "INSERT INTO osmly VALUES(?, ?, ?, ?, ?);"

    if editable:
        feature['properties']['id'] = easy_count
        edit_c.execute(statement, (easy_count, json.dumps(feature), '', '', ''))
        edit_conn.commit()
        easy_count = easy_count + 1
    else:
        feature['properties']['id'] = diff_count
        diff_c.execute(statement, (diff_count, json.dumps(feature), '', '', ''))
        diff_conn.commit()
        diff_count = diff_count + 1

    count = count + 1


print str(count) + ' items'
print str(easy_count) + ' easy, ' + str(diff_count) + ' difficult'
edit_conn.close()
diff_conn.close()<commit_msg>Back to one db, more columns<commit_after>#!/usr/bin/env python

# converts a geojson file with polygons to a sqlite database with individuals features as rows
# run: python build.py file.geojson
# creates two sqlite databases, named [file].sqlite and [file]-difficult.sqlite
    # [file] is taken from the source file
import json
import sqlite3
from shapely.geometry import asShape, mapping
from argparse import ArgumentParser

# MAX_EDITABLE_AREA = 1/float(69.11*69.11)
    # square mile
MAX_EDITABLE_AREA = 1/float(138.22*138.22)
    # half sqaure mile
    # obviously not very accurate but it's good enough

parser = ArgumentParser()

parser.add_argument(
    'source',
    help='Source geojson file to parse')
parser.add_argument(
    '--simplify',
    help='Simplification tolerance.',
    type=float,
    default=0.0001)

args = vars(parser.parse_args())


def isEditable(geo):
    # items that are easily editable for leaflet
    # mirrors isEditable() in osmly.item.js
    if geo.geom_type == 'Polygon' and geo.interiors:
        return False
    elif geo.geom_type == 'MultiPolygon':
        return False
    if geo.area > MAX_EDITABLE_AREA:
        return False
    if geo.area == 0:
        return False
    return True

data = open(args['source'])
data = json.load(data)
dbName = args['source'].split('.')[0] + '.sqlite'

db_conn = sqlite3.connect(dbName)
db_c = db_conn.cursor()
db_c.execute('DROP TABLE IF EXISTS osmly')
db_c.execute('CREATE TABLE osmly (id INTEGER PRIMARY KEY, geo TEXT, remote TEXT,' +
             'problem TEXT, done TEXT, difficulty INT, bounds TEXT, area REAL, comments TEXT)')
db_conn.commit()

count = 0
easy_count = 0
diff_count = 0

for feature in data['features']:
    geo = asShape(feature['geometry'])
    boundz = geo.bounds
    bounds = [0, 1, 2, 3]
    geoarea = geo.area
    editable = isEditable(geo)

    bounds[0] = float('{0:.5f}'.format(boundz[0]))
    bounds[1] = float('{0:.5f}'.format(boundz[1]))
    bounds[2] = float('{0:.5f}'.format(boundz[2]))
    bounds[3] = float('{0:.5f}'.format(boundz[3]))

    # we want to use simplify() with False because it's faster
    # but it occasionally deletes all nodes and that upsets mapping()
    try:
        simple = geo.simplify(0.0001, False)
        geo = mapping(simple)
    except:
        simple = geo.simplify(0.0001, True)
        geo = mapping(simple)

    feature['properties']['bounds'] = bounds
    feature['geometry']['coordinates'] = geo['coordinates']
    feature['properties']['id'] = count
    statement = 'INSERT INTO osmly VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?);'

    if editable:
        difficulty = 0
        easy_count = easy_count + 1
    else:
        difficulty = 1
        diff_count = diff_count + 1

    db_c.execute(statement, (count, json.dumps(feature), '', '', '', difficulty, json.dumps(bounds), geoarea, ''))
    db_conn.commit()
        # need to bulk these up, transaction
    count = count + 1

print str(count) + ' items'
print str(easy_count) + ' easy, ' + str(diff_count) + ' difficult'
db_conn.close()
{'fraction_non_alphanumeric': 0.1296145714750366, 'fraction_numerical': 0.01853959993494877, 'mean_word_length': 3.4727272727272727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5535608', 'n_tokens_mistral': 2131, 'n_tokens_neox': 1948, 'n_words': 551}