<commit_before># A little python script that uses mltools to train a pool detector and classify
# polygons in a given overlay

import json
import os

import pool_detector as pd
import crowdsourcing as cr
import json_tools as jt

# suppress annoying warnings
import warnings
warnings.filterwarnings("ignore")

# parameters
credentials = {'host':'mapperdb.cj6xoak5f54o.us-east-1.rds.amazonaws.com',
               'db':'tomnod', 
               'user':'tomnod', 
               'password':'<PASSWORD>'}

schema = 'adelaide_pools_2016'
cat_id = '1040010015D38800'
image_file = '14316.tif'
max_pools_samples = 300
ratio_train_pools = 0.333
max_nopools_samples = 5000
ratio_train_nopools = 0.02
train_file = 'train.geojson'
target_file = 'target.geojson'
output_file = 'classified.geojson'
max_area = 1000                           # max area in m2

# classifier params
n_estimators = 200<commit_msg>define classification job by json file<commit_after># A little python script that uses mltools to train a pool detector and classify
# polygons in a given overlay

import json
import os
import sys


import pool_detector as pd
import crowdsourcing as cr
import json_tools as jt

# suppress annoying warnings
import warnings
warnings.filterwarnings("ignore")

# tomnod parameters
credentials = {'host':'mapperdb.cj6xoak5f54o.us-east-1.rds.amazonaws.com',
               'db':'tomnod', 
               'user':'tomnod', 
               'password':'<PASSWORD>'}


with open('class_job.json', 'r') as f:
    class_job = json.load(f)

# get parameters from command line
schema = class_job['schema']
cat_id = class_job['cat_id']
image_file = class_job['image_file']
max_pools_samples = class_job['max_pools_samples']
ratio_train_pools = class_job['ratio_train_pools']
max_nopools_samples = class_job['max_nopools_samples']
ratio_train_nopools = class_job['ratio_train_pools']
train_file = class_job['train_file']
target_file = class_job['target_file']
output_file = class_job['output_file']
max_area = class_job['max_area']                           # max area in m2

# classifier params
n_estimators = 200
{'fraction_non_alphanumeric': 0.09460105112279026, 'fraction_numerical': 0.02866698518872432, 'mean_word_length': 4.208955223880597, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7178339', 'n_tokens_mistral': 733, 'n_tokens_neox': 661, 'n_words': 175}