<commit_before>"""
This script executes the task of estimating the number of inhabitants of a neighborhood to be under or over the
median of all neighborhoods, based solely on the geometry for that neighborhood. The data for this script can be
generated by running the prep/get-data.sh and prep/preprocess-neighborhoods.py scripts, which will take about
an hour or two.

The script itself will run for about one and a half hours depending on your hardware, if you have at least a recent i7 or
comparable
"""

import multiprocessing
import os
from time import time
from datetime import datetime, timedelta

import numpy as np
import sys

from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedShuffleSplit, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler

PACKAGE_PARENT = '..'
SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))))
sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))

from topoml_util.slack_send import notify

SCRIPT_VERSION = '0.0.1'
SCRIPT_NAME = os.path.basename(__file__)
TIMESTAMP = str(datetime.now()).replace(':', '.')
NUM_CPUS = multiprocessing.cpu_count() - 1 if multiprocessing.cpu_count() > 1 else 1
DATA_FOLDER = SCRIPT_DIR + '/../../files/archaeology/'
FILENAME_PREFIX = 'archaeology_order_30_train'
REPEAT_ACCURACY_TEST = 10
SCRIPT_START = time()

if __name__ == '__main__':  # this is to squelch warnings on scikit-learn multithreaded grid search
    # Load training data
    training_files = []
    for file in os.listdir(DATA_FOLDER):
        if file.startswith(FILENAME_PREFIX) and file.endswith('.npz'):
            training_files.append(file)

    train_fourier_descriptors = np.array([])
    train_labels = np.array([])

    for index, file in enumerate(training_files):  # load and concatenate the training files
        train_loaded = np.load(DATA_FOLDER + file)

        if index == 0:
            train_fourier_descriptors = train_loaded['fourier_descriptors']
            train_labels = train_loaded['feature_type']
        else:
            train_fourier_descriptors = \
                np.append(train_fourier_descriptors, train_loaded['fourier_descriptors'], axis=0)
            train_labels = \
                np.append(train_labels, train_loaded['feature_type'], axis=0)

    scaler = StandardScaler().fit(train_fourier_descriptors)
    train_fourier_descriptors = scaler.transform(train_fourier_descriptors)

    # Grid search
    C_range = [1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3]
    param_grid = dict(C=C_range)
    cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
    grid = GridSearchCV(
        LogisticRegression(verbose=True),
        n_jobs=NUM_CPUS,
        param_grid=param_grid, cv=cv)

    print('Performing grid search on model...')
    print('Using %i threads for grid search' % NUM_CPUS)
    grid.fit(train_fourier_descriptors, train_labels)

    print("The best parameters are %s with a score of %0.3f"
          % (grid.best_params_, grid.best_score_))

    clf = LogisticRegression(C=grid.best_params_['C'])

    print('Fitting data to model...')
    scores = cross_val_score(clf, train_fourier_descriptors, train_labels, cv=10, n_jobs=NUM_CPUS)
    print('Cross-validation scores:', scores)
    clf.fit(train_fourier_descriptors, train_labels)

    # Run predictions on unseen test data to verify generalization
    TEST_DATA_FILE = '../../files/archaeology/archaeology_order_30_test.npz'
    test_loaded = np.load(TEST_DATA_FILE)
    test_fourier_descriptors = test_loaded['fourier_descriptors']
    test_feature_type = np.asarray(test_loaded['feature_type'], dtype=int)
    test_fourier_descriptors = scaler.transform(test_fourier_descriptors)

    print('Run on test data...')
    predictions = clf.predict(test_fourier_descriptors)
    accuracy = accuracy_score(test_feature_type, predictions)
    print('Test accuracy: %0.3f' % accuracy)

    runtime = time() - SCRIPT_START
    message = 'test accuracy of {} with C: {} in {}'.format(str(accuracy), grid.best_params_['C'],
                                                            timedelta(seconds=runtime))
    notify(SCRIPT_NAME, message)
    print(SCRIPT_NAME, 'finished successfully with', message)<commit_msg>refactor to grid search fourier descriptor order<commit_after>"""
This script executes the task of estimating the number of inhabitants of a neighborhood to be under or over the
median of all neighborhoods, based solely on the geometry for that neighborhood. The data for this script can be
generated by running the prep/get-data.sh and prep/preprocess-neighborhoods.py scripts, which will take about
an hour or two.

The script itself will run for about two hours depending on your hardware, if you have at least a recent i7 or
comparable
"""

import multiprocessing
import os
from time import time
from datetime import datetime, timedelta

import numpy as np
import sys

from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedShuffleSplit, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler

PACKAGE_PARENT = '..'
SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))))
sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))

from topoml_util.slack_send import notify

SCRIPT_VERSION = '1.0.0'
SCRIPT_NAME = os.path.basename(__file__)
TIMESTAMP = str(datetime.now()).replace(':', '.')
NUM_CPUS = multiprocessing.cpu_count() - 1 if multiprocessing.cpu_count() > 1 else 1
DATA_FOLDER = SCRIPT_DIR + '/../../files/archaeology/'
FILENAME_PREFIX = 'archaeology_order_30_train'
EFD_ORDERS = [1, 2, 3, 4, 6, 8, 12, 16, 20, 24]
SCRIPT_START = time()

if __name__ == '__main__':  # this is to squelch warnings on scikit-learn multithreaded grid search
    # Load training data
    training_files = []
    for file in os.listdir(DATA_FOLDER):
        if file.startswith(FILENAME_PREFIX) and file.endswith('.npz'):
            training_files.append(file)

    train_fourier_descriptors = np.array([])
    train_labels = np.array([])

    for index, file in enumerate(training_files):  # load and concatenate the training files
        train_loaded = np.load(DATA_FOLDER + file)

        if index == 0:
            train_fourier_descriptors = train_loaded['fourier_descriptors']
            train_labels = train_loaded['feature_type']
        else:
            train_fourier_descriptors = \
                np.append(train_fourier_descriptors, train_loaded['fourier_descriptors'], axis=0)
            train_labels = \
                np.append(train_labels, train_loaded['feature_type'], axis=0)

    scaler = StandardScaler().fit(train_fourier_descriptors)
    train_fourier_descriptors = scaler.transform(train_fourier_descriptors)

    # Grid search
    C_range = [1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3]
    param_grid = dict(C=C_range)
    cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
    grid = GridSearchCV(
        LogisticRegression(),
        n_jobs=NUM_CPUS,
        param_grid=param_grid,
        verbose=2,
        cv=cv)

    print('Performing grid search on model...')
    print('Using %i threads for grid search' % NUM_CPUS)
    print('Searching {} elliptic fourier descriptor orders'.format(EFD_ORDERS))

    best_order = 0
    best_score = 0
    best_params = {}

    for order in EFD_ORDERS:
        print('Fitting order {} fourier descriptors'.format(order))
        stop_position = 3 + (order * 8)
        grid.fit(train_fourier_descriptors[:, :stop_position], train_labels)
        print("The best parameters for order {} are {} with a score of {}\n".format(
            order, grid.best_params_, grid.best_score_))
        if grid.best_score_ > best_score:
            best_score = grid.best_score_
            best_order = order
            best_params = grid.best_params_

    print('Training model on order {} with best parameters {}'.format(
        best_order, best_params))
    stop_position = 3 + (best_order * 8)
    clf = LogisticRegression(C=best_params['C'])
    scores = cross_val_score(clf, train_fourier_descriptors[:, :stop_position], train_labels, cv=10, n_jobs=NUM_CPUS)
    print('Cross-validation scores:', scores)
    clf.fit(train_fourier_descriptors[:, :stop_position], train_labels)

    # Run predictions on unseen test data to verify generalization
    TEST_DATA_FILE = '../../files/archaeology/archaeology_order_30_test.npz'
    test_loaded = np.load(TEST_DATA_FILE)
    test_fourier_descriptors = test_loaded['fourier_descriptors']
    test_feature_type = np.asarray(test_loaded['feature_type'], dtype=int)
    test_fourier_descriptors = scaler.transform(test_fourier_descriptors)

    print('Run on test data...')
    predictions = clf.predict(test_fourier_descriptors[:, :stop_position])
    test_accuracy = accuracy_score(test_feature_type, predictions)
    print('Test accuracy: %0.3f' % test_accuracy)

    runtime = time() - SCRIPT_START
    print('')
    message = 'Test accuracy of {} for fourier descriptor order {} with {} in {}'.format(
        test_accuracy, best_order, best_params, timedelta(seconds=runtime))
    print(message)
    notify(SCRIPT_NAME, message)
    print(SCRIPT_NAME, 'finished successfully with', message)
{'fraction_non_alphanumeric': 0.09707306801431881, 'fraction_numerical': 0.01000210570646452, 'mean_word_length': 3.949973944762897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5829048', 'n_tokens_mistral': 3088, 'n_tokens_neox': 2953, 'n_words': 738}