<commit_before>import os

from datetime import datetime

import multiprocessing
import numpy as np
from sklearn.model_selection import StratifiedShuffleSplit, GridSearchCV
from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer
from sklearn.svm import SVC

# This script executes the task of estimating the number of inhabitants of a neighborhood to be under or over the median
# of all neighborhoods, based solely on the geometry for that neighborhood.
# The data for this script can be generated by running the prep/get-data.sh and prep/preprocess-neighborhoods.py scripts, which
# will take about an hour or two.

# The script itself will run for about two hours depending on your hardware, if you have at least a recent i7 or
# comparable

SCRIPT_VERSION = '0.0.2'
SCRIPT_NAME = os.path.basename(__file__)
TIMESTAMP = str(datetime.now()).replace(':', '.')
TRAINING_DATA_FILE = '../files/neighborhoods/neighborhoods_train.npz'

if __name__ == '__main__':  # this is to squelch warnings on scikit-learn multithreaded grid search
    num_cpus = multiprocessing.cpu_count() - 1 if multiprocessing.cpu_count() > 1 else 1

    train_loaded = np.load(TRAINING_DATA_FILE)
    train_fourier_descriptors = train_loaded['fourier_descriptors']
    train_above_or_below_median = train_loaded['above_or_below_median'][:, 0]
    train_above_or_below_median = np.reshape(train_above_or_below_median, (train_above_or_below_median.shape[0]))

    scaler = StandardScaler().fit(train_fourier_descriptors)
    train_fourier_descriptors = scaler.transform(train_fourier_descriptors)

    C_range = [1e-3, 1e-2, 1e-1, 1e0, 1e1]
    param_grid = dict(C=C_range)
    cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
    grid = GridSearchCV(
        SVC(kernel='linear', verbose=True),
        n_jobs=num_cpus,
        param_grid=param_grid, cv=cv)

    print('Performing grid search on model...')
    print('Using %i threads for grid search' % num_cpus)
    grid.fit(X=train_fourier_descriptors, y=train_above_or_below_median)

    print("The best parameters are %s with a score of %0.2f"
          % (grid.best_params_, grid.best_score_))

    clf = SVC(kernel='linear', C=grid.best_params_['C'], verbose=True)
    clf.fit(X=train_fourier_descriptors, y=train_above_or_below_median)

    # Run predictions on unseen test data to verify generalization
    TEST_DATA_FILE = '../files/neighborhoods/neighborhoods_test.npz'
    test_loaded = np.load(TEST_DATA_FILE)
    test_fourier_descriptors = test_loaded['fourier_descriptors']
    test_above_or_below_median = test_loaded['above_or_below_median'][:, 0]
    test_fourier_descriptors = scaler.transform(test_fourier_descriptors)
    test_above_or_below_median = np.reshape(test_above_or_below_median, (test_above_or_below_median.shape[0]))

    predictions = clf.predict(test_fourier_descriptors)

    correct = 0
    for prediction, expected in zip(predictions, test_above_or_below_median):
        if prediction == expected:
            correct += 1

    accuracy = correct / len(predictions)
    print('Test accuracy: %0.2f' % accuracy)
<commit_msg>docstring, data paths, C-range, notify<commit_after>"""
This script executes the task of estimating the number of inhabitants of a neighborhood to be under or over the
median of all neighborhoods, based solely on the geometry for that neighborhood. The data for this script can be
generated by running the prep/get-data.sh and prep/preprocess-neighborhoods.py scripts, which will take about an hour
or two.

This script itself will run for about six hours depending on your hardware, if you have at least a recent i7 or
comparable
"""

import os

from datetime import datetime

import multiprocessing
import numpy as np
import sys
from sklearn.model_selection import StratifiedShuffleSplit, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC

PACKAGE_PARENT = '..'
SCRIPT_DIR = os.path.dirname(os.path.realpath(os.path.join(os.getcwd(), os.path.expanduser(__file__))))
sys.path.append(os.path.normpath(os.path.join(SCRIPT_DIR, PACKAGE_PARENT)))

from topoml_util.slack_send import notify

SCRIPT_VERSION = '0.0.3'
SCRIPT_NAME = os.path.basename(__file__)
TIMESTAMP = str(datetime.now()).replace(':', '.')
TRAINING_DATA_FILE = '../../files/neighborhoods/neighborhoods_train.npz'

if __name__ == '__main__':  # this is to squelch warnings on scikit-learn multithreaded grid search
    num_cpus = multiprocessing.cpu_count() - 1 if multiprocessing.cpu_count() > 1 else 1

    train_loaded = np.load(TRAINING_DATA_FILE)
    train_fourier_descriptors = train_loaded['fourier_descriptors']
    train_above_or_below_median = train_loaded['above_or_below_median'][:, 0]
    train_above_or_below_median = np.reshape(train_above_or_below_median, (train_above_or_below_median.shape[0]))

    scaler = StandardScaler().fit(train_fourier_descriptors)
    train_fourier_descriptors = scaler.transform(train_fourier_descriptors)

    C_range = [1e-2, 1e-1, 1e0, 1e1, 1e2, 1e3]
    param_grid = dict(C=C_range)
    cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=42)
    grid = GridSearchCV(
        SVC(kernel='linear', verbose=True),
        n_jobs=num_cpus,
        param_grid=param_grid, cv=cv)

    print('Performing grid search on model...')
    print('Using %i threads for grid search' % num_cpus)
    grid.fit(X=train_fourier_descriptors, y=train_above_or_below_median)

    print("The best parameters are %s with a score of %0.3f"
          % (grid.best_params_, grid.best_score_))

    clf = SVC(kernel='linear', C=grid.best_params_['C'], verbose=True)
    clf.fit(X=train_fourier_descriptors, y=train_above_or_below_median)

    # Run predictions on unseen test data to verify generalization
    TEST_DATA_FILE = '../../files/neighborhoods/neighborhoods_test.npz'
    test_loaded = np.load(TEST_DATA_FILE)
    test_fourier_descriptors = test_loaded['fourier_descriptors']
    test_above_or_below_median = test_loaded['above_or_below_median'][:, 0]
    test_fourier_descriptors = scaler.transform(test_fourier_descriptors)
    test_above_or_below_median = np.reshape(test_above_or_below_median, (test_above_or_below_median.shape[0]))

    predictions = clf.predict(test_fourier_descriptors)

    correct = 0
    for prediction, expected in zip(predictions, test_above_or_below_median):
        if prediction == expected:
            correct += 1

    accuracy = correct / len(predictions)
    print('Test accuracy: %0.3f' % accuracy)

    message = 'test accuracy of {0} with C: {1} '.format(str(accuracy), grid.best_params_['C'])
    notify(SCRIPT_NAME, message)
    print(SCRIPT_NAME, 'finished successfully')

{'fraction_non_alphanumeric': 0.09373598445208552, 'fraction_numerical': 0.01016594408730752, 'mean_word_length': 4.878734622144113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4736049', 'n_tokens_mistral': 2340, 'n_tokens_neox': 2186, 'n_words': 519}