<commit_before># This code is supporting material for the book
# Building Machine Learning Systems with Python
# by <NAME> and <NAME>
# published by PACKT Publishing
#
# It is made available under the MIT License

import numpy as np


# This function was called ``learn_model`` in the first edition
def fit_model(features, labels):
    '''Learn a simple threshold model'''
    best_acc = -1.0
    # Loop over all the features:
    for fi in range(features.shape[1]):
        thresh = features[:, fi].copy()
        # test all feature values in order:
        thresh.sort()
        for t in thresh:
            pred = (features[:, fi] > t)

            # Measure the accuracy of this 
            acc = (pred == labels).mean()
            if acc > best_acc:
                best_acc = acc
                best_fi = fi
                best_t = t

    # A model is a threshold and an index
    return best_t, best_fi


# This function was called ``apply_model`` in the first edition
def predict(features, model):
    '''Apply a learned model'''
    # A model is a pair as returned by fit_model
    t, fi = model
    return features[:, fi] > t

def accuracy(features, labels, model):
    '''Compute the accuracy of the model'''
    preds = predict(features, model)<commit_msg>ENH Test reversed threshold<commit_after># This code is supporting material for the book
# Building Machine Learning Systems with Python
# by <NAME> and <NAME>
# published by PACKT Publishing
#
# It is made available under the MIT License

import numpy as np


# This function was called ``learn_model`` in the first edition
def fit_model(features, labels):
    '''Learn a simple threshold model'''
    best_acc = -1.0
    # Loop over all the features:
    for fi in range(features.shape[1]):
        thresh = features[:, fi].copy()
        # test all feature values in order:
        thresh.sort()
        for t in thresh:
            pred = (features[:, fi] > t)

            # Measure the accuracy of this 
            acc = (pred == labels).mean()

            rev_acc = (pred == ~labels).mean()
            if rev_acc > acc:
                acc = rev_acc
                reverse = True
            else:
                reverse = False
            if acc > best_acc:
                best_acc = acc
                best_fi = fi
                best_t = t
                best_reverse = reverse

    # A model is a threshold and an index
    return best_t, best_fi, best_reverse


# This function was called ``apply_model`` in the first edition
def predict(features, model):
    '''Apply a learned model'''
    # A model is a pair as returned by fit_model
    t, fi, reverse = model
    if reverse:
        return features[:, fi] <= t
    else:
        return features[:, fi] > t

def accuracy(features, labels, model):
    '''Compute the accuracy of the model'''
    preds = predict(features, model)
{'fraction_non_alphanumeric': 0.08904347826086957, 'fraction_numerical': 0.0020869565217391303, 'mean_word_length': 2.2098214285714284, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3050931', 'n_tokens_mistral': 792, 'n_tokens_neox': 767, 'n_words': 334}