<commit_before># -*- encoding: utf-8 -*-
import numpy as np

import sklearn.model_selection
import sklearn.datasets
import sklearn.metrics

import autosklearn.classification
import autosklearn.metrics

try:
    import openml
except ImportError:
    print("#"*80 + """
    To run this example you need to install openml-python:

    git+https://github.com/renatopp/liac-arff
    # OpenML is currently not on pypi, use an old version to not depend on
    # scikit-learn 0.18
    requests
    xmltodict
    git+https://github.com/renatopp/liac-arff
    git+https://github.com/openml/""" +
    "openml-python@0b9009b0436fda77d9f7c701bd116aff4158d5e1\n""" +
          "#"*80)
    raise


def accuracy(solution, prediction):
    return np.mean(solution == prediction)

def accuracy_with_kwargs(solution, prediction, )

def main():
    # Load adult dataset from openml.org, see https://www.openml.org/t/2117
    openml.config.apikey = '610344db6388d9ba34f6db45a3cf71de'

    task = openml.tasks.get_task(2117)
    train_indices, test_indices = task.get_train_test_split_indices()
    X, y = task.get_X_and_y()

    X_train = X[train_indices]
    y_train = y[train_indices]
    X_test = X[test_indices]
    y_test = y[test_indices]

    dataset = task.get_dataset()
    _, _, categorical_indicator = dataset.\
        get_data(target=task.target_name, return_categorical_indicator=True)

    # Create feature type list from openml.org indicator and run autosklearn
    feat_type = ['categorical' if ci else 'numerical'
                 for ci in categorical_indicator]

    # Run auto-sklearn with our metric
    accuracy_scorer = autosklearn.metrics.make_scorer(name="accu_self",
                                                      score_func=accuracy,
                                                      greater_is_better=True,
                                                      needs_proba=False,
                                                      needs_threshold=False)
    cls = autosklearn.classification.\
        AutoSklearnClassifier(time_left_for_this_task=60,
                              per_run_time_limit=30)
    cls.fit(X_train, y_train, feat_type=feat_type, metric=accuracy_scorer)

    predictions = cls.predict(X_test)
    print("Accuracy score", sklearn.metrics.accuracy_score(y_test, predictions))





if __name__ == "__main__":
    main()<commit_msg>ADD example on how to use metrics<commit_after># -*- encoding: utf-8 -*-
import numpy as np

import sklearn.model_selection
import sklearn.datasets
import sklearn.metrics

import autosklearn.classification
import autosklearn.metrics

try:
    import openml
except ImportError:
    print("#"*80 + """
    To run this example you need to install openml-python:

    git+https://github.com/renatopp/liac-arff
    # OpenML is currently not on pypi, use an old version to not depend on
    # scikit-learn 0.18
    requests
    xmltodict
    git+https://github.com/renatopp/liac-arff
    git+https://github.com/openml/""" +
    "openml-python@0b9009b0436fda77d9f7c701bd116aff4158d5e1\n""" +
          "#"*80)
    raise


def accuracy(solution, prediction):
    # function defining accuracy
    return np.mean(solution == prediction)


def accuracy_wk(solution, prediction, dummy):
    # function defining accuracy and accepting an additional argument
    assert dummy is None
    return np.mean(solution == prediction)


def main():
    # Load adult dataset from openml.org, see https://www.openml.org/t/2117
    openml.config.apikey = '610344db6388d9ba34f6db45a3cf71de'

    task = openml.tasks.get_task(2117)
    train_indices, test_indices = task.get_train_test_split_indices()
    X, y = task.get_X_and_y()

    X_train = X[train_indices]
    y_train = y[train_indices]
    X_test = X[test_indices]
    y_test = y[test_indices]

    dataset = task.get_dataset()
    _, _, categorical_indicator = dataset.\
        get_data(target=task.target_name, return_categorical_indicator=True)

    # Create feature type list from openml.org indicator and run autosklearn
    feat_type = ['categorical' if ci else 'numerical'
                 for ci in categorical_indicator]

    # Print a list of available metrics
    print("Available CLASSIFICATION metrics autosklearn.metrics.*:")
    print("\t*" + "\n\t*".join(autosklearn.metrics.CLASSIFICATION_METRICS))

    print("Available REGRESSION autosklearn.metrics.*:")
    print("\t*" + "\n\t*".join(autosklearn.metrics.REGRESSION_METRICS))

    # First example: Use predefined accuracy metric
    print("#"*80)
    print("Use predefined accuracy metric")
    cls = autosklearn.classification.\
        AutoSklearnClassifier(time_left_for_this_task=60,
                              per_run_time_limit=30, seed=1)
    cls.fit(X_train, y_train, feat_type=feat_type,
            metric=autosklearn.metrics.accuracy)

    predictions = cls.predict(X_test)
    print("Accuracy score {:g} using {:s}".
          format(sklearn.metrics.accuracy_score(y_test, predictions),
                 cls._automl._automl._metric.name))

    print("#"*80)
    print("Use self defined accuracy accuracy metric")
    accuracy_scorer = autosklearn.metrics.make_scorer(name="accu",
                                                      score_func=accuracy,
                                                      greater_is_better=True,
                                                      needs_proba=False,
                                                      needs_threshold=False)
    cls = autosklearn.classification.\
        AutoSklearnClassifier(time_left_for_this_task=60,
                              per_run_time_limit=30, seed=1)
    cls.fit(X_train, y_train, feat_type=feat_type, metric=accuracy_scorer)

    predictions = cls.predict(X_test)
    print("Accuracy score {:g} using {:s}".
          format(sklearn.metrics.accuracy_score(y_test, predictions),
                 cls._automl._automl._metric.name))

    print("#"*80)
    print("Use self defined accuracy with additional argument")
    accuracy_scorer = autosklearn.metrics.make_scorer(name="accu_add",
                                                      score_func=accuracy_wk,
                                                      greater_is_better=True,
                                                      needs_proba=False,
                                                      needs_threshold=False,
                                                      dummy=None)
    cls = autosklearn.classification.\
        AutoSklearnClassifier(time_left_for_this_task=60,
                              per_run_time_limit=30, seed=1)
    cls.fit(X_train, y_train, feat_type=feat_type, metric=accuracy_scorer)

    predictions = cls.predict(X_test)
    print("Accuracy score {:g} using {:s}".
          format(sklearn.metrics.accuracy_score(y_test, predictions),
                 cls._automl._automl._metric.name))


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.09281480410582622, 'fraction_numerical': 0.020962845164088476, 'mean_word_length': 2.346879535558781, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2820268', 'n_tokens_mistral': 2152, 'n_tokens_neox': 2031, 'n_words': 406}