<commit_before>from textblob.classifiers import NaiveBayesClassifier


def get_training_data(owner):
    from djofx import models

    categorised = models.Transaction.objects.filter(
        account__owner=owner,
        category_verified=True
    )

    data = [(t.payee, t.transaction_category) for t in categorised]

    # Avoiding .distinct, because SQLite doesn't support it
    seen_payees = set()
    unique_data = []
    for payee, category in data:
        if payee in seen_payees:
            continue

        seen_payees.add(payee)
        unique_data.append((payee, category))

    return unique_data


def get_classifier(owner, training_data=None):
    if not training_data:
        training_data = get_training_data(owner)

    return NaiveBayesClassifier(training_data)


def autocategorise_transaction(transaction, classifier=None):
    training_data = get_training_data(transaction.account.owner)

    for payee, category in training_data:
        if payee == transaction.payee:
            return category

    if not classifier:
        classifier = get_classifier(transaction.account.owner, training_data)

    return classifier.classify(transaction.payee)


def classify_text(text, classifier):
    return classifier.classify(text)<commit_msg>If we can't classify a transaction, return None<commit_after>from textblob.classifiers import NaiveBayesClassifier


def get_training_data(owner):
    from djofx import models

    categorised = models.Transaction.objects.filter(
        account__owner=owner,
        category_verified=True
    )

    data = [(t.payee, t.transaction_category) for t in categorised]

    # Avoiding .distinct, because SQLite doesn't support it
    seen_payees = set()
    unique_data = []
    for payee, category in data:
        if payee in seen_payees:
            continue

        seen_payees.add(payee)
        unique_data.append((payee, category))

    return unique_data


def get_classifier(owner, training_data=None):
    if not training_data:
        training_data = get_training_data(owner)

    return NaiveBayesClassifier(training_data)


def autocategorise_transaction(transaction, classifier=None):
    training_data = get_training_data(transaction.account.owner)

    for payee, category in training_data:
        if payee == transaction.payee:
            return category

    if not classifier:
        classifier = get_classifier(transaction.account.owner, training_data)

    return classifier.classify(transaction.payee)


def classify_text(text, classifier):
    try:
        return classifier.classify(text)
    except ValueError:
        return None
{'fraction_non_alphanumeric': 0.07093821510297482, 'fraction_numerical': 0.0, 'mean_word_length': 3.4232715008431702, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '862274', 'n_tokens_mistral': 777, 'n_tokens_neox': 736, 'n_words': 179}