<commit_before>import numpy

from matrixstore.db import get_db, get_row_grouper
from matrixstore.matrix_ops import get_submatrix
from matrixstore.sql_functions import MatrixSum

from .substitution_sets import get_substitution_sets


# Defines how we determine the target PPU against which savings are calculated.
# We want to know which set of practices we are comparing with and which
# centile over those practices we are targeting.
CONFIG_TARGET_CENTILE = 10
# Note this is "standard_practice" rather than "practice" as we only want to
# include setting 4 practices (i.e. ordinary GP practices)
CONFIG_TARGET_PEER_GROUP = "standard_practice"

# Savings are calculated in two stages: first at the practice level, and then
# aggregated up into organisation savings. At both stages we apply a minimum
# threshold such that savings below that threshold are treated as zero. This is
# particularly important at practice level as it prevents "negative savings"
# from practices already performing better than the target PPU.
# (Note: all values in pence)
CONFIG_MIN_SAVINGS_FOR_ORG_TYPE = {
    "practice": 10 * 100,
    "ccg": 200 * 100,
    # Picked somewhat arbitrarily based on the CCG limit
    "all_standard_practices": 50000 * 100,
}


def get_all_savings_for_orgs(date, org_type, org_ids):
    """
    Get all available savings through presentation switches for the given orgs
    """
    results = []
    for generic_code in get_substitution_sets().keys():
        savings = get_savings_for_orgs(generic_code, date, org_type, org_ids)
        results.extend(savings)
    results.sort(key=lambda i: i["possible_savings"], reverse=True)
    return results


def get_savings_for_orgs(generic_code, date, org_type, org_ids):
    """
    Get available savings for the given orgs within a particular class of
    substitutable presentations
    """
    substitution_set = get_substitution_sets()[generic_code]

    quantities, net_costs = get_quantities_and_net_costs_at_date(
        substitution_set.presentations, date
    )

    group_by_org = get_row_grouper(org_type)
    quantities_for_orgs = group_by_org.sum(quantities, org_ids)
    # Bail early if none of the orgs have any relevant prescribing
    if not numpy.any(quantities_for_orgs):
        return []
    net_costs_for_orgs = group_by_org.sum(net_costs, org_ids)
    ppu_for_orgs = net_costs_for_orgs / quantities_for_orgs

    target_ppu = get_target_ppu(quantities, net_costs)
    practice_level_savings = get_savings(quantities, net_costs, target_ppu)
    savings_for_orgs = group_by_org.sum(practice_level_savings, org_ids)
    min_threshold = CONFIG_MIN_SAVINGS_FOR_ORG_TYPE[org_type]

    results = [
        {
            "date": date,
            "org_id": org_id,
            "price_per_unit": ppu_for_orgs[offset, 0] / 100,
            "possible_savings": savings_for_orgs[offset, 0] / 100,
            "quantity": quantities_for_orgs[offset, 0],
            "lowest_decile": target_ppu[0] / 100,
            "presentation": substitution_set.id,
            "formulation_swap": substitution_set.formulation_swaps,
            "name": substitution_set.name,
        }
        for offset, org_id in enumerate(org_ids)
        if savings_for_orgs[offset, 0] >= min_threshold
    ]

    results.sort(key=lambda i: i["possible_savings"], reverse=True)
    return results


def get_target_ppu(quantities, net_costs):
    """
    Calculate the price-per-unit achieved by the practice at the target centile
    """
    group_by_org = get_row_grouper(CONFIG_TARGET_PEER_GROUP)
    quantities = group_by_org.sum(quantities)
    net_costs = group_by_org.sum(net_costs)
    ppu = net_costs / quantities
    target_ppu = numpy.nanpercentile(ppu, axis=0, q=CONFIG_TARGET_CENTILE)
    return target_ppu

<commit_msg>Add function to get total savings for an organisation<commit_after>import numpy

from matrixstore.db import get_db, get_row_grouper
from matrixstore.matrix_ops import get_submatrix, zeros_like
from matrixstore.sql_functions import MatrixSum

from .substitution_sets import get_substitution_sets


# Defines how we determine the target PPU against which savings are calculated.
# We want to know which set of practices we are comparing with and which
# centile over those practices we are targeting.
CONFIG_TARGET_CENTILE = 10
# Note this is "standard_practice" rather than "practice" as we only want to
# include setting 4 practices (i.e. ordinary GP practices)
CONFIG_TARGET_PEER_GROUP = "standard_practice"

# Savings are calculated in two stages: first at the practice level, and then
# aggregated up into organisation savings. At both stages we apply a minimum
# threshold such that savings below that threshold are treated as zero. This is
# particularly important at practice level as it prevents "negative savings"
# from practices already performing better than the target PPU.
# (Note: all values in pence)
CONFIG_MIN_SAVINGS_FOR_ORG_TYPE = {
    "practice": 10 * 100,
    "ccg": 200 * 100,
    # Picked somewhat arbitrarily based on the CCG limit
    "all_standard_practices": 50000 * 100,
}


def get_all_savings_for_orgs(date, org_type, org_ids):
    """
    Get all available savings through presentation switches for the given orgs
    """
    results = []
    for generic_code in get_substitution_sets().keys():
        savings = get_savings_for_orgs(generic_code, date, org_type, org_ids)
        results.extend(savings)
    results.sort(key=lambda i: i["possible_savings"], reverse=True)
    return results


def get_savings_for_orgs(generic_code, date, org_type, org_ids):
    """
    Get available savings for the given orgs within a particular class of
    substitutable presentations
    """
    substitution_set = get_substitution_sets()[generic_code]

    quantities, net_costs = get_quantities_and_net_costs_at_date(
        substitution_set.presentations, date
    )

    group_by_org = get_row_grouper(org_type)
    quantities_for_orgs = group_by_org.sum(quantities, org_ids)
    # Bail early if none of the orgs have any relevant prescribing
    if not numpy.any(quantities_for_orgs):
        return []
    net_costs_for_orgs = group_by_org.sum(net_costs, org_ids)
    ppu_for_orgs = net_costs_for_orgs / quantities_for_orgs

    target_ppu = get_target_ppu(quantities, net_costs)
    practice_level_savings = get_savings(quantities, net_costs, target_ppu)
    savings_for_orgs = group_by_org.sum(practice_level_savings, org_ids)
    min_threshold = CONFIG_MIN_SAVINGS_FOR_ORG_TYPE[org_type]

    results = [
        {
            "date": date,
            "org_id": org_id,
            "price_per_unit": ppu_for_orgs[offset, 0] / 100,
            "possible_savings": savings_for_orgs[offset, 0] / 100,
            "quantity": quantities_for_orgs[offset, 0],
            "lowest_decile": target_ppu[0] / 100,
            "presentation": substitution_set.id,
            "formulation_swap": substitution_set.formulation_swaps,
            "name": substitution_set.name,
        }
        for offset, org_id in enumerate(org_ids)
        if savings_for_orgs[offset, 0] >= min_threshold
    ]

    results.sort(key=lambda i: i["possible_savings"], reverse=True)
    return results


def get_total_savings_for_org(date, org_type, org_id):
    """
    Get total available savings through presentation switches for the given org
    """
    totals = get_total_savings_for_org_type(date, org_type)
    group_by_org = get_row_grouper(org_type)
    offset = group_by_org.offsets[org_id]
    return totals[offset, 0] / 100


def get_total_savings_for_org_type(date, org_type):
    """
    Return a matrix giving total savings for all orgs of a given type

    This duplicates some of the logic in `get_savings_for_orgs` but it gives us
    much better caching behaviour to calculate savings for all orgs of a given
    type together in a single, cacheable matrix than it does to do them one by
    one.
    """
    group_by_org = get_row_grouper(org_type)
    min_threshold = CONFIG_MIN_SAVINGS_FOR_ORG_TYPE[org_type]
    totals = None
    for substitution_set in get_substitution_sets().values():
        quantities, net_costs = get_quantities_and_net_costs_at_date(
            substitution_set.presentations, date
        )
        target_ppu = get_target_ppu(quantities, net_costs)
        practice_level_savings = get_savings(quantities, net_costs, target_ppu)
        savings_for_orgs = group_by_org.sum(practice_level_savings)
        savings_above_threshold = savings_for_orgs >= min_threshold
        if totals is None:
            totals = zeros_like(savings_for_orgs)
        numpy.add(totals, savings_for_orgs, out=totals, where=savings_above_threshold)
    assert totals is not None
    return totals


def get_target_ppu(quantities, net_costs):
    """
    Calculate the price-per-unit achieved by the practice at the target centile
    """
    group_by_org = get_row_grouper(CONFIG_TARGET_PEER_GROUP)
    quantities = group_by_org.sum(quantities)
    net_costs = group_by_org.sum(net_costs)
    ppu = net_costs / quantities
    target_ppu = numpy.nanpercentile(ppu, axis=0, q=CONFIG_TARGET_CENTILE)
    return target_ppu


{'fraction_non_alphanumeric': 0.0696604600219058, 'fraction_numerical': 0.008543263964950712, 'mean_word_length': 3.8933547695605575, 'pattern_counts': {'":': 24, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1080010', 'n_tokens_mistral': 3029, 'n_tokens_neox': 2849, 'n_words': 778}