<commit_before>import pytest
from fontTools.varLib.models import VariationModel
from fontTools.varLib.varStore import OnlineVarStoreBuilder, VarStoreInstancer
from fontTools.ttLib.tables._f_v_a_r import Axis


@pytest.mark.parametrize(
    "locations, masterValues",
    [
        (
            [{}, {"a": 1}],
            [
                [10, 20],
                [100, 2000],
                [100, 22000],
            ],
        ),
        (
            [{}, {"a": 1}, {"b": 1}, {"a": 1, "b": 1}],
            [
                [10, 20, 40, 60],
                [100, 2000, 400, 6000],
                [100, 22000, 4000, 30000],
            ],
        ),
    ],
)
def test_onlineVarStoreBuilder(locations, masterValues):
    axisTags = sorted({k for loc in locations for k in loc})
    model = VariationModel(locations)
    builder = OnlineVarStoreBuilder(axisTags)
    builder.setModel(model)
    expectedDeltasAndVarIdxs = []
    for masters in masterValues:
        base, *deltas = model.getDeltas(masters)
        varIdx = builder.storeDeltas(deltas)
        expectedDeltasAndVarIdxs.append((deltas, varIdx))

    varStore = builder.finish()
    varData = varStore.VarData

    for deltas, varIdx in expectedDeltasAndVarIdxs:
        major, minor = varIdx >> 16, varIdx & 0xFFFF
        storedDeltas = varData[major].Item[minor]
        assert deltas == storedDeltas

    fvarAxes = [buildAxis(axisTag) for axisTag in axisTags]
    instancer = VarStoreInstancer(varStore, fvarAxes)
    for masters, (deltas, varIdx) in zip(masterValues, expectedDeltasAndVarIdxs):
        base, *rest = masters
        for expectedValue, loc in zip(masters, locations):
            instancer.setLocation(loc)
            value = base + instancer[varIdx]
            assert expectedValue == value


def buildAxis(axisTag):
    axis = Axis()
    axis.axisTag = axisTag
    return axis
<commit_msg>use optimizer<commit_after>import pytest
from fontTools.varLib.models import VariationModel
from fontTools.varLib.varStore import OnlineVarStoreBuilder, VarStoreInstancer
from fontTools.ttLib.tables._f_v_a_r import Axis


@pytest.mark.parametrize(
    "locations, masterValues",
    [
        (
            [{}, {"a": 1}],
            [
                [10, 20],
                [100, 2000],
                [100, 22000],
            ],
        ),
        (
            [{}, {"a": 1}, {"b": 1}, {"a": 1, "b": 1}],
            [
                [10, 20, 40, 60],
                [100, 2000, 400, 6000],
                [100, 22000, 4000, 30000],
            ],
        ),
    ],
)
def test_onlineVarStoreBuilder(locations, masterValues):
    axisTags = sorted({k for loc in locations for k in loc})
    model = VariationModel(locations)
    builder = OnlineVarStoreBuilder(axisTags)
    builder.setModel(model)
    expectedDeltasAndVarIdxs = []
    for masters in masterValues:
        base, *deltas = model.getDeltas(masters)
        varIdx = builder.storeDeltas(deltas)
        expectedDeltasAndVarIdxs.append((deltas, varIdx))

    varStore = builder.finish()
    mapping = varStore.optimize()
    expectedDeltasAndVarIdxs = [
        (deltas, mapping[varIdx]) for deltas, varIdx in expectedDeltasAndVarIdxs
    ]

    for deltas, varIdx in expectedDeltasAndVarIdxs:
        major, minor = varIdx >> 16, varIdx & 0xFFFF
        storedDeltas = varStore.VarData[major].Item[minor]
        assert deltas == storedDeltas

    fvarAxes = [buildAxis(axisTag) for axisTag in axisTags]
    instancer = VarStoreInstancer(varStore, fvarAxes)
    for masters, (deltas, varIdx) in zip(masterValues, expectedDeltasAndVarIdxs):
        base, *rest = masters
        for expectedValue, loc in zip(masters, locations):
            instancer.setLocation(loc)
            value = base + instancer[varIdx]
            assert expectedValue == value


def buildAxis(axisTag):
    axis = Axis()
    axis.axisTag = axisTag
    return axis

{'fraction_non_alphanumeric': 0.10809426229508197, 'fraction_numerical': 0.03381147540983607, 'mean_word_length': 2.349056603773585, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1854535', 'n_tokens_mistral': 1351, 'n_tokens_neox': 1210, 'n_words': 281}