<commit_before>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.processing;

import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext;
import com.yahoo.searchdefinition.expressiontransforms.TokenTransformer;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.evaluation.MapContext;
import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
import com.yahoo.tensor.Tensor;
import org.junit.Test;

import java.util.Collections;

import static org.junit.Assert.assertEquals;

public class RankingExpressionWithTransformerTokensTestCase {

    @Test
    public void testTokenInputIds() throws Exception {
        String expected = "tensor(d0[1],d1[12]):[101,1,2,102,3,4,5,102,6,7,102,0]";
        String a = "tensor(d0[2]):[1,2]";
        String b = "tensor(d0[3]):[3,4,5]";
        String c = "tensor(d0[2]):[6,7]";
        String expression = "tokenInputIds(12, a, b, c)";
        Tensor result = evaluateExpression(expression, a, b, c);
        assertEquals(Tensor.from(expected), result);
    }

    @Test
    public void testTokenTypeIds() throws Exception {
        String expected = "tensor(d0[1],d1[10]):[0,0,0,0,1,1,1,1,1,1]";
        String a = "tensor(d0[2]):[1,2]";<commit_msg>Fix unit type for token type ids<commit_after>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.searchdefinition.processing;

import com.yahoo.config.model.test.MockApplicationPackage;
import com.yahoo.search.query.profile.QueryProfileRegistry;
import com.yahoo.searchdefinition.RankProfile;
import com.yahoo.searchdefinition.RankProfileRegistry;
import com.yahoo.searchdefinition.Search;
import com.yahoo.searchdefinition.SearchBuilder;
import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext;
import com.yahoo.searchdefinition.expressiontransforms.TokenTransformer;
import com.yahoo.searchdefinition.parser.ParseException;
import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.evaluation.MapContext;
import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
import com.yahoo.tensor.Tensor;
import org.junit.Test;

import java.util.Collections;

import static org.junit.Assert.assertEquals;

public class RankingExpressionWithTransformerTokensTestCase {

    @Test
    public void testTokenInputIds() throws Exception {
        String expected = "tensor(d0[1],d1[12]):[101,1,2,102,3,4,5,102,6,7,102,0]";
        String a = "tensor(d0[2]):[1,2]";
        String b = "tensor(d0[3]):[3,4,5]";
        String c = "tensor(d0[2]):[6,7]";
        String expression = "tokenInputIds(12, a, b, c)";
        Tensor result = evaluateExpression(expression, a, b, c);
        assertEquals(Tensor.from(expected), result);
    }

    @Test
    public void testTokenTypeIds() throws Exception {
        String expected = "tensor(d0[1],d1[10]):[0,0,0,0,1,1,1,1,0,0]";
        String a = "tensor(d0[2]):[1,2]";
{'fraction_non_alphanumeric': 0.12347083926031295, 'fraction_numerical': 0.03470839260312945, 'mean_word_length': 5.245115452930729, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6967184', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1050, 'n_words': 207}