<commit_before>package no.ssb.vtl.script.visitors;

/*-
 * ========================LICENSE_START=================================
 * Java VTL
 * %%
 * Copyright (C) 2016 - 2017 <NAME>
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =========================LICENSE_END==================================
 */

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import no.ssb.vtl.model.Component;
import no.ssb.vtl.model.DataPoint;
import no.ssb.vtl.model.DataStructure;
import no.ssb.vtl.model.Dataset;
import no.ssb.vtl.model.StaticDataset;
import no.ssb.vtl.model.VTLObject;
import no.ssb.vtl.script.operations.AggregationOperation;
import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.junit.Before;
import org.junit.Test;

import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.entry;
import static org.assertj.core.api.Assertions.fail;

public class AggregationVisitorTest {

    private AggregationVisitor visitor = new AggregationVisitor();
    private Dataset datasetSingleMeasure;
    private Dataset datasetMultiMeasure;
    private DataStructure dataStructureSingleMeasure;

    @Before
    public void setUp() throws Exception {
        dataStructureSingleMeasure = DataStructure.of(
                "time", Component.Role.IDENTIFIER, String.class,
                "geo", Component.Role.IDENTIFIER, String.class,
                "m1", Component.Role.MEASURE, Long.class);

        datasetSingleMeasure = StaticDataset.create(dataStructureSingleMeasure)
                .addPoints("2010", "NO", 20L)
                .addPoints("2011", "SE", 31L)
                .addPoints("2012", "SE", 41L)
                .addPoints("2012", "NO", 72L)
                .addPoints("2010", "SE", 40L)
                .addPoints("2011", "NO", 11L)
                .addPoints("2012", "SE", 41L)
                .addPoints("2010", "DK", 60L)
                .addPoints("2011", "DK", 51L)
                .addPoints("2012", "DK", 92L)
                .build();

        DataStructure dataStructureMultiMeasure = DataStructure.of(
                "time", Component.Role.IDENTIFIER, String.class,
                "geo", Component.Role.IDENTIFIER, String.class,
                "m1", Component.Role.MEASURE, Long.class,
                "m2", Component.Role.MEASURE, Long.class);
        datasetMultiMeasure = StaticDataset.create(dataStructureMultiMeasure)
                .addPoints("2010", "NO", 20L, 2L)
                .addPoints("2010", "SE", 40L, 4L)
                .addPoints("2010", "DK", 60L, 6L)
                .addPoints("2011", "NO", 11L, 1L)
                .addPoints("2011", "SE", 31L, 3L)
                .addPoints("2011", "DK", 51L, 5L)
                .addPoints("2012", "NO", 72L, 7L)
                .addPoints("2012", "SE", 82L, 8L)
                .addPoints("2012", "DK", 92L, 9L)
                .build();


    }

    @Test
    public void testSumSingleMeasureDataSet() throws Exception {

        List<Component> components = Lists.newArrayList(datasetSingleMeasure.getDataStructure().getOrDefault("time", null));
        AggregationOperation sumOperation = visitor.getSumOperation(datasetSingleMeasure,components);
        sumOperation.getData().forEach(System.out::println);

        DataStructure dataStructure = sumOperation.getDataStructure();

        assertThat(dataStructure.getRoles()).contains(
                entry("time", Component.Role.IDENTIFIER),<commit_msg>Add test for the aggregation visitor<commit_after>package no.ssb.vtl.script.visitors;

/*-
 * ========================LICENSE_START=================================
 * Java VTL
 * %%
 * Copyright (C) 2016 - 2017 <NAME>
 * %%
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =========================LICENSE_END==================================
 */

import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import no.ssb.vtl.model.Component;
import no.ssb.vtl.model.DataPoint;
import no.ssb.vtl.model.DataStructure;
import no.ssb.vtl.model.Dataset;
import no.ssb.vtl.model.StaticDataset;
import no.ssb.vtl.model.VTLObject;
import no.ssb.vtl.parser.VTLLexer;
import no.ssb.vtl.parser.VTLParser;
import no.ssb.vtl.script.error.ContextualRuntimeException;
import no.ssb.vtl.script.operations.AggregationOperation;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.assertj.core.api.JUnitSoftAssertions;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

import javax.script.SimpleBindings;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.entry;
import static org.assertj.core.api.Assertions.fail;

public class AggregationVisitorTest {

    private AggregationVisitor visitor;
    private Dataset datasetSingleMeasure;
    private Dataset datasetMultiMeasure;
    private DataStructure dataStructureSingleMeasure;

    @Rule
    public JUnitSoftAssertions softly = new JUnitSoftAssertions();

    @Before
    public void setUp() throws Exception {

        SimpleBindings bindings = new SimpleBindings();
        visitor = new AggregationVisitor(
                new DatasetExpressionVisitor(
                        new ExpressionVisitor(
                                bindings
                        )
                )
        );

        dataStructureSingleMeasure = DataStructure.of(
                "time", Component.Role.IDENTIFIER, String.class,
                "geo", Component.Role.IDENTIFIER, String.class,
                "m1", Component.Role.MEASURE, Long.class);

        datasetSingleMeasure = StaticDataset.create(dataStructureSingleMeasure)
                .addPoints("2010", "NO", 20L)
                .addPoints("2011", "SE", 31L)
                .addPoints("2012", "SE", 41L)
                .addPoints("2012", "NO", 72L)
                .addPoints("2010", "SE", 40L)
                .addPoints("2011", "NO", 11L)
                .addPoints("2012", "SE", 41L)
                .addPoints("2010", "DK", 60L)
                .addPoints("2011", "DK", 51L)
                .addPoints("2012", "DK", 92L)
                .build();

        DataStructure dataStructureMultiMeasure = DataStructure.of(
                "time", Component.Role.IDENTIFIER, String.class,
                "geo", Component.Role.IDENTIFIER, String.class,
                "m1", Component.Role.MEASURE, Long.class,
                "m2", Component.Role.MEASURE, Long.class);
        datasetMultiMeasure = StaticDataset.create(dataStructureMultiMeasure)
                .addPoints("2010", "NO", 20L, 2L)
                .addPoints("2010", "SE", 40L, 4L)
                .addPoints("2010", "DK", 60L, 6L)
                .addPoints("2011", "NO", 11L, 1L)
                .addPoints("2011", "SE", 31L, 3L)
                .addPoints("2011", "DK", 51L, 5L)
                .addPoints("2012", "NO", 72L, 7L)
                .addPoints("2012", "SE", 82L, 8L)
                .addPoints("2012", "DK", 92L, 9L)
                .build();

        bindings.put("dsMulti", datasetMultiMeasure);
        bindings.put("dsSingle", datasetSingleMeasure);

    }

    @Test
    public void variableDifferentDataset() throws Exception {

        List<String> expressions = Arrays.asList(
                "sum(dsSingle) group by dsMulti.time",
                "sum(dsSingle) along dsMulti.time"
        );
        for (String expression : expressions) {

            softly.assertThatThrownBy(() -> visitor.visit(parse(expression)))
                    .isInstanceOf(ContextualRuntimeException.class)
                    .hasMessage("variable dsMulti.time does not belong to dataset dsSingle");
        }
    }

    @Test
    public void variableNotFound() throws Exception {

        List<String> expressions = Arrays.asList(
                "sum(notFound) group by time",
                "sum(dsSingle) group by notFound",
                "sum(dsSingle) group by dsSingle.notFound",
                "sum(dsSingle.notFound) group by time",
                "sum(notFound) along geo",
                "sum(dsSingle) along notFound",
                "sum(dsSingle) along dsSingle.notFound",
                "sum(dsSingle.notFound) along time"
        );
        for (String expression : expressions) {

            softly.assertThatThrownBy(() -> visitor.visit(parse(expression)))
                    .isInstanceOf(ContextualRuntimeException.class)
                    .hasMessage("undefined variable notFound");
        }
    }

    private VTLParser.AggregationFunctionContext parse(String expression) {
        VTLLexer lexer = new VTLLexer(CharStreams.fromString(expression));
        VTLParser parser = new VTLParser(new CommonTokenStream(lexer));
        return parser.aggregationFunction();
    }

    @Test
    public void testAggregateOnMeasure() throws Exception {

        List<String> expressions = Arrays.asList(
                "sum(dsMulti) group by m2",
                "sum(dsMulti) group by dsMulti.m2",
                "sum(dsMulti.m1) group by m2",
                "sum(dsMulti.m1) group by dsMulti.m2",
                "sum(dsMulti) along m2",
                "sum(dsMulti) along dsMulti.m2",
                "sum(dsMulti.m1) along m2",
                "sum(dsMulti.m1) along dsMulti.m2"
        );
        for (String expression : expressions) {

            softly.assertThatThrownBy(() -> visitor.visit(parse(expression)))
                    .isInstanceOf(ContextualRuntimeException.class)
                    .hasMessage("variable m2 was not an identifier");

        }
    }

    @Test
    public void testMembership() throws Exception {

        List<String> expressions = Arrays.asList(
                "sum(dsSingle) group by time",
                "sum(dsSingle) group by dsSingle.time",
                "sum(dsSingle.m1) group by time",
                "sum(dsSingle.m1) group by dsSingle.time",
                "sum(dsSingle) along geo",
                "sum(dsSingle) along dsSingle.geo",
                "sum(dsSingle.m1) along geo",
                "sum(dsSingle.m1) along dsSingle.geo"
        );
        for (String expression : expressions) {

            AggregationOperation result = visitor.visit(parse(expression));

            DataStructure dataStructure = result.getDataStructure();
            Map<String, Component.Role> roles = result.getDataStructure().getRoles();
            softly.assertThat(roles).contains(
                    entry("time", Component.Role.IDENTIFIER),
                    entry("m1", Component.Role.MEASURE)
            );

            softly.assertThat(result.getDataStructure().getTypes()).contains(
                    entry("time", String.class),
                    entry("m1", Long.class)
            );


            softly.assertThat(result.getData()).contains(
                    dataStructure.wrap(ImmutableMap.of("time", "2010", "m1", 20L+40L+60L)),
                    dataStructure.wrap(ImmutableMap.of("time", "2011", "m1", 11L+31L+51L)),
                    dataStructure.wrap(ImmutableMap.of("time", "2012", "m1", 72L+82L+92L))
            );

        }
    }

    @Test
    public void testSumSingleMeasureDataSet() throws Exception {

        List<Component> components = Lists.newArrayList(datasetSingleMeasure.getDataStructure().getOrDefault("time", null));
        AggregationOperation sumOperation = visitor.getSumOperation(datasetSingleMeasure,components);
        sumOperation.getData().forEach(System.out::println);

        DataStructure dataStructure = sumOperation.getDataStructure();

        assertThat(dataStructure.getRoles()).contains(
                entry("time", Component.Role.IDENTIFIER),
{'fraction_non_alphanumeric': 0.13038496791934007, 'fraction_numerical': 0.025358997861289336, 'mean_word_length': 2.4840340606705693, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4631150', 'n_tokens_mistral': 3967, 'n_tokens_neox': 3549, 'n_words': 803}