<commit_before>package no.ssb.vtl.script.operations;

import no.ssb.vtl.model.AbstractUnaryDatasetOperation;
import no.ssb.vtl.model.Component;
import no.ssb.vtl.model.DataPoint;
import no.ssb.vtl.model.DataStructure;
import no.ssb.vtl.model.Dataset;
import no.ssb.vtl.model.VTLExpression;

import java.lang.String;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;

import static com.google.common.base.Preconditions.*;
import static java.lang.String.format;

public class CalcOperation extends AbstractUnaryDatasetOperation {
    
    private VTLExpression componentExpression;
    private final String variableName;
    
    public CalcOperation(Dataset dataset, VTLExpression componentExpression, String identifier) {
        super(checkNotNull(dataset, "the dataset was null"));
        this.componentExpression = checkNotNull(componentExpression, "the function was null");
        
        /*
            TODO: Handle explicit and implicit component computation.
            Need to parse the role
            If implicit, error if already defined.
         */
        variableName = removeQuoteIfNeeded(identifier);
//        return new Dataset() {
//
//
//
//            @Override
//            public String toString() {
//                MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper("Calc");
//                return helper.omitNullValues().toString();
//            }
//        };
    }
    
    @Override
    protected DataStructure computeDataStructure() {
        Component.Role role = Component.Role.MEASURE;
        Class<?> type = componentExpression.getType(); //Number.class; //TODO
    
        DataStructure.Builder structureCopy = DataStructure.copyOf(getChild().getDataStructure());
        structureCopy.put(variableName, role, type);
        return structureCopy.build();
    }
    
    private static String removeQuoteIfNeeded(String key) {
        if (!key.isEmpty() && key.length() > 3) {
            if (key.charAt(0) == '\'' && key.charAt(key.length() - 1) == '\'') {
                return key.substring(1, key.length() - 1);
            }
        }
        return key;
    }

    @Override
    public Stream<DataPoint> getData() {
        return getChild().getData().map(dataPoint -> {
            dataPoint.add(getDataStructure().wrap(variableName, componentExpression.apply(dataPoint)));
            System.out.println(format("Adding value for %s to datapoint %s", variableName, dataPoint));
            return dataPoint;
        });
    }

    @Override<commit_msg>Remove debug message in calc operation<commit_after>package no.ssb.vtl.script.operations;

import no.ssb.vtl.model.AbstractUnaryDatasetOperation;
import no.ssb.vtl.model.Component;
import no.ssb.vtl.model.DataPoint;
import no.ssb.vtl.model.DataStructure;
import no.ssb.vtl.model.Dataset;
import no.ssb.vtl.model.VTLExpression;

import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;

import static com.google.common.base.Preconditions.*;

public class CalcOperation extends AbstractUnaryDatasetOperation {
    
    private VTLExpression componentExpression;
    private final String variableName;
    
    public CalcOperation(Dataset dataset, VTLExpression componentExpression, String identifier) {
        super(checkNotNull(dataset, "the dataset was null"));
        this.componentExpression = checkNotNull(componentExpression, "the function was null");
        
        /*
            TODO: Handle explicit and implicit component computation.
            Need to parse the role
            If implicit, error if already defined.
         */
        variableName = removeQuoteIfNeeded(identifier);
//        return new Dataset() {
//
//
//
//            @Override
//            public String toString() {
//                MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper("Calc");
//                return helper.omitNullValues().toString();
//            }
//        };
    }
    
    @Override
    protected DataStructure computeDataStructure() {
        Component.Role role = Component.Role.MEASURE;
        Class<?> type = componentExpression.getType(); //Number.class; //TODO
    
        DataStructure.Builder structureCopy = DataStructure.copyOf(getChild().getDataStructure());
        structureCopy.put(variableName, role, type);
        return structureCopy.build();
    }
    
    private static String removeQuoteIfNeeded(String key) {
        if (!key.isEmpty() && key.length() > 3) {
            if (key.charAt(0) == '\'' && key.charAt(key.length() - 1) == '\'') {
                return key.substring(1, key.length() - 1);
            }
        }
        return key;
    }

    @Override
    public Stream<DataPoint> getData() {
        return getChild().getData().map(dataPoint -> {
            dataPoint.add(getDataStructure().wrap(variableName, componentExpression.apply(dataPoint)));
            return dataPoint;
        });
    }

    @Override
{'fraction_non_alphanumeric': 0.11418196489812386, 'fraction_numerical': 0.0020173492031470646, 'mean_word_length': 2.5901520637219404, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3112454', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1330, 'n_words': 288}