<commit_before>package io.virtdata.core;

import io.virtdata.api.DataMapperLibrary;
import io.virtdata.api.ValueType;
import io.virtdata.api.VirtDataFunctionLibrary;
import io.virtdata.api.composers.FunctionAssembly;
import io.virtdata.ast.FunctionCall;
import io.virtdata.ast.VirtDataFlow;
import io.virtdata.parser.VirtDataDSL;
import org.apache.commons.lang3.ClassUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.invoke.MethodHandles;
import java.util.*;
import java.util.stream.Collectors;

/**
 * <H2>Synopsis</H2>
 * <p>This library implements the ability to compose a lambda function from a sequence of other functions.
 * The resulting lambda will use the specialized primitive function interfaces, such as LongUnaryOperator, LongFunction, etc.
 * Where there are two functions which do not have matching input and output types, the most obvious conversion is made.
 * This means that while you are able to compose a LongUnaryOperator with a LongUnaryOperator for maximum
 * efficiency, you can also compose LongUnaryOperator with an IntFunction, and a best effort attempt will be made to
 * do a reasonable conversion in between.</p>
 *
 * <H2>Limitations</H2>
 * <P>Due to type erasure, it is not possible to know the generic type parameters for non-primitive functional types.
 * These include IntFunction&lt;?&gt;, LongFunction&lt;?&gt;, and in the worst case, Function&lt;?,?&gt;.
 * For these types, annotations are provided to better inform the runtime lambda compositor.</P>
 *
 * <H2>Multiple Paths</H2>
 * <P>The library allows for there to be multiple functions which match the spec, possibly because multiple
 * functions have the same name, but exist in different libraries or in different packages within the same library.
 * This means that the composer library must find a connecting path between the functions that can match at each stage,
 * disregarding all but one.</P>
 *
 * <H2>Path Finding</H2>
 * <P>The rule for finding the best path among the available functions is as follows, at each pairing between
 * adjacent stages of functions:</P>
 * <OL>
 * <li>The co-compatible output and input types between the functions are mapped. Functions sharing the co-compatible
 * types are kept in the list. Functions not sharing them are removed.</li>
 * <li>As long as functions can be removed in this way, the process iterates through the chain, starting again
 * at the front of the list.</li>
 * <li>When no functions can be removed due to lack of co-compatible types, each stage is selected according to
 * type preferences as represented in {@link ValueType}</li>
 *
 * <LI>If the next (outer) function does not have a compatible input type, move it down on the list.
 * If, after this step, there are functions which do have matching signatures, all others are removed.</LI>
 * </OL>
 */
public class VirtDataComposer {

    private final static String PREAMBLE = "compose ";
    private final static Logger logger = LoggerFactory.getLogger(DataMapperLibrary.class);
    private final static MethodHandles.Lookup lookup = MethodHandles.publicLookup();

    private final VirtDataFunctionLibrary functionLibrary;
    private final Map<String,Object> customElements = new HashMap<>();

    public VirtDataComposer(VirtDataFunctionLibrary functionLibrary) {
        this.functionLibrary = functionLibrary;
    }

    public VirtDataComposer() {
        this.functionLibrary = VirtDataLibraries.get();
    }


    public Optional<ResolvedFunction> resolveFunctionFlow(String flowspec) {

        String strictSpec = flowspec.startsWith("compose ") ? flowspec.substring(8) : flowspec;
        VirtDataDSL.ParseResult parseResult = VirtDataDSL.parse(strictSpec);
        if (parseResult.throwable != null) {
            throw new RuntimeException(parseResult.throwable);
        }
        VirtDataFlow flow = parseResult.flow;

        return resolveFunctionFlow(flow);
    }

    public ResolverDiagnostics resolveDiagnosticFunctionFlow(String flowspec) {
        String strictSpec = flowspec.startsWith("compose ") ? flowspec.substring(8) : flowspec;
        VirtDataDSL.ParseResult parseResult = VirtDataDSL.parse(strictSpec);
        if (parseResult.throwable != null) {
            throw new RuntimeException(parseResult.throwable);
        }
        VirtDataFlow flow = parseResult.flow;

        return resolveDiagnosticFunctionFlow(flow);
    }

    public ResolverDiagnostics resolveDiagnosticFunctionFlow(VirtDataFlow flow) {
        ResolverDiagnostics diagnostics = new ResolverDiagnostics();
        diagnostics.trace("processing flow " + flow.toString() + " from output to input");

        LinkedList<List<ResolvedFunction>> funcs = new LinkedList<>();

        LinkedList<Set<Class<?>>> nextFunctionInputTypes = new LinkedList<>();
        Optional<Class<?>> finalValueTypeOption =
                Optional.ofNullable(flow.getLastExpression().getCall().getOutputType())
                        .map(ValueType::valueOfClassName).map(ValueType::getValueClass);

        nextFunctionInputTypes.add(new HashSet<>());
        finalValueTypeOption.ifPresent(t -> nextFunctionInputTypes.get(0).add(t));

        diagnostics.trace("working backwards from " + (flow.getExpressions().size()-1));

        for (int i = flow.getExpressions().size() - 1; i >= 0; i--) {
            FunctionCall call = flow.getExpressions().get(i).getCall();
            diagnostics.trace("resolving args for " + call.toString());

            List<ResolvedFunction> nodeFunctions = new LinkedList<>();

            String funcName = call.getFunctionName();
            Class<?> inputType = ValueType.classOfType(call.getInputType());
            Class<?> outputType = ValueType.classOfType(call.getOutputType());
            Object[] args = call.getArguments();
            try {
                args = populateFunctions(diagnostics, args, this.customElements);
            } catch (Exception e) {
                return diagnostics.error(e);
            }

            diagnostics.trace("resolved args: ");
            for (Object arg : args) {
                diagnostics.trace(" " + arg.getClass().getSimpleName() + ": " + arg.toString());
            }

            List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, this.customElements,args);
            if (resolved.size() == 0) {
                return diagnostics.error(new RuntimeException("Unable to find even one function for " + call));
            }
            diagnostics.trace(" resolved functions:");
            diagnostics.trace(summarize(resolved));

            nodeFunctions.addAll(resolved);
            funcs.addFirst(nodeFunctions);

            Set<Class<?>> inputTypes = nodeFunctions.stream().map(ResolvedFunction::getInputClass).collect(Collectors.toSet());
            nextFunctionInputTypes.addFirst(inputTypes);
        }

        if (!nextFunctionInputTypes.peekFirst().contains(Long.TYPE)) {
            return diagnostics.error(new RuntimeException("There is no initial function which accepts a long input. Function chain, after type filtering: \n" + summarizeBulk(funcs)));
        }
        removeNonLongFunctions(funcs.getFirst());

        List<ResolvedFunction> flattenedFuncs = optimizePath(funcs, ValueType.classOfType(flow.getLastExpression().getCall().getOutputType()));

        if (flattenedFuncs.size() == 1) {
            diagnostics.trace("FUNCTION resolution succeeded (single): '" + flow.toString() + "'");
            return diagnostics.setResolvedFunction(flattenedFuncs.get(0));
        }

        FunctionAssembly assembly = new FunctionAssembly();
        diagnostics.trace("composed summary: " + summarize(flattenedFuncs));

        boolean isThreadSafe = true;
        diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs) + "'");
        for (ResolvedFunction resolvedFunction : flattenedFuncs) {
            try {
                Object functionObject = resolvedFunction.getFunctionObject();
                assembly.andThen(functionObject);
                if (!resolvedFunction.isThreadSafe()) {
                    isThreadSafe = false;
                }
            } catch (Exception e) {
                return diagnostics.error(new RuntimeException("FUNCTION resolution failed: '" + flow.toString() + "': " + e.toString()));
            }
        }
        ResolvedFunction composedFunction = assembly.getResolvedFunction(isThreadSafe);
        diagnostics.trace("FUNCTION resolution succeeded (lambda): '" + flow.toString() + "'");
        return diagnostics.setResolvedFunction(composedFunction);
    }

    public Optional<ResolvedFunction> resolveFunctionFlow(VirtDataFlow flow) {
        ResolverDiagnostics resolverDiagnostics = resolveDiagnosticFunctionFlow(flow);
        return resolverDiagnostics.getResolvedFunction();
    }

    private Object[] populateFunctions(ResolverDiagnostics diagnostics, Object[] args, Map<String,?> cconfig) {
        for (int i = 0; i < args.length; i++) {
            Object o = args[i];
            if (o instanceof FunctionCall) {<commit_msg>avoid NPE in exception handler<commit_after>package io.virtdata.core;

import io.virtdata.api.DataMapperLibrary;
import io.virtdata.api.ValueType;
import io.virtdata.api.VirtDataFunctionLibrary;
import io.virtdata.api.composers.FunctionAssembly;
import io.virtdata.ast.FunctionCall;
import io.virtdata.ast.VirtDataFlow;
import io.virtdata.parser.VirtDataDSL;
import org.apache.commons.lang3.ClassUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.lang.invoke.MethodHandles;
import java.util.*;
import java.util.stream.Collectors;

/**
 * <H2>Synopsis</H2>
 * <p>This library implements the ability to compose a lambda function from a sequence of other functions.
 * The resulting lambda will use the specialized primitive function interfaces, such as LongUnaryOperator, LongFunction, etc.
 * Where there are two functions which do not have matching input and output types, the most obvious conversion is made.
 * This means that while you are able to compose a LongUnaryOperator with a LongUnaryOperator for maximum
 * efficiency, you can also compose LongUnaryOperator with an IntFunction, and a best effort attempt will be made to
 * do a reasonable conversion in between.</p>
 *
 * <H2>Limitations</H2>
 * <P>Due to type erasure, it is not possible to know the generic type parameters for non-primitive functional types.
 * These include IntFunction&lt;?&gt;, LongFunction&lt;?&gt;, and in the worst case, Function&lt;?,?&gt;.
 * For these types, annotations are provided to better inform the runtime lambda compositor.</P>
 *
 * <H2>Multiple Paths</H2>
 * <P>The library allows for there to be multiple functions which match the spec, possibly because multiple
 * functions have the same name, but exist in different libraries or in different packages within the same library.
 * This means that the composer library must find a connecting path between the functions that can match at each stage,
 * disregarding all but one.</P>
 *
 * <H2>Path Finding</H2>
 * <P>The rule for finding the best path among the available functions is as follows, at each pairing between
 * adjacent stages of functions:</P>
 * <OL>
 * <li>The co-compatible output and input types between the functions are mapped. Functions sharing the co-compatible
 * types are kept in the list. Functions not sharing them are removed.</li>
 * <li>As long as functions can be removed in this way, the process iterates through the chain, starting again
 * at the front of the list.</li>
 * <li>When no functions can be removed due to lack of co-compatible types, each stage is selected according to
 * type preferences as represented in {@link ValueType}</li>
 *
 * <LI>If the next (outer) function does not have a compatible input type, move it down on the list.
 * If, after this step, there are functions which do have matching signatures, all others are removed.</LI>
 * </OL>
 */
public class VirtDataComposer {

    private final static String PREAMBLE = "compose ";
    private final static Logger logger = LoggerFactory.getLogger(DataMapperLibrary.class);
    private final static MethodHandles.Lookup lookup = MethodHandles.publicLookup();

    private final VirtDataFunctionLibrary functionLibrary;
    private final Map<String,Object> customElements = new HashMap<>();

    public VirtDataComposer(VirtDataFunctionLibrary functionLibrary) {
        this.functionLibrary = functionLibrary;
    }

    public VirtDataComposer() {
        this.functionLibrary = VirtDataLibraries.get();
    }


    public Optional<ResolvedFunction> resolveFunctionFlow(String flowspec) {

        String strictSpec = flowspec.startsWith("compose ") ? flowspec.substring(8) : flowspec;
        VirtDataDSL.ParseResult parseResult = VirtDataDSL.parse(strictSpec);
        if (parseResult.throwable != null) {
            throw new RuntimeException(parseResult.throwable);
        }
        VirtDataFlow flow = parseResult.flow;

        return resolveFunctionFlow(flow);
    }

    public ResolverDiagnostics resolveDiagnosticFunctionFlow(String flowspec) {
        String strictSpec = flowspec.startsWith("compose ") ? flowspec.substring(8) : flowspec;
        VirtDataDSL.ParseResult parseResult = VirtDataDSL.parse(strictSpec);
        if (parseResult.throwable != null) {
            throw new RuntimeException(parseResult.throwable);
        }
        VirtDataFlow flow = parseResult.flow;

        return resolveDiagnosticFunctionFlow(flow);
    }

    public ResolverDiagnostics resolveDiagnosticFunctionFlow(VirtDataFlow flow) {
        ResolverDiagnostics diagnostics = new ResolverDiagnostics();
        diagnostics.trace("processing flow " + flow.toString() + " from output to input");

        LinkedList<List<ResolvedFunction>> funcs = new LinkedList<>();

        LinkedList<Set<Class<?>>> nextFunctionInputTypes = new LinkedList<>();
        Optional<Class<?>> finalValueTypeOption =
                Optional.ofNullable(flow.getLastExpression().getCall().getOutputType())
                        .map(ValueType::valueOfClassName).map(ValueType::getValueClass);

        nextFunctionInputTypes.add(new HashSet<>());
        finalValueTypeOption.ifPresent(t -> nextFunctionInputTypes.get(0).add(t));

        diagnostics.trace("working backwards from " + (flow.getExpressions().size()-1));

        for (int i = flow.getExpressions().size() - 1; i >= 0; i--) {
            FunctionCall call = flow.getExpressions().get(i).getCall();
            diagnostics.trace("resolving args for " + call.toString());

            List<ResolvedFunction> nodeFunctions = new LinkedList<>();

            String funcName = call.getFunctionName();
            Class<?> inputType = ValueType.classOfType(call.getInputType());
            Class<?> outputType = ValueType.classOfType(call.getOutputType());
            Object[] args = call.getArguments();
            try {
                args = populateFunctions(diagnostics, args, this.customElements);
            } catch (Exception e) {
                return diagnostics.error(e);
            }

            diagnostics.trace("resolved args: ");
            for (Object arg : args) {
                diagnostics.trace(" " + arg.getClass().getSimpleName() + ": " + arg.toString());
            }

            List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, this.customElements,args);
            if (resolved.size() == 0) {
                return diagnostics.error(new RuntimeException("Unable to find even one function for " + call));
            }
            diagnostics.trace(" resolved functions:");
            diagnostics.trace(summarize(resolved));

            nodeFunctions.addAll(resolved);
            funcs.addFirst(nodeFunctions);

            Set<Class<?>> inputTypes = nodeFunctions.stream().map(ResolvedFunction::getInputClass).collect(Collectors.toSet());
            nextFunctionInputTypes.addFirst(inputTypes);
        }

        if (!nextFunctionInputTypes.peekFirst().contains(Long.TYPE)) {
            return diagnostics.error(new RuntimeException("There is no initial function which accepts a long input. Function chain, after type filtering: \n" + summarizeBulk(funcs)));
        }
        removeNonLongFunctions(funcs.getFirst());

        List<ResolvedFunction> flattenedFuncs = optimizePath(funcs, ValueType.classOfType(flow.getLastExpression().getCall().getOutputType()));

        if (flattenedFuncs.size() == 1) {
            diagnostics.trace("FUNCTION resolution succeeded (single): '" + flow.toString() + "'");
            return diagnostics.setResolvedFunction(flattenedFuncs.get(0));
        }

        FunctionAssembly assembly = new FunctionAssembly();
        diagnostics.trace("composed summary: " + summarize(flattenedFuncs));

        boolean isThreadSafe = true;
        diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs) + "'");
        for (ResolvedFunction resolvedFunction : flattenedFuncs) {
            try {
                Object functionObject = resolvedFunction.getFunctionObject();
                assembly.andThen(functionObject);
                if (!resolvedFunction.isThreadSafe()) {
                    isThreadSafe = false;
                }
            } catch (Exception e) {
                String flowdata = flow!=null? flow.toString() : "undefined";
                return diagnostics.error(new RuntimeException("FUNCTION resolution failed: '" + flowdata + "': " + e.toString()));
            }
        }
        ResolvedFunction composedFunction = assembly.getResolvedFunction(isThreadSafe);
        diagnostics.trace("FUNCTION resolution succeeded (lambda): '" + flow.toString() + "'");
        return diagnostics.setResolvedFunction(composedFunction);
    }

    public Optional<ResolvedFunction> resolveFunctionFlow(VirtDataFlow flow) {
        ResolverDiagnostics resolverDiagnostics = resolveDiagnosticFunctionFlow(flow);
        return resolverDiagnostics.getResolvedFunction();
    }

    private Object[] populateFunctions(ResolverDiagnostics diagnostics, Object[] args, Map<String,?> cconfig) {
        for (int i = 0; i < args.length; i++) {
            Object o = args[i];
            if (o instanceof FunctionCall) {
{'fraction_non_alphanumeric': 0.09952580803401101, 'fraction_numerical': 0.002289202594429607, 'mean_word_length': 3.3810888252148996, 'pattern_counts': {'":': 2, '<': 103, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3402668', 'n_tokens_mistral': 4732, 'n_tokens_neox': 4579, 'n_words': 1355}