<commit_before>package org.smoothbuild.parse;

import static java.util.Arrays.asList;
import static java.util.stream.Collectors.toList;
import static org.smoothbuild.lang.function.base.Name.name;
import static org.smoothbuild.lang.function.base.Parameter.parameter;
import static org.smoothbuild.lang.function.base.Parameter.parametersToString;
import static org.smoothbuild.lang.function.base.Parameters.parametersToNames;
import static org.smoothbuild.lang.type.Conversions.canConvert;
import static org.smoothbuild.lang.type.Types.BLOB;
import static org.smoothbuild.lang.type.Types.FILE;
import static org.smoothbuild.lang.type.Types.NIL;
import static org.smoothbuild.lang.type.Types.STRING;
import static org.smoothbuild.lang.type.Types.allTypes;
import static org.smoothbuild.parse.LocationHelpers.locationOf;
import static org.smoothbuild.parse.arg.Argument.argument;
import static org.smoothbuild.util.Lists.map;
import static org.smoothbuild.util.Maybe.error;
import static org.smoothbuild.util.Maybe.errors;
import static org.smoothbuild.util.Maybe.invoke;
import static org.smoothbuild.util.Maybe.invokeWrap;
import static org.smoothbuild.util.Maybe.pullUp;
import static org.smoothbuild.util.Maybe.value;
import static org.smoothbuild.util.StringUnescaper.unescaped;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.antlr.v4.runtime.tree.TerminalNode;
import org.smoothbuild.antlr.SmoothParser.ArgContext;
import org.smoothbuild.antlr.SmoothParser.ArgListContext;
import org.smoothbuild.antlr.SmoothParser.ArrayContext;
import org.smoothbuild.antlr.SmoothParser.CallContext;
import org.smoothbuild.antlr.SmoothParser.ExprContext;
import org.smoothbuild.antlr.SmoothParser.NameContext;
import org.smoothbuild.lang.expr.ArrayExpression;
import org.smoothbuild.lang.expr.DefaultValueExpression;
import org.smoothbuild.lang.expr.Expression;
import org.smoothbuild.lang.expr.StringLiteralExpression;
import org.smoothbuild.lang.function.Functions;
import org.smoothbuild.lang.function.base.Function;
import org.smoothbuild.lang.function.base.Name;
import org.smoothbuild.lang.function.base.Parameter;
import org.smoothbuild.lang.function.base.Signature;
import org.smoothbuild.lang.function.def.DefinedFunction;
import org.smoothbuild.lang.message.CodeLocation;
import org.smoothbuild.lang.type.ArrayType;
import org.smoothbuild.lang.type.Conversions;
import org.smoothbuild.lang.type.Type;
import org.smoothbuild.lang.type.Types;
import org.smoothbuild.lang.value.Value;
import org.smoothbuild.parse.arg.Argument;
import org.smoothbuild.parse.arg.MapToString;
import org.smoothbuild.parse.arg.ParametersPool;
import org.smoothbuild.parse.arg.TypedParametersPool;
import org.smoothbuild.parse.ast.ArgNode;
import org.smoothbuild.parse.ast.ArrayNode;
import org.smoothbuild.parse.ast.CallNode;
import org.smoothbuild.parse.ast.ContextExprNode;
import org.smoothbuild.parse.ast.ExprNode;
import org.smoothbuild.parse.ast.FunctionNode;
import org.smoothbuild.parse.ast.ParamNode;
import org.smoothbuild.parse.ast.StringNode;
import org.smoothbuild.util.Maybe;
import org.smoothbuild.util.UnescapingFailedException;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;

public class DefinedFunctionLoader {

  public static Maybe<DefinedFunction> loadDefinedFunction(Functions loadedFunctions,
      FunctionNode functionNode) {
    return new Worker(loadedFunctions).loadFunction(functionNode);
  }

  private static class Worker {
    private final Functions loadedFunctions;

    public Worker(Functions loadedFunctions) {
      this.loadedFunctions = loadedFunctions;
    }

    public Maybe<DefinedFunction> loadFunction(FunctionNode node) {
      List<Parameter> parameters = createParameters(node.params());
      Maybe<Expression> expression = createExpression(node.expr());
      return invokeWrap(expression, e -> createFunction(node, e));
    }

    private static List<Parameter> createParameters(List<ParamNode> params) {
      return params
          .stream()
          .map(p -> parameter(p.type(), p.name()))
          .collect(toList());
    }

    private static DefinedFunction createFunction(FunctionNode node, Expression expression) {
      Signature signature = new Signature(expression.type(), node.name(), asList());
      return new DefinedFunction(signature, expression);
    }

    private Maybe<List<Expression>> parseExpressionList(List<ExprContext> contexts) {
      return pullUp(map(contexts, this::parseExpression));
    }

    private Maybe<Expression> createExpression(ExprNode node) {
      if (node instanceof CallNode) {
        return createCall((CallNode) node);
      }
      if (node instanceof StringNode) {
        return createStringLiteral((StringNode) node);
      }
      if (node instanceof ArrayNode) {
        return createArray((ArrayNode) node);
      }
      if (node instanceof ContextExprNode) {
        return parseExpression(((ContextExprNode) node).expr());
      }
      throw new RuntimeException("Illegal parse tree: " + ExprContext.class.getSimpleName()
          + " without children.");
    }

    private Maybe<Expression> parseExpression(ExprContext context) {
      if (context.array() != null) {
        return parseArray(context.array());
      }
      if (context.call() != null) {
        return parseCall(context.call());
      }
      if (context.STRING() != null) {
        return parseStringLiteral(context.STRING());
      }
      throw new RuntimeException("Illegal parse tree: " + ExprContext.class.getSimpleName()
          + " without children.");
    }

    private Maybe<Expression> createArray(ArrayNode node) {
      Maybe<List<Expression>> exprList = pullUp(map(node.elements(), this::createExpression));
      CodeLocation location = node.codeLocation();
      Maybe<ArrayType> arrayType = invoke(exprList, es -> arrayType(es, location));
      return invokeWrap(arrayType, at -> createArray(at, exprList.value(), location));
    }

    private Maybe<Expression> parseArray(ArrayContext context) {
      List<ExprContext> elems = context.expr();
      Maybe<List<Expression>> expressions = parseExpressionList(elems);
      CodeLocation location = locationOf(context);
      Maybe<ArrayType> arrayType = invoke(expressions, es -> arrayType(es, location));
      return invokeWrap(arrayType, at -> createArray(at, expressions.value(), location));
    }

    private Expression createArray(ArrayType type, List<Expression> elements,
        CodeLocation location) {
      List<Expression> converted = map(elements, e -> implicitConversion(type.elemType(), e));
      return new ArrayExpression(type, converted, location);
    }

    private Maybe<ArrayType> arrayType(List<Expression> expressions, CodeLocation location) {
      if (expressions.isEmpty()) {
        return value(NIL);
      }
      Type firstType = expressions.get(0).type();
      Type superType = firstType;

      for (int i = 1; i < expressions.size(); i++) {
        Type type = expressions.get(i).type();
        superType = commonSuperType(superType, type);

        if (superType == null) {
          return error(new ParseError(location,
              "Array cannot contain elements of incompatible types.\n"
                  + "First element has type " + firstType + " while element at index " + i
                  + " has type " + type + "."));
        }
      }
      ArrayType arrayType = Types.arrayOf(superType);
      if (arrayType == null) {
        return error(new ParseError(location, "Array cannot contain element with type "
            + superType + ". Only following types are allowed: " + Types.basicTypes()
            + "."));
      }

      return value(arrayType);
    }

    private static Type commonSuperType(Type type1, Type type2) {
      if (type1 == STRING) {
        if (type2 == STRING) {
          return STRING;
        } else {
          return null;
        }
      } else if (type1 == BLOB) {
        if (type2 == BLOB || type2 == FILE) {
          return BLOB;
        } else {
          return null;
        }
      } else if (type1 == FILE) {
        if (type2 == FILE) {
          return FILE;
        } else if (type2 == BLOB) {
          return BLOB;
        } else {
          return null;
        }
      }
      return null;
    }

    private Maybe<Expression> createCall(CallNode node) {
      Maybe<List<Argument>> args = convertArgNodesToArguments(node.args());
      Function function = loadedFunctions.get(name(node.name()));
      Maybe<List<Expression>> expressions = invoke(args,
          a -> createArgExprs(node.codeLocation(), function, a));
      return invokeWrap(expressions,
          es -> function.createCallExpression(es, false, node.codeLocation()));
    }

    private Maybe<List<Argument>> convertArgNodesToArguments(List<ArgNode> args) {
      List<Maybe<Argument>> result = new ArrayList<Maybe<Argument>>();
      for (ArgNode argNode : args) {
        Maybe<Expression> expression = createExpression(argNode.expr());
        Maybe<Argument> argument = invokeWrap(expression,
            e -> argument(argNode.number(), argNode.name(), e, argNode.codeLocation()));
        result.add(argument);
      }
      return Maybe.pullUp(result);

    }

    private Maybe<Expression> parseCall(CallContext context) {
      Maybe<List<Argument>> arguments = parseArgumentList(context.argList());
      return invoke(arguments, as -> parseCall(context, as));
    }

    private Maybe<Expression> parseCall(CallContext context, List<Argument> arguments) {
      Function function = loadedFunctions.get(name(context.name().getText()));
      CodeLocation codeLocation = locationOf(context.name());
      Maybe<List<Expression>> expressions = createArgExprs(codeLocation, function, arguments);
      return invokeWrap(expressions, es -> function.createCallExpression(es, false, codeLocation));
    }

    private Maybe<List<Argument>> parseArgumentList(ArgListContext context) {
      List<Maybe<Argument>> result = new ArrayList<>();
      if (context != null) {
        List<ArgContext> argContexts = context.arg();
        for (int i = 0; i < argContexts.size(); i++) {
          result.add(parseArgument(i, argContexts.get(i)));
        }
      }
      return pullUp(result);
    }

    private Maybe<Argument> parseArgument(int index, ArgContext context) {
      Maybe<Expression> expression = parseExpression(context.expr());
      return invokeWrap(expression, e -> createArgument(index, context, e));
    }

    private Argument createArgument(int index, ArgContext context, Expression expression) {
      NameContext paramName = context.name();
      String name = paramName == null ? null : paramName.getText();
      return argument(index, name, expression, locationOf(context));
    }

    private Maybe<Expression> parseStringLiteral(TerminalNode stringToken) {
      String quotedString = stringToken.getText();
      String string = quotedString.substring(1, quotedString.length() - 1);
      CodeLocation location = locationOf(stringToken.getSymbol());
      try {
        return value(new StringLiteralExpression(unescaped(string), location));
      } catch (UnescapingFailedException e) {
        return error(new ParseError(location, e.getMessage()));
      }
    }

    private Maybe<Expression> createStringLiteral(StringNode node) {
      try {
        return value(new StringLiteralExpression(
            unescaped(node.value()), node.codeLocation()));
      } catch (UnescapingFailedException e) {
        return error(new ParseError(node.codeLocation(), e.getMessage()));
      }
    }

    public Maybe<List<Expression>> createArgExprs(CodeLocation codeLocation, Function function,
        List<Argument> arguments) {
      ParametersPool parametersPool = new ParametersPool(function.parameters());
      List<Argument> namedArguments = Argument.filterNamed(arguments);

      List<Object> errors = duplicatedAndUnknownArgumentNames(function, namedArguments);
      if (!errors.isEmpty()) {
        return errors(errors);
      }

      Map<Parameter, Argument> argumentMap = new HashMap<>();
      errors = processNamedArguments(parametersPool, argumentMap, namedArguments);
      if (!errors.isEmpty()) {
        return errors(errors);
      }

      errors = processNamelessArguments(function, arguments, parametersPool, argumentMap,
          codeLocation);<commit_msg>removed dead code from DefinedFunctionLoader<commit_after>package org.smoothbuild.parse;

import static java.util.Arrays.asList;
import static java.util.stream.Collectors.toList;
import static org.smoothbuild.lang.function.base.Name.name;
import static org.smoothbuild.lang.function.base.Parameter.parameter;
import static org.smoothbuild.lang.function.base.Parameter.parametersToString;
import static org.smoothbuild.lang.function.base.Parameters.parametersToNames;
import static org.smoothbuild.lang.type.Conversions.canConvert;
import static org.smoothbuild.lang.type.Types.BLOB;
import static org.smoothbuild.lang.type.Types.FILE;
import static org.smoothbuild.lang.type.Types.NIL;
import static org.smoothbuild.lang.type.Types.STRING;
import static org.smoothbuild.lang.type.Types.allTypes;
import static org.smoothbuild.parse.arg.Argument.argument;
import static org.smoothbuild.util.Lists.map;
import static org.smoothbuild.util.Maybe.error;
import static org.smoothbuild.util.Maybe.errors;
import static org.smoothbuild.util.Maybe.invoke;
import static org.smoothbuild.util.Maybe.invokeWrap;
import static org.smoothbuild.util.Maybe.pullUp;
import static org.smoothbuild.util.Maybe.value;
import static org.smoothbuild.util.StringUnescaper.unescaped;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.smoothbuild.antlr.SmoothParser.ExprContext;
import org.smoothbuild.lang.expr.ArrayExpression;
import org.smoothbuild.lang.expr.DefaultValueExpression;
import org.smoothbuild.lang.expr.Expression;
import org.smoothbuild.lang.expr.StringLiteralExpression;
import org.smoothbuild.lang.function.Functions;
import org.smoothbuild.lang.function.base.Function;
import org.smoothbuild.lang.function.base.Name;
import org.smoothbuild.lang.function.base.Parameter;
import org.smoothbuild.lang.function.base.Signature;
import org.smoothbuild.lang.function.def.DefinedFunction;
import org.smoothbuild.lang.message.CodeLocation;
import org.smoothbuild.lang.type.ArrayType;
import org.smoothbuild.lang.type.Conversions;
import org.smoothbuild.lang.type.Type;
import org.smoothbuild.lang.type.Types;
import org.smoothbuild.lang.value.Value;
import org.smoothbuild.parse.arg.Argument;
import org.smoothbuild.parse.arg.MapToString;
import org.smoothbuild.parse.arg.ParametersPool;
import org.smoothbuild.parse.arg.TypedParametersPool;
import org.smoothbuild.parse.ast.ArgNode;
import org.smoothbuild.parse.ast.ArrayNode;
import org.smoothbuild.parse.ast.CallNode;
import org.smoothbuild.parse.ast.ExprNode;
import org.smoothbuild.parse.ast.FunctionNode;
import org.smoothbuild.parse.ast.ParamNode;
import org.smoothbuild.parse.ast.StringNode;
import org.smoothbuild.util.Maybe;
import org.smoothbuild.util.UnescapingFailedException;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMultimap;

public class DefinedFunctionLoader {

  public static Maybe<DefinedFunction> loadDefinedFunction(Functions loadedFunctions,
      FunctionNode functionNode) {
    return new Worker(loadedFunctions).loadFunction(functionNode);
  }

  private static class Worker {
    private final Functions loadedFunctions;

    public Worker(Functions loadedFunctions) {
      this.loadedFunctions = loadedFunctions;
    }

    public Maybe<DefinedFunction> loadFunction(FunctionNode node) {
      List<Parameter> parameters = createParameters(node.params());
      Maybe<Expression> expression = createExpression(node.expr());
      return invokeWrap(expression, e -> createFunction(node, e));
    }

    private static List<Parameter> createParameters(List<ParamNode> params) {
      return params
          .stream()
          .map(p -> parameter(p.type(), p.name()))
          .collect(toList());
    }

    private static DefinedFunction createFunction(FunctionNode node, Expression expression) {
      Signature signature = new Signature(expression.type(), node.name(), asList());
      return new DefinedFunction(signature, expression);
    }

    private Maybe<Expression> createExpression(ExprNode node) {
      if (node instanceof CallNode) {
        return createCall((CallNode) node);
      }
      if (node instanceof StringNode) {
        return createStringLiteral((StringNode) node);
      }
      if (node instanceof ArrayNode) {
        return createArray((ArrayNode) node);
      }
      throw new RuntimeException("Illegal parse tree: " + ExprContext.class.getSimpleName()
          + " without children.");
    }

    private Maybe<Expression> createArray(ArrayNode node) {
      Maybe<List<Expression>> exprList = pullUp(map(node.elements(), this::createExpression));
      CodeLocation location = node.codeLocation();
      Maybe<ArrayType> arrayType = invoke(exprList, es -> arrayType(es, location));
      return invokeWrap(arrayType, at -> createArray(at, exprList.value(), location));
    }

    private Expression createArray(ArrayType type, List<Expression> elements,
        CodeLocation location) {
      List<Expression> converted = map(elements, e -> implicitConversion(type.elemType(), e));
      return new ArrayExpression(type, converted, location);
    }

    private Maybe<ArrayType> arrayType(List<Expression> expressions, CodeLocation location) {
      if (expressions.isEmpty()) {
        return value(NIL);
      }
      Type firstType = expressions.get(0).type();
      Type superType = firstType;

      for (int i = 1; i < expressions.size(); i++) {
        Type type = expressions.get(i).type();
        superType = commonSuperType(superType, type);

        if (superType == null) {
          return error(new ParseError(location,
              "Array cannot contain elements of incompatible types.\n"
                  + "First element has type " + firstType + " while element at index " + i
                  + " has type " + type + "."));
        }
      }
      ArrayType arrayType = Types.arrayOf(superType);
      if (arrayType == null) {
        return error(new ParseError(location, "Array cannot contain element with type "
            + superType + ". Only following types are allowed: " + Types.basicTypes()
            + "."));
      }

      return value(arrayType);
    }

    private static Type commonSuperType(Type type1, Type type2) {
      if (type1 == STRING) {
        if (type2 == STRING) {
          return STRING;
        } else {
          return null;
        }
      } else if (type1 == BLOB) {
        if (type2 == BLOB || type2 == FILE) {
          return BLOB;
        } else {
          return null;
        }
      } else if (type1 == FILE) {
        if (type2 == FILE) {
          return FILE;
        } else if (type2 == BLOB) {
          return BLOB;
        } else {
          return null;
        }
      }
      return null;
    }

    private Maybe<Expression> createCall(CallNode node) {
      Maybe<List<Argument>> args = convertArgNodesToArguments(node.args());
      Function function = loadedFunctions.get(name(node.name()));
      Maybe<List<Expression>> expressions = invoke(args,
          a -> createArgExprs(node.codeLocation(), function, a));
      return invokeWrap(expressions,
          es -> function.createCallExpression(es, false, node.codeLocation()));
    }

    private Maybe<List<Argument>> convertArgNodesToArguments(List<ArgNode> args) {
      List<Maybe<Argument>> result = new ArrayList<Maybe<Argument>>();
      for (ArgNode argNode : args) {
        Maybe<Expression> expression = createExpression(argNode.expr());
        Maybe<Argument> argument = invokeWrap(expression,
            e -> argument(argNode.number(), argNode.name(), e, argNode.codeLocation()));
        result.add(argument);
      }
      return Maybe.pullUp(result);

    }

    private Maybe<Expression> createStringLiteral(StringNode node) {
      try {
        return value(new StringLiteralExpression(
            unescaped(node.value()), node.codeLocation()));
      } catch (UnescapingFailedException e) {
        return error(new ParseError(node.codeLocation(), e.getMessage()));
      }
    }

    public Maybe<List<Expression>> createArgExprs(CodeLocation codeLocation, Function function,
        List<Argument> arguments) {
      ParametersPool parametersPool = new ParametersPool(function.parameters());
      List<Argument> namedArguments = Argument.filterNamed(arguments);

      List<Object> errors = duplicatedAndUnknownArgumentNames(function, namedArguments);
      if (!errors.isEmpty()) {
        return errors(errors);
      }

      Map<Parameter, Argument> argumentMap = new HashMap<>();
      errors = processNamedArguments(parametersPool, argumentMap, namedArguments);
      if (!errors.isEmpty()) {
        return errors(errors);
      }

      errors = processNamelessArguments(function, arguments, parametersPool, argumentMap,
          codeLocation);
{'fraction_non_alphanumeric': 0.11106442577030812, 'fraction_numerical': 0.00130718954248366, 'mean_word_length': 3.6577516851489453, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2559901', 'n_tokens_mistral': 5894, 'n_tokens_neox': 5717, 'n_words': 1268}