<commit_before>//Basicly, this high-level program generator generate Lambdas in a loop:

//First, it try to use options(i.e patterns, like join,split,map,reduce..) that takes the initial input (Note that FPatterns also need a Lambda as input)
//Each round, it will generate :
//A FunCall
//A Lambda using the funcall
//A param that have the same type with the lambda
//A param that unpack the result array(only when 1.the result type is an ArrayType 2.MapStrictMatchUnpack or ReduceStrictMatchUnpack or both of them are set as True)

//For example: if we have a Param x with x.t == ArrayType(Float,64),then it can generate:
//1. Split(8) $ x
//2. fun(ArrayType(Float,64),x =>{ Split(8) $ x})
//3. Param(ArrayType(ArrayType(Float,8),8)
//4. Param(ArrayType(8))

//Second, it will use the Params(and Lambdas for FPatterns) that comes form initial input or the first step, to generate deeper programs

//Then, repeatedly generate Lambdas using the Params and Lambdas we have, until reach the loop limit(also the depth limit)

//Finally, refine the result,deal with the Params that comes from FunCall or unpack

//The final result is stored in RefinedResult:ArrayBuffer[Lambda]

//Now we have Join, Split,UserFun, Zip,Get,Map,Reduce, and can generate the features of Matrix Mult
//It's simple to support more patterns.. Just tell me

//Then the usage of the controllers:


//1.LoopNum : Int                => The loop limit (max depth)
//2.ConsequentUserFun: Boolean   => allow for UserFun() o UserFun()
//3.ReduceOnOneElement: Boolean  => allow for reduction on only one element. The compiler have a problem with that.
//4.AllowJoinASplit: Boolean     => allow for Join()o Split(). It is identical
//5.MustContainsUserFun: Boolean => filter the result that does not contains any userfun
//6.MustContainsMap:Boolean      => filter the result that does not contains any maps
//7.MapStrictMatchUnpack:Boolean => When generate Map, like:
//                                  Map(Lambda1) o Param2, the input of Lambda1 is Param1
//                                  If MapStrictMatchUnpack is set to True, then it will ensure: Param1 comes form the unpack of Param2
//8.ReduceStrictMatchUnpack:Boolean  => similar with (7)
//9.LimitNum:Int                 => the max number of Lambda generated each cycle

//10.ZipLimit:Int                => Max number of Param to zip
//11.GenJoin/GenSplit... :Boolean=> Whether generate a certain pattern


package ir.hlGenerator

import java.io.PrintWriter

import ir._
import ir.ast._
import ir.interpreter.Interpreter
import opencl.executor.{Compile, Eval, Execute}
import opencl.ir._
import opencl.ir.pattern.ReduceSeq
import rewriting.{EnabledMappings, Lower, Rewrite}

import scala.collection.mutable._

class hlGenerator {

  val RefinedResult: ArrayBuffer[Lambda] = new ArrayBuffer[Lambda]()
  val ParamList: ArrayBuffer[Param] = new ArrayBuffer[Param]()
  //var FunCallList: ArrayBuffer[FunCall] = new ArrayBuffer[FunCall]()
  val LambdaList: ArrayBuffer[Lambda] = new ArrayBuffer[Lambda]()
  val ParamToFunCall = collection.mutable.Map[Param, FunCall]()
  val UnpackedToExpr = collection.mutable.Map[Param,Expr]()

  //Used for debug
  var AssignedChoiceNum = 0
  var PassCounter = 0
  val PassParamUpPossibility = 0.0
  val PrintDebugInfo = true


  //controllers for run programs
  val GlobalSize = 512
  val LocalSize = 32
  val RunInterpreter = false
  val useRandomRewrite = false
  val rewriteDepth = 3

  //controllers for generate programs
  val LoopNum = 30
  val ConsequentUserFun = false
  //val ReduceInitToGlobal = false
  val ReduceOnOneElement = false
  val AllowJoinASplit = false
  val MustContainsUserFun = true
  val MustContainsMap = true
  val MapStrictMatchUnpack = true
  val ReduceStrictMatchUnpack = true
  var LimitNum = 40

  //controllers for patterns
  val ZipLimit = 2
  val SplitChunkSize = 4
  val GenJoin = true
  val GenSplit = true
  val GenUserFun = true
  val GenZip = true
  val GenGet = true
  val GenMap = true
  val GenReduce = true


  //Avoid for redundant
  //Join
  var Join_P = 0

  //Split
  var Split_P = 0

  //Reduce
  val Reduce_L_PI_PE = scala.collection.mutable.Set[(Int, Int, Int)]()
  val Reduce_Lambda_Check = scala.collection.mutable.Set[Int]()

  //UserFun
  val Add_Check = scala.collection.mutable.Set[(Int, Int)]()

  //Map
  val Map_L_E = scala.collection.mutable.Set[(Int, Int)]()
  var Map_Checked = 0

  //Zip
  var Zip_P = 0

  //Get
  var Get_P = 0

  //UnpackParam
  var UnPack_P = 0


  private def tryRewriting(l:Lambda,w:PrintWriter):scala.Seq[Lambda] ={
    if (useRandomRewrite) {
      try {
        //Rewrite.rewriteWithoutLowering(l,rewriting.allRulesWithoutLowering,rewriteDepth)
        Rewrite.rewriteWithoutLowering(l, rewriting.allRulesWithoutMapsLowering(1, 1, 1), rewriteDepth)
      }
      catch {
        case e: Throwable =>
          println("catch a exception in random-rewrite-by-user")
          writeln(w, "catch a exception in random-rewrite-by-user")

          if (PrintDebugInfo) {
            e.printStackTrace(w)
            e.printStackTrace()
          }
          scala.Seq()
      }
    } else {
      scala.Seq(l)
    }
  }

  private def tryLowering(l:Lambda,w:PrintWriter):List[Lambda] = {
    try {
      Lower.mapCombinations(l, EnabledMappings(global0 = true, global01 = true, global10 = true, group0 = true, group01 = true, group10 = true))
    }
    catch {
      case e: Throwable =>
        if(PrintDebugInfo) {
          println("catch a exception in lower-parser-by-user")
          writeln(w, "catch a exception in lower-parser-by-user")
          e.printStackTrace(w)
          e.printStackTrace ()
        }
        List[Lambda]()
    }
  }

  private def tryCompile(l:Lambda,w:PrintWriter):(String,Boolean)={
    try {<commit_msg>Minor change to imports<commit_after>//Basicly, this high-level program generator generate Lambdas in a loop:

//First, it try to use options(i.e patterns, like join,split,map,reduce..) that takes the initial input (Note that FPatterns also need a Lambda as input)
//Each round, it will generate :
//A FunCall
//A Lambda using the funcall
//A param that have the same type with the lambda
//A param that unpack the result array(only when 1.the result type is an ArrayType 2.MapStrictMatchUnpack or ReduceStrictMatchUnpack or both of them are set as True)

//For example: if we have a Param x with x.t == ArrayType(Float,64),then it can generate:
//1. Split(8) $ x
//2. fun(ArrayType(Float,64),x =>{ Split(8) $ x})
//3. Param(ArrayType(ArrayType(Float,8),8)
//4. Param(ArrayType(8))

//Second, it will use the Params(and Lambdas for FPatterns) that comes form initial input or the first step, to generate deeper programs

//Then, repeatedly generate Lambdas using the Params and Lambdas we have, until reach the loop limit(also the depth limit)

//Finally, refine the result,deal with the Params that comes from FunCall or unpack

//The final result is stored in RefinedResult:ArrayBuffer[Lambda]

//Now we have Join, Split,UserFun, Zip,Get,Map,Reduce, and can generate the features of Matrix Mult
//It's simple to support more patterns.. Just tell me

//Then the usage of the controllers:


//1.LoopNum : Int                => The loop limit (max depth)
//2.ConsequentUserFun: Boolean   => allow for UserFun() o UserFun()
//3.ReduceOnOneElement: Boolean  => allow for reduction on only one element. The compiler have a problem with that.
//4.AllowJoinASplit: Boolean     => allow for Join()o Split(). It is identical
//5.MustContainsUserFun: Boolean => filter the result that does not contains any userfun
//6.MustContainsMap:Boolean      => filter the result that does not contains any maps
//7.MapStrictMatchUnpack:Boolean => When generate Map, like:
//                                  Map(Lambda1) o Param2, the input of Lambda1 is Param1
//                                  If MapStrictMatchUnpack is set to True, then it will ensure: Param1 comes form the unpack of Param2
//8.ReduceStrictMatchUnpack:Boolean  => similar with (7)
//9.LimitNum:Int                 => the max number of Lambda generated each cycle

//10.ZipLimit:Int                => Max number of Param to zip
//11.GenJoin/GenSplit... :Boolean=> Whether generate a certain pattern


package ir.hlGenerator

import java.io.PrintWriter

import ir._
import ir.ast._
import ir.interpreter.Interpreter
import opencl.executor.{Compile, Eval, Execute}
import opencl.ir._
import opencl.ir.pattern.ReduceSeq
import rewriting.{EnabledMappings, Lower, Rewrite}

import scala.collection.mutable.ArrayBuffer

class hlGenerator {

  val RefinedResult = ArrayBuffer[Lambda]()
  val ParamList = ArrayBuffer[Param]()
  val LambdaList = ArrayBuffer[Lambda]()
  val ParamToFunCall = collection.mutable.Map[Param, FunCall]()
  val UnpackedToExpr = collection.mutable.Map[Param,Expr]()

  //Used for debug
  var AssignedChoiceNum = 0
  var PassCounter = 0
  val PassParamUpPossibility = 0.0
  val PrintDebugInfo = true


  //controllers for run programs
  val GlobalSize = 512
  val LocalSize = 32
  val RunInterpreter = false
  val useRandomRewrite = false
  val rewriteDepth = 3

  //controllers for generate programs
  val LoopNum = 30
  val ConsequentUserFun = false
  //val ReduceInitToGlobal = false
  val ReduceOnOneElement = false
  val AllowJoinASplit = false
  val MustContainsUserFun = true
  val MustContainsMap = true
  val MapStrictMatchUnpack = true
  val ReduceStrictMatchUnpack = true
  var LimitNum = 40

  //controllers for patterns
  val ZipLimit = 2
  val SplitChunkSize = 4
  val GenJoin = true
  val GenSplit = true
  val GenUserFun = true
  val GenZip = true
  val GenGet = true
  val GenMap = true
  val GenReduce = true


  //Avoid for redundant
  //Join
  var Join_P = 0

  //Split
  var Split_P = 0

  //Reduce
  val Reduce_L_PI_PE = scala.collection.mutable.Set[(Int, Int, Int)]()
  val Reduce_Lambda_Check = scala.collection.mutable.Set[Int]()

  //UserFun
  val Add_Check = scala.collection.mutable.Set[(Int, Int)]()

  //Map
  val Map_L_E = scala.collection.mutable.Set[(Int, Int)]()
  var Map_Checked = 0

  //Zip
  var Zip_P = 0

  //Get
  var Get_P = 0

  //UnpackParam
  var UnPack_P = 0


  private def tryRewriting(l:Lambda,w:PrintWriter):Seq[Lambda] ={
    if (useRandomRewrite) {
      try {
        //Rewrite.rewriteWithoutLowering(l,rewriting.allRulesWithoutLowering,rewriteDepth)
        Rewrite.rewriteWithoutLowering(l, rewriting.allRulesWithoutMapsLowering(1, 1, 1), rewriteDepth)
      }
      catch {
        case e: Throwable =>
          println("catch a exception in random-rewrite-by-user")
          writeln(w, "catch a exception in random-rewrite-by-user")

          if (PrintDebugInfo) {
            e.printStackTrace(w)
            e.printStackTrace()
          }
          Seq()
      }
    } else {
      Seq(l)
    }
  }

  private def tryLowering(l:Lambda,w:PrintWriter):List[Lambda] = {
    try {
      Lower.mapCombinations(l, EnabledMappings(global0 = true, global01 = true, global10 = true, group0 = true, group01 = true, group10 = true))
    }
    catch {
      case e: Throwable =>
        if(PrintDebugInfo) {
          println("catch a exception in lower-parser-by-user")
          writeln(w, "catch a exception in lower-parser-by-user")
          e.printStackTrace(w)
          e.printStackTrace ()
        }
        List[Lambda]()
    }
  }

  private def tryCompile(l:Lambda,w:PrintWriter):(String,Boolean)={
    try {
{'fraction_non_alphanumeric': 0.09522975929978118, 'fraction_numerical': 0.01225382932166302, 'mean_word_length': 3.1473684210526316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2127632', 'n_tokens_mistral': 3644, 'n_tokens_neox': 3474, 'n_words': 1158}