<commit_before>package io.joern.console.cpgcreation

import better.files.File
import io.shiftleft.codepropertygraph.Cpg
import io.shiftleft.codepropertygraph.generated.Languages
import io.joern.console.workspacehandling.Project
import overflowdb.traversal.help.Table

import scala.util.Try

class ImportCode[T <: Project](console: io.joern.console.Console[T]) {
  import io.joern.console.Console._

  private val config = console.config
  private val workspace = console.workspace
  protected val generatorFactory = new CpgGeneratorFactory(config)

  /**
    * This is the `importCode(...)` method exposed on the console. It attempts
    * to find a suitable CPG generator first by looking at the `language`
    * parameter and if no generator is found for the language, looking the
    * contents at `inputPath` to determine heuristically which generator to use.
    * */
  def apply(inputPath: String,
            projectName: String = "",
            namespaces: List[String] = List(),
            language: String = ""): Option[Cpg] = {

    if (language != "") {
      val generator = generatorFactory.forLanguage(language)
      if (generator.isEmpty) {
        report("No CPG generator exists for language: " + language)
        return None
      }
      generator.flatMap { frontend =>
        apply(frontend, inputPath, projectName, namespaces)
      }
    } else {
      val generator = generatorFactory.forCodeAt(inputPath)
      if (generator.isEmpty) {
        report("No suitable CPG generator found for: " + inputPath)
        return None
      }
      generator.flatMap { frontend =>
        apply(frontend, inputPath, projectName, namespaces)
      }
    }
  }

  def oldc: CFrontend = new CFrontend()
  def c: CFrontend = new CFrontend(Languages.NEWC, "Eclipse CDT Based Frontend for C/C++")
  def llvm: Frontend = new Frontend(Languages.LLVM, "LLVM Bitcode Frontend")
  def java: Frontend = new Frontend(Languages.JAVA, "Java/Dalvik Bytecode Frontend")
  def javasrc: Frontend = new Frontend(Languages.JAVASRC, "Java Source Frontend")
  def jimple: Frontend = new Frontend(Languages.JAVA, "Java Bytecode Frontend using soot's Jimple IR")
  def golang: Frontend = new Frontend(Languages.GOLANG, "Golang Source Frontend")
  def javascript: Frontend = new Frontend(Languages.JAVASCRIPT, "Javascript Source Frontend")
  def csharp: Frontend = new Frontend(Languages.CSHARP, "C# Source Frontend (Roslyn)")
  def python: Frontend = new Frontend(Languages.PYTHON, "Python Source Frontend")
  def php: Frontend = new Frontend(Languages.PHP, "PHP bytecode frontend")
  def ghidra: Frontend = new Frontend(Languages.GHIDRA, "ghidra reverse engineering frontend")

  class Frontend(val language: String, val description: String = "") {
    def isAvailable: Boolean = {
      cpgGeneratorForLanguage(language, config.frontend, config.install.rootPath.path, args = Nil).get.isAvailable
    }

    def apply(inputPath: String,
              projectName: String = "",
              namespaces: List[String] = List(),
              args: List[String] = List()): Option[Cpg] = {
      val frontend = {
        cpgGeneratorForLanguage(language, config.frontend, config.install.rootPath.path, args)
      }
      new ImportCode(console)(frontend.get, inputPath, projectName, namespaces)
    }
  }

  class CFrontend(language: String = Languages.C, description: String = "Fuzzy Parser for C/C++")
      extends Frontend(language, description) {
    def fromString(str: String): Option[Cpg] = {
      withCodeInTmpFile(str, "tmp.c") { dir =>
        apply(dir.path.toString)
      }
    }
  }

  private def withCodeInTmpFile(str: String, filename: String)(f: File => Option[Cpg]): Option[Cpg] = {
    val dir = File.newTemporaryDirectory("console")
    val result = Try {
      (dir / filename).write(str)
      f(dir)
    }.toOption.flatten
    dir.delete()
    result
  }

  private def allFrontends: List[Frontend] = List(
    oldc,
    csharp,
    golang,
    java,
    javasrc,
    javascript,
    jimple,
    llvm,
    python,
    ghidra,
  )

  /**
    * Provide an overview of the available CPG generators (frontends)
    * */
  override def toString: String = {
    val cols = List("name", "description", "available")
    val rows = allFrontends.map { frontend =>
      List(frontend.language.toLowerCase, frontend.description, frontend.isAvailable.toString)
    }
    "Type `importCode.<language>` to run a specific language frontend\n" +
      "\n" + Table(cols, rows).render
  }

  private def apply(frontend: CpgGenerator,
                    inputPath: String,
                    projectName: String,
                    namespaces: List[String]): Option[Cpg] = {
    val name =
      Option(projectName).filter(_.nonEmpty).getOrElse(deriveNameFromInputPath(inputPath, workspace))
    report(s"Creating project `$name` for code at `$inputPath`")
    val pathToProject = workspace.createProject(inputPath, name)
    val frontendCpgOutFileOpt = pathToProject.map(_.resolve(nameOfLegacyCpgInProject))

    if (frontendCpgOutFileOpt.isEmpty) {
      report(s"Error creating project for input path: `$inputPath`")
    }

    val result = frontendCpgOutFileOpt.flatMap { frontendCpgOutFile =>
      Some(frontend).flatMap { frontend =>
        generatorFactory
          .runGenerator(
            frontend,
            inputPath,
            frontendCpgOutFile.toString,
            namespaces
          )
          .flatMap(_ => console.open(name).flatMap(_.cpg))
          .map { c =>
            console.applyDefaultOverlays(c)
          }
      }
    }
    if (result.isDefined) {
      report(
        """|Code successfully imported. You can now query it using `cpg`.
           |For an overview of all imported code, type `workspace`.""".stripMargin
      )
    }
    result
  }

}
<commit_msg>Cleanup `importCode` (#702)<commit_after>package io.joern.console.cpgcreation

import better.files.File
import io.shiftleft.codepropertygraph.Cpg
import io.shiftleft.codepropertygraph.generated.Languages
import io.joern.console.workspacehandling.Project
import overflowdb.traversal.help.Table

import scala.util.Try

class ImportCode[T <: Project](console: io.joern.console.Console[T]) {
  import io.joern.console.Console._

  private val config = console.config
  private val workspace = console.workspace
  protected val generatorFactory = new CpgGeneratorFactory(config)

  /**
    * This is the `importCode(...)` method exposed on the console. It attempts
    * to find a suitable CPG generator first by looking at the `language`
    * parameter and if no generator is found for the language, looking the
    * contents at `inputPath` to determine heuristically which generator to use.
    * */
  def apply(inputPath: String,
            projectName: String = "",
            namespaces: List[String] = List(),
            language: String = ""): Option[Cpg] = {

    if (language != "") {
      val generator = generatorFactory.forLanguage(language)
      if (generator.isEmpty) {
        report("No CPG generator exists for language: " + language)
        return None
      }
      generator.flatMap { frontend =>
        apply(frontend, inputPath, projectName, namespaces)
      }
    } else {
      val generator = generatorFactory.forCodeAt(inputPath)
      if (generator.isEmpty) {
        report("No suitable CPG generator found for: " + inputPath)
        return None
      }
      generator.flatMap { frontend =>
        apply(frontend, inputPath, projectName, namespaces)
      }
    }
  }

  def oldc: SourceBasedFrontend = new SourceBasedFrontend("oldc", Languages.C)
  def c: SourceBasedFrontend = new SourceBasedFrontend("c", Languages.NEWC, "Eclipse CDT Based Frontend for C/C++")
  def cpp: SourceBasedFrontend =
    new SourceBasedFrontend("cpp", Languages.NEWC, "Eclipse CDT Based Frontend for C/C++", "cpp")
  def java: SourceBasedFrontend = new SourceBasedFrontend("java", Languages.JAVASRC, "Java Source Frontend", "java")

  def jvm: Frontend = new Frontend("jvm", Languages.JAVA, "Java/Dalvik Bytecode Frontend (based on SOOT's jimple)")
  def ghidra: Frontend = new Frontend("ghidra", Languages.GHIDRA, "ghidra reverse engineering frontend")

  def python: Frontend = new Frontend("python", Languages.PYTHON, "Python Source Frontend")
  def golang: Frontend = new Frontend("golang", Languages.GOLANG, "Golang Source Frontend")
  def javascript: Frontend = new Frontend("javascript", Languages.JAVASCRIPT, "Javascript Source Frontend")
  def csharp: Frontend = new Frontend("csharp", Languages.CSHARP, "C# Source Frontend (Roslyn)")

  def llvm: Frontend = new Frontend("llvm", Languages.LLVM, "LLVM Bitcode Frontend")
  def php: Frontend = new Frontend("php", Languages.PHP, "PHP bytecode frontend")

  class Frontend(val name: String, val language: String, val description: String = "") {
    def isAvailable: Boolean = {
      cpgGeneratorForLanguage(language, config.frontend, config.install.rootPath.path, args = Nil).get.isAvailable
    }

    def apply(inputPath: String,
              projectName: String = "",
              namespaces: List[String] = List(),
              args: List[String] = List()): Option[Cpg] = {
      val frontend = cpgGeneratorForLanguage(language, config.frontend, config.install.rootPath.path, args)
      new ImportCode(console)(frontend.get, inputPath, projectName, namespaces)
    }
  }

  class SourceBasedFrontend(name: String,
                            language: String = Languages.C,
                            description: String = "Fuzzy Parser for C/C++",
                            extension: String = "c")
      extends Frontend(name, language, description) {
    def fromString(str: String): Option[Cpg] = {
      withCodeInTmpFile(str, "tmp." + extension) { dir =>
        apply(dir.path.toString)
      }
    }
  }

  private def withCodeInTmpFile(str: String, filename: String)(f: File => Option[Cpg]): Option[Cpg] = {
    val dir = File.newTemporaryDirectory("console")
    val result = Try {
      (dir / filename).write(str)
      f(dir)
    }.toOption.flatten
    dir.delete()
    result
  }

  private def allFrontends: List[Frontend] = List(
    c,
    cpp,
    ghidra,
    java,
    jvm,
    javascript,
    golang,
    llvm,
    oldc,
    python,
    csharp,
  )

  /**
    * Provide an overview of the available CPG generators (frontends)
    * */
  override def toString: String = {
    val cols = List("name", "description", "available")
    val rows = allFrontends.map { frontend =>
      List(frontend.name, frontend.description, frontend.isAvailable.toString)
    }
    "Type `importCode.<language>` to run a specific language frontend\n" +
      "\n" + Table(cols, rows).render
  }

  private def apply(frontend: CpgGenerator,
                    inputPath: String,
                    projectName: String,
                    namespaces: List[String]): Option[Cpg] = {
    val name =
      Option(projectName).filter(_.nonEmpty).getOrElse(deriveNameFromInputPath(inputPath, workspace))
    report(s"Creating project `$name` for code at `$inputPath`")
    val pathToProject = workspace.createProject(inputPath, name)
    val frontendCpgOutFileOpt = pathToProject.map(_.resolve(nameOfLegacyCpgInProject))

    if (frontendCpgOutFileOpt.isEmpty) {
      report(s"Error creating project for input path: `$inputPath`")
    }

    val result = frontendCpgOutFileOpt.flatMap { frontendCpgOutFile =>
      Some(frontend).flatMap { frontend =>
        generatorFactory
          .runGenerator(
            frontend,
            inputPath,
            frontendCpgOutFile.toString,
            namespaces
          )
          .flatMap(_ => console.open(name).flatMap(_.cpg))
          .map { c =>
            console.applyDefaultOverlays(c)
          }
      }
    }
    if (result.isDefined) {
      report(
        """|Code successfully imported. You can now query it using `cpg`.
           |For an overview of all imported code, type `workspace`.""".stripMargin
      )
    }
    result
  }

}

{'fraction_non_alphanumeric': 0.10770256752250751, 'fraction_numerical': 0.00025008336112037347, 'mean_word_length': 2.97910447761194, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6816888', 'n_tokens_mistral': 3628, 'n_tokens_neox': 3539, 'n_words': 916}