<commit_before>package com.mnubo
package dbschemas.migration

import java.io.{PrintStream, FileInputStream, InputStreamReader, File}
import java.text.SimpleDateFormat
import java.util.Date

import com.google.gson.{JsonObject, JsonElement, JsonParser}

import collection.JavaConverters._

object Boot extends App {
  val parser = new JsonParser
  val Ingestion = "ingestion"
  val Enrichment = "enrichment"
  val Analytics = "analytics"
  val GlobalConfig = "mnuboglobalconfig"
  val projects = List(Ingestion, Enrichment, Analytics, GlobalConfig)
  val TableSpec = """([a-z_\-0-9\.]+)"""
  val CreateTable1Regex = s"""(?i)create table if not exists $TableSpec .+""".r
  val CreateTable2Regex = s"""(?i)create table $TableSpec .+""".r
  val UpdateRegex = s"""(?i)update $TableSpec .+""".r
  val TruncateRegex = s"""(?i)truncate $TableSpec;""".r
  val AlterTableRegex = s"""(?i)alter table $TableSpec .+""".r
  val InsertIntoRegex = s"""(?i)insert into $TableSpec .+""".r
  val DropTable1Regex = s"""(?i)drop table $TableSpec;""".r
  val DropTable2Regex = s"""(?i)drop table if exists $TableSpec;""".r
  val DropTable3Regex = s"""(?i)drop $TableSpec;""".r
  val DeleteRegex = s"""(?i)delete from $TableSpec .+""".r
  val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'")
  val now = df.format(new Date())

  val migrationLegacyFiles = new File("../mnubo-main/platform/schemamanager/app/src/main/resources/mnuboSchemas")
    .listFiles()
    .toSeq
    .filter(_.getName.endsWith(".json"))
    .sortBy(_.getName)

  val projectTableMapping = Map(
    "rta_per_state" -> Analytics,
    "objects_per_state" -> Analytics,
    "object_state" -> Analytics,
    "rta_numeric_results" -> Analytics,
    "rta_groupby_results" -> Analytics,
    "analytics_objects_with_imported_attributes" -> Enrichment
  ).withDefaultValue(Ingestion)

  val versions = migrationLegacyFiles.map { migrationFile =>
    val json = using(new InputStreamReader(new FileInputStream(migrationFile)))(parser.parse)
    val upgradeScript = getScript(json.getAsJsonObject, "UpgradeDelta")
    val downgradeScript = getScript(json.getAsJsonObject, "DowngradeDelta")
    var schemaScripts = Map(Ingestion -> Migration(), Enrichment -> Migration(), Analytics -> Migration(), GlobalConfig -> Migration())

    def splitInProjects(script: Seq[String], addCommand: (Migration, String) => Migration) =
      script.foreach { cmd =>
        val tableName = parseTableName(cmd)
        val project =
          if (tableName.isEmpty)
            Ingestion
          else if (tableName.startsWith("mnuboglobalconfig."))
            GlobalConfig
          else
            projectTableMapping(tableName)<commit_msg>MM-2477: moved a table to analytics.<commit_after>package com.mnubo
package dbschemas.migration

import java.io.{PrintStream, FileInputStream, InputStreamReader, File}
import java.text.SimpleDateFormat
import java.util.Date

import com.google.gson.{JsonObject, JsonElement, JsonParser}

import collection.JavaConverters._

object Boot extends App {
  val parser = new JsonParser
  val Ingestion = "ingestion"
  val Enrichment = "enrichment"
  val Analytics = "analytics"
  val GlobalConfig = "mnuboglobalconfig"
  val projects = List(Ingestion, Enrichment, Analytics, GlobalConfig)
  val TableSpec = """([a-z_\-0-9\.]+)"""
  val CreateTable1Regex = s"""(?i)create table if not exists $TableSpec .+""".r
  val CreateTable2Regex = s"""(?i)create table $TableSpec .+""".r
  val UpdateRegex = s"""(?i)update $TableSpec .+""".r
  val TruncateRegex = s"""(?i)truncate $TableSpec;""".r
  val AlterTableRegex = s"""(?i)alter table $TableSpec .+""".r
  val InsertIntoRegex = s"""(?i)insert into $TableSpec .+""".r
  val DropTable1Regex = s"""(?i)drop table $TableSpec;""".r
  val DropTable2Regex = s"""(?i)drop table if exists $TableSpec;""".r
  val DropTable3Regex = s"""(?i)drop $TableSpec;""".r
  val DeleteRegex = s"""(?i)delete from $TableSpec .+""".r
  val df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm'Z'")
  val now = df.format(new Date())

  val migrationLegacyFiles = new File("../mnubo-main/platform/schemamanager/app/src/main/resources/mnuboSchemas")
    .listFiles()
    .toSeq
    .filter(_.getName.endsWith(".json"))
    .sortBy(_.getName)

  val projectTableMapping = Map(
    "rta_per_state" -> Analytics,
    "objects_per_state" -> Analytics,
    "object_state" -> Analytics,
    "rta_numeric_results" -> Analytics,
    "rta_groupby_results" -> Analytics,
    "rta_event_count_results" -> Analytics,
    "analytics_objects_with_imported_attributes" -> Enrichment
  ).withDefaultValue(Ingestion)

  val versions = migrationLegacyFiles.map { migrationFile =>
    val json = using(new InputStreamReader(new FileInputStream(migrationFile)))(parser.parse)
    val upgradeScript = getScript(json.getAsJsonObject, "UpgradeDelta")
    val downgradeScript = getScript(json.getAsJsonObject, "DowngradeDelta")
    var schemaScripts = Map(Ingestion -> Migration(), Enrichment -> Migration(), Analytics -> Migration(), GlobalConfig -> Migration())

    def splitInProjects(script: Seq[String], addCommand: (Migration, String) => Migration) =
      script.foreach { cmd =>
        val tableName = parseTableName(cmd)
        val project =
          if (tableName.isEmpty)
            Ingestion
          else if (tableName.startsWith("mnuboglobalconfig."))
            GlobalConfig
          else
            projectTableMapping(tableName)
{'fraction_non_alphanumeric': 0.1388631734863914, 'fraction_numerical': 0.0033327161636733937, 'mean_word_length': 4.214285714285714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3871782', 'n_tokens_mistral': 1766, 'n_tokens_neox': 1715, 'n_words': 382}