<commit_before>import com.github.siasia.{WebPlugin, PluginKeys}
import sbt._
import Keys._

object AntScoutBuild extends Build {

  val mapDirectory = "maps"
  val mapName = "hamburg.osm"
  val mapExtension = ".pbf"
  val originalMap = file("%s/%s%s" format (mapDirectory, mapName, mapExtension))
  val preprocessedMaps = Seq(
    (originalMap, (9.9362, 53.55131, 9.94756, 53.5468), file("%s/%s" format (mapDirectory, "Altona-Kreis.osm"))),
    (originalMap, (9.69, 53.585, 9.95, 53.545), file("%s/%s" format (mapDirectory, "Altona-Wedel.osm")))
  )

  val downloadMap = TaskKey[Unit]("download-map")
  val preprocessMap = TaskKey[Unit]("preprocess-map")

  val downloadMapTask = downloadMap <<= streams.map { s: TaskStreams =>
    if (!originalMap.exists) {
      val mapUrl = url("http://download.geofabrik.de/osm/europe/germany/%s%s" format (mapName, mapExtension))
      s.log.info("Downloading map from %s to %s" format (mapUrl, originalMap))
      sbt.IO.download(mapUrl, originalMap)
      s.log.success("Map downloaded")
    }
  }

  val preprocessMapTask = preprocessMap <<= streams.map { s: TaskStreams =>
    preprocessedMaps foreach {
      case (originalMap, boundingBox, preprocessedMap) =>
        if (!preprocessedMap.exists) {
          s.log.info("Preprocessing map %s" format preprocessedMap)
          val command = "maps/osmosis-0.40.1/bin/osmosis" + (if (sys.props("os.name").startsWith("Win")) ".bat" else "")
          val (left, top, right, bottom) = boundingBox
          val arguments = Seq[String](
            "-q",
            "--read-pbf", "file=" + originalMap,
            "--bounding-box", "left=" + left, "top=" + top, "right=" + right, "bottom=" + bottom, "completeWays=yes",
            "--tag-filter", "accept-ways", "highway=motorway,motorway_link,trunk,trunk_link,primary,primary_link," +
              "secondary,secondary_link,tertiary,tertiary_link,residential,living_street,unclassified",
            "--tag-filter", "reject-relations",
            "--used-node",
            "--write-xml", preprocessedMap.toString
          )
          (Process(command, arguments) !) match {
            case 0 => s.log.success("Map %s preprocessed" format preprocessedMap)
            case _ => s.log.error("Preprocessing map %s failed" format preprocessedMap)
          }
        }
    }
  }

  lazy val antScout = Project("AntScout", file("."), settings = Project.defaultSettings ++ WebPlugin.webSettings ++
    Seq(downloadMapTask, preprocessMapTask, PluginKeys.start in WebPlugin.container.Configuration <<= (PluginKeys.start
      in WebPlugin.container.Configuration).dependsOn(preprocessMap), preprocessMap <<= preprocessMap.dependsOn
      (downloadMap)))
}<commit_msg>Osmosis wird vor dem Karten-Preprocessing auf "ausführbar" gesetzt.<commit_after>import com.github.siasia.{WebPlugin, PluginKeys}
import sbt._
import Keys._

object AntScoutBuild extends Build {

  val mapDirectory = "maps"
  val mapName = "hamburg.osm"
  val mapExtension = ".pbf"
  val originalMap = file("%s/%s%s" format (mapDirectory, mapName, mapExtension))
  val preprocessedMaps = Seq(
    (originalMap, (9.9362, 53.55131, 9.94756, 53.5468), file("%s/%s" format (mapDirectory, "Altona-Kreis.osm"))),
    (originalMap, (9.69, 53.585, 9.95, 53.545), file("%s/%s" format (mapDirectory, "Altona-Wedel.osm")))
  )

  val downloadMap = TaskKey[Unit]("download-map")
  val preprocessMap = TaskKey[Unit]("preprocess-map")

  val downloadMapTask = downloadMap <<= streams.map { s: TaskStreams =>
    if (!originalMap.exists) {
      val mapUrl = url("http://download.geofabrik.de/osm/europe/germany/%s%s" format (mapName, mapExtension))
      s.log.info("Downloading map from %s to %s" format (mapUrl, originalMap))
      sbt.IO.download(mapUrl, originalMap)
      s.log.success("Map downloaded")
    }
  }

  val preprocessMapTask = preprocessMap <<= streams.map { s: TaskStreams =>
    preprocessedMaps foreach {
      case (originalMap, boundingBox, preprocessedMap) =>
        if (!preprocessedMap.exists) {
          s.log.info("Preprocessing map %s" format preprocessedMap)
          val command = "maps/osmosis-0.40.1/bin/osmosis" + (if (sys.props("os.name").startsWith("Win")) ".bat" else "")
          file(command).setExecutable(true)
          val (left, top, right, bottom) = boundingBox
          val arguments = Seq[String](
            "-q",
            "--read-pbf", "file=" + originalMap,
            "--bounding-box", "left=" + left, "top=" + top, "right=" + right, "bottom=" + bottom, "completeWays=yes",
            "--tag-filter", "accept-ways", "highway=motorway,motorway_link,trunk,trunk_link,primary,primary_link," +
              "secondary,secondary_link,tertiary,tertiary_link,residential,living_street,unclassified",
            "--tag-filter", "reject-relations",
            "--used-node",
            "--write-xml", preprocessedMap.toString
          )
          (Process(command, arguments) !) match {
            case 0 => s.log.success("Map %s preprocessed" format preprocessedMap)
            case _ => s.log.error("Preprocessing map %s failed" format preprocessedMap)
          }
        }
    }
  }

  lazy val antScout = Project("AntScout", file("."), settings = Project.defaultSettings ++ WebPlugin.webSettings ++
    Seq(downloadMapTask, preprocessMapTask, PluginKeys.start in WebPlugin.container.Configuration <<= (PluginKeys.start
      in WebPlugin.container.Configuration).dependsOn(preprocessMap), preprocessMap <<= preprocessMap.dependsOn
      (downloadMap)))
}
{'fraction_non_alphanumeric': 0.1492564381574175, 'fraction_numerical': 0.01632208922742111, 'mean_word_length': 3.322100313479624, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '725044', 'n_tokens_mistral': 1851, 'n_tokens_neox': 1744, 'n_words': 358}