<commit_before>package blended.file

import java.io._
import java.nio.file.{DirectoryStream, Files, Path, StandardCopyOption}
import java.text.SimpleDateFormat
import java.util.Date
import java.util.zip.{GZIPInputStream, ZipInputStream}

import akka.actor.{Actor, ActorRef}
import akka.util.ByteString
import blended.util.logging.Logger

import scala.util.{Failure, Success, Try}
import scala.util.control.NonFatal
import StandardCopyOption._

import scala.collection.JavaConverters._

object FileDropCommand {
  val tsPattern : SimpleDateFormat = new SimpleDateFormat("yyyyMMdd.HHmmssSSS")
}

case class FileDropCommand(
  id : String,
  content: ByteString,
  directory: String,
  fileName: String,
  compressed: Boolean,
  append: Boolean,
  timestamp: Long,
  properties: Map[String, Any],
  log : Logger
) {

  val trimmedFileName : String = fileName.trim()

  override def equals(obj: Any): Boolean = obj match {
    case cmd : FileDropCommand =>
      content.sameElements(cmd.content) &&
      directory.equals(cmd.directory) &&
      fileName.equals(cmd.fileName) &&
      compressed == cmd.compressed &&
      append == cmd.append &&
      timestamp == cmd.timestamp &&
      properties.equals(cmd.properties)
    case _ => false
  }

  override def hashCode(): Int = toString().hashCode()

  val timestampAsString : String = FileDropCommand.tsPattern.format(new Date(timestamp))

  override def toString: String = {

    s"FileDropCommand[$id](dir = [$directory], fileName = [$fileName], compressed = $compressed, append = $append, " +
    s"timestamp = [$timestampAsString], content-size = ${content.length}), properties=${properties.mkString("[", ",", "]")}"
  }

  // determine the final file name for a file drop
  val finalFile : File = {

    val file = new File(directory, trimmedFileName)

    if (!append) {
      if (file.exists()) {
        // In case we need to generate a new file name
        new File(directory, fileName.lastIndexOf('.') match {
          case -1 => s"dup_${timestampAsString}_${fileName}"
          case pos => s"${fileName.substring(0, pos)}.dup_${timestampAsString}${fileName.substring(pos)}"
        })<commit_msg>Adding warning to log file when the filename is trimmed.<commit_after>package blended.file

import java.io._
import java.nio.file.{DirectoryStream, Files, Path, StandardCopyOption}
import java.text.SimpleDateFormat
import java.util.Date
import java.util.zip.{GZIPInputStream, ZipInputStream}

import akka.actor.{Actor, ActorRef}
import akka.util.ByteString
import blended.util.logging.Logger

import scala.util.{Failure, Success, Try}
import scala.util.control.NonFatal
import StandardCopyOption._

import scala.collection.JavaConverters._

object FileDropCommand {
  val tsPattern : SimpleDateFormat = new SimpleDateFormat("yyyyMMdd.HHmmssSSS")
}

case class FileDropCommand(
  id : String,
  content: ByteString,
  directory: String,
  fileName: String,
  compressed: Boolean,
  append: Boolean,
  timestamp: Long,
  properties: Map[String, Any],
  log : Logger
) {

  val trimmedFileName : String = fileName.trim()

  override def equals(obj: Any): Boolean = obj match {
    case cmd : FileDropCommand =>
      content.sameElements(cmd.content) &&
      directory.equals(cmd.directory) &&
      fileName.equals(cmd.fileName) &&
      compressed == cmd.compressed &&
      append == cmd.append &&
      timestamp == cmd.timestamp &&
      properties.equals(cmd.properties)
    case _ => false
  }

  override def hashCode(): Int = toString().hashCode()

  val timestampAsString : String = FileDropCommand.tsPattern.format(new Date(timestamp))

  override def toString: String = {

    s"FileDropCommand[$id](dir = [$directory], fileName = [$fileName], compressed = $compressed, append = $append, " +
    s"timestamp = [$timestampAsString], content-size = ${content.length}), properties=${properties.mkString("[", ",", "]")}"
  }

  // determine the final file name for a file drop
  val finalFile : File = {

    if (trimmedFileName.length() < fileName.length()) {
      log.warn(s"Using trimmed file name [$trimmedFileName] for [${toString()}]")
    }

    val file = new File(directory, trimmedFileName)

    if (!append) {
      if (file.exists()) {
        // In case we need to generate a new file name
        new File(directory, fileName.lastIndexOf('.') match {
          case -1 => s"dup_${timestampAsString}_${fileName}"
          case pos => s"${fileName.substring(0, pos)}.dup_${timestampAsString}${fileName.substring(pos)}"
        })
{'fraction_non_alphanumeric': 0.12749445676274945, 'fraction_numerical': 0.0008869179600886918, 'mean_word_length': 3.9139433551198257, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4738422', 'n_tokens_mistral': 1332, 'n_tokens_neox': 1313, 'n_words': 333}