<commit_before>/*
 * Copyright (C) 2012 reuillon
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.openmole.plugin.environment.batch.storage

import java.io._
import java.net.{ SocketTimeoutException, URI }
import java.util.concurrent.{ Callable, TimeUnit, TimeoutException }

import com.google.common.cache.CacheBuilder
import fr.iscpif.gridscale.storage._
import org.openmole.core.communication.storage._
import org.openmole.core.fileservice.{ FileDeleter, FileService }
import org.openmole.core.preference.{ ConfigurationLocation, Preference }
import org.openmole.core.replication.{ ReplicaCatalog, ReplicationStorage }
import org.openmole.core.serializer._
import org.openmole.core.threadprovider.{ ThreadProvider, Updater }
import org.openmole.core.workspace.{ NewFile, Workspace }
import org.openmole.plugin.environment.batch.control._
import org.openmole.plugin.environment.batch.environment._
import org.openmole.plugin.environment.batch.refresh._
import org.openmole.tool.cache._
import org.openmole.tool.logger.Logger
import squants.time.TimeConversions._

import scala.ref.WeakReference

object StorageService extends Logger {
  val DirRegenerate = ConfigurationLocation("StorageService", "DirRegenerate", Some(1 hours))
  val TmpDirRemoval = ConfigurationLocation("StorageService", "TmpDirRemoval", Some(30 days))
  val persistent = "persistent/"
  val tmp = "tmp/"

  def startGC(storage: StorageService)(implicit threadProvider: ThreadProvider, preference: Preference) =
    Updater.delay(new StoragesGC(WeakReference(storage)), preference(BatchEnvironment.StoragesGCUpdateInterval))

  implicit def replicationStorage(implicit token: AccessToken): ReplicationStorage[StorageService] = new ReplicationStorage[StorageService] {
    override def backgroundRmFile(storage: StorageService, path: String): Unit = storage.backgroundRmFile(path)
    override def exists(storage: StorageService, path: String): Boolean = storage.exists(path)
    override def id(storage: StorageService): String = storage.id
  }
}

import org.openmole.plugin.environment.batch.storage.StorageService.Log._
import org.openmole.plugin.environment.batch.storage.StorageService._

trait StorageService extends BatchService with Storage {

  def url: URI
  def id: String

  val remoteStorage: RemoteStorage

  import environment.services
  import environment.services._

  val _directoryCache = Cache {
    CacheBuilder.
      newBuilder().
      expireAfterWrite(preference(StorageService.DirRegenerate).millis, TimeUnit.MILLISECONDS).
      build[String, String]()
  }

  val _serializedRemoteStorage = Cache {
    val file = newFile.newFile("remoteStorage", ".xml")
    fileService.deleteWhenGarbageCollected(file)
    serializerService.serialiseAndArchiveFiles(remoteStorage, file)
    file
  }

  def serializedRemoteStorage = _serializedRemoteStorage()
  def directoryCache = _directoryCache()

  protected implicit def callable[T](f: () ⇒ T): Callable[T] = new Callable[T]() { def call() = f() }

  def clean(implicit token: AccessToken) = {
    replicaCatalog.deleteReplicas(this)
    rmDir(baseDir)
    directoryCache.invalidateAll
  }

  def baseDir(implicit token: AccessToken): String =
    unwrap { directoryCache.get("baseDir", () ⇒ createBasePath) }

  protected def createBasePath(implicit token: AccessToken) = {
    val rootPath = mkRootDir
    val basePath = child(rootPath, baseDirName)
    if (!exists(basePath)) makeDir(basePath)
    basePath
  }

  protected def mkRootDir(implicit token: AccessToken): String = synchronized {
    val paths = Iterator.iterate[Option[String]](Some(root))(p ⇒ p.flatMap(parent)).takeWhile(_.isDefined).toSeq.reverse.flatten
<commit_msg>[Plugin] fix: make basedir creation more robust.<commit_after>/*
 * Copyright (C) 2012 reuillon
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package org.openmole.plugin.environment.batch.storage

import java.io._
import java.net.{ SocketTimeoutException, URI }
import java.util.concurrent.{ Callable, TimeUnit, TimeoutException }

import com.google.common.cache.CacheBuilder
import fr.iscpif.gridscale.storage._
import org.openmole.core.communication.storage._
import org.openmole.core.fileservice.{ FileDeleter, FileService }
import org.openmole.core.preference.{ ConfigurationLocation, Preference }
import org.openmole.core.replication.{ ReplicaCatalog, ReplicationStorage }
import org.openmole.core.serializer._
import org.openmole.core.threadprovider.{ ThreadProvider, Updater }
import org.openmole.core.workspace.{ NewFile, Workspace }
import org.openmole.plugin.environment.batch.control._
import org.openmole.plugin.environment.batch.environment._
import org.openmole.plugin.environment.batch.refresh._
import org.openmole.tool.cache._
import org.openmole.tool.logger.Logger
import squants.time.TimeConversions._

import scala.ref.WeakReference

object StorageService extends Logger {
  val DirRegenerate = ConfigurationLocation("StorageService", "DirRegenerate", Some(1 hours))
  val TmpDirRemoval = ConfigurationLocation("StorageService", "TmpDirRemoval", Some(30 days))
  val persistent = "persistent/"
  val tmp = "tmp/"

  def startGC(storage: StorageService)(implicit threadProvider: ThreadProvider, preference: Preference) =
    Updater.delay(new StoragesGC(WeakReference(storage)), preference(BatchEnvironment.StoragesGCUpdateInterval))

  implicit def replicationStorage(implicit token: AccessToken): ReplicationStorage[StorageService] = new ReplicationStorage[StorageService] {
    override def backgroundRmFile(storage: StorageService, path: String): Unit = storage.backgroundRmFile(path)
    override def exists(storage: StorageService, path: String): Boolean = storage.exists(path)
    override def id(storage: StorageService): String = storage.id
  }
}

import org.openmole.plugin.environment.batch.storage.StorageService.Log._
import org.openmole.plugin.environment.batch.storage.StorageService._

trait StorageService extends BatchService with Storage {

  def url: URI
  def id: String

  val remoteStorage: RemoteStorage

  import environment.services
  import environment.services._

  val _directoryCache = Cache {
    CacheBuilder.
      newBuilder().
      expireAfterWrite(preference(StorageService.DirRegenerate).millis, TimeUnit.MILLISECONDS).
      build[String, String]()
  }

  val _serializedRemoteStorage = Cache {
    val file = newFile.newFile("remoteStorage", ".xml")
    fileService.deleteWhenGarbageCollected(file)
    serializerService.serialiseAndArchiveFiles(remoteStorage, file)
    file
  }

  def serializedRemoteStorage = _serializedRemoteStorage()
  def directoryCache = _directoryCache()

  protected implicit def callable[T](f: () ⇒ T): Callable[T] = new Callable[T]() { def call() = f() }

  def clean(implicit token: AccessToken) = {
    replicaCatalog.deleteReplicas(this)
    rmDir(baseDir)
    directoryCache.invalidateAll
  }

  def baseDir(implicit token: AccessToken): String =
    unwrap { directoryCache.get("baseDir", () ⇒ createBasePath) }

  protected def createBasePath(implicit token: AccessToken) = {
    val rootPath = mkRootDir
    val basePath = child(rootPath, baseDirName)
    util.Try(makeDir(basePath)) match {
      case util.Success(_) ⇒ basePath
      case util.Failure(e) ⇒
        if (exists(basePath)) basePath else throw e
    }
  }

  protected def mkRootDir(implicit token: AccessToken): String = synchronized {
    val paths = Iterator.iterate[Option[String]](Some(root))(p ⇒ p.flatMap(parent)).takeWhile(_.isDefined).toSeq.reverse.flatten

{'fraction_non_alphanumeric': 0.09443043777217511, 'fraction_numerical': 0.0018336007334402934, 'mean_word_length': 4.98559670781893, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3409696', 'n_tokens_mistral': 2496, 'n_tokens_neox': 2466, 'n_words': 669}