<commit_before>////////////////////////////////////////////////////////////////////////////////
/// @brief Recovery state
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author <NAME>
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

#include "RecoverState.h"
#include "Basics/FileUtils.h"
#include "Basics/conversions.h"
#include "Basics/files.h"
#include "VocBase/collection.h"
#include "VocBase/replication-applier.h"
#include "VocBase/voc-shaper.h"
#include "Wal/LogfileManager.h"
#include "Wal/Slots.h"
#include "Utils/Exception.h"

using namespace triagens::wal;

// -----------------------------------------------------------------------------
// --SECTION--                                                  helper functions
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not a collection is volatile
////////////////////////////////////////////////////////////////////////////////

static inline bool IsVolatile (TRI_transaction_collection_t const* trxCollection) {
  return trxCollection->_collection->_collection->_info._isVolatile;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief get the directory for a database
////////////////////////////////////////////////////////////////////////////////

static std::string GetDatabaseDirectory (TRI_server_t* server,
                                         TRI_voc_tick_t databaseId) {
  char* idString = TRI_StringUInt64(databaseId);
  char* dname = TRI_Concatenate2String("database-", idString);
  TRI_FreeString(TRI_CORE_MEM_ZONE, idString);
  char* filename = TRI_Concatenate2File(server->_databasePath, dname);
  TRI_FreeString(TRI_CORE_MEM_ZONE, dname);

  std::string result(filename);
  TRI_FreeString(TRI_CORE_MEM_ZONE, filename);

  return result;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief get the directory for a collection
////////////////////////////////////////////////////////////////////////////////
  
static std::string GetCollectionDirectory (TRI_vocbase_t* vocbase,
                                           TRI_voc_cid_t collectionId) {
  char* dirname = TRI_GetDirectoryCollection(vocbase->_path,
                                             "empty", // does not matter
                                             TRI_COL_TYPE_DOCUMENT, // does not matter 
                                             collectionId);

  std::string result(dirname);
  TRI_FreeString(TRI_CORE_MEM_ZONE, dirname);

  return result;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief wait until a database directory disappears
////////////////////////////////////////////////////////////////////////////////

static int WaitForDeletion (TRI_server_t* server,
                            TRI_voc_tick_t databaseId,
                            int statusCode) {
  std::string const result = GetDatabaseDirectory(server, databaseId);
 
  int iterations = 0;
  // wait for at most 30 seconds for the directory to be removed
  while (TRI_IsDirectory(result.c_str())) {
    if (iterations == 0) {
      LOG_TRACE("waiting for deletion of database directory '%s', called with status code %d", 
                result.c_str(), 
                statusCode);
      
      if (statusCode != TRI_ERROR_FORBIDDEN &&
          (statusCode == TRI_ERROR_ARANGO_DATABASE_NOT_FOUND ||
           statusCode != TRI_ERROR_NO_ERROR)) {
        LOG_WARNING("forcefully deleting database directory '%s'", result.c_str());
        TRI_RemoveDirectory(result.c_str());
      }
    }
    else if (++iterations >= 30 * 10) {
      LOG_WARNING("unable to remove database directory '%s'", result.c_str());
      return TRI_ERROR_INTERNAL;
    }

    usleep(100000);
  }

  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief wait until a collection directory disappears
////////////////////////////////////////////////////////////////////////////////

static int WaitForDeletion (TRI_vocbase_t* vocbase,
                            TRI_voc_cid_t collectionId,
                            int statusCode) {
  std::string const result = GetCollectionDirectory(vocbase, collectionId);

  int iterations = 0;
  // wait for at most 30 seconds for the directory to be removed
  while (TRI_IsDirectory(result.c_str())) {
    if (iterations == 0) {
      LOG_TRACE("waiting for deletion of collection directory '%s', called with status code %d", 
                result.c_str(),
                statusCode);
      
      if (statusCode != TRI_ERROR_FORBIDDEN && 
          (statusCode == TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND ||
           statusCode != TRI_ERROR_NO_ERROR)) {
        LOG_WARNING("forcefully deleting collection directory '%s'", result.c_str());
        TRI_RemoveDirectory(result.c_str());
      }
    }
    else if (++iterations >= 30 * 10) {
      LOG_WARNING("unable to remove collection directory '%s'", result.c_str());
      return TRI_ERROR_INTERNAL;
    }
<commit_msg>more log output<commit_after>////////////////////////////////////////////////////////////////////////////////
/// @brief Recovery state
///
/// @file
///
/// DISCLAIMER
///
/// Copyright 2014 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author <NAME>
/// @author Copyright 2014, ArangoDB GmbH, Cologne, Germany
/// @author Copyright 2011-2013, triAGENS GmbH, Cologne, Germany
////////////////////////////////////////////////////////////////////////////////

#include "RecoverState.h"
#include "Basics/FileUtils.h"
#include "Basics/conversions.h"
#include "Basics/files.h"
#include "VocBase/collection.h"
#include "VocBase/replication-applier.h"
#include "VocBase/voc-shaper.h"
#include "Wal/LogfileManager.h"
#include "Wal/Slots.h"
#include "Utils/Exception.h"

using namespace triagens::wal;

// -----------------------------------------------------------------------------
// --SECTION--                                                  helper functions
// -----------------------------------------------------------------------------

////////////////////////////////////////////////////////////////////////////////
/// @brief whether or not a collection is volatile
////////////////////////////////////////////////////////////////////////////////

static inline bool IsVolatile (TRI_transaction_collection_t const* trxCollection) {
  return trxCollection->_collection->_collection->_info._isVolatile;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief get the directory for a database
////////////////////////////////////////////////////////////////////////////////

static std::string GetDatabaseDirectory (TRI_server_t* server,
                                         TRI_voc_tick_t databaseId) {
  char* idString = TRI_StringUInt64(databaseId);
  char* dname = TRI_Concatenate2String("database-", idString);
  TRI_FreeString(TRI_CORE_MEM_ZONE, idString);
  char* filename = TRI_Concatenate2File(server->_databasePath, dname);
  TRI_FreeString(TRI_CORE_MEM_ZONE, dname);

  std::string result(filename);
  TRI_FreeString(TRI_CORE_MEM_ZONE, filename);

  return result;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief get the directory for a collection
////////////////////////////////////////////////////////////////////////////////
  
static std::string GetCollectionDirectory (TRI_vocbase_t* vocbase,
                                           TRI_voc_cid_t collectionId) {
  char* dirname = TRI_GetDirectoryCollection(vocbase->_path,
                                             "empty", // does not matter
                                             TRI_COL_TYPE_DOCUMENT, // does not matter 
                                             collectionId);

  std::string result(dirname);
  TRI_FreeString(TRI_CORE_MEM_ZONE, dirname);

  return result;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief wait until a database directory disappears
////////////////////////////////////////////////////////////////////////////////

static int WaitForDeletion (TRI_server_t* server,
                            TRI_voc_tick_t databaseId,
                            int statusCode) {
  std::string const result = GetDatabaseDirectory(server, databaseId);
 
  int iterations = 0;
  // wait for at most 30 seconds for the directory to be removed
  while (TRI_IsDirectory(result.c_str())) {
    if (iterations == 0) {
      LOG_TRACE("waiting for deletion of database directory '%s', called with status code %d", 
                result.c_str(), 
                statusCode);
      
      if (statusCode != TRI_ERROR_FORBIDDEN &&
          (statusCode == TRI_ERROR_ARANGO_DATABASE_NOT_FOUND ||
           statusCode != TRI_ERROR_NO_ERROR)) {
        LOG_WARNING("forcefully deleting database directory '%s'", result.c_str());
        TRI_RemoveDirectory(result.c_str());
      }
    }
    else if (iterations >= 30 * 10) {
      LOG_WARNING("unable to remove database directory '%s'", result.c_str());
      return TRI_ERROR_INTERNAL;
    }

    if (iterations == 5 * 10) {
      LOG_INFO("waiting for deletion of database directory '%s'", result.c_str());
    }

    ++iterations;
    usleep(100000);
  }

  return TRI_ERROR_NO_ERROR;
}

////////////////////////////////////////////////////////////////////////////////
/// @brief wait until a collection directory disappears
////////////////////////////////////////////////////////////////////////////////

static int WaitForDeletion (TRI_vocbase_t* vocbase,
                            TRI_voc_cid_t collectionId,
                            int statusCode) {
  std::string const result = GetCollectionDirectory(vocbase, collectionId);

  int iterations = 0;
  // wait for at most 30 seconds for the directory to be removed
  while (TRI_IsDirectory(result.c_str())) {
    if (iterations == 0) {
      LOG_TRACE("waiting for deletion of collection directory '%s', called with status code %d", 
                result.c_str(),
                statusCode);
      
      if (statusCode != TRI_ERROR_FORBIDDEN && 
          (statusCode == TRI_ERROR_ARANGO_COLLECTION_NOT_FOUND ||
           statusCode != TRI_ERROR_NO_ERROR)) {
        LOG_WARNING("forcefully deleting collection directory '%s'", result.c_str());
        TRI_RemoveDirectory(result.c_str());
      }
    }
    else if (iterations >= 30 * 10) {
      LOG_WARNING("unable to remove collection directory '%s'", result.c_str());
      return TRI_ERROR_INTERNAL;
    }
    
    if (iterations == 5 * 10) {
      LOG_INFO("waiting for deletion of collection directory '%s'", result.c_str());
    }

    ++iterations;
{'fraction_non_alphanumeric': 0.2659677419354839, 'fraction_numerical': 0.009193548387096774, 'mean_word_length': 1.4590521514971246, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4910574', 'n_tokens_mistral': 3134, 'n_tokens_neox': 2921, 'n_words': 824}