<commit_before>/**
 * This file is part of the CernVM File System
 */

#define __STDC_FORMAT_MACROS

#include "sync_union_tarball.h"

#include <pthread.h>
#include <stdio.h>
#include <cassert>
#include <set>
#include <string>
#include <vector>

#include "duplex_libarchive.h"
#include "fs_traversal.h"
#include "smalloc.h"
#include "sync_item.h"
#include "sync_item_dummy.h"
#include "sync_item_tar.h"
#include "sync_mediator.h"
#include "sync_union.h"
#include "util/posix.h"
#include "util_concurrency.h"

namespace publish {

SyncUnionTarball::SyncUnionTarball(AbstractSyncMediator *mediator,
                                   const std::string &rdonly_path,
                                   const std::string &union_path,
                                   const std::string &scratch_path,
                                   const std::string &tarball_path,
                                   const std::string &base_directory,
                                   const std::string &to_delete)
    : SyncUnion(mediator, rdonly_path, union_path, scratch_path),
      tarball_path_(tarball_path),
      base_directory_(base_directory),
      to_delete_(to_delete),
      read_archive_signal_(new Signal) {
  read_archive_signal_->Wakeup();
}

SyncUnionTarball::~SyncUnionTarball() { delete read_archive_signal_; }

bool SyncUnionTarball::Initialize() {
  bool result;

  src = archive_read_new();
  assert(ARCHIVE_OK == archive_read_support_format_tar(src));
  assert(ARCHIVE_OK == archive_read_support_format_empty(src));

  if (tarball_path_ == "--") {
    result = archive_read_open_filename(src, NULL, 4096);
  } else {
    std::string tarball_absolute_path = GetAbsolutePath(tarball_path_);
    result =
        archive_read_open_filename(src, tarball_absolute_path.c_str(), 4096);
  }

  if (result != ARCHIVE_OK) {
    LogCvmfs(kLogUnionFs, kLogStderr, "Impossible to open the archive.");
    return false;
  }

  return SyncUnion::Initialize();
}

void SyncUnionTarball::Traverse() {
  assert(this->IsInitialized());
  struct archive_entry *entry = archive_entry_new();

  /*
   * As first step we eliminate the directories we are request.
   */
  if (to_delete_ != "") {
    vector<std::string> to_eliminate_vec = SplitString(to_delete_, ':');

    for (vector<string>::iterator s = to_eliminate_vec.begin();
         s != to_eliminate_vec.end(); ++s) {
      std::string parent_path;
      std::string filename;
      SplitPath(*s, &parent_path, &filename);
      if (parent_path == ".") parent_path = "";
      SharedPtr<SyncItem> sync_entry =
          CreateSyncItem(parent_path, filename, kItemDir);
      mediator_->Remove(sync_entry);
    }
  }

  while (true) {

    /* Get the lock, wait if lock is not available yet */
    read_archive_signal_->Wait();

    int result = archive_read_next_header2(src, entry);

    switch (result) {
      case ARCHIVE_FATAL: {
        LogCvmfs(kLogUnionFs, kLogStderr,
                 "Fatal error in reading the archive.");
        return;
        break; /* Only exit point with error */
      }

      case ARCHIVE_RETRY: {
        LogCvmfs(kLogUnionFs, kLogStderr,
                 "Error in reading the header, retrying. \n %s",
                 archive_error_string(src));
        continue;
        break;
      }

      case ARCHIVE_EOF: {
        for (set<string>::iterator dir = to_create_catalog_dirs_.begin();<commit_msg>print retry on stdout and not on stderr<commit_after>/**
 * This file is part of the CernVM File System
 */

#define __STDC_FORMAT_MACROS

#include "sync_union_tarball.h"

#include <pthread.h>
#include <stdio.h>
#include <cassert>
#include <set>
#include <string>
#include <vector>

#include "duplex_libarchive.h"
#include "fs_traversal.h"
#include "smalloc.h"
#include "sync_item.h"
#include "sync_item_dummy.h"
#include "sync_item_tar.h"
#include "sync_mediator.h"
#include "sync_union.h"
#include "util/posix.h"
#include "util_concurrency.h"

namespace publish {

SyncUnionTarball::SyncUnionTarball(AbstractSyncMediator *mediator,
                                   const std::string &rdonly_path,
                                   const std::string &union_path,
                                   const std::string &scratch_path,
                                   const std::string &tarball_path,
                                   const std::string &base_directory,
                                   const std::string &to_delete)
    : SyncUnion(mediator, rdonly_path, union_path, scratch_path),
      tarball_path_(tarball_path),
      base_directory_(base_directory),
      to_delete_(to_delete),
      read_archive_signal_(new Signal) {
  read_archive_signal_->Wakeup();
}

SyncUnionTarball::~SyncUnionTarball() { delete read_archive_signal_; }

bool SyncUnionTarball::Initialize() {
  bool result;

  src = archive_read_new();
  assert(ARCHIVE_OK == archive_read_support_format_tar(src));
  assert(ARCHIVE_OK == archive_read_support_format_empty(src));

  if (tarball_path_ == "--") {
    result = archive_read_open_filename(src, NULL, 4096);
  } else {
    std::string tarball_absolute_path = GetAbsolutePath(tarball_path_);
    result =
        archive_read_open_filename(src, tarball_absolute_path.c_str(), 4096);
  }

  if (result != ARCHIVE_OK) {
    LogCvmfs(kLogUnionFs, kLogStderr, "Impossible to open the archive.");
    return false;
  }

  return SyncUnion::Initialize();
}

void SyncUnionTarball::Traverse() {
  assert(this->IsInitialized());
  struct archive_entry *entry = archive_entry_new();

  /*
   * As first step we eliminate the directories we are request.
   */
  if (to_delete_ != "") {
    vector<std::string> to_eliminate_vec = SplitString(to_delete_, ':');

    for (vector<string>::iterator s = to_eliminate_vec.begin();
         s != to_eliminate_vec.end(); ++s) {
      std::string parent_path;
      std::string filename;
      SplitPath(*s, &parent_path, &filename);
      if (parent_path == ".") parent_path = "";
      SharedPtr<SyncItem> sync_entry =
          CreateSyncItem(parent_path, filename, kItemDir);
      mediator_->Remove(sync_entry);
    }
  }

  while (true) {

    /* Get the lock, wait if lock is not available yet */
    read_archive_signal_->Wait();

    int result = archive_read_next_header2(src, entry);

    switch (result) {
      case ARCHIVE_FATAL: {
        LogCvmfs(kLogUnionFs, kLogStderr,
                 "Fatal error in reading the archive.");
        return;
        break; /* Only exit point with error */
      }

      case ARCHIVE_RETRY: {
        LogCvmfs(kLogUnionFs, kLogStdout,
                 "Error in reading the header, retrying. \n %s",
                 archive_error_string(src));
        continue;
        break;
      }

      case ARCHIVE_EOF: {
        for (set<string>::iterator dir = to_create_catalog_dirs_.begin();
{'fraction_non_alphanumeric': 0.11375661375661375, 'fraction_numerical': 0.0026455026455026454, 'mean_word_length': 2.91542002301496, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5972746', 'n_tokens_mistral': 2214, 'n_tokens_neox': 2145, 'n_words': 448}