<commit_before>/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "Copyright (c) 2012-2013 Tokutek Inc.  All rights reserved."
#ident "$Id$"

#include "backup_debug.h"
#include "copier.h"
#include "file_hash_table.h"
#include "manager.h"
#include "mutex.h"
#include "real_syscalls.h"
#include "source_file.h"

#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#include <vector>

template class std::vector<char *>;

#if DEBUG_HOTBACKUP
#define WARN(string, arg) HotBackup::CopyWarn(string, arg)
#define TRACE(string, arg) HotBackup::CopyTrace(string, arg)
#define ERROR(string, arg) HotBackup::CopyError(string, arg)
#else
#define WARN(string, arg) 
#define TRACE(string, arg) 
#define ERROR(string, arg) 
#endif

#if PAUSE_POINTS_ON
#define PAUSE(int) while(HotBackup::should_pause(int)) { sched_yield(); }
#else
#define PAUSE(int)
#endif

////////////////////////////////////////////////////////////////////////////////
//
// is_dot() -
//
// Description: 
//
//     Helper function that returns true if the given directory 
// entry is either of the special cases: ".." or ".".
//
static bool is_dot(struct dirent const *entry)
{
    if (strcmp(entry->d_name, "..") == 0 || strcmp(entry->d_name, ".") == 0) {
        return true;
    }
    return false;
}

pthread_mutex_t copier::m_todo_mutex = PTHREAD_MUTEX_INITIALIZER;

////////////////////////////////////////////////////////////////////////////////
//
// copier() - 
//
// Description:
//
//     Constructor for this copier object.
//
copier::copier(backup_callbacks *calls, file_hash_table * const table) throw()
    : m_source(NULL), 
      m_dest(NULL), 
      m_calls(calls), 
      m_table(table),
      m_total_bytes_backed_up(0),
      m_total_files_backed_up(0)
{}

////////////////////////////////////////////////////////////////////////////////
//
// set_directories() -
//
// Description: 
//
//     Adds a directory heirarchy to be copied from the given source
// to the given destination.
//
void copier::set_directories(const char *source, const char *dest) throw() {
    m_source = source;
    m_dest = dest;
}

////////////////////////////////////////////////////////////////////////////////
//
// start_copy() -
//
// Description: 
//
//     Loops through all files and subdirectories of the current 
// directory that has been selected for backup.
//
int copier::do_copy(void) throw() {
    int r = 0;
    char *fname = 0;
    size_t n_known = 0;
    pmutex_lock(&m_todo_mutex);
    // Start with "."
    m_todo.push_back(strdup("."));
    n_known = m_todo.size();
    pmutex_unlock(&m_todo_mutex);
    while (n_known != 0) {

        if (!the_manager.copy_is_enabled()) goto out;
        
        pmutex_lock(&m_todo_mutex);
        fname = m_todo.back();
        pmutex_unlock(&m_todo_mutex);
        TRACE("Copying: ", fname);
        
        char *msg = malloc_snprintf(strlen(fname)+100, "Backup progress %ld bytes, %ld files.  %ld files known of. Copying file %s",  m_total_bytes_backed_up, m_total_files_backed_up, n_known, fname);
        // Use n_done/n_files.   We need to do a better estimate involving n_bytes_copied/n_bytes_total
        // This one is very wrongu
        r = m_calls->poll(0, msg);
        free(msg);
        if (r != 0) {
            fprintf(stderr, "%s:%d r=%d\n", __FILE__, __LINE__, r);
            goto out;
        }

        pmutex_lock(&m_todo_mutex);
        m_todo.pop_back();
        pmutex_unlock(&m_todo_mutex);
        
        r = this->copy_stripped_file(fname);
        free((void*)fname);
        fname = NULL;
        if(r != 0) {
            fprintf(stderr, "%s:%d r=%d\n", __FILE__, __LINE__, r);
            goto out;
        }
        m_total_files_backed_up++;
        
        pmutex_lock(&m_todo_mutex);
        n_known = m_todo.size();
        pmutex_unlock(&m_todo_mutex);
    }

out:
    this->cleanup();
    return r;
}


static void pathcat(char *dest, size_t destlen, const char *a, int alen, const char *b) throw()
// Effect: Concatenate paths A and B (insert a / between if needed) into dest.  If a ends with a / and b starts with a / then put only 1 / in.
// Requires: destlen is big enough.  A is nonempty.
{
    bool a_has_slash_at_end = (alen>0) && (a[alen-1]=='/');
    if (b[0]=='/') b++;
    int r = snprintf(dest, destlen, "%s%s%s", a, a_has_slash_at_end ? "" : "/", b);
    if (r>=(int)destlen) {
        static bool ecount=0;
        ecount++;
        if (ecount==0) fprintf(stderr, "pathcat length computation error in backup\n");
    }
}

////////////////////////////////////////////////////////////////////////////////
//
// copy_stripped_file() -
//
// Description: 
//
//     Copy's the given file, using this copier object's source and
// destination directory members to determine the exact location
// of the file in both the original and backup locations.
//
int copier::copy_stripped_file(const char *file) throw() {
    int r = 0;
    bool is_dot = (strcmp(file, ".") == 0);<commit_msg>Refs #6699.  Add check to code that shouldn't fail.  (Our code should call check() when we really think it shouldn't fail.)<commit_after>/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
#ident "Copyright (c) 2012-2013 Tokutek Inc.  All rights reserved."
#ident "$Id$"

#include "backup_debug.h"
#include "check.h"
#include "copier.h"
#include "file_hash_table.h"
#include "manager.h"
#include "mutex.h"
#include "real_syscalls.h"
#include "source_file.h"

#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
#include <vector>

template class std::vector<char *>;

#if DEBUG_HOTBACKUP
#define WARN(string, arg) HotBackup::CopyWarn(string, arg)
#define TRACE(string, arg) HotBackup::CopyTrace(string, arg)
#define ERROR(string, arg) HotBackup::CopyError(string, arg)
#else
#define WARN(string, arg) 
#define TRACE(string, arg) 
#define ERROR(string, arg) 
#endif

#if PAUSE_POINTS_ON
#define PAUSE(int) while(HotBackup::should_pause(int)) { sched_yield(); }
#else
#define PAUSE(int)
#endif

////////////////////////////////////////////////////////////////////////////////
//
// is_dot() -
//
// Description: 
//
//     Helper function that returns true if the given directory 
// entry is either of the special cases: ".." or ".".
//
static bool is_dot(struct dirent const *entry)
{
    if (strcmp(entry->d_name, "..") == 0 || strcmp(entry->d_name, ".") == 0) {
        return true;
    }
    return false;
}

pthread_mutex_t copier::m_todo_mutex = PTHREAD_MUTEX_INITIALIZER;

////////////////////////////////////////////////////////////////////////////////
//
// copier() - 
//
// Description:
//
//     Constructor for this copier object.
//
copier::copier(backup_callbacks *calls, file_hash_table * const table) throw()
    : m_source(NULL), 
      m_dest(NULL), 
      m_calls(calls), 
      m_table(table),
      m_total_bytes_backed_up(0),
      m_total_files_backed_up(0)
{}

////////////////////////////////////////////////////////////////////////////////
//
// set_directories() -
//
// Description: 
//
//     Adds a directory heirarchy to be copied from the given source
// to the given destination.
//
void copier::set_directories(const char *source, const char *dest) throw() {
    m_source = source;
    m_dest = dest;
}

////////////////////////////////////////////////////////////////////////////////
//
// start_copy() -
//
// Description: 
//
//     Loops through all files and subdirectories of the current 
// directory that has been selected for backup.
//
int copier::do_copy(void) throw() {
    int r = 0;
    char *fname = 0;
    size_t n_known = 0;
    pmutex_lock(&m_todo_mutex);
    // Start with "."
    m_todo.push_back(strdup("."));
    n_known = m_todo.size();
    pmutex_unlock(&m_todo_mutex);
    while (n_known != 0) {

        if (!the_manager.copy_is_enabled()) goto out;
        
        pmutex_lock(&m_todo_mutex);
        fname = m_todo.back();
        pmutex_unlock(&m_todo_mutex);
        TRACE("Copying: ", fname);
        
        char *msg = malloc_snprintf(strlen(fname)+100, "Backup progress %ld bytes, %ld files.  %ld files known of. Copying file %s",  m_total_bytes_backed_up, m_total_files_backed_up, n_known, fname);
        // Use n_done/n_files.   We need to do a better estimate involving n_bytes_copied/n_bytes_total
        // This one is very wrongu
        r = m_calls->poll(0, msg);
        free(msg);
        if (r != 0) {
            fprintf(stderr, "%s:%d r=%d\n", __FILE__, __LINE__, r);
            goto out;
        }

        pmutex_lock(&m_todo_mutex);
        m_todo.pop_back();
        pmutex_unlock(&m_todo_mutex);
        
        r = this->copy_stripped_file(fname);
        free((void*)fname);
        fname = NULL;
        if(r != 0) {
            fprintf(stderr, "%s:%d r=%d\n", __FILE__, __LINE__, r);
            goto out;
        }
        m_total_files_backed_up++;
        
        pmutex_lock(&m_todo_mutex);
        n_known = m_todo.size();
        pmutex_unlock(&m_todo_mutex);
    }

out:
    this->cleanup();
    return r;
}


static void pathcat(char *dest, size_t destlen, const char *a, int alen, const char *b) throw()
// Effect: Concatenate paths A and B (insert a / between if needed) into dest.  If a ends with a / and b starts with a / then put only 1 / in.
// Requires: destlen is big enough.  A is nonempty.
{
    bool a_has_slash_at_end = (alen>0) && (a[alen-1]=='/');
    if (b[0]=='/') b++;
    int r = snprintf(dest, destlen, "%s%s%s", a, a_has_slash_at_end ? "" : "/", b);
    check(r<(int)destlen);
}

////////////////////////////////////////////////////////////////////////////////
//
// copy_stripped_file() -
//
// Description: 
//
//     Copy's the given file, using this copier object's source and
// destination directory members to determine the exact location
// of the file in both the original and backup locations.
//
int copier::copy_stripped_file(const char *file) throw() {
    int r = 0;
    bool is_dot = (strcmp(file, ".") == 0);
{'fraction_non_alphanumeric': 0.21632613537329332, 'fraction_numerical': 0.006778348019754043, 'mean_word_length': 2.1975232198142414, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2576863', 'n_tokens_mistral': 3517, 'n_tokens_neox': 3458, 'n_words': 873}