<commit_before>#ifndef SLASH_ENV_H_
#define SLASH_ENV_H_

#include <string>
#include <vector>
#include <unistd.h>

#include "slash_status.h"

namespace slash {

class WritableFile;
class SequentialFile;
class RWFile;
class RandomRWFile;

/*
 * Set size of initial mmap size
 */
void SetMmapBoundSize(size_t size);

static const size_t kPageSize = getpagesize();


/*
 * File Operations
 */
int IsDir(const std::string& path);
int DeleteDir(const std::string& path);
bool DeleteDirIfExist(const std::string& path);
int CreateDir(const std::string& path);
int CreatePath(const std::string& path, mode_t mode = 0755);
uint64_t Du(const std::string& path);

/*
 * Rsync
 *
 */
const std::string kRsyncConfFile = "slash_rsync.conf";
const std::string kRsyncLogFile = "slash_rsync.log";
const std::string kRsyncPidFile = "slash_rsync.pid";
const std::string kRsyncLockFile = "slash_rsync.lock";
const std::string kRsyncSubDir = "rsync";
struct RsyncRemote {
  std::string host;
  int port;
  std::string module;
  int kbps; //speed limit
  RsyncRemote(const std::string& _host, const int _port,
      const std::string& _module, const int _kbps)
  : host(_host), port(_port), module(_module), kbps(_kbps) {}
};
int StartRsync(const std::string& rsync_path, const std::string& module, const int port);
int StopRsync(const std::string& path);
int RsyncSendFile(const std::string& local_file_path, const std::string& remote_file_path,
    const RsyncRemote& remote);
int RsyncSendClearTarget(const std::string& local_dir_path, const std::string& remote_dir_path,
    const RsyncRemote& remote);

/*
 * Whether the file is exist
 * If exist return true, else return false
 */
int FileExists(const std::string& path);

Status DeleteFile(const std::string& fname);

int RenameFile(const std::string& oldname, const std::string& newname);

int GetChildren(const std::string& dir, std::vector<std::string>& result);
bool GetDescendant(const std::string& dir, std::vector<std::string>& result);


uint64_t NowMicros();
void SleepForMicroseconds(int micros);

Status NewSequentialFile(const std::string& fname, SequentialFile** result);

Status NewWritableFile(const std::string& fname, WritableFile** result);

Status NewRWFile(const std::string& fname, RWFile** result);

Status AppendSequentialFile(const std::string& fname, SequentialFile** result);

Status AppendWritableFile(const std::string& fname, WritableFile** result, uint64_t write_len = 0);

Status NewRandomRWFile(const std::string& fname, RandomRWFile** result);

// A file abstraction for sequential writing.  The implementation
// must provide buffering since callers may append small fragments
// at a time to the file.
class WritableFile {
 public:
  WritableFile() { }
  virtual ~WritableFile();

  virtual Status Append(const Slice& data) = 0;
  virtual Status Close() = 0;
  virtual Status Flush() = 0;
  virtual Status Sync() = 0;
  virtual uint64_t Filesize() = 0;

 private:
  // No copying allowed
  WritableFile(const WritableFile&);
  void operator=(const WritableFile&);
};

// A abstract for the sequential readable file
class SequentialFile {
 public:
  SequentialFile() {};
  virtual ~SequentialFile();
  //virtual Status Read(size_t n, char *&result, char *scratch) = 0;
  virtual Status Read(size_t n, Slice* result, char* scratch) = 0;
  virtual Status Skip(uint64_t n) = 0;
  //virtual Status Close() = 0;
  virtual char *ReadLine(char *buf, int n) = 0;
};

class RWFile {
public:
  RWFile() { }
  virtual ~RWFile();
  virtual char* GetData() = 0;

private:
  // No copying allowed
  RWFile(const RWFile&);
  void operator=(const RWFile&);
};

// A file abstraction for random reading and writing.
class RandomRWFile {
 public:
  RandomRWFile() {}
  virtual ~RandomRWFile() {}

  // Write data from Slice data to file starting from offset
  // Returns IOError on failure, but does not guarantee
  // atomicity of a write.  Returns OK status on success.
  //
  // Safe for concurrent use.
  virtual Status Write(uint64_t offset, const Slice& data) = 0;
  // Read up to "n" bytes from the file starting at "offset".
  // "scratch[0..n-1]" may be written by this routine.  Sets "*result"
  // to the data that was read (including if fewer than "n" bytes were
  // successfully read).  May set "*result" to point at data in
  // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
  // "*result" is used.  If an error was encountered, returns a non-OK
  // status.
  //
  // Safe for concurrent use by multiple threads.
  virtual Status Read(uint64_t offset, size_t n, Slice* result,
                      char* scratch) const = 0;
  virtual Status Close() = 0; // closes the file
  virtual Status Sync() = 0; // sync data

  /*
   * Sync data and/or metadata as well.
   * By default, sync only data.
   * Override this method for environments where we need to sync
   * metadata as well.
   */
  virtual Status Fsync() {
    return Sync();
  }

  /*
   * Pre-allocate space for a file.
   */
  virtual Status Allocate(off_t offset, off_t len) {
    return Status::OK();
  }

 private:
  // No copying allowed
  RandomRWFile(const RandomRWFile&);
  void operator=(const RandomRWFile&);
};


}   // namespace slash
#endif  // SLASH_ENV_H_<commit_msg>silent warning of unused params for trival Allocate<commit_after>#ifndef SLASH_ENV_H_
#define SLASH_ENV_H_

#include <string>
#include <vector>
#include <unistd.h>

#include "slash_status.h"

namespace slash {

class WritableFile;
class SequentialFile;
class RWFile;
class RandomRWFile;

/*
 * Set size of initial mmap size
 */
void SetMmapBoundSize(size_t size);

static const size_t kPageSize = getpagesize();


/*
 * File Operations
 */
int IsDir(const std::string& path);
int DeleteDir(const std::string& path);
bool DeleteDirIfExist(const std::string& path);
int CreateDir(const std::string& path);
int CreatePath(const std::string& path, mode_t mode = 0755);
uint64_t Du(const std::string& path);

/*
 * Rsync
 *
 */
const std::string kRsyncConfFile = "slash_rsync.conf";
const std::string kRsyncLogFile = "slash_rsync.log";
const std::string kRsyncPidFile = "slash_rsync.pid";
const std::string kRsyncLockFile = "slash_rsync.lock";
const std::string kRsyncSubDir = "rsync";
struct RsyncRemote {
  std::string host;
  int port;
  std::string module;
  int kbps; //speed limit
  RsyncRemote(const std::string& _host, const int _port,
      const std::string& _module, const int _kbps)
  : host(_host), port(_port), module(_module), kbps(_kbps) {}
};
int StartRsync(const std::string& rsync_path, const std::string& module, const int port);
int StopRsync(const std::string& path);
int RsyncSendFile(const std::string& local_file_path, const std::string& remote_file_path,
    const RsyncRemote& remote);
int RsyncSendClearTarget(const std::string& local_dir_path, const std::string& remote_dir_path,
    const RsyncRemote& remote);

/*
 * Whether the file is exist
 * If exist return true, else return false
 */
int FileExists(const std::string& path);

Status DeleteFile(const std::string& fname);

int RenameFile(const std::string& oldname, const std::string& newname);

int GetChildren(const std::string& dir, std::vector<std::string>& result);
bool GetDescendant(const std::string& dir, std::vector<std::string>& result);


uint64_t NowMicros();
void SleepForMicroseconds(int micros);

Status NewSequentialFile(const std::string& fname, SequentialFile** result);

Status NewWritableFile(const std::string& fname, WritableFile** result);

Status NewRWFile(const std::string& fname, RWFile** result);

Status AppendSequentialFile(const std::string& fname, SequentialFile** result);

Status AppendWritableFile(const std::string& fname, WritableFile** result, uint64_t write_len = 0);

Status NewRandomRWFile(const std::string& fname, RandomRWFile** result);

// A file abstraction for sequential writing.  The implementation
// must provide buffering since callers may append small fragments
// at a time to the file.
class WritableFile {
 public:
  WritableFile() { }
  virtual ~WritableFile();

  virtual Status Append(const Slice& data) = 0;
  virtual Status Close() = 0;
  virtual Status Flush() = 0;
  virtual Status Sync() = 0;
  virtual uint64_t Filesize() = 0;

 private:
  // No copying allowed
  WritableFile(const WritableFile&);
  void operator=(const WritableFile&);
};

// A abstract for the sequential readable file
class SequentialFile {
 public:
  SequentialFile() {};
  virtual ~SequentialFile();
  //virtual Status Read(size_t n, char *&result, char *scratch) = 0;
  virtual Status Read(size_t n, Slice* result, char* scratch) = 0;
  virtual Status Skip(uint64_t n) = 0;
  //virtual Status Close() = 0;
  virtual char *ReadLine(char *buf, int n) = 0;
};

class RWFile {
public:
  RWFile() { }
  virtual ~RWFile();
  virtual char* GetData() = 0;

private:
  // No copying allowed
  RWFile(const RWFile&);
  void operator=(const RWFile&);
};

// A file abstraction for random reading and writing.
class RandomRWFile {
 public:
  RandomRWFile() {}
  virtual ~RandomRWFile() {}

  // Write data from Slice data to file starting from offset
  // Returns IOError on failure, but does not guarantee
  // atomicity of a write.  Returns OK status on success.
  //
  // Safe for concurrent use.
  virtual Status Write(uint64_t offset, const Slice& data) = 0;
  // Read up to "n" bytes from the file starting at "offset".
  // "scratch[0..n-1]" may be written by this routine.  Sets "*result"
  // to the data that was read (including if fewer than "n" bytes were
  // successfully read).  May set "*result" to point at data in
  // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
  // "*result" is used.  If an error was encountered, returns a non-OK
  // status.
  //
  // Safe for concurrent use by multiple threads.
  virtual Status Read(uint64_t offset, size_t n, Slice* result,
                      char* scratch) const = 0;
  virtual Status Close() = 0; // closes the file
  virtual Status Sync() = 0; // sync data

  /*
   * Sync data and/or metadata as well.
   * By default, sync only data.
   * Override this method for environments where we need to sync
   * metadata as well.
   */
  virtual Status Fsync() {
    return Sync();
  }

  /*
   * Pre-allocate space for a file.
   */
  virtual Status Allocate(off_t offset, off_t len) {
    (void)offset;
    (void)len;
    return Status::OK();
  }

 private:
  // No copying allowed
  RandomRWFile(const RandomRWFile&);
  void operator=(const RandomRWFile&);
};


}   // namespace slash
#endif  // SLASH_ENV_H_
{'fraction_non_alphanumeric': 0.11980602833507654, 'fraction_numerical': 0.007606731957782637, 'mean_word_length': 4.227634194831014, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4388084', 'n_tokens_mistral': 3440, 'n_tokens_neox': 3326, 'n_words': 1126}