<commit_before>#include <cmath>
#include <string>
#include <cctype>
#include <vector>
#include <sstream>
#include <iostream>
#include <stdexcept>

#include <leveldb/db.h>
#include <leveldb/write_batch.h>

#include "riba.h"

namespace {
/* A single leveldb instance */
leveldb::DB* ldb = NULL;

/* A batch of updates */
bool should_batch = false;
leveldb::WriteBatch lbatch;

/* A DB snapshot */
bool exists_snapshot = false;
leveldb::ReadOptions loptions;

void clear_db_state()
{
  if(!ldb) return;

  // Clear any pending batches
  if(should_batch)
  {
    should_batch = false;
    lbatch.Clear();
  }
  
  // Clear any snapshots
  if(exists_snapshot)
  {
    exists_snapshot = false;
    ldb->ReleaseSnapshot(loptions.snapshot);
  }
}

bool get_confirmation(const std::string& question,
                      const std::string& yes,
                      const std::string& no)
{
  std::string answer;
  std::cout << question;
  std::cin >> answer;
  return (answer == yes);
}

/* Is this slice a string? */
bool is_string(const char* str, size_t len)
{
  if(!str) return false;
  for(size_t i=0; i<len; i++)
  {
    if( !isalpha( str[i]) )
      return false;
  }
  return true;
}

char from_hex(char hbyte)
{
  char outbyte = -1;
  
  if(isalpha(hbyte))
  {
    hbyte = tolower(hbyte);
    outbyte = 10 + (hbyte - 'a');
  }
  else if(isdigit(hbyte))
  {
    outbyte = hbyte - '0';
  }

  return outbyte;
}

// Note: The caller needs to free the underlying memory after
// using the returned slice
leveldb::Slice make_slice(const char* sdata)
{
  int i=0, len = strlen(sdata);
  
  // If the data is a string, create a string slice out of it
  for(i=0; i<len && isalpha(sdata[i]); i++);
  if(i == len)
  {
    return leveldb::Slice( strdup(sdata) );
  }

  // If the data is a hex string, create the appropriate bytes
  if(len > 2 && sdata[0] == '0' && (sdata[1] == 'x' || sdata[1] == 'X'))
  {
    int hlen = 0, j = 0;
    char* hdata;
    
    sdata += 2, len -=2; // Skip past 0x
    hlen = (int)ceil(len/2.0);

    hdata = (char*)malloc(hlen);
    memset(hdata, 0, hlen);
    for(int i=len-1; i>=0; i-=2)
    {
      char nibble_low = from_hex(sdata[i]);
      char nibble_high = (i>0) ? from_hex(sdata[i-1]) : 0;
      // If neither character is a valid hex character, bail out
      if(nibble_low == -1 || nibble_high == -1)
      {
        free(hdata);
        throw(std::runtime_error("Please enter a valid hex string"));
      }
      hdata[j++] = nibble_high*16 + nibble_low;
    }
    return leveldb::Slice(hdata, hlen);
  }

  // Else the data is just a number
  std::stringstream ss;
  int *number = (int*)malloc(sizeof(int));
  ss << sdata;
  ss >> (*number);
  return leveldb::Slice((const char*)number, sizeof(int));
}

std::string to_print(const char* stream, size_t len)
{
  static char hexmap[] = { '0', '1', '2', '3', '4', '5', '6', '7',
                           '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};

  if(is_string(stream, len))
    return std::string(stream, len);

  // Treat it as a byte stream
  std::string outs("0x");
  for(int i=len-1; i>=0; i--)
  {
    char hb = stream[i];
    outs += hexmap[(hb & 0xf0) >> 4];
    outs += hexmap[hb & 0x0f];
  }
  return outs;
}

std::ostream& operator <<(std::ostream& os, leveldb::Slice slice)
{
  os << to_print(slice.data(), slice.size());
  return os;
}

}

void leveldb_open(const char* name)
{<commit_msg>handle all printable characters within a string<commit_after>#include <cmath>
#include <string>
#include <cctype>
#include <vector>
#include <sstream>
#include <iostream>
#include <stdexcept>

#include <leveldb/db.h>
#include <leveldb/write_batch.h>

#include "riba.h"

namespace {
/* A single leveldb instance */
leveldb::DB* ldb = NULL;

/* A batch of updates */
bool should_batch = false;
leveldb::WriteBatch lbatch;

/* A DB snapshot */
bool exists_snapshot = false;
leveldb::ReadOptions loptions;

void clear_db_state()
{
  if(!ldb) return;

  // Clear any pending batches
  if(should_batch)
  {
    should_batch = false;
    lbatch.Clear();
  }
  
  // Clear any snapshots
  if(exists_snapshot)
  {
    exists_snapshot = false;
    ldb->ReleaseSnapshot(loptions.snapshot);
  }
}

bool get_confirmation(const std::string& question,
                      const std::string& yes,
                      const std::string& no)
{
  std::string answer;
  std::cout << question;
  std::cin >> answer;
  return (answer == yes);
}

/* Is this slice a string? */
bool is_string(const char* str, size_t len)
{
  if(!str) return false;
  for(size_t i=0; i<len; i++)
  {
    if( !isprint( str[i]) )
      return false;
  }
  return true;
}

char from_hex(char hbyte)
{
  char outbyte = -1;
  
  if(isalpha(hbyte))
  {
    hbyte = tolower(hbyte);
    outbyte = 10 + (hbyte - 'a');
  }
  else if(isdigit(hbyte))
  {
    outbyte = hbyte - '0';
  }

  return outbyte;
}

// Note: The caller needs to free the underlying memory after
// using the returned slice
leveldb::Slice make_slice(const char* sdata)
{
  int i=0, len = strlen(sdata);

  // If the data is an integer, handle it accordingly
  for(i=0; i<len && isdigit(sdata[i]); i++);
  if(i == len)
  {
    std::stringstream ss;
    int *number = (int*)malloc(sizeof(int));
    ss << sdata;
    ss >> (*number);
    return leveldb::Slice((const char*)number, sizeof(int));
  }

  // If the data is a hex string, create the appropriate bytes
  if(len > 2 && sdata[0] == '0' && (sdata[1] == 'x' || sdata[1] == 'X'))
  {
    int hlen = 0, j = 0;
    char* hdata;
    
    sdata += 2, len -=2; // Skip past 0x
    hlen = (int)ceil(len/2.0);

    hdata = (char*)malloc(hlen);
    memset(hdata, 0, hlen);
    for(int i=len-1; i>=0; i-=2)
    {
      char nibble_low = from_hex(sdata[i]);
      char nibble_high = (i>0) ? from_hex(sdata[i-1]) : 0;
      // If neither character is a valid hex character, bail out
      if(nibble_low == -1 || nibble_high == -1)
      {
        free(hdata);
        throw(std::runtime_error("Please enter a valid hex string"));
      }
      hdata[j++] = nibble_high*16 + nibble_low;
    }
    return leveldb::Slice(hdata, hlen);
  }

  // Else the data is just a string
  return leveldb::Slice( strdup(sdata) );
}

std::string to_print(const char* stream, size_t len)
{
  static char hexmap[] = { '0', '1', '2', '3', '4', '5', '6', '7',
                           '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};

  if(is_string(stream, len))
    return std::string(stream, len);

  // Treat it as a byte stream
  std::string outs("0x");
  for(int i=len-1; i>=0; i--)
  {
    char hb = stream[i];
    outs += hexmap[(hb & 0xf0) >> 4];
    outs += hexmap[hb & 0x0f];
  }
  return outs;
}

std::ostream& operator <<(std::ostream& os, leveldb::Slice slice)
{
  os << to_print(slice.data(), slice.size());
  return os;
}

}

void leveldb_open(const char* name)
{
{'fraction_non_alphanumeric': 0.16862514688601646, 'fraction_numerical': 0.01410105757931845, 'mean_word_length': 2.9154686601495112, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1604845', 'n_tokens_mistral': 2614, 'n_tokens_neox': 2557, 'n_words': 682}