<commit_before>/*************************************************************************************************
 * The command line utility of the B+ tree database API
 *                                                               Copyright (C) 2006-2012 FAL Labs
 * This file is part of Tokyo Cabinet.
 * Tokyo Cabinet is free software; you can redistribute it and/or modify it under the terms of
 * the GNU Lesser General Public License as published by the Free Software Foundation; either
 * version 2.1 of the License or any later version.  Tokyo Cabinet is distributed in the hope
 * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 * You should have received a copy of the GNU Lesser General Public License along with Tokyo
 * Cabinet; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307 USA.
 *************************************************************************************************/


#include <tcutil.h>
#include <tcbdb.h>
#include "myconf.h"


/* global variables */
const char *g_progname;                  // program name
int g_dbgfd;                             // debugging output


/* function prototypes */
int main(int argc, char **argv);
static void usage(void);
static void printerr(TCBDB *bdb);
static int printdata(const char *ptr, int size, bool px);
static char *mygetline(FILE *ifp);
static int mycmpfunc(const char *aptr, int asiz, const char *bptr, int bsiz, void *op);
static int write_all(int fd, const char *data, size_t len);
static int read_all(int fd, char *data, size_t len);
static int runcreate(int argc, char **argv);
static int runinform(int argc, char **argv);
static int runput(int argc, char **argv);
static int runout(int argc, char **argv);
static int runget(int argc, char **argv);
static int runlist(int argc, char **argv);
static int runoptimize(int argc, char **argv);
static int runimporttsv(int argc, char **argv);
static int runversion(int argc, char **argv);
static int rundump(int argc, char **argv);
static int runimport(int argc, char **argv);
static int proccreate(const char *path, int lmemb, int nmemb,
                      int bnum, int apow, int fpow, TCCMP cmp, int opts);
static int procinform(const char *path, int omode);
static int procput(const char *path, const char *kbuf, int ksiz, const char *vbuf, int vsiz,
                   TCCMP cmp, int omode, int dmode);
static int procout(const char *path, const char *kbuf, int ksiz, TCCMP cmp, int omode);
static int procget(const char *path, const char *kbuf, int ksiz, TCCMP cmp, int omode,
                   bool px, bool pz);
static int proclist(const char *path, TCCMP cmp, int omode, int max, bool pv, bool px, bool bk,
                    const char *jstr, const char *bstr, const char *estr, const char *fmstr);
static int procoptimize(const char *path, int lmemb, int nmemb,
                        int bnum, int apow, int fpow, TCCMP cmp, int opts, int omode, bool df);
static int procimporttsv(const char *path, const char *file, int omode, bool sc);
static int procversion(void);
static int procdump(const char *path, TCCMP cmp, int omode);
static int procimport(const char *path, int omode);


/* main routine */
int main(int argc, char **argv){
  g_progname = argv[0];
  g_dbgfd = -1;
  const char *ebuf = getenv("TCDBGFD");
  if(ebuf) g_dbgfd = tcatoix(ebuf);
  if(argc < 2) usage();
  int rv = 0;
  if(!strcmp(argv[1], "create")){
    rv = runcreate(argc, argv);
  } else if(!strcmp(argv[1], "inform")){
    rv = runinform(argc, argv);
  } else if(!strcmp(argv[1], "put")){
    rv = runput(argc, argv);
  } else if(!strcmp(argv[1], "out")){
    rv = runout(argc, argv);
  } else if(!strcmp(argv[1], "get")){
    rv = runget(argc, argv);
  } else if(!strcmp(argv[1], "list")){
    rv = runlist(argc, argv);
  } else if(!strcmp(argv[1], "optimize")){
    rv = runoptimize(argc, argv);
  } else if(!strcmp(argv[1], "importtsv")){
    rv = runimporttsv(argc, argv);
  } else if(!strcmp(argv[1], "version") || !strcmp(argv[1], "--version")){
    rv = runversion(argc, argv);
  } else if(!strcmp(argv[1], "dump")){
    rv = rundump(argc, argv);
  } else if(!strcmp(argv[1], "import")){
    rv = runimport(argc, argv);
  } else {
    usage();
  }
  return rv;
}


/* print the usage and exit */
static void usage(void){
  fprintf(stderr, "%s: the command line utility of the B+ tree database API\n", g_progname);
  fprintf(stderr, "\n");
  fprintf(stderr, "usage:\n");
  fprintf(stderr, "  %s create [-cd|-ci|-cj] [-tl] [-td|-tb|-tt|-tx] path"
          " [lmemb [nmemb [bnum [apow [fpow]]]]]\n", g_progname);
  fprintf(stderr, "  %s inform [-nl|-nb] path\n", g_progname);
  fprintf(stderr, "  %s put [-cd|-ci|-cj] [-nl|-nb] [-sx] [-dk|-dc|-dd|-db|-dai|-dad] path"
          " key value\n", g_progname);
  fprintf(stderr, "  %s out [-cd|-ci|-cj] [-nl|-nb] [-sx] path key\n", g_progname);
  fprintf(stderr, "  %s get [-cd|-ci|-cj] [-nl|-nb] [-sx] [-px] [-pz] path key\n", g_progname);
  fprintf(stderr, "  %s list [-cd|-ci|-cj] [-nl|-nb] [-m num] [-bk] [-pv] [-px] [-j str]"
          " [-rb bkey ekey] [-fm str] path\n", g_progname);
  fprintf(stderr, "  %s optimize [-cd|-ci|-cj] [-tl] [-td|-tb|-tt|-tx] [-tz] [-nl|-nb] [-df]"
          " path [lmemb [nmemb [bnum [apow [fpow]]]]]\n", g_progname);
  fprintf(stderr, "  %s importtsv [-nl|-nb] [-sc] path [file]\n", g_progname);
  fprintf(stderr, "  %s version\n", g_progname);
  fprintf(stderr, "  %s dump [-cd|-ci|-cj] [-nl|-nb] path\n", g_progname);
  fprintf(stderr, "  %s import [-nl|-nb] path\n", g_progname);
  fprintf(stderr, "\n");
  exit(1);
}


/* print error information */
static void printerr(TCBDB *bdb){
  const char *path = tcbdbpath(bdb);
  int ecode = tcbdbecode(bdb);
  fprintf(stderr, "%s: %s: %d: %s\n", g_progname, path ? path : "-", ecode, tcbdberrmsg(ecode));
}


/* print record data */
static int printdata(const char *ptr, int size, bool px){
  int len = 0;
  while(size-- > 0){
    if(px){
      if(len > 0) putchar(' ');
      len += printf("%02X", *(unsigned char *)ptr);
    } else {
      putchar(*ptr);
      len++;
    }
    ptr++;
  }
  return len;
}


/* read a line from a file descriptor */
static char *mygetline(FILE *ifp){
  int len = 0;
  int blen = 1024;
  char *buf = tcmalloc(blen + 1);
  bool end = true;
  int c;
  while((c = fgetc(ifp)) != EOF){
    end = false;
    if(c == '\0') continue;
    if(blen <= len){
      blen *= 2;
      buf = tcrealloc(buf, blen + 1);
    }
    if(c == '\n' || c == '\r') c = '\0';
    buf[len++] = c;
    if(c == '\0') break;
  }
  if(end){
    tcfree(buf);
    return NULL;
  }
  buf[len] = '\0';
  return buf;
}


/* dummy comparison function */
static int mycmpfunc(const char *aptr, int asiz, const char *bptr, int bsiz, void *op){
  return 0;
}

/* safe 'write' */
static int write_all(int fd, const char *data, size_t len){
  ssize_t r = 0;
  while(len > 0){
    r = write(fd, data, len);
    if(r < 0){
      if(errno == EAGAIN || errno == EINTR){
        continue;
      }
      perror("write");
      return -1;
    }
    len -= r;
  }
  return 0;
}

/* safe 'read' */
static int read_all(int fd, char *data, size_t len){
  ssize_t r = 0;
  while(len > 0){
    r = read(fd, data, len);
    if(r == 0){
      return -1;
    }
    if(r < 0){
      if(errno == EAGAIN || errno == EINTR){
        continue;
      }
      perror("read");
      return -1;
    }<commit_msg>BUG tcbmgr: Fix write_all<commit_after>/*************************************************************************************************
 * The command line utility of the B+ tree database API
 *                                                               Copyright (C) 2006-2012 FAL Labs
 * This file is part of Tokyo Cabinet.
 * Tokyo Cabinet is free software; you can redistribute it and/or modify it under the terms of
 * the GNU Lesser General Public License as published by the Free Software Foundation; either
 * version 2.1 of the License or any later version.  Tokyo Cabinet is distributed in the hope
 * that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
 * License for more details.
 * You should have received a copy of the GNU Lesser General Public License along with Tokyo
 * Cabinet; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
 * Boston, MA 02111-1307 USA.
 *************************************************************************************************/


#include <tcutil.h>
#include <tcbdb.h>
#include "myconf.h"


/* global variables */
const char *g_progname;                  // program name
int g_dbgfd;                             // debugging output


/* function prototypes */
int main(int argc, char **argv);
static void usage(void);
static void printerr(TCBDB *bdb);
static int printdata(const char *ptr, int size, bool px);
static char *mygetline(FILE *ifp);
static int mycmpfunc(const char *aptr, int asiz, const char *bptr, int bsiz, void *op);
static int write_all(int fd, const char *data, size_t len);
static int read_all(int fd, char *data, size_t len);
static int runcreate(int argc, char **argv);
static int runinform(int argc, char **argv);
static int runput(int argc, char **argv);
static int runout(int argc, char **argv);
static int runget(int argc, char **argv);
static int runlist(int argc, char **argv);
static int runoptimize(int argc, char **argv);
static int runimporttsv(int argc, char **argv);
static int runversion(int argc, char **argv);
static int rundump(int argc, char **argv);
static int runimport(int argc, char **argv);
static int proccreate(const char *path, int lmemb, int nmemb,
                      int bnum, int apow, int fpow, TCCMP cmp, int opts);
static int procinform(const char *path, int omode);
static int procput(const char *path, const char *kbuf, int ksiz, const char *vbuf, int vsiz,
                   TCCMP cmp, int omode, int dmode);
static int procout(const char *path, const char *kbuf, int ksiz, TCCMP cmp, int omode);
static int procget(const char *path, const char *kbuf, int ksiz, TCCMP cmp, int omode,
                   bool px, bool pz);
static int proclist(const char *path, TCCMP cmp, int omode, int max, bool pv, bool px, bool bk,
                    const char *jstr, const char *bstr, const char *estr, const char *fmstr);
static int procoptimize(const char *path, int lmemb, int nmemb,
                        int bnum, int apow, int fpow, TCCMP cmp, int opts, int omode, bool df);
static int procimporttsv(const char *path, const char *file, int omode, bool sc);
static int procversion(void);
static int procdump(const char *path, TCCMP cmp, int omode);
static int procimport(const char *path, int omode);


/* main routine */
int main(int argc, char **argv){
  g_progname = argv[0];
  g_dbgfd = -1;
  const char *ebuf = getenv("TCDBGFD");
  if(ebuf) g_dbgfd = tcatoix(ebuf);
  if(argc < 2) usage();
  int rv = 0;
  if(!strcmp(argv[1], "create")){
    rv = runcreate(argc, argv);
  } else if(!strcmp(argv[1], "inform")){
    rv = runinform(argc, argv);
  } else if(!strcmp(argv[1], "put")){
    rv = runput(argc, argv);
  } else if(!strcmp(argv[1], "out")){
    rv = runout(argc, argv);
  } else if(!strcmp(argv[1], "get")){
    rv = runget(argc, argv);
  } else if(!strcmp(argv[1], "list")){
    rv = runlist(argc, argv);
  } else if(!strcmp(argv[1], "optimize")){
    rv = runoptimize(argc, argv);
  } else if(!strcmp(argv[1], "importtsv")){
    rv = runimporttsv(argc, argv);
  } else if(!strcmp(argv[1], "version") || !strcmp(argv[1], "--version")){
    rv = runversion(argc, argv);
  } else if(!strcmp(argv[1], "dump")){
    rv = rundump(argc, argv);
  } else if(!strcmp(argv[1], "import")){
    rv = runimport(argc, argv);
  } else {
    usage();
  }
  return rv;
}


/* print the usage and exit */
static void usage(void){
  fprintf(stderr, "%s: the command line utility of the B+ tree database API\n", g_progname);
  fprintf(stderr, "\n");
  fprintf(stderr, "usage:\n");
  fprintf(stderr, "  %s create [-cd|-ci|-cj] [-tl] [-td|-tb|-tt|-tx] path"
          " [lmemb [nmemb [bnum [apow [fpow]]]]]\n", g_progname);
  fprintf(stderr, "  %s inform [-nl|-nb] path\n", g_progname);
  fprintf(stderr, "  %s put [-cd|-ci|-cj] [-nl|-nb] [-sx] [-dk|-dc|-dd|-db|-dai|-dad] path"
          " key value\n", g_progname);
  fprintf(stderr, "  %s out [-cd|-ci|-cj] [-nl|-nb] [-sx] path key\n", g_progname);
  fprintf(stderr, "  %s get [-cd|-ci|-cj] [-nl|-nb] [-sx] [-px] [-pz] path key\n", g_progname);
  fprintf(stderr, "  %s list [-cd|-ci|-cj] [-nl|-nb] [-m num] [-bk] [-pv] [-px] [-j str]"
          " [-rb bkey ekey] [-fm str] path\n", g_progname);
  fprintf(stderr, "  %s optimize [-cd|-ci|-cj] [-tl] [-td|-tb|-tt|-tx] [-tz] [-nl|-nb] [-df]"
          " path [lmemb [nmemb [bnum [apow [fpow]]]]]\n", g_progname);
  fprintf(stderr, "  %s importtsv [-nl|-nb] [-sc] path [file]\n", g_progname);
  fprintf(stderr, "  %s version\n", g_progname);
  fprintf(stderr, "  %s dump [-cd|-ci|-cj] [-nl|-nb] path\n", g_progname);
  fprintf(stderr, "  %s import [-nl|-nb] path\n", g_progname);
  fprintf(stderr, "\n");
  exit(1);
}


/* print error information */
static void printerr(TCBDB *bdb){
  const char *path = tcbdbpath(bdb);
  int ecode = tcbdbecode(bdb);
  fprintf(stderr, "%s: %s: %d: %s\n", g_progname, path ? path : "-", ecode, tcbdberrmsg(ecode));
}


/* print record data */
static int printdata(const char *ptr, int size, bool px){
  int len = 0;
  while(size-- > 0){
    if(px){
      if(len > 0) putchar(' ');
      len += printf("%02X", *(unsigned char *)ptr);
    } else {
      putchar(*ptr);
      len++;
    }
    ptr++;
  }
  return len;
}


/* read a line from a file descriptor */
static char *mygetline(FILE *ifp){
  int len = 0;
  int blen = 1024;
  char *buf = tcmalloc(blen + 1);
  bool end = true;
  int c;
  while((c = fgetc(ifp)) != EOF){
    end = false;
    if(c == '\0') continue;
    if(blen <= len){
      blen *= 2;
      buf = tcrealloc(buf, blen + 1);
    }
    if(c == '\n' || c == '\r') c = '\0';
    buf[len++] = c;
    if(c == '\0') break;
  }
  if(end){
    tcfree(buf);
    return NULL;
  }
  buf[len] = '\0';
  return buf;
}


/* dummy comparison function */
static int mycmpfunc(const char *aptr, int asiz, const char *bptr, int bsiz, void *op){
  return 0;
}

/* safe 'write' */
static int write_all(int fd, const char *data, size_t len){
  ssize_t r = 0;
  while(len > 0){
    r = write(fd, data, len);
    if(r < 0){
      if(errno == EAGAIN || errno == EINTR){
        continue;
      }
      perror("write");
      return -1;
    }
    len -= r;
    data += r;
  }
  return 0;
}

/* safe 'read' */
static int read_all(int fd, char *data, size_t len){
  ssize_t r = 0;
  while(len > 0){
    r = read(fd, data, len);
    if(r == 0){
      return -1;
    }
    if(r < 0){
      if(errno == EAGAIN || errno == EINTR){
        continue;
      }
      perror("read");
      return -1;
    }
{'fraction_non_alphanumeric': 0.18550571447446654, 'fraction_numerical': 0.009248860408271123, 'mean_word_length': 3.139458572600492, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5460428', 'n_tokens_mistral': 5471, 'n_tokens_neox': 5282, 'n_words': 1755}