<commit_before>#include "opt.h"

static int count=0;
static int width=74;
static int content_length=0;

static size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
  int written = fwrite(ptr, size, nmemb, (FILE *)stream);
  static char* last_showd=NULL;
  char* w=q("");
  last_showd=last_showd?last_showd:q("");
  count+=written*size;
  w=s_cat2(w,q("\r"));
  if(content_length) {
    int i;
    for(i=0;i<width;++i)
      w=s_cat2(w,q((i>=(count/(content_length/(width)))?" ":"#")));
    w=s_cat2(w,qsprintf(8," %3d%%",(count/(content_length/(99)))));
  }else {
    int current,aux;
    aux=1024>count?' ':1024*1024>count?(current=count/1024,'K'):
      1024*1024*1024>count?(current=count/(1024*1024),'M'):(current=count/(1024*1024*1024),'G');
    w=s_cat2(w,qsprintf(20,"%4d%c downloaded.",current,aux));
  }
  if(strcmp(w,last_showd) !=0){
    fprintf(stderr, "%s", w);
    fflush(stderr);
    s(last_showd),last_showd=q(w);
  }
  s(w);
  return written;
}

static size_t header_callback(char *buffer, size_t size,size_t nitems, int *verbose) {
  int pos=-1,pos2,code=0;
  if(strncmp("HTTP",buffer,nitems<4?nitems:4)==0) {
    pos=position_char(" ",buffer);<commit_msg>better progress bar.<commit_after>#include "opt.h"

static int count=0;
static int width=74;
static int content_length=0;

static size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) {
  int written = fwrite(ptr, size, nmemb, (FILE *)stream);
  static char* last_showd=NULL;
  char* w=q("");
  last_showd=last_showd?last_showd:q("");
  count+=written*size;
  w=s_cat2(w,q("\r"));
  if(content_length) {
    int i;
    for(i=0;i<width;++i)
      w=s_cat2(w,q((i>=(count/(content_length/(width)))?" ":"#")));
    w=s_cat2(w,qsprintf(8," %3d%%",(100*(count/100))/(content_length/100)));
  }else {
    int current,aux;
    aux=1024>count?' ':1024*1024>count?(current=count/1024,'K'):
      1024*1024*1024>count?(current=count/(1024*1024),'M'):(current=count/(1024*1024*1024),'G');
    w=s_cat2(w,qsprintf(20,"%4d%c downloaded.",current,aux));
  }
  if(strcmp(w,last_showd) !=0){
    fprintf(stderr, "%s", w);
    fflush(stderr);
    s(last_showd),last_showd=q(w);
  }
  s(w);
  return written;
}

static size_t header_callback(char *buffer, size_t size,size_t nitems, int *verbose) {
  int pos=-1,pos2,code=0;
  if(strncmp("HTTP",buffer,nitems<4?nitems:4)==0) {
    pos=position_char(" ",buffer);
{'fraction_non_alphanumeric': 0.21273938384679433, 'fraction_numerical': 0.06203164029975021, 'mean_word_length': 5.583561643835616, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1014831', 'n_tokens_mistral': 1077, 'n_tokens_neox': 1007, 'n_words': 151}