<commit_before>#include "util.h"

static int count=0;
static int block=10240;
static int fold=90;

static size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
  int written = fwrite(ptr, size, nmemb, (FILE *)stream);
  int current = count/block;
  int dots;
  int i;
  count+=written;
  for(i=current;i<count/block;++i){
    printf(".");
    fflush(stdout);
    if(i%fold==0)
      printf("\n");
  }
  return written;
}

int download_simple (char* uri,char* path,int verbose) {
#ifndef HAVE_WINDOWS_H
  CURL *curl;
  CURLcode res;
  FILE *bodyfile;
  char* path_partial=cat(path,".partial",NULL);

  curl = curl_easy_init();
  if(curl) {
    curl_easy_setopt(curl, CURLOPT_URL, uri);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);

    bodyfile = fopen(path_partial,"wb");
    if (bodyfile == NULL) {
      curl_easy_cleanup(curl);
      s(path_partial);
      return -1;
    }
    curl_easy_setopt(curl,CURLOPT_WRITEDATA,bodyfile);
    res = curl_easy_perform(curl);<commit_msg>better progress bar when it using wininet<commit_after>#include "util.h"

static int count=0;
static int block=10240;
static int fold=90;

static size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream)
{
  int written = fwrite(ptr, size, nmemb, (FILE *)stream);
  int current = count/block;
  int dots;
  int i;
  count+=written*size;
  for(i=current;i<count/block;++i){
    printf(".");
    fflush(stdout);
    if(i%fold==0)
      printf("\n");
  }
  return written;
}

int download_simple (char* uri,char* path,int verbose) {
#ifndef HAVE_WINDOWS_H
  CURL *curl;
  CURLcode res;
  FILE *bodyfile;
  char* path_partial=cat(path,".partial",NULL);

  curl = curl_easy_init();
  if(curl) {
    curl_easy_setopt(curl, CURLOPT_URL, uri);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);

    bodyfile = fopen(path_partial,"wb");
    if (bodyfile == NULL) {
      curl_easy_cleanup(curl);
      s(path_partial);
      return -1;
    }
    curl_easy_setopt(curl,CURLOPT_WRITEDATA,bodyfile);
    res = curl_easy_perform(curl);
{'fraction_non_alphanumeric': 0.13575701794753797, 'fraction_numerical': 0.010124252185918085, 'mean_word_length': 4.263922518159807, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2588152', 'n_tokens_mistral': 875, 'n_tokens_neox': 811, 'n_words': 171}