<commit_before>
#include <archive.h>
#include <archive_entry.h>
#include "jxereader.h"
#include "toml/tparser.h"
#include "toml/files.h"
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <dlfcn.h>

#define CHUNK   16384

/* This extractor was an example from lib archive complete extractor
 * It works surprisingly well and returns 0 for success, any other number for failure
 * All I added to this was to extract to a specific folder
 * The copy data is a subroutine for extract and ought not be used anywhere else
 */
static int
copy_data(struct archive *ar, struct archive *aw)
{
  int r;
  const void *buff;
  size_t size;
  off_t offset;

  for (;;) {
    r = archive_read_data_block(ar, &buff, &size, &offset);
    if (r == ARCHIVE_EOF)
      return (ARCHIVE_OK);
    if (r < ARCHIVE_OK)
      return (r);
    r = archive_write_data_block(aw, buff, size, offset);
    if (r < ARCHIVE_OK) {
      fprintf(stderr, "%s\n", archive_error_string(aw));
      return (r);
    }
  }
}

/* This function will extract the zip file from filename to a filename_jam_run folder
 * Everythig will be handled by this
 */

static int extract(const char *filename, jxe_file * j)
{
  char cwd[1024];
  char path[1024];
  int err = 0;
  struct archive *a;
  struct archive *ext;
  struct archive_entry *entry;
  int flags;
  int r, i = 0;
  int file_index = 0;
  char * currentFile;

  //* make some directory
  if(getcwd(cwd, 1024) == NULL)
    return -1;
  
  /* Select which attributes we want to restore. */
  flags = ARCHIVE_EXTRACT_TIME;
  flags |= ARCHIVE_EXTRACT_PERM;
  flags |= ARCHIVE_EXTRACT_ACL;
  flags |= ARCHIVE_EXTRACT_FFLAGS;

  a = archive_read_new();
  archive_read_support_format_all(a);
  archive_read_support_compression_all(a);
  ext = archive_write_disk_new();
  archive_write_disk_set_options(ext, flags);
  archive_write_disk_set_standard_lookup(ext);
  //We need to decide the folder name
  //We take the filename without the extensions
  for(int i = 0; i < 512; i++){ 
    if(filename[i] == '.'){
        file_index = i;
        break;
    }
  }
  err = snprintf(j->dir_name, 512, "%s", filename);
  if(err != strlen(filename)){
    printf("File name too long 1\n");
    exit(1);
  }

  err = snprintf(j->dir_name + file_index, 512 - file_index, "_jam_run");
   if(err != strlen("_jam_run")){
    printf("File name too long\n");
    exit(1);
  }
  //With this we have our directory name which we will be extracting to

  if ((r = archive_read_open_filename(a, filename, CHUNK)))
    return 1;
  for (;;) {
    r = archive_read_next_header(a, &entry);<commit_msg>removed a warning<commit_after>
#include <archive.h>
#include <archive_entry.h>
#include "jxereader.h"
#include "toml/tparser.h"
#include "toml/files.h"
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <dlfcn.h>

#define CHUNK   16384

/* This extractor was an example from lib archive complete extractor
 * It works surprisingly well and returns 0 for success, any other number for failure
 * All I added to this was to extract to a specific folder
 * The copy data is a subroutine for extract and ought not be used anywhere else
 */
static int
copy_data(struct archive *ar, struct archive *aw)
{
  int r;
  const void *buff;
  size_t size;
  off_t offset;

  for (;;) {
    r = archive_read_data_block(ar, &buff, &size, &offset);
    if (r == ARCHIVE_EOF)
      return (ARCHIVE_OK);
    if (r < ARCHIVE_OK)
      return (r);
    r = archive_write_data_block(aw, buff, size, offset);
    if (r < ARCHIVE_OK) {
      fprintf(stderr, "%s\n", archive_error_string(aw));
      return (r);
    }
  }
}

/* This function will extract the zip file from filename to a filename_jam_run folder
 * Everythig will be handled by this
 */

static int extract(const char *filename, jxe_file * j)
{
  char cwd[1024];
  char path[1024];
  int err = 0;
  struct archive *a;
  struct archive *ext;
  struct archive_entry *entry;
  int flags;
  int r, i = 0;
  int file_index = 0;
  char * currentFile;

  //* make some directory
  if(getcwd(cwd, 1024) == NULL)
    return -1;
  
  /* Select which attributes we want to restore. */
  flags = ARCHIVE_EXTRACT_TIME;
  flags |= ARCHIVE_EXTRACT_PERM;
  flags |= ARCHIVE_EXTRACT_ACL;
  flags |= ARCHIVE_EXTRACT_FFLAGS;

  a = archive_read_new();
  archive_read_support_format_all(a);
  archive_read_support_filter_all(a);
  ext = archive_write_disk_new();
  archive_write_disk_set_options(ext, flags);
  archive_write_disk_set_standard_lookup(ext);
  //We need to decide the folder name
  //We take the filename without the extensions
  for(int i = 0; i < 512; i++){ 
    if(filename[i] == '.'){
        file_index = i;
        break;
    }
  }
  err = snprintf(j->dir_name, 512, "%s", filename);
  if(err != strlen(filename)){
    printf("File name too long 1\n");
    exit(1);
  }

  err = snprintf(j->dir_name + file_index, 512 - file_index, "_jam_run");
   if(err != strlen("_jam_run")){
    printf("File name too long\n");
    exit(1);
  }
  //With this we have our directory name which we will be extracting to

  if ((r = archive_read_open_filename(a, filename, CHUNK)))
    return 1;
  for (;;) {
    r = archive_read_next_header(a, &entry);
{'fraction_non_alphanumeric': 0.11948352283677009, 'fraction_numerical': 0.013875505877818462, 'mean_word_length': 3.5248474280732345, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '883515', 'n_tokens_mistral': 1902, 'n_tokens_neox': 1824, 'n_words': 557}