<commit_before>#define _BSD_SOURCE 
#include <endian.h>

#include <unistd.h>
#include <stdint.h>
#include <stdio.h>
#include <malloc.h>
#include <openssl/bio.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <openssl/ssl.h>
#include <openssl/sha.h>

static const char *key_file_name, *in_file_name, *out_file_name;
static int verbose;

char * read_file( const char *fname )
{
    FILE *fp;
    long fsize;
    char *ret;
    size_t ns;

    fp = fopen( fname, "r" );
    if (!fp)
	return 0;

    fseek( fp, 0L, SEEK_END);
    fsize = ftell( fp );
    fseek( fp, 0L, SEEK_SET);

    ret = (char *) malloc( fsize + 1 );
    if (!ret)
	return 0;

    ns = fread( ret, 1, fsize, fp );
    ret[ ns ] = '\0';

    fclose( fp );

    return ret;
}

RSA *read_PEM_pub_key(const char *fname)
{   
  char *skey;
  BIO *bio;   
  RSA *key;
  int rt = 0;

  skey = read_file( fname );
  if (!skey) {
    fprintf( stderr, "Error: Failed to read file %s\n" , fname );
    return 0;
  }    

  bio = BIO_new(BIO_f_buffer());
  if (!bio) {
    return 0;
  }      

  if (BIO_set_buffer_read_data(bio, (void *) skey, strlen( skey )) == 1) {
    key = (void *) PEM_read_bio_RSA_PUBKEY( bio, NULL, NULL, NULL);
    if (key == 0) {
	rt = -1;
    }
  } else {
    rt = -1;
  }	
            
  BIO_free_all(bio);
  free( skey );

  return key;
}

RSA *read_PEM_priv_key(const char *fname)
{   
  char *skey;
  BIO *bio;   
  RSA *key;
  int rt = 0;

  skey = read_file( fname );
  if (!skey) {
    return 0;
  }    

  bio = BIO_new(BIO_f_buffer());
  if (!bio) {
    return 0;
  }      

  if (BIO_set_buffer_read_data(bio, (void *) skey, strlen( skey )) == 1) {
    key = (void *) PEM_read_bio_RSAPrivateKey( bio, NULL, NULL, NULL);
    if (key == 0) {
	rt = -1;
    }
  } else {
    rt = -1;
  }	
            
  BIO_free_all(bio);
  free( skey );

  return key;
}

#define PKCS_PADDING_SIZE 42

#pragma pack(1)
typedef struct {
  uint32_t file_magic;
  uint32_t file_version;
  uint64_t data_length;

} ENC_HEADER;
#pragma pack()

#define FILE_MAGIC	0xABCDEF
#define FILE_VERSION_1  0x1

uint32_t fill_header( FILE *fp, ENC_HEADER *hdr )
{
   //fpos_t  flen;
   uint32_t length;

   fseek( fp, 0L, SEEK_END);
   //fgetpos( fp, &flen );
   length = ftell( fp );
   fseek( fp, 0L, SEEK_SET);

   hdr->file_magic =   htobe32( FILE_MAGIC );
   hdr->file_version = htobe32( FILE_VERSION_1 );
   hdr->data_length  = htobe64( length + sizeof(ENC_HEADER) );

   return length + sizeof(ENC_HEADER); 
}

void header_to_host( ENC_HEADER *hdr )
{
   hdr->file_magic =   htobe32( hdr->file_magic );
   hdr->file_version = htobe32( hdr->file_version );
   hdr->data_length  = htobe64( hdr->data_length );
}

int encrypt_file( RSA *key, FILE *fpin, FILE *fpout )
{
   int rt = 0;
   uint32_t key_size = RSA_size(key);
   uint8_t *in_buf, *out_buf;
   uint32_t in_buf_size, in_read, out_len, file_size, file_pos = 0;
   SHA_CTX sha;
   uint8_t hash[SHA_DIGEST_LENGTH];
   size_t nwritten;
   int first = 1;
   int prev_percent = 0, new_percent;

   SHA1_Init(&sha);

   in_buf_size = key_size - PKCS_PADDING_SIZE; 
   in_buf = (uint8_t *) malloc( in_buf_size ); 
   if (!in_buf) {
     return -1;
   }<commit_msg>Ups, got byte order wrong.<commit_after>#define _BSD_SOURCE 
#include <endian.h>

#include <unistd.h>
#include <stdint.h>
#include <stdio.h>
#include <malloc.h>
#include <openssl/bio.h>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <openssl/ssl.h>
#include <openssl/sha.h>

static const char *key_file_name, *in_file_name, *out_file_name;
static int verbose;

char * read_file( const char *fname )
{
    FILE *fp;
    long fsize;
    char *ret;
    size_t ns;

    fp = fopen( fname, "r" );
    if (!fp)
	return 0;

    fseek( fp, 0L, SEEK_END);
    fsize = ftell( fp );
    fseek( fp, 0L, SEEK_SET);

    ret = (char *) malloc( fsize + 1 );
    if (!ret)
	return 0;

    ns = fread( ret, 1, fsize, fp );
    ret[ ns ] = '\0';

    fclose( fp );

    return ret;
}

RSA *read_PEM_pub_key(const char *fname)
{   
  char *skey;
  BIO *bio;   
  RSA *key;
  int rt = 0;

  skey = read_file( fname );
  if (!skey) {
    fprintf( stderr, "Error: Failed to read file %s\n" , fname );
    return 0;
  }    

  bio = BIO_new(BIO_f_buffer());
  if (!bio) {
    return 0;
  }      

  if (BIO_set_buffer_read_data(bio, (void *) skey, strlen( skey )) == 1) {
    key = (void *) PEM_read_bio_RSA_PUBKEY( bio, NULL, NULL, NULL);
    if (key == 0) {
	rt = -1;
    }
  } else {
    rt = -1;
  }	
            
  BIO_free_all(bio);
  free( skey );

  return key;
}

RSA *read_PEM_priv_key(const char *fname)
{   
  char *skey;
  BIO *bio;   
  RSA *key;
  int rt = 0;

  skey = read_file( fname );
  if (!skey) {
    return 0;
  }    

  bio = BIO_new(BIO_f_buffer());
  if (!bio) {
    return 0;
  }      

  if (BIO_set_buffer_read_data(bio, (void *) skey, strlen( skey )) == 1) {
    key = (void *) PEM_read_bio_RSAPrivateKey( bio, NULL, NULL, NULL);
    if (key == 0) {
	rt = -1;
    }
  } else {
    rt = -1;
  }	
            
  BIO_free_all(bio);
  free( skey );

  return key;
}

#define PKCS_PADDING_SIZE 42

#pragma pack(1)
typedef struct {
  uint32_t file_magic;
  uint32_t file_version;
  uint64_t data_length;

} ENC_HEADER;
#pragma pack()

#define FILE_MAGIC	0xABCDEF
#define FILE_VERSION_1  0x1

uint32_t fill_header( FILE *fp, ENC_HEADER *hdr )
{
   //fpos_t  flen;
   uint32_t length;

   fseek( fp, 0L, SEEK_END);
   //fgetpos( fp, &flen );
   length = ftell( fp );
   fseek( fp, 0L, SEEK_SET);

   hdr->file_magic =   htobe32( FILE_MAGIC );
   hdr->file_version = htobe32( FILE_VERSION_1 );
   hdr->data_length  = htobe64( length + sizeof(ENC_HEADER) );

   return length + sizeof(ENC_HEADER); 
}

void header_to_host( ENC_HEADER *hdr )
{
   hdr->file_magic =   be32toh( hdr->file_magic );
   hdr->file_version = be32toh( hdr->file_version );
   hdr->data_length  = be64toh( hdr->data_length );
}

int encrypt_file( RSA *key, FILE *fpin, FILE *fpout )
{
   int rt = 0;
   uint32_t key_size = RSA_size(key);
   uint8_t *in_buf, *out_buf;
   uint32_t in_buf_size, in_read, out_len, file_size, file_pos = 0;
   SHA_CTX sha;
   uint8_t hash[SHA_DIGEST_LENGTH];
   size_t nwritten;
   int first = 1;
   int prev_percent = 0, new_percent;

   SHA1_Init(&sha);

   in_buf_size = key_size - PKCS_PADDING_SIZE; 
   in_buf = (uint8_t *) malloc( in_buf_size ); 
   if (!in_buf) {
     return -1;
   }
{'fraction_non_alphanumeric': 0.14203034569059908, 'fraction_numerical': 0.020647583294228063, 'mean_word_length': 2.7969121140142517, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1823672', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2700, 'n_words': 646}