<commit_before>#include <stdio.h>
#include <stdlib.h>

#include <png.h>

#include "n64graphics.h"
#include "utils.h"

// upscale 5-bit integer to 8-bit
#define SCALE_5_8(VAL_) (((VAL_) * 0xFF) / 0x1F)
#define SCALE_8_5(VAL_) (((VAL_) * 0x1F) / 0xFF)
#define SCALE_4_8(VAL_) ((VAL_) * 0x11)
#define SCALE_8_4(VAL_) ((VAL_) / 0x11)
#define SCALE_3_8(VAL_) ((VAL_) * 0x24)
#define SCALE_8_3(VAL_) ((VAL_) / 0x24)

typedef enum
{
   IMG_FORMAT_RGBA,
   IMG_FORMAT_IA,
   IMG_FORMAT_SKYBOX,
} img_format;

rgba *file2rgba(char *filename, int offset, int width, int height, int depth)
{
   FILE *fp;
   rgba *img = NULL;
   char *raw;
   unsigned size;
   unsigned img_size;
   int i;

   fp = fopen(filename, "rb");
   if (!fp) {
      return NULL;
   }

   if (fseek(fp, offset, SEEK_SET)) {
      ERROR("Error seeking to 0x%X in file '%s'\n", offset, filename);
      goto file2rgba_close;
   }

   size = width * height * depth / 8;
   raw = malloc(size);
   if (!raw) {
      ERROR("Error allocating %u bytes\n", size);
      goto file2rgba_close;
   }
   if (fread(raw, 1, size, fp) != size) {
      ERROR("Error reading 0x%X bytes at 0x%X from '%s'\n", size, offset, filename);
      goto file2rgba_free;
   }

   img_size = width * height * sizeof(*img);
   img = malloc(img_size);
   if (!img) {
      ERROR("Error allocating %u bytes\n", img_size);
      goto file2rgba_free;
   }

   if (depth == 16) {
      for (i = 0; i < width * height; i++) {
         img[i].red   = SCALE_5_8((raw[i*2] & 0xF8) >> 3);
         img[i].green = SCALE_5_8(((raw[i*2] & 0x07) << 2) | ((raw[i*2+1] & 0xC0) >> 6));
         img[i].blue  = SCALE_5_8((raw[i*2+1] & 0x3E) >> 1);
         img[i].alpha = (raw[i*2+1] & 0x01) ? 0xFF : 0x00;
      }
   } else if (depth == 32) {
      for (i = 0; i < width * height; i++) {
         img[i].red   = raw[i*4];
         img[i].green = raw[i*4+1];
         img[i].blue  = raw[i*4+2];
         img[i].alpha = raw[i*4+3];
      }
   }

file2rgba_free:
   free(raw);
file2rgba_close:
   fclose(fp);

   return img;
}

ia *file2ia(char *filename, int offset, int width, int height, int depth)
{
   FILE *fp;
   ia *img = NULL;
   char *raw;
   unsigned size;
   unsigned img_size;
   int i;

   fp = fopen(filename, "rb");
   if (!fp) {
      return NULL;
   }

   if (fseek(fp, offset, SEEK_SET)) {
      ERROR("Error seeking to 0x%X in file '%s'\n", offset, filename);
      goto file2ia_close;
   }

   size = width * height * 2; // 16-bit
   raw = malloc(size);
   if (!raw) {
      ERROR("Error allocating %u bytes\n", size);
      goto file2ia_close;
   }
   if (fread(raw, 1, size, fp) != size) {
      ERROR("Error reading 0x%X bytes at 0x%X from '%s'\n", size, offset, filename);
      goto file2ia_free;
   }

   img_size = width * height * sizeof(*img);
   img = malloc(img_size);
   if (!img) {
      ERROR("Error allocating %u bytes\n", img_size);
      goto file2ia_free;
   }

   switch (depth) {
      case 16:
         for (i = 0; i < width * height; i++) {
            img[i].intensity = raw[i*2];
            img[i].alpha     = raw[i*2+1];
         }
         break;
      case 8:
         for (i = 0; i < width * height; i++) {
            img[i].intensity = SCALE_4_8((raw[i] & 0xF0) >> 4);
            img[i].alpha     = SCALE_4_8(raw[i] & 0x0F);
         }
         break;
      case 4:
         for (i = 0; i < width * height; i++) {
            unsigned char bits;
            bits = raw[i/2];
            if (i % 2) {
               bits &= 0xF;
            } else {
               bits >>= 4;
            }
            img[i].intensity = SCALE_3_8((bits >> 1) & 0x07);
            img[i].alpha     = (bits & 0x01) ? 0xFF : 0x00;
         }
         break;
      case 1:
         for (i = 0; i < width * height; i++) {
            unsigned char bits;
            unsigned char mask;
            bits = raw[i/8];
            mask = 1 << (7 - (i % 8)); // MSb->LSb
            bits = (bits & mask) ? 0xFF : 0x00;
            img[i].intensity = bits;
            img[i].alpha     = bits;
         }
         break;
      default:
         ERROR("Error invalid depth %d\n", depth);
         break;
   }

file2ia_free:
   free(raw);
file2ia_close:
   fclose(fp);

   return img;
}

int rgba2file(rgba *img, int offset, int width, int height, int depth, char *filename)
{
   FILE *fp;
   char *raw;
   unsigned size;
   int i;

   fp = fopen(filename, "r+b");
   if (!fp) {
      return -1;
   }

   if (fseek(fp, offset, SEEK_SET)) {
      ERROR("Error seeking to 0x%X in file '%s'\n", offset, filename);
      goto rgba2file_close;
   }

   size = width * height * depth / 8;
   raw = malloc(size);
   if (!raw) {
      ERROR("Error allocating %u bytes\n", size);
      goto rgba2file_close;
   }

   if (depth == 16) {
      for (i = 0; i < width * height; i++) {
         char r, g, b, a;
         r = SCALE_8_5(img[i].red);
         g = SCALE_8_5(img[i].green);
         b = SCALE_8_5(img[i].blue);
         a = img[i].alpha ? 0x1 : 0x0;
         raw[i*2]   = (r << 3) | (g >> 2);
         raw[i*2+1] = ((g & 0x3) << 6) | (b << 1) | a;
      }
   } else if (depth == 32) {
      for (i = 0; i < width * height; i++) {
         raw[i*4]   = img[i].red;
         raw[i*4+1] = img[i].green;
         raw[i*4+2] = img[i].blue;
         raw[i*4+3] = img[i].alpha;
      }
   }
   if (fwrite(raw, 1, size, fp) != size) {
      ERROR("Error writing %u bytes to output file '%s'\n", size, filename);
   }

   free(raw);
rgba2file_close:
   fclose(fp);

   return 0;
}

int ia2file(ia *img, int offset, int width, int height, int depth, char *filename)
{
   FILE *fp;
   char *raw;
   unsigned size = 0;
   int i;

   fp = fopen(filename, "r+b");
   if (!fp) {
      return -1;
   }

   if (fseek(fp, offset, SEEK_SET)) {
      ERROR("Error seeking to 0x%X in file '%s'\n", offset, filename);
      goto ia2file_close;
   }

   switch (depth) {
      case 16:
         size = width * height * 2; // 16-bit
         raw = malloc(size);
         if (!raw) {
            ERROR("Error allocating %u bytes\n", size);
            goto ia2file_close;
         }
         for (i = 0; i < width * height; i++) {
            raw[i*2]   = img[i].intensity;
            raw[i*2+1] = img[i].alpha;
         }
         break;
      case 8:
         size = width * height; // 8-bit
         raw = malloc(size);
         if (!raw) {
            ERROR("Error allocating %u bytes\n", size);
            goto ia2file_close;
         }
         for (i = 0; i < width * height; i++) {
            unsigned char val = SCALE_8_4(img[i].intensity);
            unsigned char alpha = SCALE_8_4(img[i].alpha);
            raw[i] = (val << 4) | alpha;
         }
         break;
      case 4:
         size = width * height / 2; // 4-bit
         raw = malloc(size);
         if (!raw) {
            ERROR("Error allocating %u bytes\n", size);
            goto ia2file_close;
         }
         for (i = 0; i < width * height; i++) {
            unsigned char val = SCALE_8_3(img[i].intensity);
            unsigned char alpha = img[i].alpha ? 0x01 : 0x00;
            unsigned char old = raw[i/2];
            if (i % 2) {
               raw[i/2] = (old & 0xF0) | (val << 1) | alpha;
            } else {
               raw[i/2] = (old & 0x0F) | (((val << 1) | alpha) << 4);
            }
         }
         break;
      case 1:
         size = width * height / 8; // 1-bit
         raw = malloc(size);
         if (!raw) {
            ERROR("Error allocating %u bytes\n", size);
            goto ia2file_close;
         }
         for (i = 0; i < width * height; i++) {
            unsigned char val = img[i].intensity;
            unsigned char old = raw[i/8];
            unsigned char bit = val ? (1 << (7 - (i % 8))) : 0x00;
            raw[i/8] = (old & (~bit)) | bit;
         }
         break;
      default:
         ERROR("Error invalid depth %d\n", depth);
         break;<commit_msg>Fix raw IA 8/4/1 file processing bug<commit_after>#include <stdio.h>
#include <stdlib.h>

#include <png.h>

#include "n64graphics.h"
#include "utils.h"

// upscale 5-bit integer to 8-bit
#define SCALE_5_8(VAL_) (((VAL_) * 0xFF) / 0x1F)
#define SCALE_8_5(VAL_) (((VAL_) * 0x1F) / 0xFF)
#define SCALE_4_8(VAL_) ((VAL_) * 0x11)
#define SCALE_8_4(VAL_) ((VAL_) / 0x11)
#define SCALE_3_8(VAL_) ((VAL_) * 0x24)
#define SCALE_8_3(VAL_) ((VAL_) / 0x24)

typedef enum
{
   IMG_FORMAT_RGBA,
   IMG_FORMAT_IA,
   IMG_FORMAT_SKYBOX,
} img_format;

rgba *file2rgba(char *filename, int offset, int width, int height, int depth)
{
   FILE *fp;
   rgba *img = NULL;
   char *raw;
   unsigned size;
   unsigned img_size;
   int i;

   fp = fopen(filename, "rb");
   if (!fp) {
      return NULL;
   }

   if (fseek(fp, offset, SEEK_SET)) {
      ERROR("Error seeking to 0x%X in file '%s'\n", offset, filename);
      goto file2rgba_close;
   }

   size = width * height * depth / 8;
   raw = malloc(size);
   if (!raw) {
      ERROR("Error allocating %u bytes\n", size);
      goto file2rgba_close;
   }
   if (fread(raw, 1, size, fp) != size) {
      ERROR("Error reading 0x%X bytes at 0x%X from '%s'\n", size, offset, filename);
      goto file2rgba_free;
   }

   img_size = width * height * sizeof(*img);
   img = malloc(img_size);
   if (!img) {
      ERROR("Error allocating %u bytes\n", img_size);
      goto file2rgba_free;
   }

   if (depth == 16) {
      for (i = 0; i < width * height; i++) {
         img[i].red   = SCALE_5_8((raw[i*2] & 0xF8) >> 3);
         img[i].green = SCALE_5_8(((raw[i*2] & 0x07) << 2) | ((raw[i*2+1] & 0xC0) >> 6));
         img[i].blue  = SCALE_5_8((raw[i*2+1] & 0x3E) >> 1);
         img[i].alpha = (raw[i*2+1] & 0x01) ? 0xFF : 0x00;
      }
   } else if (depth == 32) {
      for (i = 0; i < width * height; i++) {
         img[i].red   = raw[i*4];
         img[i].green = raw[i*4+1];
         img[i].blue  = raw[i*4+2];
         img[i].alpha = raw[i*4+3];
      }
   }

file2rgba_free:
   free(raw);
file2rgba_close:
   fclose(fp);

   return img;
}

ia *file2ia(char *filename, int offset, int width, int height, int depth)
{
   FILE *fp;
   ia *img = NULL;
   char *raw;
   unsigned size;
   unsigned img_size;
   int i;

   fp = fopen(filename, "rb");
   if (!fp) {
      return NULL;
   }

   if (fseek(fp, offset, SEEK_SET)) {
      ERROR("Error seeking to 0x%X in file '%s'\n", offset, filename);
      goto file2ia_close;
   }

   size = width * height * depth / 8;
   raw = malloc(size);
   if (!raw) {
      ERROR("Error allocating %u bytes\n", size);
      goto file2ia_close;
   }
   if (fread(raw, 1, size, fp) != size) {
      ERROR("Error reading 0x%X bytes at 0x%X from '%s'\n", size, offset, filename);
      goto file2ia_free;
   }

   img_size = width * height * sizeof(*img);
   img = malloc(img_size);
   if (!img) {
      ERROR("Error allocating %u bytes\n", img_size);
      goto file2ia_free;
   }

   switch (depth) {
      case 16:
         for (i = 0; i < width * height; i++) {
            img[i].intensity = raw[i*2];
            img[i].alpha     = raw[i*2+1];
         }
         break;
      case 8:
         for (i = 0; i < width * height; i++) {
            img[i].intensity = SCALE_4_8((raw[i] & 0xF0) >> 4);
            img[i].alpha     = SCALE_4_8(raw[i] & 0x0F);
         }
         break;
      case 4:
         for (i = 0; i < width * height; i++) {
            unsigned char bits;
            bits = raw[i/2];
            if (i % 2) {
               bits &= 0xF;
            } else {
               bits >>= 4;
            }
            img[i].intensity = SCALE_3_8((bits >> 1) & 0x07);
            img[i].alpha     = (bits & 0x01) ? 0xFF : 0x00;
         }
         break;
      case 1:
         for (i = 0; i < width * height; i++) {
            unsigned char bits;
            unsigned char mask;
            bits = raw[i/8];
            mask = 1 << (7 - (i % 8)); // MSb->LSb
            bits = (bits & mask) ? 0xFF : 0x00;
            img[i].intensity = bits;
            img[i].alpha     = bits;
         }
         break;
      default:
         ERROR("Error invalid depth %d\n", depth);
         break;
   }

file2ia_free:
   free(raw);
file2ia_close:
   fclose(fp);

   return img;
}

int rgba2file(rgba *img, int offset, int width, int height, int depth, char *filename)
{
   FILE *fp;
   char *raw;
   unsigned size;
   int i;

   fp = fopen(filename, "r+b");
   if (!fp) {
      return -1;
   }

   if (fseek(fp, offset, SEEK_SET)) {
      ERROR("Error seeking to 0x%X in file '%s'\n", offset, filename);
      goto rgba2file_close;
   }

   size = width * height * depth / 8;
   raw = malloc(size);
   if (!raw) {
      ERROR("Error allocating %u bytes\n", size);
      goto rgba2file_close;
   }

   if (depth == 16) {
      for (i = 0; i < width * height; i++) {
         char r, g, b, a;
         r = SCALE_8_5(img[i].red);
         g = SCALE_8_5(img[i].green);
         b = SCALE_8_5(img[i].blue);
         a = img[i].alpha ? 0x1 : 0x0;
         raw[i*2]   = (r << 3) | (g >> 2);
         raw[i*2+1] = ((g & 0x3) << 6) | (b << 1) | a;
      }
   } else if (depth == 32) {
      for (i = 0; i < width * height; i++) {
         raw[i*4]   = img[i].red;
         raw[i*4+1] = img[i].green;
         raw[i*4+2] = img[i].blue;
         raw[i*4+3] = img[i].alpha;
      }
   }
   if (fwrite(raw, 1, size, fp) != size) {
      ERROR("Error writing %u bytes to output file '%s'\n", size, filename);
   }

   free(raw);
rgba2file_close:
   fclose(fp);

   return 0;
}

int ia2file(ia *img, int offset, int width, int height, int depth, char *filename)
{
   FILE *fp;
   char *raw;
   unsigned size;
   int i;

   fp = fopen(filename, "r+b");
   if (!fp) {
      return -1;
   }

   if (fseek(fp, offset, SEEK_SET)) {
      ERROR("Error seeking to 0x%X in file '%s'\n", offset, filename);
      goto ia2file_close;
   }

   size = width * height * depth / 8;
   raw = malloc(size);
   if (!raw) {
      ERROR("Error allocating %u bytes\n", size);
      goto ia2file_close;
   }

   switch (depth) {
      case 16:
         for (i = 0; i < width * height; i++) {
            raw[i*2]   = img[i].intensity;
            raw[i*2+1] = img[i].alpha;
         }
         break;
      case 8:
         for (i = 0; i < width * height; i++) {
            unsigned char val = SCALE_8_4(img[i].intensity);
            unsigned char alpha = SCALE_8_4(img[i].alpha);
            raw[i] = (val << 4) | alpha;
         }
         break;
      case 4:
         for (i = 0; i < width * height; i++) {
            unsigned char val = SCALE_8_3(img[i].intensity);
            unsigned char alpha = img[i].alpha ? 0x01 : 0x00;
            unsigned char old = raw[i/2];
            if (i % 2) {
               raw[i/2] = (old & 0xF0) | (val << 1) | alpha;
            } else {
               raw[i/2] = (old & 0x0F) | (((val << 1) | alpha) << 4);
            }
         }
         break;
      case 1:
         for (i = 0; i < width * height; i++) {
            unsigned char val = img[i].intensity;
            unsigned char old = raw[i/8];
            unsigned char bit = val ? (1 << (7 - (i % 8))) : 0x00;
            raw[i/8] = (old & (~bit)) | bit;
         }
         break;
      default:
         ERROR("Error invalid depth %d\n", depth);
         break;
{'fraction_non_alphanumeric': 0.1640604467805519, 'fraction_numerical': 0.0328515111695138, 'mean_word_length': 1.7984923699209414, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2169017', 'n_tokens_mistral': 6022, 'n_tokens_neox': 5674, 'n_words': 1487}