<commit_before>/*
 *  Wholeslide, a library for reading whole slide image files
 *
 *  Copyright (c) 2007-2008 Carnegie Mellon University
 *  All rights reserved.
 *
 *  Wholeslide is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, version 2.
 *
 *  Wholeslide is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Wholeslide. If not, see <http://www.gnu.org/licenses/>.
 *
 *  Linking Wholeslide statically or dynamically with other modules is
 *  making a combined work based on Wholeslide. Thus, the terms and
 *  conditions of the GNU General Public License cover the whole
 *  combination.
 */

/*
 * Part of this file is:
 *
 * Copyright (C) 1994-1996, <NAME>.
 * This file is part of the Independent JPEG Group's software.
 * For conditions of distribution and use, see the accompanying README file.
 */

#include "config.h"

#include <glib.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <jpeglib.h>
#include <jerror.h>
#include <inttypes.h>

#include <sys/types.h>   // for off_t ?

#include "wholeslide-private.h"

struct one_jpeg {
  struct jpeg_decompress_struct cinfo;
  struct jpeg_error_mgr jerr;

  FILE *f;

  uint64_t mcu_starts_count;
  int64_t *mcu_starts;

  uint32_t tile_width;
  uint32_t tile_height;

  uint32_t width;
  uint32_t height;

  char *comment;
};

struct layer {
  struct one_jpeg **layer_jpegs; // count given by jpeg_w * jpeg_h

  // total size, premultipled by 1/scale_denom
  int64_t pixel_w;
  int64_t pixel_h;

  uint32_t jpeg_w;       // how many distinct jpeg files across?
  uint32_t jpeg_h;       // how many distinct jpeg files down?

  uint32_t scale_denom;
};

struct jpegops_data {
  uint32_t jpeg_count;
  struct one_jpeg *all_jpegs;

  // layer_count is in the wsd struct
  struct layer *layers;
};


static bool is_zxy_successor(int64_t pz, int64_t px, int64_t py,
			     int64_t z, int64_t x, int64_t y) {
  g_debug("p_zxy: (%" PRId64 ",%" PRId64 ",%" PRId64 "), zxy: (%"
	  PRId64 ",%" PRId64 ",%" PRId64 ")",
	  pz, px, py, z, x, y);
  if (z == pz + 1) {
    return x == 0 && y == 0;
  }
  if (z != pz) {
    return false;
  }

  // z == pz

  if (y == py + 1) {
    return x == 0;
  }
  if (y != py) {
    return false;
  }

  // y == py

  return x == px + 1;
}

static guint int64_hash(gconstpointer v) {
  int64_t i = *((const int64_t *) v);
  return i ^ (i >> 32);
}

static gboolean int64_equal(gconstpointer v1, gconstpointer v2) {
  return *((int64_t *) v1) == *((int64_t *) v2);
}

static void int64_free(gpointer data) {
  g_slice_free(int64_t, data);
}

static void layer_free(gpointer data) {
  //  g_debug("layer_free: %p", data);

  struct layer *l = data;

  //  g_debug("g_free(%p)", (void *) l->layer_jpegs);
  g_free(l->layer_jpegs);
  g_slice_free(struct layer, l);
}

static void print_wlmap_entry(gpointer key, gpointer value,
			      gpointer user_data) {
  int64_t k = *((int64_t *) key);
  struct layer *v = (struct layer *) value;

  g_debug("%" PRId64 " -> ( pw: %" PRId64 ", ph: %" PRId64
	  ", jw: %" PRId32 ", jh: %" PRId32 ", scale_denom: %" PRId32 " )",
	  k, v->pixel_w, v->pixel_h, v->jpeg_w, v->jpeg_h, v->scale_denom);
}

static void generate_layers_into_map(GSList *jpegs,
				     uint32_t jpeg_w, uint32_t jpeg_h,
				     int64_t pixel_w, int64_t pixel_h,
				     GHashTable *width_to_layer_map) {
  // JPEG files can give us 1/1, 1/2, 1/4, 1/8 downsamples, so we
  // need to create 4 layers per set of JPEGs

  uint32_t num_jpegs = jpeg_w * jpeg_h;

  int scale_denom = 1;
  while (scale_denom <= 8) {
    // create layer
    struct layer *l = g_slice_new0(struct layer);
    l->jpeg_w = jpeg_w;
    l->jpeg_h = jpeg_h;
    l->pixel_w = pixel_w / scale_denom;
    l->pixel_h = pixel_h / scale_denom;
    l->scale_denom = scale_denom;

    // create array and copy
    l->layer_jpegs = g_new(struct one_jpeg *, num_jpegs);
    //    g_debug("g_new(struct one_jpeg *) -> %p", (void *) l->layer_jpegs);
    GSList *jj = jpegs;
    for (uint32_t i = 0; i < num_jpegs; i++) {
      g_assert(jj);
      l->layer_jpegs[i] = (struct one_jpeg *) jj->data;
      jj = jj->next;<commit_msg>Rename jpeg_w and jpeg_h to jpegs_across and jpegs_down<commit_after>/*
 *  Wholeslide, a library for reading whole slide image files
 *
 *  Copyright (c) 2007-2008 Carnegie Mellon University
 *  All rights reserved.
 *
 *  Wholeslide is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, version 2.
 *
 *  Wholeslide is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Wholeslide. If not, see <http://www.gnu.org/licenses/>.
 *
 *  Linking Wholeslide statically or dynamically with other modules is
 *  making a combined work based on Wholeslide. Thus, the terms and
 *  conditions of the GNU General Public License cover the whole
 *  combination.
 */

/*
 * Part of this file is:
 *
 * Copyright (C) 1994-1996, <NAME>.
 * This file is part of the Independent JPEG Group's software.
 * For conditions of distribution and use, see the accompanying README file.
 */

#include "config.h"

#include <glib.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <jpeglib.h>
#include <jerror.h>
#include <inttypes.h>

#include <sys/types.h>   // for off_t ?

#include "wholeslide-private.h"

struct one_jpeg {
  struct jpeg_decompress_struct cinfo;
  struct jpeg_error_mgr jerr;

  FILE *f;

  uint64_t mcu_starts_count;
  int64_t *mcu_starts;

  uint32_t tile_width;
  uint32_t tile_height;

  uint32_t width;
  uint32_t height;

  char *comment;
};

struct layer {
  struct one_jpeg **layer_jpegs; // count given by jpeg_w * jpeg_h

  // total size, premultipled by 1/scale_denom
  int64_t pixel_w;
  int64_t pixel_h;

  uint32_t jpegs_across;       // how many distinct jpeg files across?
  uint32_t jpegs_down;         // how many distinct jpeg files down?

  uint32_t scale_denom;
};

struct jpegops_data {
  uint32_t jpeg_count;
  struct one_jpeg *all_jpegs;

  // layer_count is in the wsd struct
  struct layer *layers;
};


static bool is_zxy_successor(int64_t pz, int64_t px, int64_t py,
			     int64_t z, int64_t x, int64_t y) {
  g_debug("p_zxy: (%" PRId64 ",%" PRId64 ",%" PRId64 "), zxy: (%"
	  PRId64 ",%" PRId64 ",%" PRId64 ")",
	  pz, px, py, z, x, y);
  if (z == pz + 1) {
    return x == 0 && y == 0;
  }
  if (z != pz) {
    return false;
  }

  // z == pz

  if (y == py + 1) {
    return x == 0;
  }
  if (y != py) {
    return false;
  }

  // y == py

  return x == px + 1;
}

static guint int64_hash(gconstpointer v) {
  int64_t i = *((const int64_t *) v);
  return i ^ (i >> 32);
}

static gboolean int64_equal(gconstpointer v1, gconstpointer v2) {
  return *((int64_t *) v1) == *((int64_t *) v2);
}

static void int64_free(gpointer data) {
  g_slice_free(int64_t, data);
}

static void layer_free(gpointer data) {
  //  g_debug("layer_free: %p", data);

  struct layer *l = data;

  //  g_debug("g_free(%p)", (void *) l->layer_jpegs);
  g_free(l->layer_jpegs);
  g_slice_free(struct layer, l);
}

static void print_wlmap_entry(gpointer key, gpointer value,
			      gpointer user_data) {
  int64_t k = *((int64_t *) key);
  struct layer *v = (struct layer *) value;

  g_debug("%" PRId64 " -> ( pw: %" PRId64 ", ph: %" PRId64
	  ", jw: %" PRId32 ", jh: %" PRId32 ", scale_denom: %" PRId32 " )",
	  k, v->pixel_w, v->pixel_h, v->jpegs_across, v->jpegs_down, v->scale_denom);
}

static void generate_layers_into_map(GSList *jpegs,
				     uint32_t jpegs_across, uint32_t jpegs_down,
				     int64_t pixel_w, int64_t pixel_h,
				     GHashTable *width_to_layer_map) {
  // JPEG files can give us 1/1, 1/2, 1/4, 1/8 downsamples, so we
  // need to create 4 layers per set of JPEGs

  uint32_t num_jpegs = jpegs_across * jpegs_down;

  int scale_denom = 1;
  while (scale_denom <= 8) {
    // create layer
    struct layer *l = g_slice_new0(struct layer);
    l->jpegs_across = jpegs_across;
    l->jpegs_down = jpegs_down;
    l->pixel_w = pixel_w / scale_denom;
    l->pixel_h = pixel_h / scale_denom;
    l->scale_denom = scale_denom;

    // create array and copy
    l->layer_jpegs = g_new(struct one_jpeg *, num_jpegs);
    //    g_debug("g_new(struct one_jpeg *) -> %p", (void *) l->layer_jpegs);
    GSList *jj = jpegs;
    for (uint32_t i = 0; i < num_jpegs; i++) {
      g_assert(jj);
      l->layer_jpegs[i] = (struct one_jpeg *) jj->data;
      jj = jj->next;
{'fraction_non_alphanumeric': 0.12155305537171851, 'fraction_numerical': 0.029781601588352084, 'mean_word_length': 3.186057248384118, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2819284', 'n_tokens_mistral': 3724, 'n_tokens_neox': 3450, 'n_words': 1014}