<commit_before>// Sprite packing method borrowed from glorp engine and heavily modified.
// For license safety, just run this as a build tool, don't build it into your game/program.
// https://github.com/zorbathut/glorp

// data we need to provide:
// sx, sy
// dx, dy
// ox, oy
// wx

// line height
// dist-per-pixel

#include <png.h>
#include <ft2build.h>
#include FT_FREETYPE_H
#include <freetype/ftbitmap.h>
#include <set>
#include <map>
#include <vector>
#include <algorithm>
#include <string>
#include <cmath>

#include "gfx/texture_atlas.h"

#include "image/png_load.h"
#include "image/zim_save.h"

#include "kanjifilter.h"
#include "util/text/utf8.h"

#define CHECK(x) if (!(x)) { printf("%i: CHECK failed on this line\n", __LINE__); exit(1); }

using namespace std;
static int global_id;
static bool highcolor = false;




typedef unsigned short u16;

struct CharRange : public AtlasCharRange {
	std::set<u16> filter;
};

enum Effect {
  FX_COPY = 0,
  FX_RED_TO_ALPHA_SOLID_WHITE = 1,   // for alpha fonts
  FX_RED_TO_INTENSITY_ALPHA_255 = 2,
  FX_PREMULTIPLY_ALPHA = 3,
  FX_PINK_TO_ALPHA = 4,   // for alpha fonts
  FX_INVALID = 5,
};

const char *effect_str[5] = {
  "copy", "r2a", "r2i", "pre", "p2a",
};

Effect GetEffect(const char *text) {
  for (int i = 0; i < 5; i++) {
    if (!strcmp(text, effect_str[i])) {
      return (Effect)i;
    }
  }
  return FX_INVALID;
}

struct FontReference {
	FontReference(string name, string file, vector<CharRange> ranges, int pixheight)
		: name_(name), file_(file), ranges_(ranges), size_(pixheight) {
	}

	string name_;
	string file_;
	vector<CharRange> ranges_;
	int size_;
};

typedef vector<FontReference> FontReferenceList;

template<class T>
struct Image {
  vector<vector<T> > dat;
  void resize(int x, int y) {
    dat.resize(y);
    for(int i = 0; i < y; i++)
      dat[i].resize(x);
  }
  int width() const {
    return (int)dat[0].size();
  }
  int height() const {
    return (int)dat.size();
  }
  void copyfrom(const Image &img, int ox, int oy, int effect) {
    CHECK(img.dat[0].size() + ox <= dat[0].size());
    CHECK(img.dat.size() + oy <= dat.size());
    for (int y = 0; y < (int)img.dat.size(); y++) {
      for (int x = 0; x < (int)img.dat[y].size(); x++) {
        switch (effect) {
        case FX_COPY:
          dat[y + oy][ox + x] = img.dat[y][x];
          break;
        case FX_RED_TO_ALPHA_SOLID_WHITE:
          dat[y + oy][ox + x] = 0x00FFFFFF | (img.dat[y][x] << 24);
          break;
        case FX_RED_TO_INTENSITY_ALPHA_255:
          dat[y + oy][ox + x] = 0xFF000000 | img.dat[y][x] | (img.dat[y][x] << 8) | (img.dat[y][x] << 16);
          break;
        case FX_PREMULTIPLY_ALPHA:
        {
          unsigned int color = img.dat[y][x];
          unsigned int a = color >> 24;
          unsigned int r = (color & 0xFF) * a >> 8, g = (color & 0xFF00) * a>> 8, b = (color & 0xFF0000) * a >> 8;
          color = (color & 0xFF000000) | (r & 0xFF) | (g & 0xFF00) | (b & 0xFF0000);
          // Simulate 4444
          color = color & 0xF0F0F0F0;
          color |= color >> 4;
          dat[y + oy][ox + x] = color;
          break;
        }
        case FX_PINK_TO_ALPHA:
          dat[y + oy][ox + x] = ((img.dat[y][x]&0xFFFFFF) == 0xFF00FF) ? 0x00FFFFFF : (img.dat[y][x] | 0xFF000000);
          break;
        default:
          dat[y + oy][ox + x] = 0xFFFF00FF;
          break;
        }
      }
    }
  }
  void set(int sx, int sy, int ex, int ey, unsigned char fil) {
    for(int y = sy; y < ey; y++)
      fill(dat[y].begin() + sx, dat[y].begin() + ex, fil);
  }
  bool LoadPNG(const char *png_name) {
    unsigned char *img_data;
    int w, h;
    if (1 != pngLoad(png_name, &w, &h, &img_data, false)) {
      printf("Failed to load %s\n", png_name);
      exit(1);
      return false;
    }
    dat.resize(h);
    for (int y = 0; y < h; y++) {
      dat[y].resize(w);
      memcpy(&dat[y][0], img_data + 4 * y * w, 4 * w);
    }
    free(img_data);
    return true;
  }
  void SavePNG(const char *png_name) {
    // Save PNG
    FILE *fil = fopen(png_name, "wb");
    png_structp  png_ptr;
    png_infop  info_ptr;
    png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
    CHECK(png_ptr);
    info_ptr = png_create_info_struct(png_ptr);
    CHECK(info_ptr);
    png_init_io(png_ptr, fil);
    //png_set_compression_level(png_ptr, Z_BEST_COMPRESSION);
    png_set_IHDR(png_ptr, info_ptr, dat[0].size(), dat.size(), 8, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
    png_write_info(png_ptr, info_ptr);
    for(int y = 0; y < (int)dat.size(); y++) {
      png_write_row(png_ptr, (png_byte*)&dat[y][0]);
    }
    png_write_end(png_ptr, NULL);
    png_destroy_write_struct(&png_ptr, &info_ptr);
  }
  void SaveZIM(const char *zim_name, int zim_format) {
    uint8 *image_data = new uint8[width() * height() * 4];
    for (int y = 0; y < height(); y++) {
      memcpy(image_data + y * width() * 4, &dat[y][0], width() * 4);
    }
    ::SaveZIM(zim_name, width(), height(), width() * 4, zim_format | ZIM_DITHER, image_data);
  }
};

template<class S, class T>
bool operator<(const Image<S> &lhs, const Image<T> &rhs) {
  return lhs.dat.size() * lhs.dat[0].size() > rhs.dat.size() * rhs.dat[0].size();
}

struct Data {
  // item ID
  int id;
  // dimensions of its spot in the world
  int sx, sy, ex, ey;
  // offset from the origin
  float ox, oy;
  // distance to move the origin forward
  float wx;

  int effect;
	int charNum;
};

bool operator<(const Data &lhs, const Data &rhs) {
  return lhs.id < rhs.id; // should be unique
}

string out_prefix;

int NextPowerOf2(int x) {
  int powof2 = 1;
  // Double powof2 until >= val
  while (powof2 < x) powof2 <<= 1;
  return powof2;
}

struct Bucket {
  vector<pair<Image<unsigned int>, Data> > items;
  void AddItem(const Image<unsigned int> &img, const Data &dat) {
    items.push_back(make_pair(img, dat));
  }
  vector<Data> Resolve(int image_width, Image<unsigned int> &dest) {
    // Place all the little images - whatever they are.
    // Uses greedy fill algorithm. Slow but works surprisingly well, CPUs are fast.
    Image<unsigned char> masq;
    masq.resize(image_width, 1);
    dest.resize(image_width, 1);
    sort(items.begin(), items.end());
    for (int i = 0; i < (int)items.size(); i++) {
			if ((i + 1) % 200 == 0) {
				printf("Resolving (%i / %i)\n", i, (int)items.size());
			}
      int idx = items[i].first.dat[0].size();
      int idy = items[i].first.dat.size();
			if (idx > 1 && idy > 1) {
				CHECK(idx <= image_width);
				bool found = false;
				for (int ty = 0; ty < 2047 && !found; ty++) {
					if(ty + idy + 1 > (int)dest.dat.size()) {
						masq.resize(image_width, ty + idy + 16);
						dest.resize(image_width, ty + idy + 16);
					}
					// Brute force packing.
					int sz = (int)items[i].first.dat[0].size();
					for (int tx = 0; tx < image_width - sz && !found; tx++) {
						bool valid = !(masq.dat[ty][tx] || masq.dat[ty + idy - 1][tx] || masq.dat[ty][tx + idx - 1] || masq.dat[ty + idy - 1][tx + idx - 1]);
						if (valid) {
							for(int ity = 0; ity < idy && valid; ity++) {
								for(int itx = 0; itx < idx && valid; itx++) {
									if(masq.dat[ty + ity][tx + itx]) {
										goto skip;
									}
								}
							}
							dest.copyfrom(items[i].first, tx, ty, items[i].second.effect);
							masq.set(tx, ty, tx + idx + 1, ty + idy + 1, 255);

							items[i].second.sx = tx;
							items[i].second.sy = ty;

							items[i].second.ex = tx + idx;
							items[i].second.ey = ty + idy;

							found = true;

							// printf("Placed %d at %dx%d-%dx%d\n", items[i].second.id, tx, ty, tx + idx, ty + idy);
						}
skip:
						;
					}
				}
      }
    }

    if ((int)dest.dat.size() > image_width * 2) {
      printf("PACKING FAIL : height=%i", (int)dest.dat.size());
      exit(1);
    }
    dest.resize(image_width, NextPowerOf2(dest.dat.size()));

    // Output the glyph data.
    vector<Data> dats;
    for(int i = 0; i < (int)items.size(); i++)
      dats.push_back(items[i].second);
    return dats;
  }
};

const int supersample = 16;
const int distmult = 64 * 3;  // this is "one pixel in the final version equals 64 difference". reduce this number to increase the "blur" radius, increase it to make things "sharper"
const int maxsearch = (128 * supersample + distmult - 1) / distmult;

struct Closest {
  FT_Bitmap bmp;
  Closest(FT_Bitmap bmp) : bmp(bmp) { }
  float find_closest(int x, int y, char search) {
    int best = 1 << 30;
    for(int i = 1; i <= maxsearch; i++) {
      if(i * i >= best)
        break;
      for(int f = -i; f < i; f++) {
        int dist = i * i + f * f;
        if(dist >= best) continue;
        if(safe_access(x + i, y + f) == search || safe_access(x - f, y + i) == search || safe_access(x - i, y - f) == search || safe_access(x + f, y - i) == search)<commit_msg>Shave 23% of time off of atlastool.<commit_after>// Sprite packing method borrowed from glorp engine and heavily modified.
// For license safety, just run this as a build tool, don't build it into your game/program.
// https://github.com/zorbathut/glorp

// data we need to provide:
// sx, sy
// dx, dy
// ox, oy
// wx

// line height
// dist-per-pixel

#include <png.h>
#include <ft2build.h>
#include FT_FREETYPE_H
#include <freetype/ftbitmap.h>
#include <set>
#include <map>
#include <vector>
#include <algorithm>
#include <string>
#include <cmath>

#include "gfx/texture_atlas.h"

#include "image/png_load.h"
#include "image/zim_save.h"

#include "kanjifilter.h"
#include "util/text/utf8.h"

#define CHECK(x) if (!(x)) { printf("%i: CHECK failed on this line\n", __LINE__); exit(1); }

using namespace std;
static int global_id;
static bool highcolor = false;




typedef unsigned short u16;

struct CharRange : public AtlasCharRange {
	std::set<u16> filter;
};

enum Effect {
  FX_COPY = 0,
  FX_RED_TO_ALPHA_SOLID_WHITE = 1,   // for alpha fonts
  FX_RED_TO_INTENSITY_ALPHA_255 = 2,
  FX_PREMULTIPLY_ALPHA = 3,
  FX_PINK_TO_ALPHA = 4,   // for alpha fonts
  FX_INVALID = 5,
};

const char *effect_str[5] = {
  "copy", "r2a", "r2i", "pre", "p2a",
};

Effect GetEffect(const char *text) {
  for (int i = 0; i < 5; i++) {
    if (!strcmp(text, effect_str[i])) {
      return (Effect)i;
    }
  }
  return FX_INVALID;
}

struct FontReference {
	FontReference(string name, string file, vector<CharRange> ranges, int pixheight)
		: name_(name), file_(file), ranges_(ranges), size_(pixheight) {
	}

	string name_;
	string file_;
	vector<CharRange> ranges_;
	int size_;
};

typedef vector<FontReference> FontReferenceList;

template<class T>
struct Image {
  vector<vector<T> > dat;
  void resize(int x, int y) {
    dat.resize(y);
    for(int i = 0; i < y; i++)
      dat[i].resize(x);
  }
  int width() const {
    return (int)dat[0].size();
  }
  int height() const {
    return (int)dat.size();
  }
  void copyfrom(const Image &img, int ox, int oy, int effect) {
    CHECK(img.dat[0].size() + ox <= dat[0].size());
    CHECK(img.dat.size() + oy <= dat.size());
    for (int y = 0; y < (int)img.dat.size(); y++) {
      for (int x = 0; x < (int)img.dat[y].size(); x++) {
        switch (effect) {
        case FX_COPY:
          dat[y + oy][ox + x] = img.dat[y][x];
          break;
        case FX_RED_TO_ALPHA_SOLID_WHITE:
          dat[y + oy][ox + x] = 0x00FFFFFF | (img.dat[y][x] << 24);
          break;
        case FX_RED_TO_INTENSITY_ALPHA_255:
          dat[y + oy][ox + x] = 0xFF000000 | img.dat[y][x] | (img.dat[y][x] << 8) | (img.dat[y][x] << 16);
          break;
        case FX_PREMULTIPLY_ALPHA:
        {
          unsigned int color = img.dat[y][x];
          unsigned int a = color >> 24;
          unsigned int r = (color & 0xFF) * a >> 8, g = (color & 0xFF00) * a>> 8, b = (color & 0xFF0000) * a >> 8;
          color = (color & 0xFF000000) | (r & 0xFF) | (g & 0xFF00) | (b & 0xFF0000);
          // Simulate 4444
          color = color & 0xF0F0F0F0;
          color |= color >> 4;
          dat[y + oy][ox + x] = color;
          break;
        }
        case FX_PINK_TO_ALPHA:
          dat[y + oy][ox + x] = ((img.dat[y][x]&0xFFFFFF) == 0xFF00FF) ? 0x00FFFFFF : (img.dat[y][x] | 0xFF000000);
          break;
        default:
          dat[y + oy][ox + x] = 0xFFFF00FF;
          break;
        }
      }
    }
  }
  void set(int sx, int sy, int ex, int ey, unsigned char fil) {
    for(int y = sy; y < ey; y++)
      fill(dat[y].begin() + sx, dat[y].begin() + ex, fil);
  }
  bool LoadPNG(const char *png_name) {
    unsigned char *img_data;
    int w, h;
    if (1 != pngLoad(png_name, &w, &h, &img_data, false)) {
      printf("Failed to load %s\n", png_name);
      exit(1);
      return false;
    }
    dat.resize(h);
    for (int y = 0; y < h; y++) {
      dat[y].resize(w);
      memcpy(&dat[y][0], img_data + 4 * y * w, 4 * w);
    }
    free(img_data);
    return true;
  }
  void SavePNG(const char *png_name) {
    // Save PNG
    FILE *fil = fopen(png_name, "wb");
    png_structp  png_ptr;
    png_infop  info_ptr;
    png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
    CHECK(png_ptr);
    info_ptr = png_create_info_struct(png_ptr);
    CHECK(info_ptr);
    png_init_io(png_ptr, fil);
    //png_set_compression_level(png_ptr, Z_BEST_COMPRESSION);
    png_set_IHDR(png_ptr, info_ptr, dat[0].size(), dat.size(), 8, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT);
    png_write_info(png_ptr, info_ptr);
    for(int y = 0; y < (int)dat.size(); y++) {
      png_write_row(png_ptr, (png_byte*)&dat[y][0]);
    }
    png_write_end(png_ptr, NULL);
    png_destroy_write_struct(&png_ptr, &info_ptr);
  }
  void SaveZIM(const char *zim_name, int zim_format) {
    uint8 *image_data = new uint8[width() * height() * 4];
    for (int y = 0; y < height(); y++) {
      memcpy(image_data + y * width() * 4, &dat[y][0], width() * 4);
    }
    ::SaveZIM(zim_name, width(), height(), width() * 4, zim_format | ZIM_DITHER, image_data);
  }
};

template<class S, class T>
bool operator<(const Image<S> &lhs, const Image<T> &rhs) {
  return lhs.dat.size() * lhs.dat[0].size() > rhs.dat.size() * rhs.dat[0].size();
}

struct Data {
  // item ID
  int id;
  // dimensions of its spot in the world
  int sx, sy, ex, ey;
  // offset from the origin
  float ox, oy;
  // distance to move the origin forward
  float wx;

  int effect;
	int charNum;
};

bool operator<(const Data &lhs, const Data &rhs) {
  return lhs.id < rhs.id; // should be unique
}

string out_prefix;

int NextPowerOf2(int x) {
  int powof2 = 1;
  // Double powof2 until >= val
  while (powof2 < x) powof2 <<= 1;
  return powof2;
}

struct Bucket {
	vector<pair<Image<unsigned int>, Data> > items;
	void AddItem(const Image<unsigned int> &img, const Data &dat) {
		items.push_back(make_pair(img, dat));
	}
	vector<Data> Resolve(int image_width, Image<unsigned int> &dest) {
		// Place all the little images - whatever they are.
		// Uses greedy fill algorithm. Slow but works surprisingly well, CPUs are fast.
		Image<unsigned char> masq;
		masq.resize(image_width, 1);
		dest.resize(image_width, 1);
		sort(items.begin(), items.end());
		for (int i = 0; i < (int)items.size(); i++) {
			if ((i + 1) % 200 == 0) {
				printf("Resolving (%i / %i)\n", i, (int)items.size());
			}
			int idx = items[i].first.dat[0].size();
			int idy = items[i].first.dat.size();
			if (idx > 1 && idy > 1) {
				CHECK(idx <= image_width);
				for (int ty = 0; ty < 2047; ty++) {
					if(ty + idy + 1 > (int)dest.dat.size()) {
						masq.resize(image_width, ty + idy + 16);
						dest.resize(image_width, ty + idy + 16);
					}
					// Brute force packing.
					int sz = (int)items[i].first.dat[0].size();
					auto &masq_ty = masq.dat[ty];
					auto &masq_idy = masq.dat[ty + idy - 1];
					for (int tx = 0; tx < image_width - sz; tx++) {
						bool valid = !(masq_ty[tx] || masq_idy[tx] || masq_ty[tx + idx - 1] || masq_idy[tx + idx - 1]);
						if (valid) {
							for(int ity = 0; ity < idy && valid; ity++) {
								for(int itx = 0; itx < idx && valid; itx++) {
									if(masq.dat[ty + ity][tx + itx]) {
										goto skip;
									}
								}
							}
							dest.copyfrom(items[i].first, tx, ty, items[i].second.effect);
							masq.set(tx, ty, tx + idx + 1, ty + idy + 1, 255);

							items[i].second.sx = tx;
							items[i].second.sy = ty;

							items[i].second.ex = tx + idx;
							items[i].second.ey = ty + idy;

							// printf("Placed %d at %dx%d-%dx%d\n", items[i].second.id, tx, ty, tx + idx, ty + idy);
							goto found;
						}
skip:
						;
					}
				}
found:
				;
			}
		}

    if ((int)dest.dat.size() > image_width * 2) {
      printf("PACKING FAIL : height=%i", (int)dest.dat.size());
      exit(1);
    }
    dest.resize(image_width, NextPowerOf2(dest.dat.size()));

    // Output the glyph data.
    vector<Data> dats;
    for(int i = 0; i < (int)items.size(); i++)
      dats.push_back(items[i].second);
    return dats;
  }
};

const int supersample = 16;
const int distmult = 64 * 3;  // this is "one pixel in the final version equals 64 difference". reduce this number to increase the "blur" radius, increase it to make things "sharper"
const int maxsearch = (128 * supersample + distmult - 1) / distmult;

struct Closest {
  FT_Bitmap bmp;
  Closest(FT_Bitmap bmp) : bmp(bmp) { }
  float find_closest(int x, int y, char search) {
    int best = 1 << 30;
    for(int i = 1; i <= maxsearch; i++) {
      if(i * i >= best)
        break;
      for(int f = -i; f < i; f++) {
        int dist = i * i + f * f;
        if(dist >= best) continue;
        if(safe_access(x + i, y + f) == search || safe_access(x - f, y + i) == search || safe_access(x - i, y - f) == search || safe_access(x + f, y - i) == search)
{'fraction_non_alphanumeric': 0.16625435735972113, 'fraction_numerical': 0.0207466546722141, 'mean_word_length': 2.6689356435643563, 'pattern_counts': {'":': 0, '<': 123, '<?xml version=': 0, '>': 93, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6845442', 'n_tokens_mistral': 7579, 'n_tokens_neox': 6819, 'n_words': 1785}