<commit_before>/*-
 * Copyright (c) 2014 <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the author nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <glew.h>

#include "../frac.h"
#include "../defs.h"
#include "../alloc.h"
#include "../rand.h"
#include "../gl/marshal.h"
#include "../gl/shaders.h"
#include "canvas.h"
#include "linear-paint-tile.h"
#include "glbrush.h"

static GLuint noise_texture;
static GLuint perlin_texture;
#define TEXSZ 256

struct glbrush_handle_s {
  glm_slab_group* glmsg_line, * glmsg_point;
  GLuint pallet_texture;
  float decay, noise;
};

static void glbrush_activate_line(glbrush_handle*);
static void glbrush_activate_point(glbrush_handle*);
static void glbrush_deactivate_point(glbrush_handle*);

void glbrush_load(void) {
  canvas* canv, * brush;
  unsigned char monochrome[TEXSZ*TEXSZ];
  canvas_pixel pallet[2];
  unsigned x, y, p;
  angle theta;

  /* Render a bluescale version of the texture */
  pallet[0] = argb(0,0,0,0);
  pallet[1] = argb(0,0,0,255);

  canv = canvas_new(TEXSZ, TEXSZ);
  brush = canvas_new(TEXSZ, TEXSZ);
  linear_paint_tile_render(canv->px, canv->w, canv->h, 4, 32,
                           pallet, lenof(pallet));

  /* Reduce to 8-bit monochrome texture to send to GL */
  for (p = 0; p < lenof(monochrome); ++p)
    monochrome[p] = get_blue(canv->px[p]);

  glGenTextures(1, &noise_texture);
  glBindTexture(GL_TEXTURE_2D, noise_texture);
  glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
  glTexImage2D(GL_TEXTURE_2D, 0, GL_RED,
               TEXSZ, TEXSZ, 0,
               GL_RED, GL_UNSIGNED_BYTE, monochrome);

  memset(canv->px, 0, sizeof(canvas_pixel) * canv->w * canv->h);
  perlin_noise(canv->px, canv->w, canv->h, 16, 64, 0);
  perlin_noise(canv->px, canv->w, canv->h, 32, 32, 1);
  perlin_noise(canv->px, canv->w, canv->h, 64, 16, 2);
  perlin_noise(canv->px, canv->w, canv->h, 128, 16, 2);

  pallet[1] = argb(0,0,0,128);
  linear_paint_tile_render(brush->px, brush->w, brush->h, 16, 1,
                           pallet, lenof(pallet));
  for (y = 0; y < brush->h; ++y) {
    for (x = 0; x < brush->w; ++x) {
      p = x + TEXSZ * y;
      theta = x * 65536 * 2 / TEXSZ;
      theta += canv->px[p] * 128;
      canv->px[p] += brush->px[
        x + TEXSZ *
        ((TEXSZ - 1) & (y + zo_cosms(theta, TEXSZ)/10))];
    }
  }

  for (p = 0; p < lenof(monochrome); ++p)
    monochrome[p] = get_blue(canv->px[p]);

  glGenTextures(1, &perlin_texture);
  glBindTexture(GL_TEXTURE_2D, perlin_texture);
  glTexImage2D(GL_TEXTURE_2D, 0, GL_RED,
               TEXSZ, TEXSZ, 0,
               GL_RED, GL_UNSIGNED_BYTE, monochrome);

  canvas_delete(canv);
  canvas_delete(brush);
}

glbrush_handle* glbrush_hnew(const glbrush_handle_info* info) {
  glbrush_handle* handle = xmalloc(sizeof(glbrush_handle));
  glGenTextures(1, &handle->pallet_texture);
  handle->glmsg_line = glm_slab_group_new((void(*)(void*))glbrush_activate_line,
                                          NULL, handle,
                                          shader_brush_configure_vbo,
                                          sizeof(shader_brush_vertex));
  handle->glmsg_point = glm_slab_group_new(
    (void(*)(void*))glbrush_activate_point,
    (void(*)(void*))glbrush_deactivate_point, handle,
    shader_splotch_configure_vbo,
    sizeof(shader_splotch_vertex));
  glm_slab_group_set_primitive(handle->glmsg_point, GL_POINTS);
  glm_slab_group_set_indices_enabled(handle->glmsg_point, 0);

  glbrush_hconfig(handle, info);
  return handle;
}

void glbrush_hconfig(glbrush_handle* handle, const glbrush_handle_info* info) {
  /* Even though this is a one-dimensional texture, we need to make it 2D,
   * since the activation of GL_TEXTURE_2D supercedes the use of 1D textures.
   */
  glBindTexture(GL_TEXTURE_2D, handle->pallet_texture);
  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA,
               info->pallet_size, 1, 0,
               GL_BGRA, GL_UNSIGNED_BYTE, info->pallet);

  handle->decay = info->decay;
  handle->noise = info->noise;
}

void glbrush_hset(glbrush_handle** handle,
                  const glbrush_handle_info* info,
                  int permit_refresh) {
  if (!*handle)
    *handle = glbrush_hnew(info);
  else if (permit_refresh)
    glbrush_hconfig(*handle, info);
}

void glbrush_hdelete(glbrush_handle* handle) {
  glm_slab_group_delete(handle->glmsg_line);
  glm_slab_group_delete(handle->glmsg_point);
  glDeleteTextures(1, &handle->pallet_texture);
  free(handle);
}

void glbrush_init(glbrush_spec* this, glbrush_handle* handle) {
  this->meth.draw_point = (drawing_method_draw_point_t)glbrush_draw_point;
  this->meth.draw_line  = (drawing_method_draw_line_t) glbrush_draw_line;
  this->meth.flush      = (drawing_method_flush_t)     glbrush_flush;
  this->line_slab = glm_slab_get(handle->glmsg_line);
  this->point_slab = glm_slab_get(handle->glmsg_point);
}

static void glbrush_activate_point(glbrush_handle* handle) {
  shader_splotch_uniform uniform;

  glBindTexture(GL_TEXTURE_2D, perlin_texture);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
  glActiveTexture(GL_TEXTURE1);
  glBindTexture(GL_TEXTURE_2D, handle->pallet_texture);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP);
  glActiveTexture(GL_TEXTURE0);

  glPushAttrib(GL_ENABLE_BIT | GL_POINT_BIT);
  glEnable(GL_POINT_SPRITE);
  glEnable(GL_PROGRAM_POINT_SIZE);
  /* Point vertices are clipped with respect to the internal point-size setting
   * and their location alone, regardless of how big the vertex shader says
   * they are. Since we can't vary this point-size per vertex, set one bigger
   * than any point we'll ever draw so that only the quad produced by the
   * point can be clipped away. (It's still faster than drawing quads
   * manually.)
   */
  glPointSize(65536);

  uniform.tex = 0;
  uniform.pallet = 1;
  uniform.noise = handle->noise;
  shader_splotch_activate(&uniform);
}

static void glbrush_deactivate_point(glbrush_handle* handle) {
  glPopAttrib();
}

void glbrush_draw_point(glbrush_accum* accum, const glbrush_spec* spec,
                        const vo3 where, zo_scaling_factor weight) {
  shader_splotch_vertex* vertices;
  signed size;
  float txxoff, txyoff;

  txxoff = lcgrand(&accum->rand) / 65536.0f;
  txyoff = lcgrand(&accum->rand) / 65536.0f;

  size = zo_scale(spec->screen_width, weight);
  if (!size || size > 65536) return;

  (void)GLM_ALLOC(&vertices, NULL, spec->point_slab, 1, 0);
  vertices[0].v[0] = where[0];
  vertices[0].v[1] = where[1];
  vertices[0].v[2] = where[2];
  vertices[0].parms[0] = txxoff;
  vertices[0].parms[1] = txyoff;
  vertices[0].parms[2] = size;
}

static void glbrush_activate_line(glbrush_handle* handle) {
  shader_brush_uniform uniform;

  glBindTexture(GL_TEXTURE_2D, noise_texture);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
  glActiveTexture(GL_TEXTURE1);
  glBindTexture(GL_TEXTURE_2D, handle->pallet_texture);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP);
  glActiveTexture(GL_TEXTURE0);<commit_msg>Restore "brushing away" effect of very large brush splotches.<commit_after>/*-
 * Copyright (c) 2014 <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the author nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <glew.h>

#include "../frac.h"
#include "../defs.h"
#include "../alloc.h"
#include "../rand.h"
#include "../gl/marshal.h"
#include "../gl/shaders.h"
#include "canvas.h"
#include "linear-paint-tile.h"
#include "glbrush.h"

static GLuint noise_texture;
static GLuint perlin_texture;
#define TEXSZ 256

struct glbrush_handle_s {
  glm_slab_group* glmsg_line, * glmsg_point;
  GLuint pallet_texture;
  float decay, noise;
};

static void glbrush_activate_line(glbrush_handle*);
static void glbrush_activate_point(glbrush_handle*);
static void glbrush_deactivate_point(glbrush_handle*);

void glbrush_load(void) {
  canvas* canv, * brush;
  unsigned char monochrome[TEXSZ*TEXSZ];
  canvas_pixel pallet[2];
  unsigned x, y, p;
  angle theta;

  /* Render a bluescale version of the texture */
  pallet[0] = argb(0,0,0,0);
  pallet[1] = argb(0,0,0,255);

  canv = canvas_new(TEXSZ, TEXSZ);
  brush = canvas_new(TEXSZ, TEXSZ);
  linear_paint_tile_render(canv->px, canv->w, canv->h, 4, 32,
                           pallet, lenof(pallet));

  /* Reduce to 8-bit monochrome texture to send to GL */
  for (p = 0; p < lenof(monochrome); ++p)
    monochrome[p] = get_blue(canv->px[p]);

  glGenTextures(1, &noise_texture);
  glBindTexture(GL_TEXTURE_2D, noise_texture);
  glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
  glTexImage2D(GL_TEXTURE_2D, 0, GL_RED,
               TEXSZ, TEXSZ, 0,
               GL_RED, GL_UNSIGNED_BYTE, monochrome);

  memset(canv->px, 0, sizeof(canvas_pixel) * canv->w * canv->h);
  perlin_noise(canv->px, canv->w, canv->h, 16, 64, 0);
  perlin_noise(canv->px, canv->w, canv->h, 32, 32, 1);
  perlin_noise(canv->px, canv->w, canv->h, 64, 16, 2);
  perlin_noise(canv->px, canv->w, canv->h, 128, 16, 2);

  pallet[1] = argb(0,0,0,128);
  linear_paint_tile_render(brush->px, brush->w, brush->h, 16, 1,
                           pallet, lenof(pallet));
  for (y = 0; y < brush->h; ++y) {
    for (x = 0; x < brush->w; ++x) {
      p = x + TEXSZ * y;
      theta = x * 65536 * 2 / TEXSZ;
      theta += canv->px[p] * 128;
      canv->px[p] += brush->px[
        x + TEXSZ *
        ((TEXSZ - 1) & (y + zo_cosms(theta, TEXSZ)/10))];
    }
  }

  for (p = 0; p < lenof(monochrome); ++p)
    monochrome[p] = get_blue(canv->px[p]);

  glGenTextures(1, &perlin_texture);
  glBindTexture(GL_TEXTURE_2D, perlin_texture);
  glTexImage2D(GL_TEXTURE_2D, 0, GL_RED,
               TEXSZ, TEXSZ, 0,
               GL_RED, GL_UNSIGNED_BYTE, monochrome);

  canvas_delete(canv);
  canvas_delete(brush);
}

glbrush_handle* glbrush_hnew(const glbrush_handle_info* info) {
  glbrush_handle* handle = xmalloc(sizeof(glbrush_handle));
  glGenTextures(1, &handle->pallet_texture);
  handle->glmsg_line = glm_slab_group_new((void(*)(void*))glbrush_activate_line,
                                          NULL, handle,
                                          shader_brush_configure_vbo,
                                          sizeof(shader_brush_vertex));
  handle->glmsg_point = glm_slab_group_new(
    (void(*)(void*))glbrush_activate_point,
    (void(*)(void*))glbrush_deactivate_point, handle,
    shader_splotch_configure_vbo,
    sizeof(shader_splotch_vertex));
  glm_slab_group_set_primitive(handle->glmsg_point, GL_POINTS);
  glm_slab_group_set_indices_enabled(handle->glmsg_point, 0);

  glbrush_hconfig(handle, info);
  return handle;
}

void glbrush_hconfig(glbrush_handle* handle, const glbrush_handle_info* info) {
  /* Even though this is a one-dimensional texture, we need to make it 2D,
   * since the activation of GL_TEXTURE_2D supercedes the use of 1D textures.
   */
  glBindTexture(GL_TEXTURE_2D, handle->pallet_texture);
  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA,
               info->pallet_size, 1, 0,
               GL_BGRA, GL_UNSIGNED_BYTE, info->pallet);

  handle->decay = info->decay;
  handle->noise = info->noise;
}

void glbrush_hset(glbrush_handle** handle,
                  const glbrush_handle_info* info,
                  int permit_refresh) {
  if (!*handle)
    *handle = glbrush_hnew(info);
  else if (permit_refresh)
    glbrush_hconfig(*handle, info);
}

void glbrush_hdelete(glbrush_handle* handle) {
  glm_slab_group_delete(handle->glmsg_line);
  glm_slab_group_delete(handle->glmsg_point);
  glDeleteTextures(1, &handle->pallet_texture);
  free(handle);
}

void glbrush_init(glbrush_spec* this, glbrush_handle* handle) {
  this->meth.draw_point = (drawing_method_draw_point_t)glbrush_draw_point;
  this->meth.draw_line  = (drawing_method_draw_line_t) glbrush_draw_line;
  this->meth.flush      = (drawing_method_flush_t)     glbrush_flush;
  this->line_slab = glm_slab_get(handle->glmsg_line);
  this->point_slab = glm_slab_get(handle->glmsg_point);
}

static void glbrush_activate_point(glbrush_handle* handle) {
  shader_splotch_uniform uniform;

  glBindTexture(GL_TEXTURE_2D, perlin_texture);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
  glActiveTexture(GL_TEXTURE1);
  glBindTexture(GL_TEXTURE_2D, handle->pallet_texture);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP);
  glActiveTexture(GL_TEXTURE0);

  glPushAttrib(GL_ENABLE_BIT | GL_POINT_BIT);
  glEnable(GL_POINT_SPRITE);
  glEnable(GL_PROGRAM_POINT_SIZE);
  /* Point vertices are clipped with respect to the internal point-size setting
   * and their location alone, regardless of how big the vertex shader says
   * they are. Since we can't vary this point-size per vertex, set one bigger
   * than any point we'll ever draw so that only the quad produced by the
   * point can be clipped away. (It's still faster than drawing quads
   * manually.)
   */
  glPointSize(65536);

  uniform.tex = 0;
  uniform.pallet = 1;
  uniform.noise = handle->noise;
  shader_splotch_activate(&uniform);
}

static void glbrush_deactivate_point(glbrush_handle* handle) {
  glPopAttrib();
}

void glbrush_draw_point(glbrush_accum* accum, const glbrush_spec* spec,
                        const vo3 where, zo_scaling_factor weight) {
  shader_splotch_vertex* vertices;
  signed size;
  float txxoff, txyoff;

  txxoff = lcgrand(&accum->rand) / 65536.0f;
  txyoff = lcgrand(&accum->rand) / 65536.0f;

  /* Size here will actually become negative for large objects when the camera
   * becomes very close to them. An interesting effect of this is that as the
   * camera gets closer, the object gets *smaller*. This gives a nice "pushing
   * out of the way effect" and also means that the object doesn't suddenly
   * vanish, so handle the case appropriately and adapt to OpenGL by taking the
   * absolute value before we pass it in.
   */
  size = zo_scale(spec->screen_width, weight);
  if (!size || size > 65536) return;

  (void)GLM_ALLOC(&vertices, NULL, spec->point_slab, 1, 0);
  vertices[0].v[0] = where[0];
  vertices[0].v[1] = where[1];
  vertices[0].v[2] = where[2];
  vertices[0].parms[0] = txxoff;
  vertices[0].parms[1] = txyoff;
  vertices[0].parms[2] = abs(size);
}

static void glbrush_activate_line(glbrush_handle* handle) {
  shader_brush_uniform uniform;

  glBindTexture(GL_TEXTURE_2D, noise_texture);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
  glActiveTexture(GL_TEXTURE1);
  glBindTexture(GL_TEXTURE_2D, handle->pallet_texture);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP);
  glActiveTexture(GL_TEXTURE0);
{'fraction_non_alphanumeric': 0.10799978906291198, 'fraction_numerical': 0.017507778305120496, 'mean_word_length': 4.240121580547113, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 125, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4702365', 'n_tokens_mistral': 7226, 'n_tokens_neox': 6640, 'n_words': 1584}