<commit_before>#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "mem.h"
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <stdint.h>

void compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size) {
  ZSTD_inBuffer in = { data, size, 0 };
  while (in.pos < in.size) {
    ZSTD_outBuffer tmp = out;
    const size_t rc = ZSTD_compressStream(ctx, &tmp, &in);
    if (ZSTD_isError(rc)) {
      exit(5);
    }
  }
  ZSTD_outBuffer tmp = out;
  const size_t rc = ZSTD_flushStream(ctx, &tmp);
  if (rc != 0) { exit(6); }
}

int main() {
  ZSTD_CStream *ctx;
  ZSTD_parameters params = {};
  size_t rc;
  unsigned windowLog;
  /* Create stream */
  ctx = ZSTD_createCStream();
  if (!ctx) { return 1; }
  /* Set parameters */
  params.cParams.windowLog = 18;
  params.cParams.chainLog = 13;
  params.cParams.hashLog = 14;
  params.cParams.searchLog = 1;
  params.cParams.searchLength = 7;
  params.cParams.targetLength = 16;
  params.cParams.strategy = ZSTD_fast;
  windowLog = params.cParams.windowLog;
  /* Initialize stream */
  rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0);
  if (ZSTD_isError(rc)) { return 2; }
  {
    U64 compressed = 0;
    const U64 toCompress = ((U64)1) << 33;
    const size_t size = 1 << windowLog;
    size_t pos = 0;
    char *srcBuffer = (char*) malloc(1 << windowLog);
    char *dstBuffer = (char*) malloc(ZSTD_compressBound(1 << windowLog));
    ZSTD_outBuffer out = { dstBuffer, ZSTD_compressBound(1 << windowLog), 0 };
    const char match[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
    const size_t randomData = (1 << windowLog) - 2*sizeof(match);
    for (size_t i = 0; i < sizeof(match); ++i) {
      srcBuffer[i] = match[i];
    }
    for (size_t i = 0; i < randomData; ++i) {
      srcBuffer[sizeof(match) + i] = (char)(rand() & 0xFF);
    }
    for (size_t i = 0; i < sizeof(match); ++i) {
      srcBuffer[sizeof(match) + randomData + i] = match[i];
    }
    compress(ctx, out, srcBuffer, size);
    compressed += size;
    while (compressed < toCompress) {
      const size_t block = rand() % (size - pos + 1);
      if (pos == size) { pos = 0; }
      compress(ctx, out, srcBuffer + pos, block);
      pos += block;
      compressed += block;
    }
    free(srcBuffer);
    free(dstBuffer);
  }
}<commit_msg>Fix longmatch test build errors.<commit_after>#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "mem.h"
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <stdint.h>

int compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size) {
  ZSTD_inBuffer in = { data, size, 0 };
  while (in.pos < in.size) {
    ZSTD_outBuffer tmp = out;
    const size_t rc = ZSTD_compressStream(ctx, &tmp, &in);
    if (ZSTD_isError(rc)) {
      return 1;
    }
  }
  {
    ZSTD_outBuffer tmp = out;
    const size_t rc = ZSTD_flushStream(ctx, &tmp);
    if (rc != 0) { return 1; }
  }
  return 0;
}

int main(int argc, const char** argv) {
  ZSTD_CStream *ctx;
  ZSTD_parameters params;
  size_t rc;
  unsigned windowLog;
  (void)argc;
  (void)argv;
  /* Create stream */
  ctx = ZSTD_createCStream();
  if (!ctx) { return 1; }
  /* Set parameters */
  memset(&params, 0, sizeof(params));
  params.cParams.windowLog = 18;
  params.cParams.chainLog = 13;
  params.cParams.hashLog = 14;
  params.cParams.searchLog = 1;
  params.cParams.searchLength = 7;
  params.cParams.targetLength = 16;
  params.cParams.strategy = ZSTD_fast;
  windowLog = params.cParams.windowLog;
  /* Initialize stream */
  rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0);
  if (ZSTD_isError(rc)) { return 2; }
  {
    U64 compressed = 0;
    const U64 toCompress = ((U64)1) << 33;
    const size_t size = 1 << windowLog;
    size_t pos = 0;
    char *srcBuffer = (char*) malloc(1 << windowLog);
    char *dstBuffer = (char*) malloc(ZSTD_compressBound(1 << windowLog));
    ZSTD_outBuffer out = { dstBuffer, ZSTD_compressBound(1 << windowLog), 0 };
    const char match[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
    const size_t randomData = (1 << windowLog) - 2*sizeof(match);
    size_t i;
    for (i = 0; i < sizeof(match); ++i) {
      srcBuffer[i] = match[i];
    }
    for (i = 0; i < randomData; ++i) {
      srcBuffer[sizeof(match) + i] = (char)(rand() & 0xFF);
    }
    for (i = 0; i < sizeof(match); ++i) {
      srcBuffer[sizeof(match) + randomData + i] = match[i];
    }
    if (compress(ctx, out, srcBuffer, size)) {
      return 1;
    }
    compressed += size;
    while (compressed < toCompress) {
      const size_t block = rand() % (size - pos + 1);
      if (pos == size) { pos = 0; }
      if (compress(ctx, out, srcBuffer + pos, block)) {
        return 1;
      }
      pos += block;
      compressed += block;
    }
    free(srcBuffer);
    free(dstBuffer);
  }
  return 0;
}
{'fraction_non_alphanumeric': 0.147584789311408, 'fraction_numerical': 0.022404933196300103, 'mean_word_length': 3.166095890410959, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6465830', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1809, 'n_words': 442}