<commit_before><?php

$documentLines = file('README.md');
$indexLines    = [];

$copyStart     = null;
$copyEnd       = null;
$i = 1;

$testLines     = [];
$testPass      = true;
$testMessages  = [];


foreach ($documentLines as $index => $line) {
  $line = rtrim($line);
  preg_match('/^(?:##)\s+(?:.+)$/i', $line, $matches);

  if (empty($matches)) {
    continue;
  }

  if (null === $copyStart) {
    $copyStart = $index;
    continue;
  }

  $topic = ltrim($documentLines[$copyStart], '# ');
  $testLines[$topic] = array_slice($documentLines, $copyStart, $index-$copyStart);
  $copyStart = $index;
}

foreach ($testLines as $topic => $lines) {
  $topic = trim($topic);
  $original = [];
  $sorted   = [];
<commit_msg>Cleanup test<commit_after><?php

$documentLines = file('README.md');
$indexLines    = [];

$copyStart     = null;

$testLines     = [];
$testPass      = true;
$testMessages  = [];

foreach ($documentLines as $index => $line) {
  $line = rtrim($line);
  preg_match('/^(?:##)\s+(?:.+)$/i', $line, $matches);

  if (empty($matches)) {
    continue;
  }

  if (null === $copyStart) {
    $copyStart = $index;
    continue;
  }

  $topic = ltrim($documentLines[$copyStart], '# ');
  $testLines[$topic] = array_slice($documentLines, $copyStart, $index-$copyStart);
  $copyStart = $index;
}

foreach ($testLines as $topic => $lines) {
  $topic = trim($topic);
  $original = [];
  $sorted   = [];

{'fraction_non_alphanumeric': 0.221119773210489, 'fraction_numerical': 0.0007087172218284905, 'mean_word_length': 3.599348534201954, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3882071', 'n_tokens_mistral': 515, 'n_tokens_neox': 508, 'n_words': 97}