<commit_before><?php
error_reporting(E_ALL);

if (!defined('HAVE_PHP_SQL_PARSER')) {

   class PHPSQLParser {
      var $reserved = array();
      var $functions = array();

      public function __construct($sql = false, $calcPositions = false) {
         $this->load_reserved_words();
         if ($sql) {
            $this->parse($sql, $calcPositions);
         }
      }

      public function parse($sql, $calcPositions = false) {
         $original = $this->replaceSpecialCharacters($sql);

         #lex the SQL statement
         $inputArray = $this->split_sql(trim($original));

         #This is the highest level lexical analysis.  This is the part of the
         #code which finds UNION and UNION ALL query parts
         $queries = $this->processUnion($inputArray);

         # If there was no UNION or UNION ALL in the query, then the query is
         # stored at $queries[0].
         if (!$this->isUnion($queries)) {
            $queries = $this->process_sql($queries[0]);
         }

         # calc the positions of some important tokens
         if ($calcPositions) {
            $queries = $this->calculatePositionsWithinSQL($original, $queries);
         }

         # store the parsed queries
         $this->parsed = $queries;
         return $this->parsed;
      }

      private function preprint($s, $return = false) {
         $x = "<pre>";
         $x .= print_r($s, 1);
         $x .= "</pre>";
         if ($return) {
            return $x;
         } else {
            if (isset($_ENV['DEBUG'])) {
               print $x . "\n";
            }
         }
      }

      private function processUnion($inputArray) {
         $outputArray = array();

         #sometimes the parser needs to skip ahead until a particular
         #token is found
         $skipUntilToken = false;

         #This is the last type of union used (UNION or UNION ALL)
         #indicates a) presence of at least one union in this query
         #          b) the type of union if this is the first or last query
         $unionType = false;

         #Sometimes a "query" consists of more than one query (like a UNION query)
         #this array holds all the queries
         $queries = array();

         foreach ($inputArray as $key => $token) {
            $trim = trim($token);

            # overread all tokens till that given token
            if ($skipUntilToken) {
               if ($trim === "") {
                  continue; # read the next token
               }
               if (strtoupper($trim) === $skipUntilToken) {
                  $skipUntilToken = false;
                  continue; # read the next token
               }
            }

            if (strtoupper($trim) !== "UNION") {
               $outputArray[] = $token; # here we get empty tokens, if we remove these, we get problems in parse_sql()
               continue;
            }

            $unionType = 'UNION';

            # we are looking for an ALL token right after UNION
            for ($i = $key + 1; $i < count($inputArray); ++$i) {
               if (trim($inputArray[$i]) === "") {
                  continue;
               }
               if (strtoupper($inputArray[$i]) !== 'ALL') {
                  break;
               }
               # the other for-loop should overread till "ALL"
               $skipUntilToken = 'ALL';
               $unionType = 'UNION ALL';
            }

            # store the tokens related to the unionType
            $queries[$unionType][] = $outputArray;
            $outputArray = array();
         }

         # the query tokens after the last UNION or UNION ALL
         # or we don't have an UNION/UNION ALL
         if (!empty($outputArray)) {
            if ($unionType) {
               $queries[$unionType][] = $outputArray;
            } else {
               $queries[] = $outputArray;
            }
         }

         return $this->processMySQLUnion($queries);
      }

      /** MySQL supports a special form of UNION:
       * (select ...)
       * union
       * (select ...)
       *
       * This function handles this query syntax.  Only one such subquery
       * is supported in each UNION block.  (select)(select)union(select) is not legal.
       * The extra queries will be silently ignored.
       */
      private function processMySQLUnion($queries) {
         $unionTypes = array('UNION', 'UNION ALL');
         foreach ($unionTypes as $unionType) {

            if (empty($queries[$unionType])) {
               continue;
            }

            foreach ($queries[$unionType] as $key => $tokenList) {
               foreach ($tokenList as $z => $token) {
                  $token = trim($token);
                  if ($token === "") {
                     continue;
                  }

                  # starts with "(select"
                  if (preg_match('/^\\(\\s*select\\s*/i', $token)) {
                     $queries[$unionType][$key] = $this->parse($this->removeParenthesisFromStart($token));
                     break;
                  }

                  $queries[$unionType][$key] = $this->process_sql($queries[$unionType][$key]);
                  break;
               }
            }
         }
         # it can be parsed or not
         return $queries;
      }

      private function isUnion($queries) {
         $unionTypes = array('UNION', 'UNION ALL');
         foreach ($unionTypes as $unionType) {
            if (!empty($queries[$unionType])) {
               return true;
            }
         }
         return false;
      }

      private function calculatePositionsWithinSQL($sql, $parsed) {
         $charPos = 0;
         $backtracking = array();
         $this->lookForBaseExpression($sql, $charPos, $parsed, 0, $backtracking);
         return $parsed;
      }

      # helper method to debug lookForBaseExpression
      private function printPos($text, $sql, $charPos, $key, $parsed, $backtracking) {
         if (!isset($_ENV['DEBUG'])) {
            return;
         }

         $spaces = "";
         $caller = debug_backtrace();
         $i = 1;
         while ($caller[$i]['function'] === 'lookForBaseExpression') {
            $spaces .= "   ";
            $i++;
         }
         $holdem = substr($sql, 0, $charPos) . "^" . substr($sql, $charPos);
         echo $spaces . $text . " key:" . $key . "  parsed:" . $parsed . " back:" . serialize($backtracking) . " "
         . $holdem . "\n";
      }

      private function findPositionWithinString($sql, $value, $expr_type) {

         $allowedOnOperator = array('\t', '\n', '\r', ' ', ',', '(', ')', '_', '\'');
         $allowedOnOthers = array('\t', '\n', '\r', ' ', ',', '(', ')', '<', '>', '*', '+', '-', '/', '|', '&', '=',
                                     '!', ';');

         $offset = 0;
         $ok = false;
         while (true) {

            $pos = strpos($sql, $value, $offset);
            if ($pos === false) {
               break;
            }

            $before = "";
            if ($pos > 0) {
               $before = $sql[$pos - 1];
            }

            $after = "";
            if (isset($sql[$pos + strlen($value)])) {
               $after = $sql[$pos + strlen($value)];
            }

            # if we have an operator, it should be surrounded by
            # whitespace, comma, parenthesis, digit or letter, end_of_string
            # an operator should not be surrounded by another operator

            if ($expr_type === 'operator') {

               $ok = ($before === "" || in_array($before, $allowedOnOperator))
               || (strtolower($before) >= 'a' && strtolower($before) <= 'z')
               || ($before >= '0' && $before <= '9');
               $ok = $ok
               && ($after === "" || in_array($after, $allowedOnOperator)
               || (strtolower($after) >= 'a' && strtolower($after) <= 'z')
               || ($after >= '0' && $after <= '9'));

               if (!$ok) {
                  $offset = $pos + 1;
                  continue;
               }

               break;
            }

            # in all other cases we accept
            # whitespace, comma, operators, parenthesis and end_of_string

            $ok = ($before === "" || in_array($before, $allowedOnOthers));
            $ok = $ok && ($after === "" || in_array($after, $allowedOnOthers));

            if ($ok) {
               break;
            }

            $offset = $pos + 1;
         }

         return $pos;
      }

      private function lookForBaseExpression($sql, &$charPos, &$parsed, $key, &$backtracking) {
         if (!is_numeric($key)) {
            if (in_array($key, array('UNION', 'UNION ALL'), true)
            || ($key === 'expr_type' && $parsed === 'expression')
            || ($key === 'expr_type' && $parsed === 'subquery')
            || ($key === 'expr_type' && $parsed === 'table_expression')
            || ($key === 'alias' && $parsed !== false)) {
               $backtracking[] = $charPos; # on the next base_expr we set the pointer back to this value

            } elseif (($key === 'ref_clause' || $key === 'columns') && $parsed !== false) {
               $backtracking[] = $charPos;
               for ($i = 1; $i < count($parsed); $i++) {
                  $backtracking[] = false; # backtracking only after n base_expr!
               }
            } else {
               # SELECT, WHERE, INSERT etc.
               if (in_array($key, $this->reserved)) {
                  $charPos = stripos($sql, $key, $charPos);
                  $charPos += strlen($key);
               }
            }
         }

         if (!is_array($parsed)) {
            return;
         }

         foreach ($parsed as $key => $value) {
            if ($key === 'base_expr') {

               $subject = substr($sql, $charPos);
               $pos = $this->findPositionWithinString($subject, $value,
               isset($parsed['expr_type']) ? $parsed['expr_type'] : 'alias');
               if ($pos === false) {
                  die("cannot calculate position of " . $value . " within " . $subject);
               }

               $parsed['position'] = $charPos + $pos;
               $charPos += $pos + strlen($value);

               $oldPos = array_pop($backtracking);
               if (isset($oldPos) && $oldPos !== false) {
                  $charPos = $oldPos;
               }

            } else {
               $this->lookForBaseExpression($sql, $charPos, $parsed[$key], $key, $backtracking);
            }
         }

      }

      private function replaceSpecialCharacters($sql) {
         # replace always the same number of characters, so we have the same positions
         # within the original string

         # issue 21: replace tabs within the query string
         # TODO: this removes special characters also from inside of Strings (is that a problem?)
         return str_replace(array('\\\'', '\\"', "\r\n", "\n", "\t", "()"), array("''", '""', "  ", " ", " ", "  "),
         $sql);
      }

      #This is the lexer
      #this function splits up a SQL statement into easy to "parse"
      #tokens for the SQL processor
      private function split_sql($sql) {

         if (!is_string($sql)) {
            echo "SQL:\n";
            print_r($sql);
            exit;
         }

         # added some code from issue 11 comment 3
         $regex = <<<EOREGEX
/(`(?:[^`]|``)`|[@A-Za-z0-9_.`-]+(?:\(\s*\)){0,1})
|(\+|-|\*|\/|!=|>=|<=|<>|>|<|&&|\|\||=|\^|\(|\))
|('(?:[^']|'')*'+)
|("(?:[^"]|"")*"+)
|([^ ,]+)
/ix
EOREGEX
         ;

         $tokens = preg_split($regex, $sql, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
         $tokens = $this->balanceParenthesis($tokens);

         return $tokens;
      }

      private function balanceParenthesis($tokens) {

         $token_count = count($tokens);
          
         $i = 0;
         while ($i < $token_count) {

            if ($tokens[$i] !== '(') {
               $i++;
               continue;
            }

            $count = 1;
            for ($n = $i+1; $n < $token_count; $n++) {

               if ($tokens[$n] ==="") {
                  continue;
               }

               $token = $tokens[$n];

               if ($token === '(') {
                  $count++;
               }
               if ($token === ')') {
                  $count--;
               }

               $tokens[$i] .= $token;
               unset($tokens[$n]);

               if ($count === 0) {
                  $i = $n + 1;
                  break;
               }
            }
         }
         return array_values($tokens);
      }

      /* This function breaks up the SQL statement into logical sections.
       Some sections are then further handled by specialized functions.
       */
      private function process_sql(&$tokens, $start_at = 0, $stop_at = false) {
         $prev_category = "";
         $start = microtime(true);
         $token_category = "";

         $skip_next = false;
         $token_count = count($tokens);

         if (!$stop_at) {
            $stop_at = $token_count;
         }

         $out = false;

         for ($token_number = $start_at; $token_number < $stop_at; ++$token_number) {
            $token = trim($tokens[$token_number]);
            # if it starts with an "(", it should follow a SELECT
            if ($token !== "" && $token[0] == '(' && $token_category == "") {
               $token_category = 'SELECT';
            }
<commit_msg>FIX: backticks are not balanced with the new regex, so we have recreated the balancing function.<commit_after><?php
error_reporting(E_ALL);

if (!defined('HAVE_PHP_SQL_PARSER')) {

   class PHPSQLParser {
      var $reserved = array();
      var $functions = array();

      public function __construct($sql = false, $calcPositions = false) {
         $this->load_reserved_words();
         if ($sql) {
            $this->parse($sql, $calcPositions);
         }
      }

      public function parse($sql, $calcPositions = false) {
         $original = $this->replaceSpecialCharacters($sql);

         #lex the SQL statement
         $inputArray = $this->split_sql(trim($original));

         #This is the highest level lexical analysis.  This is the part of the
         #code which finds UNION and UNION ALL query parts
         $queries = $this->processUnion($inputArray);

         # If there was no UNION or UNION ALL in the query, then the query is
         # stored at $queries[0].
         if (!$this->isUnion($queries)) {
            $queries = $this->process_sql($queries[0]);
         }

         # calc the positions of some important tokens
         if ($calcPositions) {
            $queries = $this->calculatePositionsWithinSQL($original, $queries);
         }

         # store the parsed queries
         $this->parsed = $queries;
         return $this->parsed;
      }

      private function preprint($s, $return = false) {
         $x = "<pre>";
         $x .= print_r($s, 1);
         $x .= "</pre>";
         if ($return) {
            return $x;
         } else {
            if (isset($_ENV['DEBUG'])) {
               print $x . "\n";
            }
         }
      }

      private function processUnion($inputArray) {
         $outputArray = array();

         #sometimes the parser needs to skip ahead until a particular
         #token is found
         $skipUntilToken = false;

         #This is the last type of union used (UNION or UNION ALL)
         #indicates a) presence of at least one union in this query
         #          b) the type of union if this is the first or last query
         $unionType = false;

         #Sometimes a "query" consists of more than one query (like a UNION query)
         #this array holds all the queries
         $queries = array();

         foreach ($inputArray as $key => $token) {
            $trim = trim($token);

            # overread all tokens till that given token
            if ($skipUntilToken) {
               if ($trim === "") {
                  continue; # read the next token
               }
               if (strtoupper($trim) === $skipUntilToken) {
                  $skipUntilToken = false;
                  continue; # read the next token
               }
            }

            if (strtoupper($trim) !== "UNION") {
               $outputArray[] = $token; # here we get empty tokens, if we remove these, we get problems in parse_sql()
               continue;
            }

            $unionType = 'UNION';

            # we are looking for an ALL token right after UNION
            for ($i = $key + 1; $i < count($inputArray); ++$i) {
               if (trim($inputArray[$i]) === "") {
                  continue;
               }
               if (strtoupper($inputArray[$i]) !== 'ALL') {
                  break;
               }
               # the other for-loop should overread till "ALL"
               $skipUntilToken = 'ALL';
               $unionType = 'UNION ALL';
            }

            # store the tokens related to the unionType
            $queries[$unionType][] = $outputArray;
            $outputArray = array();
         }

         # the query tokens after the last UNION or UNION ALL
         # or we don't have an UNION/UNION ALL
         if (!empty($outputArray)) {
            if ($unionType) {
               $queries[$unionType][] = $outputArray;
            } else {
               $queries[] = $outputArray;
            }
         }

         return $this->processMySQLUnion($queries);
      }

      /** MySQL supports a special form of UNION:
       * (select ...)
       * union
       * (select ...)
       *
       * This function handles this query syntax.  Only one such subquery
       * is supported in each UNION block.  (select)(select)union(select) is not legal.
       * The extra queries will be silently ignored.
       */
      private function processMySQLUnion($queries) {
         $unionTypes = array('UNION', 'UNION ALL');
         foreach ($unionTypes as $unionType) {

            if (empty($queries[$unionType])) {
               continue;
            }

            foreach ($queries[$unionType] as $key => $tokenList) {
               foreach ($tokenList as $z => $token) {
                  $token = trim($token);
                  if ($token === "") {
                     continue;
                  }

                  # starts with "(select"
                  if (preg_match('/^\\(\\s*select\\s*/i', $token)) {
                     $queries[$unionType][$key] = $this->parse($this->removeParenthesisFromStart($token));
                     break;
                  }

                  $queries[$unionType][$key] = $this->process_sql($queries[$unionType][$key]);
                  break;
               }
            }
         }
         # it can be parsed or not
         return $queries;
      }

      private function isUnion($queries) {
         $unionTypes = array('UNION', 'UNION ALL');
         foreach ($unionTypes as $unionType) {
            if (!empty($queries[$unionType])) {
               return true;
            }
         }
         return false;
      }

      private function calculatePositionsWithinSQL($sql, $parsed) {
         $charPos = 0;
         $backtracking = array();
         $this->lookForBaseExpression($sql, $charPos, $parsed, 0, $backtracking);
         return $parsed;
      }

      # helper method to debug lookForBaseExpression
      private function printPos($text, $sql, $charPos, $key, $parsed, $backtracking) {
         if (!isset($_ENV['DEBUG'])) {
            return;
         }

         $spaces = "";
         $caller = debug_backtrace();
         $i = 1;
         while ($caller[$i]['function'] === 'lookForBaseExpression') {
            $spaces .= "   ";
            $i++;
         }
         $holdem = substr($sql, 0, $charPos) . "^" . substr($sql, $charPos);
         echo $spaces . $text . " key:" . $key . "  parsed:" . $parsed . " back:" . serialize($backtracking) . " "
         . $holdem . "\n";
      }

      private function findPositionWithinString($sql, $value, $expr_type) {

         $allowedOnOperator = array('\t', '\n', '\r', ' ', ',', '(', ')', '_', '\'');
         $allowedOnOthers = array('\t', '\n', '\r', ' ', ',', '(', ')', '<', '>', '*', '+', '-', '/', '|', '&', '=',
                                     '!', ';');

         $offset = 0;
         $ok = false;
         while (true) {

            $pos = strpos($sql, $value, $offset);
            if ($pos === false) {
               break;
            }

            $before = "";
            if ($pos > 0) {
               $before = $sql[$pos - 1];
            }

            $after = "";
            if (isset($sql[$pos + strlen($value)])) {
               $after = $sql[$pos + strlen($value)];
            }

            # if we have an operator, it should be surrounded by
            # whitespace, comma, parenthesis, digit or letter, end_of_string
            # an operator should not be surrounded by another operator

            if ($expr_type === 'operator') {

               $ok = ($before === "" || in_array($before, $allowedOnOperator))
               || (strtolower($before) >= 'a' && strtolower($before) <= 'z')
               || ($before >= '0' && $before <= '9');
               $ok = $ok
               && ($after === "" || in_array($after, $allowedOnOperator)
               || (strtolower($after) >= 'a' && strtolower($after) <= 'z')
               || ($after >= '0' && $after <= '9'));

               if (!$ok) {
                  $offset = $pos + 1;
                  continue;
               }

               break;
            }

            # in all other cases we accept
            # whitespace, comma, operators, parenthesis and end_of_string

            $ok = ($before === "" || in_array($before, $allowedOnOthers));
            $ok = $ok && ($after === "" || in_array($after, $allowedOnOthers));

            if ($ok) {
               break;
            }

            $offset = $pos + 1;
         }

         return $pos;
      }

      private function lookForBaseExpression($sql, &$charPos, &$parsed, $key, &$backtracking) {
         if (!is_numeric($key)) {
            if (in_array($key, array('UNION', 'UNION ALL'), true)
            || ($key === 'expr_type' && $parsed === 'expression')
            || ($key === 'expr_type' && $parsed === 'subquery')
            || ($key === 'expr_type' && $parsed === 'table_expression')
            || ($key === 'alias' && $parsed !== false)) {
               $backtracking[] = $charPos; # on the next base_expr we set the pointer back to this value

            } elseif (($key === 'ref_clause' || $key === 'columns') && $parsed !== false) {
               $backtracking[] = $charPos;
               for ($i = 1; $i < count($parsed); $i++) {
                  $backtracking[] = false; # backtracking only after n base_expr!
               }
            } else {
               # SELECT, WHERE, INSERT etc.
               if (in_array($key, $this->reserved)) {
                  $charPos = stripos($sql, $key, $charPos);
                  $charPos += strlen($key);
               }
            }
         }

         if (!is_array($parsed)) {
            return;
         }

         foreach ($parsed as $key => $value) {
            if ($key === 'base_expr') {

               $subject = substr($sql, $charPos);
               $pos = $this->findPositionWithinString($subject, $value,
               isset($parsed['expr_type']) ? $parsed['expr_type'] : 'alias');
               if ($pos === false) {
                  die("cannot calculate position of " . $value . " within " . $subject);
               }

               $parsed['position'] = $charPos + $pos;
               $charPos += $pos + strlen($value);

               $oldPos = array_pop($backtracking);
               if (isset($oldPos) && $oldPos !== false) {
                  $charPos = $oldPos;
               }

            } else {
               $this->lookForBaseExpression($sql, $charPos, $parsed[$key], $key, $backtracking);
            }
         }

      }

      private function replaceSpecialCharacters($sql) {
         # replace always the same number of characters, so we have the same positions
         # within the original string

         # issue 21: replace tabs within the query string
         # TODO: this removes special characters also from inside of Strings (is that a problem?)
         return str_replace(array('\\\'', '\\"', "\r\n", "\n", "\t", "()"), array("''", '""', "  ", " ", " ", "  "),
         $sql);
      }

      #This function counts open and close backticks and
      #returns their location.  This might be faster as a regex
      private function count_backtick($token, $char) {
         $len = strlen($token);
         $cnt = 0;
         for ($i = 0; $i < $len; ++$i) {
            if ($token[$i] == $char)
            ++$cnt;
         }
         return $cnt;
      }

      # backticks are not balanced within one token, so we have
      # to re-combine some tokens
      private function balanceBackticks($tokens, $char) {
         $token_count = count($tokens);
         $i = 0;
         while ($i < $token_count) {
             
            if ($tokens[$i] === "") {
               $i++;
               continue;
            }

            $needed = $this->count_backtick($tokens[$i], $char) % 2;
            if ($needed === 0) {
               $i++;
               continue;
            }

            for ($n = $i + 1; $n<$token_count; $n++) {
               $needed = ($needed + $this->count_backtick($tokens[$n], $char)) % 2;

               $tokens[$i] .= $tokens[$n];
               unset($tokens[$n]);

               if ($needed === 0) {
                  $n++;
                  break;
               }
            }
            $i = $n;
         }
         return array_values($tokens);
      }

      #This is the lexer
      #this function splits up a SQL statement into easy to "parse"
      #tokens for the SQL processor
      private function split_sql($sql) {

         if (!is_string($sql)) {
            echo "SQL:\n";
            print_r($sql);
            exit;
         }

         # added some code from issue 11 comment 3
         $regex = <<<EOREGEX
/(`(?:[^`]|``)`|[@A-Za-z0-9_.`-]+(?:\(\s*\)){0,1})
|(\+|-|\*|\/|!=|>=|<=|<>|>|<|&&|\|\||=|\^|\(|\))
|('(?:[^']|'')*'+)
|("(?:[^"]|"")*"+)
|([^ ,]+)
/ix
EOREGEX
         ;

         $tokens = preg_split($regex, $sql, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
         $tokens = $this->balanceParenthesis($tokens);
         $tokens = $this->balanceBackticks($tokens, '`');
         $tokens = $this->balanceBackticks($tokens, '\'');
         return $tokens;
      }

      private function balanceParenthesis($tokens) {

         $token_count = count($tokens);

         $i = 0;
         while ($i < $token_count) {

            if ($tokens[$i] !== '(') {
               $i++;
               continue;
            }

            $count = 1;
            for ($n = $i+1; $n < $token_count; $n++) {

               $token = $tokens[$n];

               if ($token === '(') {
                  $count++;
               }
               if ($token === ')') {
                  $count--;
               }

               $tokens[$i] .= $token;
               unset($tokens[$n]);

               if ($count === 0) {
                  $n++;
                  break;
               }
            }
            $i = $n;
         }
          
         return array_values($tokens);
      }

      /* This function breaks up the SQL statement into logical sections.
       Some sections are then further handled by specialized functions.
       */
      private function process_sql(&$tokens, $start_at = 0, $stop_at = false) {
         $prev_category = "";
         $start = microtime(true);
         $token_category = "";

         $skip_next = false;
         $token_count = count($tokens);

         if (!$stop_at) {
            $stop_at = $token_count;
         }

         $out = false;

         for ($token_number = $start_at; $token_number < $stop_at; ++$token_number) {
            $token = trim($tokens[$token_number]);
            # if it starts with an "(", it should follow a SELECT
            if ($token !== "" && $token[0] == '(' && $token_category == "") {
               $token_category = 'SELECT';
            }

{'fraction_non_alphanumeric': 0.15464426877470355, 'fraction_numerical': 0.002717391304347826, 'mean_word_length': 1.5088092076139885, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1157100', 'n_tokens_mistral': 8075, 'n_tokens_neox': 7750, 'n_words': 2047}