<commit_before><?php
class Tail
{
  protected $path = null;
  protected $first = true;
  protected $pos = 0;

  /**
   * Follow a file
   * @param string $path
   *
   * @author <NAME> <<EMAIL>>
   */
  public function __construct($path)
  {
    $this->path = $path;
  }

  /**
   * Read a new line a long as it is possible.
   * Goes straight to the end the first time if the file exists.
   * @return mixed a string (including line return), false (EOF) or null (failure)
   *
   * @author <NAME> <<EMAIL>>
   */
  public function consume()
  {
    $first = $this->first;
    $this->first = false;
    if (!is_readable($this->path))
    {

      return null;
    }
    $fd = fopen($this->path, 'r');
    if ($first)
    {
      fseek($fd, 0, SEEK_END);
    }
    else
    {
      fseek($fd, $this->pos, SEEK_SET);
      // TODO handle file truncation
    }
    $line = fgets($fd);
    $this->pos = ftell($fd);
    fclose($fd);

    return $line;
  }
}

class MultiTail
{
  protected $tails = array();

  /**
   * Adds a Tail to watch
   * @param string $name Unique name
   * @param Tail $tail
   *
   * @author <NAME> <<EMAIL>>
   */
  public function add($name, $tail)
  {
    $this->tails[$name] = $tail;
  }
<commit_msg>Handle file truncation<commit_after><?php
class Tail
{
  protected $path = null;
  protected $first = true;
  protected $pos = 0;

  /**
   * Follow a file
   * @param string $path
   *
   * @author <NAME> <<EMAIL>>
   */
  public function __construct($path)
  {
    $this->path = $path;
  }

  /**
   * Read a new line a long as it is possible.
   * Goes straight to the end the first time if the file exists.
   * @return mixed a string (including line return), false (EOF) or null (failure)
   *
   * @author <NAME> <<EMAIL>>
   */
  public function consume()
  {
    $first = $this->first;
    $this->first = false;
    if (!is_readable($this->path))
    {

      return null;
    }
    $fd = fopen($this->path, 'r');
    if ($first)
    {
      fseek($fd, 0, SEEK_END);
    }
    else
    {
      fseek($fd, $this->pos, SEEK_SET);
    }
    $line = fgets($fd);
    $this->pos = ftell($fd);

    if ($line === false)
    {
      /* Detect file truncation.
       * There seem to be no better way, as fseek will accept
       * to go over EOF and fgets will not handle it differently either.
       */
      $stat = fstat($fd);
      if ($stat['size'] < $this->pos)
      {
        // rewind
        $this->pos = 0;
      }
    }
    fclose($fd);

    return $line;
  }
}

class MultiTail
{
  protected $tails = array();

  /**
   * Adds a Tail to watch
   * @param string $name Unique name
   * @param Tail $tail
   *
   * @author <NAME> <<EMAIL>>
   */
  public function add($name, $tail)
  {
    $this->tails[$name] = $tail;
  }

{'fraction_non_alphanumeric': 0.15688405797101448, 'fraction_numerical': 0.0018115942028985507, 'mean_word_length': 2.151826484018265, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7503882', 'n_tokens_mistral': 999, 'n_tokens_neox': 969, 'n_words': 283}