<commit_before><?php

namespace Gittern\Transport;

use Iodophor\Io\Reader;

/**
* @author <NAME>
**/
class PackfileIndex
{
  protected $reader;

  protected $size;

  const HEADER_SIZE = 8;

  public function __construct(Reader $reader)
  {
    $this->reader = $reader;
  }

  protected function readFanoutForPrefix($prefix)
  {
    $this->reader->setOffset(self::HEADER_SIZE + $prefix * 4);
    return $this->reader->readUInt32BE();
  }

  public function getSize()
  {
    if (!$this->size)
    {
      $this->size = $this->readFanoutForPrefix(0xFF);
    }
    return $this->size;
  }

  protected function getShasStart()
  {
    return self::HEADER_SIZE + 0x100*4;
  }

  protected function getShasStop()
  {
    return $this->getShasStart() + $this->getSize()*20;
  }

  protected function getCrcsStart()
  {
    return $this->getShasStop();
  }

  protected function getCrcsStop()
  {
    return $this->getCrcsStart() + $this->getSize() * 4;
  }

  protected function getSmallPackfileOffsetsStart()
  {
    return $this->getCrcsStop();
  }

  public function getShas()
  {
    $start = $this->getShasStart();
    $stop = $this->getShasStop();
    $shas = array();

    $this->reader->setOffset($start);
    while ($this->reader->getOffset() < $stop)
    {
      $shas[] = $this->reader->readHHex(20);
    }

    return $shas;
  }

  protected function getOffsetForSha($sha)
  {
    $prefix = hexdec(substr($sha, 0, 2));
    $index = $this->readFanoutForPrefix($prefix)-1;

    $shas_start = $this->getShasStart();
    $start = $shas_start + $index*20;
    $shas_stop = $this->getShasStop();

    $counter = $index;
    $this->reader->setOffset($start);
    do {
      $read_sha = $this->reader->readHHex(20);
      if ($read_sha == $sha)
      {
        return $counter;
      }

      $read_prefix = hexdec(substr($read_sha, 0, 2));
      $counter++;
    } while ($read_prefix <= $prefix && $this->reader->getOffset() < $shas_stop);

    throw new \RuntimeException("SHA $sha is not in packfile index");
  }

  public function getCrcForSha($sha)
  {
    $offset = $this->getCrcsStart() + $this->getOffsetForSha($sha)*4;
    $this->reader->setOffset($offset);
    return $this->reader->readHHex(4);
  }

  public function getPackfileOffsetForSha($sha)
  {
    $offset = $this->getSmallPackfileOffsetsStart() + $this->getOffsetForSha($sha)*4;
    $this->reader->setOffset($offset);
    return $this->reader->readUInt32BE(4);
  }

  public function hasSha($sha)
  {
    $shas = $this->getShas();

    return in_array($sha, $shas);
  }
}<commit_msg>Fixed a bug arising from a misunderstanding of how the git index works. The fanout table contains the number of entries having a SHA with that prefix or lower, not the offset at which the SHAs start.<commit_after><?php

namespace Gittern\Transport;

use Iodophor\Io\Reader;

/**
* @author <NAME>
**/
class PackfileIndex
{
  protected $reader;

  protected $size;

  const HEADER_SIZE = 8;

  public function __construct(Reader $reader)
  {
    $this->reader = $reader;
  }

  protected function readFanoutForPrefix($prefix)
  {
    $this->reader->setOffset(self::HEADER_SIZE + $prefix * 4);
    return $this->reader->readUInt32BE();
  }

  public function getSize()
  {
    if (!$this->size)
    {
      $this->size = $this->readFanoutForPrefix(0xFF);
    }
    return $this->size;
  }

  protected function getShasStart()
  {
    return self::HEADER_SIZE + 0x100*4;
  }

  protected function getShasStop()
  {
    return $this->getShasStart() + $this->getSize()*20;
  }

  protected function getCrcsStart()
  {
    return $this->getShasStop();
  }

  protected function getCrcsStop()
  {
    return $this->getCrcsStart() + $this->getSize() * 4;
  }

  protected function getSmallPackfileOffsetsStart()
  {
    return $this->getCrcsStop();
  }

  public function getShas()
  {
    $start = $this->getShasStart();
    $stop = $this->getShasStop();
    $shas = array();

    $this->reader->setOffset($start);
    while ($this->reader->getOffset() < $stop)
    {
      $shas[] = $this->reader->readHHex(20);
    }

    return $shas;
  }

  protected function getStartOffsetForPrefix($prefix)
  {
    if ($prefix == 0)
    {
      return 0;
    }
    else
    {
      return $this->getStopOffsetForPrefix($prefix - 1);
    }
  }

  protected function getStopOffsetForPrefix($prefix)
  {
    $fanout = $this->readFanoutForPrefix($prefix);
    return $this->getShasStart() + ($fanout * 20);
  }

  protected function getStartCounterForPrefix($prefix)
  {
    if ($prefix == 0)
    {
      return 0;
    }
    else
    {
      return $this->readFanoutForPrefix($prefix-1);
    }
  }

  protected function getOffsetForSha($sha)
  {
    $prefix = hexdec(substr($sha, 0, 2));

    $start = $this->getStartOffsetForPrefix($prefix);
    $stop = $this->getStopOffsetForPrefix($prefix);

    $counter = $this->getStartCounterForPrefix($prefix);
    $this->reader->setOffset($start);
    do {
      $read_sha = $this->reader->readHHex(20);
      if ($read_sha == $sha)
      {
        return $counter;
      }

      $read_prefix = hexdec(substr($read_sha, 0, 2));
      $counter++;
    } while ($read_prefix <= $prefix && $this->reader->getOffset() < $stop);

    throw new \RuntimeException("SHA $sha is not in packfile index");
  }

  public function getCrcForSha($sha)
  {
    $offset = $this->getCrcsStart() + $this->getOffsetForSha($sha)*4;
    $this->reader->setOffset($offset);
    return $this->reader->readHHex(4);
  }

  public function getPackfileOffsetForSha($sha)
  {
    $offset = $this->getSmallPackfileOffsetsStart() + $this->getOffsetForSha($sha)*4;
    $this->reader->setOffset($offset);
    return $this->reader->readUInt32BE(4);
  }

  public function hasSha($sha)
  {
    $shas = $this->getShas();

    return in_array($sha, $shas);
  }
}
{'fraction_non_alphanumeric': 0.16008280144902537, 'fraction_numerical': 0.011212696222183889, 'mean_word_length': 3.222869628550619, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1200030', 'n_tokens_mistral': 2075, 'n_tokens_neox': 1991, 'n_words': 377}