<commit_before><?php

class GitRepository
{
    private $dbw;
    
    public function __construct(&$dbw)
    {
        $this->dbw = $dbw;
    }
    
    public static function createBlobObject($data)
    {
        $blob = 'blob ' . strlen($data) . "\0";
        $hash_bin = hash('sha1', $blob, true);
        $hash_hex = bin2hex($hash_bin);
        return array('blob' => $blob, 'hash_hex' => $hash_hex, 'hash_bin' => $hash_bin);
    }
    
    public static function readBlobObject($blob)
    {
        sscanf($blob, "blob %d\0%s", $length, $data);
        return $data;
    }
    
    public static function createTreeObject($tree_data)
    {
        $tree;
        
        foreach ($tree_data as $entry)
        {
            switch ($entry['type'])
            {
                case 'NORMAL_FILE':
                    $tree .= '100644';
                    break;
                case 'EXEC_FILE':
                    $tree .= '100755';
                    break;
                case 'SYMLINK'
                    $tree .= '120000';
                    // Not sure where this would be used, but for completeness...
                    break;
                case 'TREE'
                    $tree .= '40000';
                    break;
            }
            
            $tree = $tree . ' ' . $entry['name'] . "\0" . $entry['hash_bin'];
        }
        
        $length = strlen($tree);
        $tree = 'tree ' . $length . "\0" . $tree;
        $hash_bin = hash('sha1', $tree, true);
        $hash_hex = bin2hex($hash_bin);
        
        return array('tree' => $tree, 'hash_hex' => $hash_hex, 'hash_bin' => $hash_bin);
    }
    
    public static function readTreeObject($tree)
    {
        sscanf($tree, "tree %d\0", $length);
        $raw_entries = substr(
            $tree,
            strpos($tree, "\0") + 1,
            $length
        );
        $raw_bytes = str_split($raw_entries);
        
        $tree_data = array();
        
        for ($i = 0; isset($raw_entries[$i]); ++$i)
        {
            /* Either the null is marking the beginning of the hash,
             * or it is part of the hash itself. However, the latter
             * case should not appear, since $i will be pushed past
             * the hash automatically upon encountering any NUL char.
             */
            if ($raw_bytes[$i] === "\0")
            {
                if (!isset($beginning_of_entry))
                {
                    /* "Rewind" to beginning of string to get type and name.
                     * This relies on a subtle difference between isset() and
                     * empty().
                     */
                    $beginning_of_entry = 0;
                }
                
                sscanf(
                    substr($raw_entries, $beginning_of_entry, $i - $beginning_of_entry),
                    "%d %s[^\t\n]",
                    $type_id,
                    $filename
                );
                
                switch ($type_id)
                {
                    case 100644:
                        $type = 'NORMAL_FILE';
                        break;
                    case 100755:
                        $type = 'EXEC_FILE';
                        break;
                    case 120000:
                        $type = 'SYMLINK';
                        break;
                    case 40000:
                        $type = 'TREE';
                        break;
                }
                
                $hash_bin = substr($raw_entries, $i + 1, 20);
                $hash_hex = bin2hex($hash_bin);
                
                $file_entry = array('type' => $type, 'name' => $filename, 'hash_bin' => $hash_bin, 'hash_hex' => $hash_hex);
                array_push($tree_data, $file_entry);
                
                $i = $i + 21; // Push $i past the hash
                $beginning_of_entry = $i;
            }
        }
        return $tree_data;
    }
}

<commit_msg>Fix blob parsing;<commit_after><?php

class GitRepository
{
    private $dbw;
    
    public function __construct(&$dbw)
    {
        $this->dbw = $dbw;
    }
    
    public static function createBlobObject($data)
    {
        $blob = 'blob ' . strlen($data) . "\0";
        $hash_bin = hash('sha1', $blob, true);
        $hash_hex = bin2hex($hash_bin);
        return array('blob' => $blob, 'hash_hex' => $hash_hex, 'hash_bin' => $hash_bin);
    }
    
    public static function readBlobObject($blob)
    {
        sscanf($blob, "blob %d\0", $length);
        $data = substr(
            $blob,
            strpos($blob, "\0") + 1,
            $length
        );
        return $data;
    }
    
    public static function createTreeObject($tree_data)
    {
        $tree;
        
        foreach ($tree_data as $entry)
        {
            switch ($entry['type'])
            {
                case 'NORMAL_FILE':
                    $tree .= '100644';
                    break;
                case 'EXEC_FILE':
                    $tree .= '100755';
                    break;
                case 'SYMLINK'
                    $tree .= '120000';
                    // Not sure where this would be used, but for completeness...
                    break;
                case 'TREE'
                    $tree .= '40000';
                    break;
            }
            
            $tree = $tree . ' ' . $entry['name'] . "\0" . $entry['hash_bin'];
        }
        
        $length = strlen($tree);
        $tree = 'tree ' . $length . "\0" . $tree;
        $hash_bin = hash('sha1', $tree, true);
        $hash_hex = bin2hex($hash_bin);
        
        return array('tree' => $tree, 'hash_hex' => $hash_hex, 'hash_bin' => $hash_bin);
    }
    
    public static function readTreeObject($tree)
    {
        sscanf($tree, "tree %d\0", $length);
        $raw_entries = substr(
            $tree,
            strpos($tree, "\0") + 1,
            $length
        );
        $raw_bytes = str_split($raw_entries);
        
        $tree_data = array();
        
        for ($i = 0; isset($raw_entries[$i]); ++$i)
        {
            /* Either the null is marking the beginning of the hash,
             * or it is part of the hash itself. However, the latter
             * case should not appear, since $i will be pushed past
             * the hash automatically upon encountering any NUL char.
             */
            if ($raw_bytes[$i] === "\0")
            {
                if (!isset($beginning_of_entry))
                {
                    /* "Rewind" to beginning of string to get type and name.
                     * This relies on a subtle difference between isset() and
                     * empty().
                     */
                    $beginning_of_entry = 0;
                }
                
                sscanf(
                    substr($raw_entries, $beginning_of_entry, $i - $beginning_of_entry),
                    "%d %s[^\t\n]",
                    $type_id,
                    $filename
                );
                
                switch ($type_id)
                {
                    case 100644:
                        $type = 'NORMAL_FILE';
                        break;
                    case 100755:
                        $type = 'EXEC_FILE';
                        break;
                    case 120000:
                        $type = 'SYMLINK';
                        break;
                    case 40000:
                        $type = 'TREE';
                        break;
                }
                
                $hash_bin = substr($raw_entries, $i + 1, 20);
                $hash_hex = bin2hex($hash_bin);
                
                $file_entry = array('type' => $type, 'name' => $filename, 'hash_bin' => $hash_bin, 'hash_hex' => $hash_hex);
                array_push($tree_data, $file_entry);
                
                $i = $i + 21; // Push $i past the hash
                $beginning_of_entry = $i;
            }
        }
        return $tree_data;
    }
}


{'fraction_non_alphanumeric': 0.11994542297196725, 'fraction_numerical': 0.01662118580997271, 'mean_word_length': 1.0532212885154062, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2868927', 'n_tokens_mistral': 2341, 'n_tokens_neox': 2162, 'n_words': 526}