<commit_before>module Duffer.Pack.Parser where

import qualified Data.ByteString      as B
import qualified Data.ByteString.Lazy as L

import Data.Attoparsec.ByteString
import Data.Bits

import Codec.Compression.Zlib (decompress)
import Control.Monad          (zipWithM)
import Data.List              (foldl')
import GHC.Word               (Word8)

import Prelude hiding (take)

import Duffer.Loose.Objects
import Duffer.Loose.Parser (parseBinRef, parseBlob, parseTree, parseCommit
                           ,parseTag)
import Duffer.Pack.Entries

hashResolved :: PackObjectType -> PackDecompressed B.ByteString -> Ref
hashResolved t = hash . parseResolved t

parseResolved :: PackObjectType -> PackDecompressed B.ByteString -> GitObject
parseResolved t (PackDecompressed _ source) =
    either error id $ parseOnly (parseObjectContent t) source

word8s :: [Word8] -> Parser [Word8]
word8s = mapM word8

parsePackIndex :: Parser [PackIndexEntry]
parsePackIndex = do
    total     <- parsePackIndexHeader *> fmap last parsePackIndexTotals
    refs      <- parsePackIndexRefs total
    crc32s    <- count total parse4Bytes
    offsets   <- count total parse4Bytes
    remaining <- takeByteString
    let (fifth, checks) = B.splitAt (B.length remaining - 40) remaining
    let fixedOffsets    = map (fixOffsets (fifthOffsets fifth)) offsets
    return $ zipWith3 PackIndexEntry fixedOffsets refs crc32s

parsePackIndexUptoRefs :: Parser [Ref]
parsePackIndexUptoRefs = do
    total <- parsePackIndexHeader *> fmap last parsePackIndexTotals
    parsePackIndexRefs total

parsePackIndexHeader :: Parser ()
parsePackIndexHeader = word8s start *> word8s version *> pure ()
    where start   = [255, 116, 79, 99]
          version = [0, 0, 0, 2]

parsePackIndexTotals :: Parser [Int]
parsePackIndexTotals = count 256 parse4Bytes

parsePackIndexRefs :: Int -> Parser [Ref]
parsePackIndexRefs = flip count parseBinRef

parse4Bytes :: (Bits t, Integral t) => Parser t
parse4Bytes = fromBytes <$> take 4

parsedIndex :: B.ByteString -> [PackIndexEntry]
parsedIndex = either error id . parseOnly parsePackIndex

parseVarInt :: (Bits t, Integral t) => Parser [t]
parseVarInt = anyWord8 >>= \byte ->
    let value = fromIntegral $ byte .&. 127
        more  = testMSB byte
    in (value:) <$> if more then parseVarInt else return []

testMSB :: Bits t => t -> Bool
testMSB = flip testBit 7

littleEndian, bigEndian :: (Bits t, Integral t) => [t] -> t
littleEndian = foldr  (\a b -> a + (b `shiftL` 7)) 0
bigEndian    = foldl' (\a b -> (a `shiftL` 7) + b) 0

parseOffset :: (Bits t, Integral t) => Parser t
parseOffset = do
    values <- parseVarInt
    let len = length values - 1
    let concatenated = bigEndian values
    return $ concatenated + if len > 0
        -- I think the addition reinstates the MSBs that are otherwise
        -- used to indicate whether there is more of the variable length
        -- integer to parse.
        then sum $ map (\i -> 2^(7*i)) [1..len]
        else 0

parseTypeLen :: (Bits t, Integral t) => Parser (PackObjectType, t)
parseTypeLen = do
    header <- anyWord8
    let packType = packObjectType header
    let initial  = fromIntegral $ header .&. 15
    size <- if testMSB header
        then do
            rest <- littleEndian <$> parseVarInt
            return $ initial + (rest `shiftL` 4)
        else
            return initial
    return (packType, size)

parseDeltaInstruction :: Parser DeltaInstruction
parseDeltaInstruction = do
    instruction <- fromIntegral <$> anyWord8
    if testMSB instruction
        then parseCopyInstruction   instruction
        else parseInsertInstruction instruction

parseInsertInstruction :: Int -> Parser DeltaInstruction<commit_msg>Less do-notation.<commit_after>module Duffer.Pack.Parser where

import qualified Data.ByteString      as B
import qualified Data.ByteString.Lazy as L

import Data.Attoparsec.ByteString
import Data.Bits

import Codec.Compression.Zlib (decompress)
import Control.Monad          (zipWithM)
import Data.List              (foldl')
import GHC.Word               (Word8)

import Prelude hiding (take)

import Duffer.Loose.Objects
import Duffer.Loose.Parser (parseBinRef, parseBlob, parseTree, parseCommit
                           ,parseTag)
import Duffer.Pack.Entries

hashResolved :: PackObjectType -> PackDecompressed B.ByteString -> Ref
hashResolved t = hash . parseResolved t

parseResolved :: PackObjectType -> PackDecompressed B.ByteString -> GitObject
parseResolved t (PackDecompressed _ source) =
    either error id $ parseOnly (parseObjectContent t) source

word8s :: [Word8] -> Parser [Word8]
word8s = mapM word8

parsePackIndex :: Parser [PackIndexEntry]
parsePackIndex = do
    total     <- parsePackIndexHeader *> fmap last parsePackIndexTotals
    refs      <- parsePackIndexRefs total
    crc32s    <- count total parse4Bytes
    offsets   <- count total parse4Bytes
    remaining <- takeByteString
    let (fifth, checks) = B.splitAt (B.length remaining - 40) remaining
    let fixedOffsets    = map (fixOffsets (fifthOffsets fifth)) offsets
    return $ zipWith3 PackIndexEntry fixedOffsets refs crc32s

parsePackIndexUptoRefs :: Parser [Ref]
parsePackIndexUptoRefs = do
    total <- parsePackIndexHeader *> fmap last parsePackIndexTotals
    parsePackIndexRefs total

parsePackIndexHeader :: Parser ()
parsePackIndexHeader = word8s start *> word8s version *> pure ()
    where start   = [255, 116, 79, 99]
          version = [0, 0, 0, 2]

parsePackIndexTotals :: Parser [Int]
parsePackIndexTotals = count 256 parse4Bytes

parsePackIndexRefs :: Int -> Parser [Ref]
parsePackIndexRefs = flip count parseBinRef

parse4Bytes :: (Bits t, Integral t) => Parser t
parse4Bytes = fromBytes <$> take 4

parsedIndex :: B.ByteString -> [PackIndexEntry]
parsedIndex = either error id . parseOnly parsePackIndex

parseVarInt :: (Bits t, Integral t) => Parser [t]
parseVarInt = anyWord8 >>= \byte ->
    let value = fromIntegral $ byte .&. 127
        more  = testMSB byte
    in (value:) <$> if more then parseVarInt else return []

testMSB :: Bits t => t -> Bool
testMSB = flip testBit 7

littleEndian, bigEndian :: (Bits t, Integral t) => [t] -> t
littleEndian = foldr  (\a b -> a + (b `shiftL` 7)) 0
bigEndian    = foldl' (\a b -> (a `shiftL` 7) + b) 0

parseOffset :: (Bits t, Integral t) => Parser t
parseOffset = do
    values <- parseVarInt
    let len = length values - 1
    let concatenated = bigEndian values
    return $ concatenated + if len > 0
        -- I think the addition reinstates the MSBs that are otherwise
        -- used to indicate whether there is more of the variable length
        -- integer to parse.
        then sum $ map (\i -> 2^(7*i)) [1..len]
        else 0

parseTypeLen :: (Bits t, Integral t) => Parser (PackObjectType, t)
parseTypeLen = do
    header <- anyWord8
    let packType = packObjectType header
    let initial  = fromIntegral $ header .&. 15
    size <- if testMSB header
        then do
            rest <- littleEndian <$> parseVarInt
            return $ initial + (rest `shiftL` 4)
        else
            return initial
    return (packType, size)

parseDeltaInstruction :: Parser DeltaInstruction
parseDeltaInstruction = fromIntegral <$> anyWord8 >>= \instruction ->
    if testMSB instruction
        then parseCopyInstruction   instruction
        else parseInsertInstruction instruction

parseInsertInstruction :: Int -> Parser DeltaInstruction
{'fraction_non_alphanumeric': 0.08771929824561403, 'fraction_numerical': 0.015654520917678813, 'mean_word_length': 3.1705120990433313, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2612254', 'n_tokens_mistral': 2324, 'n_tokens_neox': 2223, 'n_words': 733}