<commit_before>module Duffer.Pack where

import qualified Data.ByteString as B
import qualified Data.Map.Strict as Map

import GHC.Int (Int64)
import System.IO.MMap (mmapFileByteString)
import System.FilePath ((-<.>))

import Duffer.Loose.Objects (GitObject)
import Duffer.Pack.Entries
import Duffer.Pack.Parser
import Duffer.Pack.File

packFile, packIndex :: FilePath -> FilePath
packFile  path = path -<.> "pack"
packIndex path = path -<.> "idx"

region :: Map.Map Int a -> Int -> Maybe (Int64, Int)
region offsetMap offset = let
    (Just nextOffset) = Map.lookupGT offset offsetMap
    len               = fst nextOffset - offset
    in Just (fromIntegral offset, len)

packFileRegion :: FilePath -> Maybe (Int64, Int) -> IO B.ByteString
packFileRegion = mmapFileByteString

getPackFileEntry :: FilePath -> Map.Map Int B.ByteString -> Int -> IO PackEntry
getPackFileEntry packFilePath rangeMap index =
    getPackRegion <$> packFileRegion packFilePath (region rangeMap index)

indexedEntryMap :: FilePath -> IO OffsetMap
indexedEntryMap indexPath = do
    indexContent <- B.readFile indexPath
    let offsetMap = makeOffsetMap indexContent
    let indices   = Map.keys offsetMap
    let filePath  = packFile indexPath
    contentEnd <- B.length <$> B.readFile filePath
    let rangeMap  = Map.insert (contentEnd - 20) "" offsetMap
    entries <- mapM (getPackFileEntry filePath rangeMap) indices
    return $ Map.fromAscList $ zip indices entries

combinedEntryMap :: FilePath -> IO CombinedMap
combinedEntryMap indexPath = CombinedMap
    <$> indexedEntryMap indexPath
    <*> (makeRefIndex <$> B.readFile indexPath)

resolveAll :: FilePath -> IO [GitObject]
resolveAll indexPath = do
    combined <- combinedEntryMap indexPath
    let reconstitute = unpackObject . resolveDelta combined
    return $ map reconstitute $ Map.elems (getRefIndex combined)<commit_msg>Decompose packfile logic for better testing.<commit_after>module Duffer.Pack where

import qualified Data.ByteString as B
import qualified Data.Map.Strict as Map

import GHC.Int (Int64)
import System.IO.MMap (mmapFileByteString)
import System.FilePath ((-<.>))

import Duffer.Loose.Objects (GitObject)
import Duffer.Pack.Entries
import Duffer.Pack.Parser
import Duffer.Pack.File

packFile, packIndex :: FilePath -> FilePath
packFile  path = path -<.> "pack"
packIndex path = path -<.> "idx"

region :: Map.Map Int a -> Int -> Maybe (Int64, Int)
region offsetMap offset = let
    (Just nextOffset) = Map.lookupGT offset offsetMap
    len               = fst nextOffset - offset
    in Just (fromIntegral offset, len)

getPackFileEntry :: FilePath -> Map.Map Int B.ByteString -> Int -> IO PackEntry
getPackFileEntry packFilePath rangeMap index =
    parsedPackRegion <$> getPackRegion packFilePath rangeMap index

getPackRegion :: FilePath -> Map.Map Int B.ByteString -> Int -> IO B.ByteString
getPackRegion packFilePath rangeMap index =
    packFileRegion packFilePath (region rangeMap index)

packFileRegion :: FilePath -> Maybe (Int64, Int) -> IO B.ByteString
packFileRegion = mmapFileByteString

indexedEntryMap :: FilePath -> IO OffsetMap
indexedEntryMap = fmap (Map.map parsedPackRegion) . indexedByteStringMap

indexedByteStringMap :: FilePath -> IO (Map.Map Int B.ByteString)
indexedByteStringMap indexPath = do
    offsetMap  <- makeOffsetMap <$> B.readFile indexPath
    let filePath  = packFile indexPath
    contentEnd <- B.length <$> B.readFile filePath
    let indices   = Map.keys offsetMap
    let rangeMap  = Map.insert (contentEnd - 20) "" offsetMap
    entries    <- mapM (getPackRegion filePath rangeMap) indices
    return $ Map.fromAscList $ zip indices entries

combinedEntryMap :: FilePath -> IO CombinedMap
combinedEntryMap indexPath = CombinedMap
    <$> indexedEntryMap              indexPath
    <*> (makeRefIndex <$> B.readFile indexPath)

resolveAll :: FilePath -> IO [GitObject]
resolveAll indexPath = do
    combined <- combinedEntryMap indexPath
    let reconstitute = unpackObject . resolveDelta combined
    return $ map reconstitute $ Map.elems (getRefIndex combined)
{'fraction_non_alphanumeric': 0.08796068796068796, 'fraction_numerical': 0.003931203931203931, 'mean_word_length': 3.988970588235294, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3054224', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1178, 'n_words': 370}