<commit_before>﻿namespace SourceLink

open System
open System.IO
open System.Collections.Generic
open System.Text
open SourceLink.PdbModify

type PdbFile(path) =
    let srcsrv = "srcsrv"
    let fs = File.Open(path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)
    let br = new BinaryReader(fs, Encoding.UTF8, true)
    let bw = new BinaryWriter(fs, Encoding.UTF8, true)

    do // check header
        let c00 = char 0x00 // null \0 null character
        let c0D = char 0x0D // cr \r carriage return
        let c0A = char 0x0A // lf \n line feed
        let c1A = char 0x1A // ctrl+z substitute character
        let msf = sprintf "Microsoft C/C++ MSF 7.00%c%c%cDS%c%c%c" c0D c0A c1A c00 c00 c00
        if false = msf.ToUtf8.CollectionEquals (br.ReadBytes 32) then
            Ex.failwithf "pdb header didn't match"
    
    // read rest of header
    let pageByteCount = br.ReadInt32() // 0x20
    let pagesFree = br.ReadInt32() // 0x24 TODO not sure meaning
    let pageCount = br.ReadInt32() // 0x28 for file
    let rootByteCount = br.ReadInt32() // 0x2C
    do br.Skip 4 // 0
    let mutable rootPage = br.ReadInt32() // 0x34

    do // check pdb
        let length = int fs.Length
        do if length % pageByteCount <> 0 then
            Ex.failwithf "pdb length %% bytes per page <> 0, %d, %d" length pageByteCount
        if length / pageByteCount <> pageCount then
            Ex.failwithf "pdb length does not match page count, length: %d, bytes per page: %d, page count: %d" length pageByteCount pageCount

    // reading functions
    let countPages nBytes = (nBytes + pageByteCount - 1) / pageByteCount
    let goToPage n = br.Position <- n * pageByteCount
    let goToEnd() = fs.Seek(0L, SeekOrigin.End) |> ignore
    let readPage (bytes:byte[]) page offset count =
        goToPage page
        let read = br.Read(bytes, offset, count)
        if read <> count then Ex.failwithf "tried reading %d bytes at offset %d, but only read %d" count offset read
    let readStreamBytes (stream:PdbStream) =
        let bytes = Array.create stream.ByteCount 0uy
        let pages = stream.Pages
        if pages.Length <> 0 then
            for i in 0 .. pages.Length - 2 do
                readPage bytes pages.[i] (i * pageByteCount) pageByteCount
            let i = pages.Length - 1
            readPage bytes pages.[i] (i * pageByteCount) (stream.ByteCount - (i * pageByteCount))
        bytes
    let readStream stream = new MemoryStream(readStreamBytes stream)
    let streamReader stream = new BinaryReader(readStream stream)

    // read root stream
    let readRoot streamRoot =
        let root = PdbRoot()
        use brDirectory = streamReader streamRoot
        let streamCount = brDirectory.ReadInt32()
        if streamCount <> 0x0131CA0B then
            let streams = root.Streams
            for i in 0 .. streamCount - 1 do
                let stream = PdbStream()
                streams.Add stream
                let byteCount = brDirectory.ReadInt32()
                stream.ByteCount <- byteCount
                let pageCount = countPages byteCount
                stream.Pages <- Array.create pageCount 0
            for i in 0 .. streamCount - 1 do
                for j in 0 .. streams.[i].Pages.Length - 1 do
                    let page = brDirectory.ReadInt32()
                    streams.[i].Pages.[j] <- page
        root
    let rootPdbStream =
        let pdbStream = PdbStream()
        pdbStream.ByteCount <- rootByteCount
        pdbStream.Pages <- Array.create (countPages rootByteCount) 0
        goToPage rootPage
        for i in 0 .. pdbStream.Pages.Length - 1 do
            pdbStream.Pages.[i] <- br.ReadInt32()
        pdbStream
    let root = readRoot rootPdbStream

    // read info stream
    let info =
        let info = PdbInfo()
        use br = streamReader root.Streams.[1]
        info.Version <- br.ReadInt32() // 0x00 of stream
        info.Signature <- br.ReadInt32() // 0x04
        info.Age <- br.ReadInt32() // 0x08
        info.Guid <- br.ReadGuid() // 0x0C
        let namesByteCount = br.ReadInt32() // 0x16
        let namesByteStart = br.Position // 0x20
        br.Position <- namesByteStart + namesByteCount
        let nameCount = br.ReadInt32()
        info.FlagIndexMax <- br.ReadInt32()
        info.FlagCount <- br.ReadInt32()
        let flags = Array.create info.FlagCount 0 // bit flags for each nameCountMax
        for i in 0 .. flags.Length - 1 do
            flags.[i] <- br.ReadInt32() 
        let hasName i =
            let a = flags.[i / 32]
            let b = 1 <<< (i % 32)
            a &&& b <> 0
        br.Skip 4 // 0
        let positions = List<int*PdbName>(nameCount)
        for i in 0 .. info.FlagIndexMax - 1 do
            if hasName i then
                let position = br.ReadInt32()
                let name = PdbName()
                name.FlagIndex <- i
                name.Stream <- br.ReadInt32()
                positions.Add(position, name)
        if positions.Count <> nameCount then
            Ex.failwithf "names count, %d <> %d" positions.Count nameCount
        let tailByteCount = root.Streams.[1].ByteCount - br.Position
        info.Tail <- br.ReadBytes tailByteCount
        for position, name in positions do
            br.Position <- namesByteStart + position
            name.Name <- br.ReadCString()
            info.AddName name
        info

    let freePages = SortedSet<int>()
    let zerosPage = Array.create pageByteCount 0uy

    let allocPages n = 
        let pages = List<int>()
        let free = freePages |> Seq.toList
        let nFree = if n <= free.Length then n else free.Length
        let nAlloc = if n <= free.Length then 0 else n - free.Length
        for i in 0 .. nFree-1 do
            let page = free.[i]
            pages.Add page
            freePages.Remove page |> ignore
        if nAlloc > 0 then
            goToEnd()
            for i in 0 .. nAlloc-1 do
                let page = (int fs.Position) / pageByteCount
                pages.Add page
                fs.WriteBytes zerosPage
        pages

    member x.Dispose() =
        use fs = fs
        use br = br
        use bw = bw
        GC.SuppressFinalize x
    interface IDisposable with member x.Dispose() = x.Dispose() 
    override x.Finalize() = x.Dispose()

    member x.Path with get() = path<commit_msg>may fix ctaggart/SourceLink#12 error disposing PdbFile, Cannot access a closed file.<commit_after>﻿namespace SourceLink

open System
open System.IO
open System.Collections.Generic
open System.Text
open SourceLink.PdbModify

type PdbFile(path) =
    let srcsrv = "srcsrv"
    let fs = File.Open(path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite)
    let br = new BinaryReader(fs, Encoding.UTF8, true)
    let bw = new BinaryWriter(fs, Encoding.UTF8, true)

    do // check header
        let c00 = char 0x00 // null \0 null character
        let c0D = char 0x0D // cr \r carriage return
        let c0A = char 0x0A // lf \n line feed
        let c1A = char 0x1A // ctrl+z substitute character
        let msf = sprintf "Microsoft C/C++ MSF 7.00%c%c%cDS%c%c%c" c0D c0A c1A c00 c00 c00
        if false = msf.ToUtf8.CollectionEquals (br.ReadBytes 32) then
            Ex.failwithf "pdb header didn't match"
    
    // read rest of header
    let pageByteCount = br.ReadInt32() // 0x20
    let pagesFree = br.ReadInt32() // 0x24 TODO not sure meaning
    let pageCount = br.ReadInt32() // 0x28 for file
    let rootByteCount = br.ReadInt32() // 0x2C
    do br.Skip 4 // 0
    let mutable rootPage = br.ReadInt32() // 0x34

    do // check pdb
        let length = int fs.Length
        do if length % pageByteCount <> 0 then
            Ex.failwithf "pdb length %% bytes per page <> 0, %d, %d" length pageByteCount
        if length / pageByteCount <> pageCount then
            Ex.failwithf "pdb length does not match page count, length: %d, bytes per page: %d, page count: %d" length pageByteCount pageCount

    // reading functions
    let countPages nBytes = (nBytes + pageByteCount - 1) / pageByteCount
    let goToPage n = br.Position <- n * pageByteCount
    let goToEnd() = fs.Seek(0L, SeekOrigin.End) |> ignore
    let readPage (bytes:byte[]) page offset count =
        goToPage page
        let read = br.Read(bytes, offset, count)
        if read <> count then Ex.failwithf "tried reading %d bytes at offset %d, but only read %d" count offset read
    let readStreamBytes (stream:PdbStream) =
        let bytes = Array.create stream.ByteCount 0uy
        let pages = stream.Pages
        if pages.Length <> 0 then
            for i in 0 .. pages.Length - 2 do
                readPage bytes pages.[i] (i * pageByteCount) pageByteCount
            let i = pages.Length - 1
            readPage bytes pages.[i] (i * pageByteCount) (stream.ByteCount - (i * pageByteCount))
        bytes
    let readStream stream = new MemoryStream(readStreamBytes stream)
    let streamReader stream = new BinaryReader(readStream stream)

    // read root stream
    let readRoot streamRoot =
        let root = PdbRoot()
        use brDirectory = streamReader streamRoot
        let streamCount = brDirectory.ReadInt32()
        if streamCount <> 0x0131CA0B then
            let streams = root.Streams
            for i in 0 .. streamCount - 1 do
                let stream = PdbStream()
                streams.Add stream
                let byteCount = brDirectory.ReadInt32()
                stream.ByteCount <- byteCount
                let pageCount = countPages byteCount
                stream.Pages <- Array.create pageCount 0
            for i in 0 .. streamCount - 1 do
                for j in 0 .. streams.[i].Pages.Length - 1 do
                    let page = brDirectory.ReadInt32()
                    streams.[i].Pages.[j] <- page
        root
    let rootPdbStream =
        let pdbStream = PdbStream()
        pdbStream.ByteCount <- rootByteCount
        pdbStream.Pages <- Array.create (countPages rootByteCount) 0
        goToPage rootPage
        for i in 0 .. pdbStream.Pages.Length - 1 do
            pdbStream.Pages.[i] <- br.ReadInt32()
        pdbStream
    let root = readRoot rootPdbStream

    // read info stream
    let info =
        let info = PdbInfo()
        use br = streamReader root.Streams.[1]
        info.Version <- br.ReadInt32() // 0x00 of stream
        info.Signature <- br.ReadInt32() // 0x04
        info.Age <- br.ReadInt32() // 0x08
        info.Guid <- br.ReadGuid() // 0x0C
        let namesByteCount = br.ReadInt32() // 0x16
        let namesByteStart = br.Position // 0x20
        br.Position <- namesByteStart + namesByteCount
        let nameCount = br.ReadInt32()
        info.FlagIndexMax <- br.ReadInt32()
        info.FlagCount <- br.ReadInt32()
        let flags = Array.create info.FlagCount 0 // bit flags for each nameCountMax
        for i in 0 .. flags.Length - 1 do
            flags.[i] <- br.ReadInt32() 
        let hasName i =
            let a = flags.[i / 32]
            let b = 1 <<< (i % 32)
            a &&& b <> 0
        br.Skip 4 // 0
        let positions = List<int*PdbName>(nameCount)
        for i in 0 .. info.FlagIndexMax - 1 do
            if hasName i then
                let position = br.ReadInt32()
                let name = PdbName()
                name.FlagIndex <- i
                name.Stream <- br.ReadInt32()
                positions.Add(position, name)
        if positions.Count <> nameCount then
            Ex.failwithf "names count, %d <> %d" positions.Count nameCount
        let tailByteCount = root.Streams.[1].ByteCount - br.Position
        info.Tail <- br.ReadBytes tailByteCount
        for position, name in positions do
            br.Position <- namesByteStart + position
            name.Name <- br.ReadCString()
            info.AddName name
        info

    let freePages = SortedSet<int>()
    let zerosPage = Array.create pageByteCount 0uy

    let allocPages n = 
        let pages = List<int>()
        let free = freePages |> Seq.toList
        let nFree = if n <= free.Length then n else free.Length
        let nAlloc = if n <= free.Length then 0 else n - free.Length
        for i in 0 .. nFree-1 do
            let page = free.[i]
            pages.Add page
            freePages.Remove page |> ignore
        if nAlloc > 0 then
            goToEnd()
            for i in 0 .. nAlloc-1 do
                let page = (int fs.Position) / pageByteCount
                pages.Add page
                fs.WriteBytes zerosPage
        pages

    member x.Dispose() =
        use fs = fs
        GC.SuppressFinalize x
    interface IDisposable with member x.Dispose() = x.Dispose() 
//    override x.Finalize() = x.Dispose()

    member x.Path with get() = path
{'fraction_non_alphanumeric': 0.09053110535812348, 'fraction_numerical': 0.023691849062524514, 'mean_word_length': 1.8512636993961082, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1836631', 'n_tokens_mistral': 4021, 'n_tokens_neox': 3705, 'n_words': 1236}