<commit_before>/**
 * Copyright 2012 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.jsign.pe;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;

import org.bouncycastle.asn1.cms.Attribute;
import org.bouncycastle.asn1.cms.ContentInfo;
import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
import org.bouncycastle.asn1.x509.X509ObjectIdentifiers;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.cms.CMSProcessable;
import org.bouncycastle.cms.CMSSignedData;
import org.bouncycastle.cms.SignerInformation;

/**
 * Portable Executable File.
 * 
 * This class is not thread safe, read access must be synchronized.
 * 
 * @see <a href="http://msdn.microsoft.com/en-us/library/windows/hardware/gg463119.aspx">Microsoft PE and COFF Specification </a>
 * 
 * @author <NAME>
 * @since 1.0
 */
public class PEFile implements Closeable {

    /** The position of the PE header in the file */
    private long peHeaderOffset;

    private File file;
    private ExtendedRandomAccessFile raf;

    public PEFile(File file) throws IOException {
        this.file = file;
        raf = new ExtendedRandomAccessFile(file, "rw");
        
        // DOS Header
        
        byte[] buffer = new byte[2];
        raf.read(buffer);
        
        if (!Arrays.equals(buffer, "MZ".getBytes())) {
            throw new IOException("DOS header signature not found");
        }
        
        raf.seek(0x3C);
        peHeaderOffset = raf.readDWord();
        
        // PE Header
        
        raf.seek(peHeaderOffset);
        
        buffer = new byte[4];
        raf.read(buffer);
        if (!Arrays.equals(buffer, new byte[] { 'P', 'E', 0, 0})) {
            throw new IOException("PE signature not found as expected at offset 0x" + Long.toHexString(peHeaderOffset));
        }
    }

    public void close() throws IOException {
        raf.close();
    }
    
    int read(byte[] buffer, int offset) {
        try {
            raf.seek(offset);
            return raf.read(buffer);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    int read(long base, int offset) {
        try {
            raf.seek(base + offset);
            return raf.read();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    int readWord(long base, int offset) {
        try {
            raf.seek(base + offset);
            return raf.readWord();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    long readDWord(long base, int offset) {
        try {
            raf.seek(base + offset);
            return raf.readDWord();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    long readQWord(long base, int offset) {
        try {
            raf.seek(base + offset);
            return raf.readQWord();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    void write(long base, byte[] data) {
        try {
            raf.seek(base);
            raf.write(data);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    public MachineType getMachineType() {
        return MachineType.valueOf(readWord(peHeaderOffset, 4));
    }

    /**
     * The number of sections. This indicates the size of the section table,
     * which immediately follows the headers.
     */
    public int getNumberOfSections() {
        return readWord(peHeaderOffset, 6);
    }

    /**
     * The low 32 bits of the number of seconds since 00:00 January 1, 1970
     * (a C runtime time_t value), that indicates when the file was created.
     */
    public Date getTimeDateStamp() {
        return new Date(1000 * readDWord(peHeaderOffset, 8));
    }

    /**
     * The file offset of the COFF symbol table, or zero if no COFF symbol table
     * is present. This value should be zero for an image because COFF debugging
     * information is deprecated.
     */
    public long getPointerToSymbolTable() {
        return readDWord(peHeaderOffset, 12);
    }

    /**
     * The number of entries in the symbol table. This data can be used to
     * locate the string table, which immediately follows the symbol table.
     * This value should be zero for an image because COFF debugging
     * information is deprecated.
     */
    public long getNumberOfSymbols() {
        return readDWord(peHeaderOffset, 16);
    }

    /**
     * The size of the optional header, which is required for executable files
     * but not for object files. This value should be zero for an object file.
     */
    public int getSizeOfOptionalHeader() {
        return readWord(peHeaderOffset, 20);
    }

    /**
     * The flags that indicate the attributes of the file. 
     */
    public int getCharacteristics() {
        return readWord(peHeaderOffset, 22);
    }
    
    public PEFormat getFormat() {
        return PEFormat.valueOf(readWord(peHeaderOffset, 24));
    }

    /**
     * The linker major version number.
     */
    public int getMajorLinkerVersion() {
        return read(peHeaderOffset, 26);
    }

    /**
     * The linker minor version number.
     */
    public int getMinorLinkerVersion() {
        return read(peHeaderOffset, 27);
    }

    /**
     * The size of the code (text) section, or the sum of all code sections
     * if there are multiple sections.
     */
    public long getSizeOfCode() {
        return readDWord(peHeaderOffset, 28);
    }

    /**
     * The size of the initialized data section, or the sum of all such
     * sections if there are multiple data sections.
     */
    public long getSizeOfInitializedData() {
        return readDWord(peHeaderOffset, 32);
    }

    /**
     * The size of the uninitialized data section (BSS), or the sum of all such
     * sections if there are multiple BSS sections.
     */
    public long getSizeOfUninitializedData() {
        return readDWord(peHeaderOffset, 36);
    }

    /**
     * The address of the entry point relative to the image base when the
     * executable file is loaded into memory. For program images, this is the
     * starting address. For device drivers, this is the address of the
     * initialization function. An entry point is optional for DLLs. When no
     * entry point is present, this field must be zero.
     */
    public long getAddressOfEntryPoint() {
        return readDWord(peHeaderOffset, 40);
    }

    /**
     * The address that is relative to the image base of the beginning-of-code 
     * section when it is loaded into memory.
     */
    public long getBaseOfCode() {
        return readDWord(peHeaderOffset, 44);
    }

    /**
     * The address that is relative to the image base of the beginning-of-data 
     * section when it is loaded into memory (PE32 only).
     */
    public long getBaseOfData() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 48);
        } else {
            return 0;
        }
    }

    /**
     * The preferred address of the first byte of image when loaded into memory;
     * must be a multiple of 64 K. The default for DLLs is 0x10000000. The default
     * for Windows CE EXEs is 0x00010000. The default for Windows NT, Windows 2000,
     * Windows XP, Windows 95, Windows 98, and Windows Me is 0x00400000.
     */
    public long getImageBase() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 52);
        } else {
            return readQWord(peHeaderOffset, 48);
        }
    }

    /**
     * The alignment (in bytes) of sections when they are loaded into memory.
     * It must be greater than or equal to FileAlignment. The default is the
     * page size for the architecture.
     */
    public long getSectionAlignment() {
        return readDWord(peHeaderOffset, 56);
    }

    /**
     * The alignment factor (in bytes) that is used to align the raw data of
     * sections in the image file. The value should be a power of 2 between
     * 512 and 64 K, inclusive. The default is 512. If the SectionAlignment
     * is less than the architecture?s page size, then FileAlignment must
     * match SectionAlignment.
     */
    public long getFileAlignment() {
        return readDWord(peHeaderOffset, 60);
    }

    /**
     * The major version number of the required operating system.
     */
    public int getMajorOperatingSystemVersion() {
        return readWord(peHeaderOffset, 64);
    }

    /**
     * The minor version number of the required operating system.
     */
    public int getMinorOperatingSystemVersion() {
        return readWord(peHeaderOffset, 66);
    }

    /**
     * The major version number of the image.
     */
    public int getMajorImageVersion() {
        return readWord(peHeaderOffset, 68);
    }

    /**
     * The minor version number of the image.
     */
    public int getMinorImageVersion() {
        return readWord(peHeaderOffset, 70);
    }

    /**
     * The major version number of the subsystem.
     */
    public int getMajorSubsystemVersion() {
        return readWord(peHeaderOffset, 72);
    }

    /**
     * The minor version number of the subsystem.
     */
    public int getMinorSubsystemVersion() {
        return readWord(peHeaderOffset, 74);
    }

    /**
     * Reserved, must be zero.
     */
    public long getWin32VersionValue() {
        return readDWord(peHeaderOffset, 76);
    }

    /**
     * The size (in bytes) of the image, including all headers, as the image
     * is loaded in memory. It must be a multiple of SectionAlignment.
     */
    public long getSizeOfImage() {
        return readDWord(peHeaderOffset, 80);
    }

    /**
     * The combined size of an MS DOS stub, PE header, and section headers
     * rounded up to a multiple of FileAlignment.
     */
    public long getSizeOfHeaders() {
        return readDWord(peHeaderOffset, 84);
    }

    /**
     * The image file checksum. The algorithm for computing the checksum
     * is incorporated into IMAGHELP.DLL. The following are checked for
     * validation at load time: all drivers, any DLL loaded at boot time,
     * and any DLL that is loaded into a critical Windows process.
     */
    public long getCheckSum() {
        return readDWord(peHeaderOffset, 88);
    }

    /**
     * Compute the checksum of the image file.
     */
    public long computeChecksum() {
        try {
            long checksumOffset = peHeaderOffset + 88;

            long checksum = 0;
            long top = (long) Math.pow(2, 32);
            long length = raf.length();
            
            byte[] buffer = new byte[4];            

            raf.seek(0);
            for (long i = 0; i < length / 4; i++) {
                if (i == checksumOffset / 4) {
                    raf.skipBytes(4);
                    continue;
                }
                
                raf.readFully(buffer);
                
                long ch1 = buffer[0] & 0xFF;
                long ch2 = buffer[1] & 0xFF;
                long ch3 = buffer[2] & 0xFF;
                long ch4 = buffer[3] & 0xFF;
                
                long dword = ch1 + (ch2 << 8) + (ch3 << 16) + (ch4 << 24);
    
                checksum = (checksum & 0xffffffffL) + dword + (checksum >> 32);
                
                if (checksum > top) {
                    checksum = (checksum & 0xffffffffL) + (checksum >> 32);
                }
            }

            checksum = (checksum & 0xffff) + (checksum >> 16);
            checksum = checksum + (checksum >> 16);
            checksum = checksum & 0xffff;
            checksum += length;

            return checksum;
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    
    public void updateChecksum() {
        try {
            ByteBuffer buffer = ByteBuffer.allocate(4);
            buffer.order(ByteOrder.LITTLE_ENDIAN);
            buffer.putInt((int) computeChecksum());
            
            raf.seek(peHeaderOffset + 88);
            raf.write(buffer.array());
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * The subsystem that is required to run this image.
     */
    public Subsystem getSubsystem() {
        return Subsystem.valueOf(readWord(peHeaderOffset, 92));
    }

    public int getDllCharacteristics() {
        return readWord(peHeaderOffset, 94);
    }

    /**
     * The size of the stack to reserve. Only SizeOfStackCommit is committed;
     * the rest is made available one page at a time until the reserve size is reached.
     */
    public long getSizeOfStackReserve() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 96);
        } else {
            return readQWord(peHeaderOffset, 96);
        }
    }

    /**
     * The size of the stack to commit.
     */
    public long getSizeOfStackCommit() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 100);
        } else {
            return readQWord(peHeaderOffset, 104);
        }
    }

    /**
     * The size of the local heap space to reserve. Only SizeOfHeapCommit is
     * committed; the rest is made available one page at a time until the
     * reserve size is reached.
     */
    public long getSizeOfHeapReserve() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 104);
        } else {
            return readQWord(peHeaderOffset, 112);
        }
    }

    /**
     * The size of the local heap space to commit.
     */
    public long getSizeOfHeapCommit() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 108);
        } else {
            return readQWord(peHeaderOffset, 120);
        }
    }

    /**
     * Reserved, must be zero.
     */
    public long getLoaderFlags() {
        return readDWord(peHeaderOffset, PEFormat.PE32.equals(getFormat()) ? 112 : 128);
    }

    /**
     * The number of data-directory entries in the remainder of the optional
     * header. Each describes a location and size.
     */
    public int getNumberOfRvaAndSizes() {
        return (int) readDWord(peHeaderOffset, PEFormat.PE32.equals(getFormat()) ? 116 : 132);
    }

    int getDataDirectoryOffset() {
        return (int) peHeaderOffset + (PEFormat.PE32.equals(getFormat()) ? 120 : 136);
    }

    /**
     * Returns the data directory of the specified type.
     */
    public DataDirectory getDataDirectory(DataDirectoryType type) {
        if (type.ordinal() >= getNumberOfRvaAndSizes()) {
            return null;
        } else {
            return new DataDirectory(this, type.ordinal());
        }
    }

    /**
     * Writes the data directory of the specified type. The data is appended
     * at the end of the file. If a previous entry was already present it's
     * left as is in the file and is simply dereferenced from the table. 
     * 
     * @param type
     * @param data
     * @throws IOException
     */
    public void writeDataDirectory(DataDirectoryType type, byte[] data) throws IOException {
        // todo overwrite an existing entry at the end of the file
        
        // append the data directory at the end of the file
        long offset = raf.length();        
        
        raf.seek(offset);
        raf.write(data);
        
        // add the entry in the data directory table
        DataDirectory entry = getDataDirectory(type);
        entry.write(offset, data.length);
        
        updateChecksum();
    }

    /**
     * Returns the authenticode signatures on the file.
     */
    public List<CMSSignedData> getSignatures() {
        List<CMSSignedData> signatures = new ArrayList<CMSSignedData>();
        DataDirectory certificateTable = getDataDirectory(DataDirectoryType.CERTIFICATE_TABLE);
        
        if (certificateTable != null && certificateTable.getVirtualAddress() != 0 && certificateTable.getSize() != 0) {
            long position = certificateTable.getVirtualAddress();
            
            try {
                raf.seek(position);
                int size = (int) raf.readDWord();
                int revision = raf.readWord();
                int type = raf.readWord();

                if (revision == 0x0200 && type == 2) {
                    // PKCS#7 SignedData structure
                    byte[] buffer = new byte[size - 8];
                    raf.read(buffer);

                    CMSSignedData signedData = new CMSSignedData((CMSProcessable) null, ContentInfo.getInstance(buffer));
                    signatures.add(signedData);
                }
                
                // todo read the remaining signatures
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        
        return signatures;
    }

    /**
     * Print detailed informations about the PE file.
     */
    public void printInfo(OutputStream out) {
        printInfo(new PrintWriter(out, true));
    }

    /**
     * Print detailed informations about the PE file.
     */
    public void printInfo(PrintWriter out) {
        out.println("PE File");
        out.println("  Name:          " + file.getName());
        out.println("  Size:          " + file.length());
        out.println("  Last Modified: " + new Date(file.lastModified()));
        out.println();
        
        out.println("PE Header");
        out.println("  Machine:                    " + getMachineType());
        out.println("  Number of sections:         " + getNumberOfSections());
        out.println("  Timestamp:                  " + getTimeDateStamp());
        out.println("  Pointer to symbol table:    0x" + Long.toHexString(getPointerToSymbolTable()));
        out.println("  Number of symbols:          " + getNumberOfSymbols());
        out.println("  Size of optional header:    " + getSizeOfOptionalHeader());
        out.println("  Characteristics:            0x" + Long.toBinaryString(getCharacteristics()));
        out.println();
        
        out.println("Optional Header");
        PEFormat format = getFormat();
        out.println("  PE Format:                  0x" + Integer.toHexString(format.value) + " (" + format.label + ")");
        out.println("  Linker version:             " + getMajorLinkerVersion() + "." + getMinorLinkerVersion());
        out.println("  Size of code:               " + getSizeOfCode());
        out.println("  Size of initialized data:   " + getSizeOfInitializedData());
        out.println("  Size of uninitialized data: " + getSizeOfUninitializedData());
        out.println("  Address of entry point:     0x" + Long.toHexString(getAddressOfEntryPoint()));
        out.println("  Base of code:               0x" + Long.toHexString(getBaseOfCode()));
        if (PEFormat.PE32.equals(getFormat())) {
            out.println("  Base of data:               0x" + Long.toHexString(getBaseOfData()));
        }
        out.println("  Image base:                 0x" + Long.toHexString(getImageBase()));
        out.println("  Section alignment:          " + getSectionAlignment());
        out.println("  File alignment:             " + getFileAlignment());
        out.println("  Operating system version:   " + getMajorOperatingSystemVersion() + "." + getMinorOperatingSystemVersion());
        out.println("  Image version:              " + getMajorImageVersion() + "." + getMinorImageVersion());
        out.println("  Subsystem version:          " + getMajorSubsystemVersion() + "." + getMinorSubsystemVersion());
        out.println("  Size of image:              " + getSizeOfImage());
        out.println("  Size of headers:            " + getSizeOfHeaders());
        out.println("  Checksum:                   0x" + Long.toHexString(getCheckSum()));
        out.println("  Checksum (computed):        0x" + Long.toHexString(computeChecksum()));
        out.println("  Subsystem:                  " + getSubsystem());
        out.println("  DLL characteristics:        0x" + Long.toBinaryString(getDllCharacteristics()));
        out.println("  Size of stack reserve:      " + getSizeOfStackReserve());
        out.println("  Size of stack commit:       " + getSizeOfStackCommit());
        out.println("  Size of heap reserve:       " + getSizeOfHeapReserve());
        out.println("  Size of heap commit:        " + getSizeOfHeapCommit());
        out.println("  Number of RVA and sizes:    " + getNumberOfRvaAndSizes());
        out.println();
        
        out.println("Data Directory");
        for (DataDirectoryType type : DataDirectoryType.values()) {
            DataDirectory entry = getDataDirectory(type);
            if (entry != null && entry.getVirtualAddress() != 0) {
                out.printf("  %-30s 0x%08x %8d bytes\n", type, entry.getVirtualAddress(), entry.getSize());
            }
        }
        out.println();
        
        int sectionTableOffset = getDataDirectoryOffset() + 8 * getNumberOfRvaAndSizes(); 
        out.println("Sections");
        out.println("      Name     Virtual Size  Virtual Address  Raw Data Size  Raw Data Ptr  Characteristics");
        for (int i = 0; i < getNumberOfSections(); i++) {
            Section section = new Section(this, sectionTableOffset + 40 * i);
            out.printf("  #%d  %-8s     %8d       0x%08x       %8d    0x%08x  %s\n", i + 1, section.getName(), section.getVirtualSize(), section.getVirtualAddress(), section.getSizeOfRawData(), section.getPointerToRawData(), section.getCharacteristics());
        }
        out.println();
        
        List<CMSSignedData> signatures = getSignatures();
        if (!signatures.isEmpty()) {
            out.println("Signatures");
            for (CMSSignedData signedData : signatures) {
                SignerInformation signerInformation = (SignerInformation) signedData.getSignerInfos().getSigners().iterator().next();
                X509CertificateHolder certificate = (X509CertificateHolder) signedData.getCertificates().getMatches(signerInformation.getSID()).iterator().next();
                
                String commonName = certificate.getSubject().getRDNs(X509ObjectIdentifiers.commonName)[0].getFirst().getValue().toString();
                
                Attribute counterSignature = null;
                if (signerInformation.getUnsignedAttributes() != null) {
                    counterSignature = signerInformation.getUnsignedAttributes().get(PKCSObjectIdentifiers.pkcs_9_at_counterSignature);
                }
                
                out.println("  " + commonName + "   " + (counterSignature != null ? "(timestamped)" : ""));
            }
        }
    }

    /**
     * Compute the digest of the file. The checksum field, the certificate
     * directory table entry and the certificate table are excluded from
     * the digest.
     * 
     * todo Handle files > 2GB
     */
    private byte[] computeDigest(MessageDigest digest) throws IOException {
        int checksumLocation = (int) peHeaderOffset + 88;
        
        DataDirectory certificateTable = getDataDirectory(DataDirectoryType.CERTIFICATE_TABLE);
        
        // load the file in a buffer
        byte[] signedContent = new byte[(int) raf.length()];
        raf.seek(0);
        raf.read(signedContent);
        
        // digest from the beginning to the checksum field (excluded)
        digest.update(signedContent, 0, checksumLocation);
        
        // skip the checksum field
        int position = checksumLocation + 4;<commit_msg>Faster checksum<commit_after>/**
 * Copyright 2012 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.jsign.pe;

import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;

import org.bouncycastle.asn1.cms.Attribute;
import org.bouncycastle.asn1.cms.ContentInfo;
import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
import org.bouncycastle.asn1.x509.X509ObjectIdentifiers;
import org.bouncycastle.cert.X509CertificateHolder;
import org.bouncycastle.cms.CMSProcessable;
import org.bouncycastle.cms.CMSSignedData;
import org.bouncycastle.cms.SignerInformation;

/**
 * Portable Executable File.
 * 
 * This class is not thread safe, read access must be synchronized.
 * 
 * @see <a href="http://msdn.microsoft.com/en-us/library/windows/hardware/gg463119.aspx">Microsoft PE and COFF Specification </a>
 * 
 * @author <NAME>
 * @since 1.0
 */
public class PEFile implements Closeable {

    /** The position of the PE header in the file */
    private long peHeaderOffset;

    private File file;
    private ExtendedRandomAccessFile raf;

    public PEFile(File file) throws IOException {
        this.file = file;
        raf = new ExtendedRandomAccessFile(file, "rw");
        
        // DOS Header
        
        byte[] buffer = new byte[2];
        raf.read(buffer);
        
        if (!Arrays.equals(buffer, "MZ".getBytes())) {
            throw new IOException("DOS header signature not found");
        }
        
        raf.seek(0x3C);
        peHeaderOffset = raf.readDWord();
        
        // PE Header
        
        raf.seek(peHeaderOffset);
        
        buffer = new byte[4];
        raf.read(buffer);
        if (!Arrays.equals(buffer, new byte[] { 'P', 'E', 0, 0})) {
            throw new IOException("PE signature not found as expected at offset 0x" + Long.toHexString(peHeaderOffset));
        }
    }

    public void close() throws IOException {
        raf.close();
    }
    
    int read(byte[] buffer, int offset) {
        try {
            raf.seek(offset);
            return raf.read(buffer);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    int read(long base, int offset) {
        try {
            raf.seek(base + offset);
            return raf.read();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    int readWord(long base, int offset) {
        try {
            raf.seek(base + offset);
            return raf.readWord();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    long readDWord(long base, int offset) {
        try {
            raf.seek(base + offset);
            return raf.readDWord();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    long readQWord(long base, int offset) {
        try {
            raf.seek(base + offset);
            return raf.readQWord();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    void write(long base, byte[] data) {
        try {
            raf.seek(base);
            raf.write(data);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    public MachineType getMachineType() {
        return MachineType.valueOf(readWord(peHeaderOffset, 4));
    }

    /**
     * The number of sections. This indicates the size of the section table,
     * which immediately follows the headers.
     */
    public int getNumberOfSections() {
        return readWord(peHeaderOffset, 6);
    }

    /**
     * The low 32 bits of the number of seconds since 00:00 January 1, 1970
     * (a C runtime time_t value), that indicates when the file was created.
     */
    public Date getTimeDateStamp() {
        return new Date(1000 * readDWord(peHeaderOffset, 8));
    }

    /**
     * The file offset of the COFF symbol table, or zero if no COFF symbol table
     * is present. This value should be zero for an image because COFF debugging
     * information is deprecated.
     */
    public long getPointerToSymbolTable() {
        return readDWord(peHeaderOffset, 12);
    }

    /**
     * The number of entries in the symbol table. This data can be used to
     * locate the string table, which immediately follows the symbol table.
     * This value should be zero for an image because COFF debugging
     * information is deprecated.
     */
    public long getNumberOfSymbols() {
        return readDWord(peHeaderOffset, 16);
    }

    /**
     * The size of the optional header, which is required for executable files
     * but not for object files. This value should be zero for an object file.
     */
    public int getSizeOfOptionalHeader() {
        return readWord(peHeaderOffset, 20);
    }

    /**
     * The flags that indicate the attributes of the file. 
     */
    public int getCharacteristics() {
        return readWord(peHeaderOffset, 22);
    }
    
    public PEFormat getFormat() {
        return PEFormat.valueOf(readWord(peHeaderOffset, 24));
    }

    /**
     * The linker major version number.
     */
    public int getMajorLinkerVersion() {
        return read(peHeaderOffset, 26);
    }

    /**
     * The linker minor version number.
     */
    public int getMinorLinkerVersion() {
        return read(peHeaderOffset, 27);
    }

    /**
     * The size of the code (text) section, or the sum of all code sections
     * if there are multiple sections.
     */
    public long getSizeOfCode() {
        return readDWord(peHeaderOffset, 28);
    }

    /**
     * The size of the initialized data section, or the sum of all such
     * sections if there are multiple data sections.
     */
    public long getSizeOfInitializedData() {
        return readDWord(peHeaderOffset, 32);
    }

    /**
     * The size of the uninitialized data section (BSS), or the sum of all such
     * sections if there are multiple BSS sections.
     */
    public long getSizeOfUninitializedData() {
        return readDWord(peHeaderOffset, 36);
    }

    /**
     * The address of the entry point relative to the image base when the
     * executable file is loaded into memory. For program images, this is the
     * starting address. For device drivers, this is the address of the
     * initialization function. An entry point is optional for DLLs. When no
     * entry point is present, this field must be zero.
     */
    public long getAddressOfEntryPoint() {
        return readDWord(peHeaderOffset, 40);
    }

    /**
     * The address that is relative to the image base of the beginning-of-code 
     * section when it is loaded into memory.
     */
    public long getBaseOfCode() {
        return readDWord(peHeaderOffset, 44);
    }

    /**
     * The address that is relative to the image base of the beginning-of-data 
     * section when it is loaded into memory (PE32 only).
     */
    public long getBaseOfData() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 48);
        } else {
            return 0;
        }
    }

    /**
     * The preferred address of the first byte of image when loaded into memory;
     * must be a multiple of 64 K. The default for DLLs is 0x10000000. The default
     * for Windows CE EXEs is 0x00010000. The default for Windows NT, Windows 2000,
     * Windows XP, Windows 95, Windows 98, and Windows Me is 0x00400000.
     */
    public long getImageBase() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 52);
        } else {
            return readQWord(peHeaderOffset, 48);
        }
    }

    /**
     * The alignment (in bytes) of sections when they are loaded into memory.
     * It must be greater than or equal to FileAlignment. The default is the
     * page size for the architecture.
     */
    public long getSectionAlignment() {
        return readDWord(peHeaderOffset, 56);
    }

    /**
     * The alignment factor (in bytes) that is used to align the raw data of
     * sections in the image file. The value should be a power of 2 between
     * 512 and 64 K, inclusive. The default is 512. If the SectionAlignment
     * is less than the architecture?s page size, then FileAlignment must
     * match SectionAlignment.
     */
    public long getFileAlignment() {
        return readDWord(peHeaderOffset, 60);
    }

    /**
     * The major version number of the required operating system.
     */
    public int getMajorOperatingSystemVersion() {
        return readWord(peHeaderOffset, 64);
    }

    /**
     * The minor version number of the required operating system.
     */
    public int getMinorOperatingSystemVersion() {
        return readWord(peHeaderOffset, 66);
    }

    /**
     * The major version number of the image.
     */
    public int getMajorImageVersion() {
        return readWord(peHeaderOffset, 68);
    }

    /**
     * The minor version number of the image.
     */
    public int getMinorImageVersion() {
        return readWord(peHeaderOffset, 70);
    }

    /**
     * The major version number of the subsystem.
     */
    public int getMajorSubsystemVersion() {
        return readWord(peHeaderOffset, 72);
    }

    /**
     * The minor version number of the subsystem.
     */
    public int getMinorSubsystemVersion() {
        return readWord(peHeaderOffset, 74);
    }

    /**
     * Reserved, must be zero.
     */
    public long getWin32VersionValue() {
        return readDWord(peHeaderOffset, 76);
    }

    /**
     * The size (in bytes) of the image, including all headers, as the image
     * is loaded in memory. It must be a multiple of SectionAlignment.
     */
    public long getSizeOfImage() {
        return readDWord(peHeaderOffset, 80);
    }

    /**
     * The combined size of an MS DOS stub, PE header, and section headers
     * rounded up to a multiple of FileAlignment.
     */
    public long getSizeOfHeaders() {
        return readDWord(peHeaderOffset, 84);
    }

    /**
     * The image file checksum. The algorithm for computing the checksum
     * is incorporated into IMAGHELP.DLL. The following are checked for
     * validation at load time: all drivers, any DLL loaded at boot time,
     * and any DLL that is loaded into a critical Windows process.
     */
    public long getCheckSum() {
        return readDWord(peHeaderOffset, 88);
    }

    /**
     * Compute the checksum of the image file.
     */
    public long computeChecksum() {
        try {
            long checksumOffset = peHeaderOffset + 88;
            boolean checksumOffsetSkipped = false;
            
            long checksum = 0;
            long top = (long) Math.pow(2, 32);
            long length = raf.length();
            
            ByteBuffer buffer = ByteBuffer.allocate(16 * 1024);
            
            FileChannel channel = raf.getChannel();
            channel.position(0);
            
            int l;
            long offset = 0;
            
            while ((l = channel.read(buffer)) > 0) {
                buffer.rewind();
                
                for (int i = 0; i < l / 4; i++) {
                    if (!checksumOffsetSkipped && offset + 4 * i == checksumOffset) {
                        buffer.position(buffer.position() + 4);
                        checksumOffsetSkipped = true;
                        
                    } else {
                        
                        long ch1 = buffer.get() & 0xFF;
                        long ch2 = buffer.get() & 0xFF;
                        long ch3 = buffer.get() & 0xFF;
                        long ch4 = buffer.get() & 0xFF;
                        
                        long dword = ch1 + (ch2 << 8) + (ch3 << 16) + (ch4 << 24);
                        
                        checksum = (checksum & 0xffffffffL) + dword + (checksum >> 32);
                        
                        if (checksum > top) {
                            checksum = (checksum & 0xffffffffL) + (checksum >> 32);
                        }
                    }
                }
                
                buffer.rewind();
                
                offset += l;
            }

            checksum = (checksum & 0xffff) + (checksum >> 16);
            checksum = checksum + (checksum >> 16);
            checksum = checksum & 0xffff;
            checksum += length;

            return checksum;
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    
    public void updateChecksum() {
        try {
            ByteBuffer buffer = ByteBuffer.allocate(4);
            buffer.order(ByteOrder.LITTLE_ENDIAN);
            buffer.putInt((int) computeChecksum());
            
            raf.seek(peHeaderOffset + 88);
            raf.write(buffer.array());
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }

    /**
     * The subsystem that is required to run this image.
     */
    public Subsystem getSubsystem() {
        return Subsystem.valueOf(readWord(peHeaderOffset, 92));
    }

    public int getDllCharacteristics() {
        return readWord(peHeaderOffset, 94);
    }

    /**
     * The size of the stack to reserve. Only SizeOfStackCommit is committed;
     * the rest is made available one page at a time until the reserve size is reached.
     */
    public long getSizeOfStackReserve() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 96);
        } else {
            return readQWord(peHeaderOffset, 96);
        }
    }

    /**
     * The size of the stack to commit.
     */
    public long getSizeOfStackCommit() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 100);
        } else {
            return readQWord(peHeaderOffset, 104);
        }
    }

    /**
     * The size of the local heap space to reserve. Only SizeOfHeapCommit is
     * committed; the rest is made available one page at a time until the
     * reserve size is reached.
     */
    public long getSizeOfHeapReserve() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 104);
        } else {
            return readQWord(peHeaderOffset, 112);
        }
    }

    /**
     * The size of the local heap space to commit.
     */
    public long getSizeOfHeapCommit() {
        if (PEFormat.PE32.equals(getFormat())) {
            return readDWord(peHeaderOffset, 108);
        } else {
            return readQWord(peHeaderOffset, 120);
        }
    }

    /**
     * Reserved, must be zero.
     */
    public long getLoaderFlags() {
        return readDWord(peHeaderOffset, PEFormat.PE32.equals(getFormat()) ? 112 : 128);
    }

    /**
     * The number of data-directory entries in the remainder of the optional
     * header. Each describes a location and size.
     */
    public int getNumberOfRvaAndSizes() {
        return (int) readDWord(peHeaderOffset, PEFormat.PE32.equals(getFormat()) ? 116 : 132);
    }

    int getDataDirectoryOffset() {
        return (int) peHeaderOffset + (PEFormat.PE32.equals(getFormat()) ? 120 : 136);
    }

    /**
     * Returns the data directory of the specified type.
     */
    public DataDirectory getDataDirectory(DataDirectoryType type) {
        if (type.ordinal() >= getNumberOfRvaAndSizes()) {
            return null;
        } else {
            return new DataDirectory(this, type.ordinal());
        }
    }

    /**
     * Writes the data directory of the specified type. The data is appended
     * at the end of the file. If a previous entry was already present it's
     * left as is in the file and is simply dereferenced from the table. 
     * 
     * @param type
     * @param data
     * @throws IOException
     */
    public void writeDataDirectory(DataDirectoryType type, byte[] data) throws IOException {
        // todo overwrite an existing entry at the end of the file
        
        // append the data directory at the end of the file
        long offset = raf.length();        
        
        raf.seek(offset);
        raf.write(data);
        
        // add the entry in the data directory table
        DataDirectory entry = getDataDirectory(type);
        entry.write(offset, data.length);
        
        updateChecksum();
    }

    /**
     * Returns the authenticode signatures on the file.
     */
    public List<CMSSignedData> getSignatures() {
        List<CMSSignedData> signatures = new ArrayList<CMSSignedData>();
        DataDirectory certificateTable = getDataDirectory(DataDirectoryType.CERTIFICATE_TABLE);
        
        if (certificateTable != null && certificateTable.getVirtualAddress() != 0 && certificateTable.getSize() != 0) {
            long position = certificateTable.getVirtualAddress();
            
            try {
                raf.seek(position);
                int size = (int) raf.readDWord();
                int revision = raf.readWord();
                int type = raf.readWord();

                if (revision == 0x0200 && type == 2) {
                    // PKCS#7 SignedData structure
                    byte[] buffer = new byte[size - 8];
                    raf.read(buffer);

                    CMSSignedData signedData = new CMSSignedData((CMSProcessable) null, ContentInfo.getInstance(buffer));
                    signatures.add(signedData);
                }
                
                // todo read the remaining signatures
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        
        return signatures;
    }

    /**
     * Print detailed informations about the PE file.
     */
    public void printInfo(OutputStream out) {
        printInfo(new PrintWriter(out, true));
    }

    /**
     * Print detailed informations about the PE file.
     */
    public void printInfo(PrintWriter out) {
        out.println("PE File");
        out.println("  Name:          " + file.getName());
        out.println("  Size:          " + file.length());
        out.println("  Last Modified: " + new Date(file.lastModified()));
        out.println();
        
        out.println("PE Header");
        out.println("  Machine:                    " + getMachineType());
        out.println("  Number of sections:         " + getNumberOfSections());
        out.println("  Timestamp:                  " + getTimeDateStamp());
        out.println("  Pointer to symbol table:    0x" + Long.toHexString(getPointerToSymbolTable()));
        out.println("  Number of symbols:          " + getNumberOfSymbols());
        out.println("  Size of optional header:    " + getSizeOfOptionalHeader());
        out.println("  Characteristics:            0x" + Long.toBinaryString(getCharacteristics()));
        out.println();
        
        out.println("Optional Header");
        PEFormat format = getFormat();
        out.println("  PE Format:                  0x" + Integer.toHexString(format.value) + " (" + format.label + ")");
        out.println("  Linker version:             " + getMajorLinkerVersion() + "." + getMinorLinkerVersion());
        out.println("  Size of code:               " + getSizeOfCode());
        out.println("  Size of initialized data:   " + getSizeOfInitializedData());
        out.println("  Size of uninitialized data: " + getSizeOfUninitializedData());
        out.println("  Address of entry point:     0x" + Long.toHexString(getAddressOfEntryPoint()));
        out.println("  Base of code:               0x" + Long.toHexString(getBaseOfCode()));
        if (PEFormat.PE32.equals(getFormat())) {
            out.println("  Base of data:               0x" + Long.toHexString(getBaseOfData()));
        }
        out.println("  Image base:                 0x" + Long.toHexString(getImageBase()));
        out.println("  Section alignment:          " + getSectionAlignment());
        out.println("  File alignment:             " + getFileAlignment());
        out.println("  Operating system version:   " + getMajorOperatingSystemVersion() + "." + getMinorOperatingSystemVersion());
        out.println("  Image version:              " + getMajorImageVersion() + "." + getMinorImageVersion());
        out.println("  Subsystem version:          " + getMajorSubsystemVersion() + "." + getMinorSubsystemVersion());
        out.println("  Size of image:              " + getSizeOfImage());
        out.println("  Size of headers:            " + getSizeOfHeaders());
        out.println("  Checksum:                   0x" + Long.toHexString(getCheckSum()));
        out.println("  Checksum (computed):        0x" + Long.toHexString(computeChecksum()));
        out.println("  Subsystem:                  " + getSubsystem());
        out.println("  DLL characteristics:        0x" + Long.toBinaryString(getDllCharacteristics()));
        out.println("  Size of stack reserve:      " + getSizeOfStackReserve());
        out.println("  Size of stack commit:       " + getSizeOfStackCommit());
        out.println("  Size of heap reserve:       " + getSizeOfHeapReserve());
        out.println("  Size of heap commit:        " + getSizeOfHeapCommit());
        out.println("  Number of RVA and sizes:    " + getNumberOfRvaAndSizes());
        out.println();
        
        out.println("Data Directory");
        for (DataDirectoryType type : DataDirectoryType.values()) {
            DataDirectory entry = getDataDirectory(type);
            if (entry != null && entry.getVirtualAddress() != 0) {
                out.printf("  %-30s 0x%08x %8d bytes\n", type, entry.getVirtualAddress(), entry.getSize());
            }
        }
        out.println();
        
        int sectionTableOffset = getDataDirectoryOffset() + 8 * getNumberOfRvaAndSizes(); 
        out.println("Sections");
        out.println("      Name     Virtual Size  Virtual Address  Raw Data Size  Raw Data Ptr  Characteristics");
        for (int i = 0; i < getNumberOfSections(); i++) {
            Section section = new Section(this, sectionTableOffset + 40 * i);
            out.printf("  #%d  %-8s     %8d       0x%08x       %8d    0x%08x  %s\n", i + 1, section.getName(), section.getVirtualSize(), section.getVirtualAddress(), section.getSizeOfRawData(), section.getPointerToRawData(), section.getCharacteristics());
        }
        out.println();
        
        List<CMSSignedData> signatures = getSignatures();
        if (!signatures.isEmpty()) {
            out.println("Signatures");
            for (CMSSignedData signedData : signatures) {
                SignerInformation signerInformation = (SignerInformation) signedData.getSignerInfos().getSigners().iterator().next();
                X509CertificateHolder certificate = (X509CertificateHolder) signedData.getCertificates().getMatches(signerInformation.getSID()).iterator().next();
                
                String commonName = certificate.getSubject().getRDNs(X509ObjectIdentifiers.commonName)[0].getFirst().getValue().toString();
                
                Attribute counterSignature = null;
                if (signerInformation.getUnsignedAttributes() != null) {
                    counterSignature = signerInformation.getUnsignedAttributes().get(PKCSObjectIdentifiers.pkcs_9_at_counterSignature);
                }
                
                out.println("  " + commonName + "   " + (counterSignature != null ? "(timestamped)" : ""));
            }
        }
    }

    /**
     * Compute the digest of the file. The checksum field, the certificate
     * directory table entry and the certificate table are excluded from
     * the digest.
     * 
     * todo Handle files > 2GB
     */
    private byte[] computeDigest(MessageDigest digest) throws IOException {
        int checksumLocation = (int) peHeaderOffset + 88;
        
        DataDirectory certificateTable = getDataDirectory(DataDirectoryType.CERTIFICATE_TABLE);
        
        // load the file in a buffer
        byte[] signedContent = new byte[(int) raf.length()]; // todo don't load the whole file in memory, use smaller chunks
        raf.seek(0);
        raf.read(signedContent);
        
        // digest from the beginning to the checksum field (excluded)
        digest.update(signedContent, 0, checksumLocation);
        
        // skip the checksum field
        int position = checksumLocation + 4;
{'fraction_non_alphanumeric': 0.09938150126511106, 'fraction_numerical': 0.013273625446805092, 'mean_word_length': 2.0225176013595534, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3090805', 'n_tokens_mistral': 14024, 'n_tokens_neox': 13149, 'n_words': 4036}