<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cabfile provides a bare minimum implementation of a parser for the
// Microsoft Cabinet file format. Its goal is to support the feature set of
// Cabinet files produced by gcab for the LVFS project.
//
// Normative references for this implementation are [MS-CAB] for the Cabinet
// file format and [MS-MCI] for the Microsoft ZIP Compression and Decompression
// Data Structure.
//
// [MS-CAB]: http://download.microsoft.com/download/4/d/a/4da14f27-b4ef-4170-a6e6-5b1ef85b1baa/[ms-cab].pdf
// [MS-MCI]: http://interoperability.blob.core.windows.net/files/MS-MCI/[MS-MCI].pdf
package cabfile

import (
	"bufio"
	"bytes"
	"compress/flate"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
)

// Cabinet provides read-only access to Microsoft Cabinet files.
type Cabinet struct {
	r     io.ReadSeeker
	hdr   *cfHeader
	fldrs []*cfFolder
	files []*file
}

type cfHeader struct {
	Signature    [4]byte
	Reserved1    uint32
	CBCabinet    uint32 // size of this cabinet file in bytes
	Reserved2    uint32
	COFFFiles    uint32 // offset of the first CFFILE entry
	Reserved3    uint32 // reserved
	VersionMinor uint8  // cabinet file format version, minor
	VersionMajor uint8  // cabinet file format version, major
	CFolders     uint16 // number of CFFOLDER entries in this cabinet
	CFiles       uint16 // number of CFFILE entries in this cabinet
	Flags        uint16 // cabinet file option indicators
	SetID        uint16 // must be the same for all cabinets in a set
	ICabinet     uint16 // number of this cabinet file in a set
}

const (
	hdrPrevCabinet uint16 = 1 << iota
	hdrNextCabinet
	hdrReservePresent
)

type cfFolder struct {
	COFFCabStart uint32 // offset of the first CFDATA block in this folder
	CCFData      uint16 // number of CFDATA blocks in this folder
	TypeCompress uint16 // compression type indicator
}

const (
	compMask    uint16 = 0xf
	compNone           = 0x0
	compMSZIP          = 0x1
	compQuantum        = 0x2
	compLZX            = 0x3
)

type cfFile struct {
	CBFile          uint32 // uncompressed size of this file in bytes
	UOffFolderStart uint32 // uncompressed offset of this file in the folder
	IFolder         uint16 // index into the CFFOLDER area
	Date            uint16 // date stamp for this file
	Time            uint16 // time stamp for this file
	Attribs         uint16 // attribute flags for this file
}

const (
	attribReadOnly = 1 << iota // file is read-only
	attribHidden               // file is hidden
	attribSystem               // file is a system file
	_
	_
	attribArchive   // file modified since last backup
	attribExec      // run after extraction
	attribNameIsUTF // filename is UTF-encoded
)

type file struct {
	*cfFile
	name string
}

type cfData struct {
	Checksum uint32 // checksum of this CFDATA entry
	CBData   uint16 // number of compressed bytes in this block
	CBUncomp uint16 // number of uncompressed bytes in this block
}

// New returns a new Cabinet with the header structures parsed and sanity checked.
func New(r io.ReadSeeker) (*Cabinet, error) {
	if _, err := r.Seek(0, io.SeekStart); err != nil {
		return nil, fmt.Errorf("could not seek to the beginning: %v", err)
	}

	// CFHEADER
	var hdr cfHeader
	if err := binary.Read(r, binary.LittleEndian, &hdr); err != nil {
		return nil, fmt.Errorf("could not deserialize header: %v", err)
	}
	if !bytes.Equal(hdr.Signature[:], []byte("MSCF")) {
		return nil, fmt.Errorf("invalid Cabinet file signature: %v", hdr.Signature)
	}
	if hdr.Reserved1 != 0 || hdr.Reserved2 != 0 || hdr.Reserved3 != 0 {
		return nil, fmt.Errorf("reserved files must be zero: %v, %v, %v", hdr.Reserved1, hdr.Reserved2, hdr.Reserved3)
	}
	if hdr.VersionMajor != 1 || hdr.VersionMinor != 3 {
		return nil, fmt.Errorf("Cabinet file version has unsupported version %d.%d", hdr.VersionMajor, hdr.VersionMinor)
	}
	if (hdr.Flags&hdrPrevCabinet) != 0 || (hdr.Flags&hdrNextCabinet) != 0 {
		return nil, errors.New("multi-part Cabinet files are unsupported")
	}
	if (hdr.Flags & hdrReservePresent) != 0 {
		return nil, errors.New("Cabinet files with reserved fields are unsupported")
	}

	// CFFOLDER
	var fldrs []*cfFolder
	for i := uint16(0); i < hdr.CFolders; i++ {
		var fldr cfFolder
		if err := binary.Read(r, binary.LittleEndian, &fldr); err != nil {
			return nil, fmt.Errorf("could not deserialize folder %d: %v", i, err)
		}
		switch fldr.TypeCompress & compMask {
		case compNone:
		case compMSZIP:
		default:
			return nil, fmt.Errorf("folder compressed with unsupported algorithm %d", fldr.TypeCompress)
		}
		fldrs = append(fldrs, &fldr)<commit_msg>Add support for reserved fields<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cabfile provides a bare minimum implementation of a parser for the
// Microsoft Cabinet file format. Its goal is to support the feature set of
// Cabinet files produced by gcab for the LVFS project.
//
// Normative references for this implementation are [MS-CAB] for the Cabinet
// file format and [MS-MCI] for the Microsoft ZIP Compression and Decompression
// Data Structure.
//
// [MS-CAB]: http://download.microsoft.com/download/4/d/a/4da14f27-b4ef-4170-a6e6-5b1ef85b1baa/[ms-cab].pdf
// [MS-MCI]: http://interoperability.blob.core.windows.net/files/MS-MCI/[MS-MCI].pdf
package cabfile

import (
	"bufio"
	"bytes"
	"compress/flate"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
)

// Cabinet provides read-only access to Microsoft Cabinet files.
type Cabinet struct {
	r     io.ReadSeeker
	hdr   *cfHeader
	fldrs []*cfFolder
	files []*file
}

type cfHeader struct {
	Signature    [4]byte
	Reserved1    uint32
	CBCabinet    uint32 // size of this cabinet file in bytes
	Reserved2    uint32
	COFFFiles    uint32 // offset of the first CFFILE entry
	Reserved3    uint32 // reserved
	VersionMinor uint8  // cabinet file format version, minor
	VersionMajor uint8  // cabinet file format version, major
	CFolders     uint16 // number of CFFOLDER entries in this cabinet
	CFiles       uint16 // number of CFFILE entries in this cabinet
	Flags        uint16 // cabinet file option indicators
	SetID        uint16 // must be the same for all cabinets in a set
	ICabinet     uint16 // number of this cabinet file in a set
	CBCFHeader   uint16 // size of abReserve field in the header in bytes (optional)
	CBCFFolder   uint8  // size of abReserve field in each CFFolder entry in bytes (optional)
	CBCFData     uint8  // size of abReserve field in each CFData entry in bytes (optional)
}

const (
	hdrPrevCabinet uint16 = 1 << iota
	hdrNextCabinet
	hdrReservePresent
)

type cfFolder struct {
	COFFCabStart uint32 // offset of the first CFDATA block in this folder
	CCFData      uint16 // number of CFDATA blocks in this folder
	TypeCompress uint16 // compression type indicator
}

const (
	compMask    uint16 = 0xf
	compNone           = 0x0
	compMSZIP          = 0x1
	compQuantum        = 0x2
	compLZX            = 0x3
)

type cfFile struct {
	CBFile          uint32 // uncompressed size of this file in bytes
	UOffFolderStart uint32 // uncompressed offset of this file in the folder
	IFolder         uint16 // index into the CFFOLDER area
	Date            uint16 // date stamp for this file
	Time            uint16 // time stamp for this file
	Attribs         uint16 // attribute flags for this file
}

const (
	attribReadOnly = 1 << iota // file is read-only
	attribHidden               // file is hidden
	attribSystem               // file is a system file
	_
	_
	attribArchive   // file modified since last backup
	attribExec      // run after extraction
	attribNameIsUTF // filename is UTF-encoded
)

type file struct {
	*cfFile
	name string
}

type cfData struct {
	Checksum uint32 // checksum of this CFDATA entry
	CBData   uint16 // number of compressed bytes in this block
	CBUncomp uint16 // number of uncompressed bytes in this block
}

// New returns a new Cabinet with the header structures parsed and sanity checked.
func New(r io.ReadSeeker) (*Cabinet, error) {
	if _, err := r.Seek(0, io.SeekStart); err != nil {
		return nil, fmt.Errorf("could not seek to the beginning: %v", err)
	}

	// CFHEADER
	var hdr cfHeader
	if err := binary.Read(r, binary.LittleEndian, &hdr.Signature); err != nil {
		return nil, fmt.Errorf("could not deserialize header signature: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.Reserved1); err != nil {
		return nil, fmt.Errorf("could not deserialize header reserved1: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.CBCabinet); err != nil {
		return nil, fmt.Errorf("could not deserialize header cbCabinet: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.Reserved2); err != nil {
		return nil, fmt.Errorf("could not deserialize header reserved2: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.COFFFiles); err != nil {
		return nil, fmt.Errorf("could not deserialize header coffFiles: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.Reserved3); err != nil {
		return nil, fmt.Errorf("could not deserialize header reserved3: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.VersionMinor); err != nil {
		return nil, fmt.Errorf("could not deserialize header versionMinor: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.VersionMajor); err != nil {
		return nil, fmt.Errorf("could not deserialize header versionMajor: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.CFolders); err != nil {
		return nil, fmt.Errorf("could not deserialize header cFolder: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.CFiles); err != nil {
		return nil, fmt.Errorf("could not deserialize header cFiles: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.Flags); err != nil {
		return nil, fmt.Errorf("could not deserialize header flags: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.SetID); err != nil {
		return nil, fmt.Errorf("could not deserialize header setID: %w", err)
	}
	if err := binary.Read(r, binary.LittleEndian, &hdr.ICabinet); err != nil {
		return nil, fmt.Errorf("could not deserialize header iCabinet: %w", err)
	}

	// if cfhdrRESERVE_PRESENT flag is set, parse the optional abReserve size fields
	if (hdr.Flags & hdrReservePresent) != 0 {
		if err := binary.Read(r, binary.LittleEndian, &hdr.CBCFHeader); err != nil {
			return nil, fmt.Errorf("could not deserialize header cbCFHeader: %w", err)
		}
		if err := binary.Read(r, binary.LittleEndian, &hdr.CBCFFolder); err != nil {
			return nil, fmt.Errorf("could not deserialize header cbCFFolder: %w", err)
		}
		if err := binary.Read(r, binary.LittleEndian, &hdr.CBCFData); err != nil {
			return nil, fmt.Errorf("could not deserialize header cbCFData: %w", err)
		}
	}

	if !bytes.Equal(hdr.Signature[:], []byte("MSCF")) {
		return nil, fmt.Errorf("invalid Cabinet file signature: %v", hdr.Signature)
	}
	if hdr.Reserved1 != 0 || hdr.Reserved2 != 0 || hdr.Reserved3 != 0 {
		return nil, fmt.Errorf("reserved files must be zero: %v, %v, %v", hdr.Reserved1, hdr.Reserved2, hdr.Reserved3)
	}
	if hdr.VersionMajor != 1 || hdr.VersionMinor != 3 {
		return nil, fmt.Errorf("Cabinet file version has unsupported version %d.%d", hdr.VersionMajor, hdr.VersionMinor)
	}
	if (hdr.Flags&hdrPrevCabinet) != 0 || (hdr.Flags&hdrNextCabinet) != 0 {
		return nil, errors.New("multi-part Cabinet files are unsupported")
	}

	// skip abReserve by skipping ahead cbCFHeader bytes
	ioutil.ReadAll(io.LimitReader(r, int64(hdr.CBCFHeader)))

	// CFFOLDER
	var fldrs []*cfFolder
	for i := uint16(0); i < hdr.CFolders; i++ {
		var fldr cfFolder
		if err := binary.Read(r, binary.LittleEndian, &fldr); err != nil {
			return nil, fmt.Errorf("could not deserialize folder %d: %v", i, err)
		}
		switch fldr.TypeCompress & compMask {
		case compNone:
		case compMSZIP:
		default:
			return nil, fmt.Errorf("folder compressed with unsupported algorithm %d", fldr.TypeCompress)
		}
		fldrs = append(fldrs, &fldr)
{'fraction_non_alphanumeric': 0.10389709823137586, 'fraction_numerical': 0.01760967766633489, 'mean_word_length': 3.345309381237525, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1829528', 'n_tokens_mistral': 4437, 'n_tokens_neox': 4165, 'n_words': 1492}