<commit_before>// Copyright 2013 <NAME>. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mscfb

import (
	"encoding/binary"
	"io"
	"os"
	"time"
	"unicode"
	"unicode/utf16"

	"github.com/richardlehane/msoleps/types"
)

//objectType types
const (
	unknown     uint8 = 0x0 // this means unallocated - typically zeroed dir entries
	storage     uint8 = 0x1 // this means dir
	stream      uint8 = 0x2 // this means file
	rootStorage uint8 = 0x5 // this means root
)

// color flags
const (
	red   uint8 = 0x0
	black uint8 = 0x1
)

const lenDirEntry int = 64 + 4*4 + 16 + 4 + 8*2 + 4 + 8

type directoryEntryFields struct {
	rawName           [32]uint16     //64 bytes, unicode string encoded in UTF-16. If root, "Root Entry\0" w
	nameLength        uint16         //2 bytes
	objectType        uint8          //1 byte Must be one of the types specified above
	color             uint8          //1 byte Must be 0x00 RED or 0x01 BLACK
	leftSibID         uint32         //4 bytes, Dir? Stream ID of left sibling, if none set to NOSTREAM
	rightSibID        uint32         //4 bytes, Dir? Stream ID of right sibling, if none set to NOSTREAM
	childID           uint32         //4 bytes, Dir? Stream ID of child object, if none set to NOSTREAM
	clsid             types.Guid     // Contains an object class GUID (must be set to zeroes for stream object)
	stateBits         [4]byte        // user-defined flags for storage object
	create            types.FileTime // Windows FILETIME structure
	modify            types.FileTime // Windows FILETIME structure
	startingSectorLoc uint32         // if a stream object, first sector location. If root, first sector of ministream
	streamSize        [8]byte        // if a stream, size of user-defined data. If root, size of ministream
}

func makeDirEntry(b []byte) *directoryEntryFields {
	d := &directoryEntryFields{}
	for i := range d.rawName {
		d.rawName[i] = binary.LittleEndian.Uint16(b[i*2 : i*2+2])
	}
	d.nameLength = binary.LittleEndian.Uint16(b[64:66])
	d.objectType = uint8(b[66])
	d.color = uint8(b[67])
	d.leftSibID = binary.LittleEndian.Uint32(b[68:72])
	d.rightSibID = binary.LittleEndian.Uint32(b[72:76])
	d.childID = binary.LittleEndian.Uint32(b[76:80])
	d.clsid = types.MustGuid(b[80:96])
	copy(d.stateBits[:], b[96:100])
	d.create = types.MustFileTime(b[100:108])
	d.modify = types.MustFileTime(b[108:116])
	d.startingSectorLoc = binary.LittleEndian.Uint32(b[116:120])
	copy(d.streamSize[:], b[120:128])
	return d
}

func (r *Reader) setDirEntries() error {
	c := 20
	if r.header.numDirectorySectors > 0 {
		c = int(r.header.numDirectorySectors)
	}
	de := make([]*File, 0, c)
	cycles := make(map[uint32]bool)
	num := int(r.sectorSize / 128)
	sn := r.header.directorySectorLoc
	for sn != endOfChain {
		buf, err := r.readAt(fileOffset(r.sectorSize, sn), int(r.sectorSize))
		if err != nil {
			return Error{ErrRead, "directory entries read error (" + err.Error() + ")", fileOffset(r.sectorSize, sn)}
		}
		for i := 0; i < num; i++ {
			f := &File{r: r}
			f.directoryEntryFields = makeDirEntry(buf[i*128:])
			fixFile(r.header.majorVersion, f)
			f.curSector = f.startingSectorLoc
			de = append(de, f)
		}
		nsn, err := r.findNext(sn, false)
		if err != nil {
			return Error{ErrRead, "directory entries error finding sector (" + err.Error() + ")", int64(nsn)}
		}
		if nsn <= sn {
			if nsn == sn || cycles[nsn] {
				return Error{ErrRead, "directory entries sector cycle", int64(nsn)}
			}
			cycles[nsn] = true
		}
		sn = nsn
	}
	r.direntries = de
	return nil
}

func fixFile(v uint16, f *File) {
	fixName(f)
	if f.objectType != stream {
		return
	}
	// if the MSCFB major version is 4, then this can be a uint64 otherwise is a uint32 and the least signficant bits can contain junk
	if v > 3 {
		f.Size = int64(binary.LittleEndian.Uint64(f.streamSize[:]))
	} else {
		f.Size = int64(binary.LittleEndian.Uint32(f.streamSize[:4]))
	}
}

func fixName(f *File) {
	// From the spec:
	// "The length [name] MUST be a multiple of 2, and include the terminating null character in the count.
	// This length MUST NOT exceed 64, the maximum size of the Directory Entry Name field."
	if f.nameLength < 4 || f.nameLength > 64 {
		return
	}
	nlen := int(f.nameLength/2 - 1)
	f.Initial = f.rawName[0]
	var slen int
	if !unicode.IsPrint(rune(f.Initial)) {
		slen = 1
	}
	f.Name = string(utf16.Decode(f.rawName[slen:nlen]))
}

func (r *Reader) traverse() error {
	r.File = make([]*File, 0, len(r.direntries))
	var recurse func(int, []string)
	var err error
	recurse = func(i int, path []string) {
		if i < 0 || i >= len(r.direntries) {
			err = Error{ErrTraverse, "illegal traversal index", int64(i)}
			return
		}
		file := r.direntries[i]
		if file.leftSibID != noStream {
			recurse(int(file.leftSibID), path)
		}
		if len(r.File) >= cap(r.File) {
			err = Error{ErrTraverse, "traversal counter overflow", int64(i)}
			return
		}
		r.File = append(r.File, file)
		file.Path = path
		if file.childID != noStream {
			if i > 0 {
				recurse(int(file.childID), append(path, file.Name))
			} else {
				recurse(int(file.childID), path)
			}
		}
		if file.rightSibID != noStream {
			recurse(int(file.rightSibID), path)
		}
		return
	}
	recurse(0, []string{})
	return err
}

// File represents a MSCFB directory entry
type File struct {
	Name      string   // stream or directory name<commit_msg>check for self recurse to prevent stack overflow; fixes #9<commit_after>// Copyright 2013 <NAME>. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mscfb

import (
	"encoding/binary"
	"io"
	"os"
	"time"
	"unicode"
	"unicode/utf16"

	"github.com/richardlehane/msoleps/types"
)

//objectType types
const (
	unknown     uint8 = 0x0 // this means unallocated - typically zeroed dir entries
	storage     uint8 = 0x1 // this means dir
	stream      uint8 = 0x2 // this means file
	rootStorage uint8 = 0x5 // this means root
)

// color flags
const (
	red   uint8 = 0x0
	black uint8 = 0x1
)

const lenDirEntry int = 64 + 4*4 + 16 + 4 + 8*2 + 4 + 8

type directoryEntryFields struct {
	rawName           [32]uint16     //64 bytes, unicode string encoded in UTF-16. If root, "Root Entry\0" w
	nameLength        uint16         //2 bytes
	objectType        uint8          //1 byte Must be one of the types specified above
	color             uint8          //1 byte Must be 0x00 RED or 0x01 BLACK
	leftSibID         uint32         //4 bytes, Dir? Stream ID of left sibling, if none set to NOSTREAM
	rightSibID        uint32         //4 bytes, Dir? Stream ID of right sibling, if none set to NOSTREAM
	childID           uint32         //4 bytes, Dir? Stream ID of child object, if none set to NOSTREAM
	clsid             types.Guid     // Contains an object class GUID (must be set to zeroes for stream object)
	stateBits         [4]byte        // user-defined flags for storage object
	create            types.FileTime // Windows FILETIME structure
	modify            types.FileTime // Windows FILETIME structure
	startingSectorLoc uint32         // if a stream object, first sector location. If root, first sector of ministream
	streamSize        [8]byte        // if a stream, size of user-defined data. If root, size of ministream
}

func makeDirEntry(b []byte) *directoryEntryFields {
	d := &directoryEntryFields{}
	for i := range d.rawName {
		d.rawName[i] = binary.LittleEndian.Uint16(b[i*2 : i*2+2])
	}
	d.nameLength = binary.LittleEndian.Uint16(b[64:66])
	d.objectType = uint8(b[66])
	d.color = uint8(b[67])
	d.leftSibID = binary.LittleEndian.Uint32(b[68:72])
	d.rightSibID = binary.LittleEndian.Uint32(b[72:76])
	d.childID = binary.LittleEndian.Uint32(b[76:80])
	d.clsid = types.MustGuid(b[80:96])
	copy(d.stateBits[:], b[96:100])
	d.create = types.MustFileTime(b[100:108])
	d.modify = types.MustFileTime(b[108:116])
	d.startingSectorLoc = binary.LittleEndian.Uint32(b[116:120])
	copy(d.streamSize[:], b[120:128])
	return d
}

func (r *Reader) setDirEntries() error {
	c := 20
	if r.header.numDirectorySectors > 0 {
		c = int(r.header.numDirectorySectors)
	}
	de := make([]*File, 0, c)
	cycles := make(map[uint32]bool)
	num := int(r.sectorSize / 128)
	sn := r.header.directorySectorLoc
	for sn != endOfChain {
		buf, err := r.readAt(fileOffset(r.sectorSize, sn), int(r.sectorSize))
		if err != nil {
			return Error{ErrRead, "directory entries read error (" + err.Error() + ")", fileOffset(r.sectorSize, sn)}
		}
		for i := 0; i < num; i++ {
			f := &File{r: r}
			f.directoryEntryFields = makeDirEntry(buf[i*128:])
			fixFile(r.header.majorVersion, f)
			f.curSector = f.startingSectorLoc
			de = append(de, f)
		}
		nsn, err := r.findNext(sn, false)
		if err != nil {
			return Error{ErrRead, "directory entries error finding sector (" + err.Error() + ")", int64(nsn)}
		}
		if nsn <= sn {
			if nsn == sn || cycles[nsn] {
				return Error{ErrRead, "directory entries sector cycle", int64(nsn)}
			}
			cycles[nsn] = true
		}
		sn = nsn
	}
	r.direntries = de
	return nil
}

func fixFile(v uint16, f *File) {
	fixName(f)
	if f.objectType != stream {
		return
	}
	// if the MSCFB major version is 4, then this can be a uint64 otherwise is a uint32 and the least signficant bits can contain junk
	if v > 3 {
		f.Size = int64(binary.LittleEndian.Uint64(f.streamSize[:]))
	} else {
		f.Size = int64(binary.LittleEndian.Uint32(f.streamSize[:4]))
	}
}

func fixName(f *File) {
	// From the spec:
	// "The length [name] MUST be a multiple of 2, and include the terminating null character in the count.
	// This length MUST NOT exceed 64, the maximum size of the Directory Entry Name field."
	if f.nameLength < 4 || f.nameLength > 64 {
		return
	}
	nlen := int(f.nameLength/2 - 1)
	f.Initial = f.rawName[0]
	var slen int
	if !unicode.IsPrint(rune(f.Initial)) {
		slen = 1
	}
	f.Name = string(utf16.Decode(f.rawName[slen:nlen]))
}

func (r *Reader) traverse() error {
	r.File = make([]*File, 0, len(r.direntries))
	var recurse func(int, []string)
	var err error
	recurse = func(i int, path []string) {
		if i < 0 || i >= len(r.direntries) {
			err = Error{ErrTraverse, "illegal traversal index", int64(i)}
			return
		}
		file := r.direntries[i]
		if file.leftSibID != noStream && int(file.leftSibID) != i {
			recurse(int(file.leftSibID), path)
		}
		if len(r.File) >= cap(r.File) {
			err = Error{ErrTraverse, "traversal counter overflow", int64(i)}
			return
		}
		r.File = append(r.File, file)
		file.Path = path
		if file.childID != noStream {
			if i > 0 {
				recurse(int(file.childID), append(path, file.Name))
			} else {
				recurse(int(file.childID), path)
			}
		}
		if file.rightSibID != noStream {
			recurse(int(file.rightSibID), path)
		}
		return
	}
	recurse(0, []string{})
	return err
}

// File represents a MSCFB directory entry
type File struct {
	Name      string   // stream or directory name
{'fraction_non_alphanumeric': 0.12422728427470574, 'fraction_numerical': 0.033618426623761535, 'mean_word_length': 3.0738185581234907, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1800815', 'n_tokens_mistral': 4548, 'n_tokens_neox': 4108, 'n_words': 1264}