<commit_before>// Package zip implements functions for jarcat that manipulate .zip files.
package zip

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"gopkg.in/op/go-logging.v1"

	"third_party/go/zip"
)

var log = logging.MustGetLogger("zip")
var modTime = time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC)

// fileHeaderLen is the length of a file header in a zipfile.
// We need to know this to adjust alignment.
const fileHeaderLen = 30

// A File represents an output zipfile.
type File struct {
	f        io.WriteCloser
	w        *zip.Writer
	filename string
	input    string
	// Include and Exclude are prefixes of filenames to include or exclude from the zipfile.
	Include, Exclude []string
	// Strict controls whether we deny duplicate files or not.
	// Zipfiles can readily contain duplicates, if this is true we reject them unless they are identical.
	// If false we allow duplicates and leave it to someone else to handle.
	Strict bool
	// RenameDirs is a map of directories to rename, from the old name to the new one.
	RenameDirs map[string]string
	// StripPrefix is a prefix that is stripped off any files added with AddFiles.
	StripPrefix string
	// Suffix is the suffix of files that we include while scanning.
	Suffix []string
	// ExcludeSuffix is a list of suffixes that are excluded from the file scan.
	ExcludeSuffix []string
	// StoreSuffix is a list of file suffixes that will be stored instead of deflated.
	StoreSuffix []string
	// IncludeOther will make the file scan include other files that are not part of a zip file.
	IncludeOther bool
	// AddInitPy will make the writer add __init__.py files to all directories that don't already have one on close.
	AddInitPy bool
	// DirEntries makes the writer add empty directory entries.
	DirEntries bool
	// Align aligns entries to a multiple of this many bytes.
	Align int
	// files tracks the files that we've written so far.
	files map[string]fileRecord
	// concatenatedFiles tracks the files that are built up as we go.
	concatenatedFiles map[string][]byte
}

// A fileRecord records some information about a file that we use to check if they're exact duplicates.
type fileRecord struct {
	ZipFile            string
	CompressedSize64   uint64
	UncompressedSize64 uint64
	CRC32              uint32
}

// NewFile constructs and returns a new File.
func NewFile(output string, strict bool) *File {
	f, err := os.Create(output)
	if err != nil {
		log.Fatalf("Failed to open output file: %s", err)
	}
	return &File{
		f:                 f,
		w:                 zip.NewWriter(f),
		filename:          output,
		Strict:            strict,
		files:             map[string]fileRecord{},
		concatenatedFiles: map[string][]byte{},
	}
}

// Close must be called before the File is destroyed.
func (f *File) Close() {
	f.handleConcatenatedFiles()
	if f.AddInitPy {
		if err := f.AddInitPyFiles(); err != nil {
			log.Fatalf("%s", err)
		}
	}
	if err := f.w.Close(); err != nil {
		log.Fatalf("Failed to finalise zip file: %s", err)
	}
	if err := f.f.Close(); err != nil {
		log.Fatalf("Failed to close file: %s", err)
	}
}

// AddZipFile copies the contents of a zip file into the new zipfile.
func (f *File) AddZipFile(filepath string) error {
	r, err := zip.OpenReader(filepath)
	if err != nil {
		return err
	}
	defer r.Close()

	// Reopen file to get a directly readable version without decompression.
	r2, err := os.Open(filepath)
	if err != nil {
		return err
	}
	defer r2.Close()

	for _, rf := range r.File {
		log.Debug("Found file %s (from %s)", rf.Name, filepath)
		// This directory is very awkward. We need to merge the contents by concatenating them,
		// we can't replace them or leave them out.
		if strings.HasPrefix(rf.Name, "META-INF/services/") ||
			strings.HasPrefix(rf.Name, "META-INF/spring") ||
			rf.Name == "META-INF/please_sourcemap" {
			if err := f.concatenateFile(rf); err != nil {
				return err
			}
			continue
		}
		if !f.shouldInclude(rf.Name) {
			continue
		}
		hasTrailingSlash := strings.HasSuffix(rf.Name, "/")
		isDir := hasTrailingSlash || rf.FileInfo().IsDir()
		if isDir && !hasTrailingSlash {
			rf.Name = rf.Name + "/"
		}
		if existing, present := f.files[rf.Name]; present {
			// Allow duplicates of directories. Seemingly the best way to identify them is that
			// they end in a trailing slash.
			if isDir {
				continue
			}
			// Allow skipping existing files that are exactly the same as the added ones.
			// It's unnecessarily awkward to insist on not ever doubling up on a dependency.
			// TODO(pebers): Bit of a hack ignoring it when CRC is 0, would be better to add
			//               the correct CRC when added through WriteFile.
			if existing.CRC32 == rf.CRC32 || existing.CRC32 == 0 {
				log.Info("Skipping %s / %s: already added (from %s)", filepath, rf.Name, existing.ZipFile)
				continue
			}
			if f.Strict {
				log.Error("Duplicate file %s (from %s, already added from %s); crc %d / %d", rf.Name, filepath, existing.ZipFile, rf.CRC32, existing.CRC32)
				return fmt.Errorf("File %s already added to destination zip file (from %s)", rf.Name, existing.ZipFile)
			}
			continue
		}<commit_msg>Minor reordering of jarcat logic<commit_after>// Package zip implements functions for jarcat that manipulate .zip files.
package zip

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"gopkg.in/op/go-logging.v1"

	"third_party/go/zip"
)

var log = logging.MustGetLogger("zip")
var modTime = time.Date(2001, time.January, 1, 0, 0, 0, 0, time.UTC)

// fileHeaderLen is the length of a file header in a zipfile.
// We need to know this to adjust alignment.
const fileHeaderLen = 30

// A File represents an output zipfile.
type File struct {
	f        io.WriteCloser
	w        *zip.Writer
	filename string
	input    string
	// Include and Exclude are prefixes of filenames to include or exclude from the zipfile.
	Include, Exclude []string
	// Strict controls whether we deny duplicate files or not.
	// Zipfiles can readily contain duplicates, if this is true we reject them unless they are identical.
	// If false we allow duplicates and leave it to someone else to handle.
	Strict bool
	// RenameDirs is a map of directories to rename, from the old name to the new one.
	RenameDirs map[string]string
	// StripPrefix is a prefix that is stripped off any files added with AddFiles.
	StripPrefix string
	// Suffix is the suffix of files that we include while scanning.
	Suffix []string
	// ExcludeSuffix is a list of suffixes that are excluded from the file scan.
	ExcludeSuffix []string
	// StoreSuffix is a list of file suffixes that will be stored instead of deflated.
	StoreSuffix []string
	// IncludeOther will make the file scan include other files that are not part of a zip file.
	IncludeOther bool
	// AddInitPy will make the writer add __init__.py files to all directories that don't already have one on close.
	AddInitPy bool
	// DirEntries makes the writer add empty directory entries.
	DirEntries bool
	// Align aligns entries to a multiple of this many bytes.
	Align int
	// files tracks the files that we've written so far.
	files map[string]fileRecord
	// concatenatedFiles tracks the files that are built up as we go.
	concatenatedFiles map[string][]byte
}

// A fileRecord records some information about a file that we use to check if they're exact duplicates.
type fileRecord struct {
	ZipFile            string
	CompressedSize64   uint64
	UncompressedSize64 uint64
	CRC32              uint32
}

// NewFile constructs and returns a new File.
func NewFile(output string, strict bool) *File {
	f, err := os.Create(output)
	if err != nil {
		log.Fatalf("Failed to open output file: %s", err)
	}
	return &File{
		f:                 f,
		w:                 zip.NewWriter(f),
		filename:          output,
		Strict:            strict,
		files:             map[string]fileRecord{},
		concatenatedFiles: map[string][]byte{},
	}
}

// Close must be called before the File is destroyed.
func (f *File) Close() {
	f.handleConcatenatedFiles()
	if f.AddInitPy {
		if err := f.AddInitPyFiles(); err != nil {
			log.Fatalf("%s", err)
		}
	}
	if err := f.w.Close(); err != nil {
		log.Fatalf("Failed to finalise zip file: %s", err)
	}
	if err := f.f.Close(); err != nil {
		log.Fatalf("Failed to close file: %s", err)
	}
}

// AddZipFile copies the contents of a zip file into the new zipfile.
func (f *File) AddZipFile(filepath string) error {
	r, err := zip.OpenReader(filepath)
	if err != nil {
		return err
	}
	defer r.Close()

	// Reopen file to get a directly readable version without decompression.
	r2, err := os.Open(filepath)
	if err != nil {
		return err
	}
	defer r2.Close()

	for _, rf := range r.File {
		log.Debug("Found file %s (from %s)", rf.Name, filepath)
		if !f.shouldInclude(rf.Name) {
			continue
		}
		// This directory is very awkward. We need to merge the contents by concatenating them,
		// we can't replace them or leave them out.
		if strings.HasPrefix(rf.Name, "META-INF/services/") ||
			strings.HasPrefix(rf.Name, "META-INF/spring") ||
			rf.Name == "META-INF/please_sourcemap" {
			if err := f.concatenateFile(rf); err != nil {
				return err
			}
			continue
		}
		hasTrailingSlash := strings.HasSuffix(rf.Name, "/")
		isDir := hasTrailingSlash || rf.FileInfo().IsDir()
		if isDir && !hasTrailingSlash {
			rf.Name = rf.Name + "/"
		}
		if existing, present := f.files[rf.Name]; present {
			// Allow duplicates of directories. Seemingly the best way to identify them is that
			// they end in a trailing slash.
			if isDir {
				continue
			}
			// Allow skipping existing files that are exactly the same as the added ones.
			// It's unnecessarily awkward to insist on not ever doubling up on a dependency.
			// TODO(pebers): Bit of a hack ignoring it when CRC is 0, would be better to add
			//               the correct CRC when added through WriteFile.
			if existing.CRC32 == rf.CRC32 || existing.CRC32 == 0 {
				log.Info("Skipping %s / %s: already added (from %s)", filepath, rf.Name, existing.ZipFile)
				continue
			}
			if f.Strict {
				log.Error("Duplicate file %s (from %s, already added from %s); crc %d / %d", rf.Name, filepath, existing.ZipFile, rf.CRC32, existing.CRC32)
				return fmt.Errorf("File %s already added to destination zip file (from %s)", rf.Name, existing.ZipFile)
			}
			continue
		}
{'fraction_non_alphanumeric': 0.10045049362599444, 'fraction_numerical': 0.007284577782037765, 'mean_word_length': 3.0901607212857702, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6509632', 'n_tokens_mistral': 3603, 'n_tokens_neox': 3289, 'n_words': 1296}