<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package zip provides functions for creating and extracting module zip files.
//
// Module zip files have several restrictions listed below. These are necessary
// to ensure that module zip files can be extracted consistently on supported
// platforms and file systems.
//
// - All file paths within a zip file must start with "<module>@<version>/",
//   where "<module>" is the module path and "<version>" is the version.
//   The module path must be valid (see golang.org/x/mod/module.CheckPath).
//   The version must be valid and canonical (see
//   golang.org/x/mod/module.CanonicalVersion). The path must have a major
//   version suffix consistent with the version (see
//   golang.org/x/mod/module.Check). The part of the file path after the
//   "<module>@<version>/" prefix must be valid (see
//   golang.org/x/mod/module.CheckFilePath).
//
// - No two file paths may be equal under Unicode case-folding (see
//   strings.EqualFold).
//
// - A go.mod file may or may not appear in the top-level directory. If present,
//   it must be named "go.mod", not any other case. Files named "go.mod"
//   are not allowed in any other directory.
//
// - The total size in bytes of a module zip file may be at most MaxZipFile
///  bytes (500 MiB). The total uncompressed size of the files within the
//   zip may also be at most MaxZipFile bytes.
//
// - Each file's uncompressed size must match its declared 64-bit uncompressed
//   size in the zip file header.
//
// - If the zip contains files named "<module>@<version>/go.mod" or
//   "<module>@<version>/LICENSE", their sizes in bytes may be at most
//   MaxGoMod or MaxLICENSE, respectively (both are 16 MiB).
//
// - Empty directories are ignored. File permissions and timestamps are also
//   ignored.
//
// - Symbolic links and other irregular files are not allowed.
//
// Note that this package does not provide hashing functionality. See
// golang.org/x/mod/sumdb/dirhash.
package zip

import (
	"archive/zip"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"unicode"
	"unicode/utf8"

	"golang.org/x/mod/module"
)

const (
	// MaxZipFile is the maximum size in bytes of a module zip file. The
	// go command will report an error if either the zip file or its extracted
	// content is larger than this.<commit_msg>zip: improve readability of package comment<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package zip provides functions for creating and extracting module zip files.
//
// Module zip files have several restrictions listed below. These are necessary
// to ensure that module zip files can be extracted consistently on supported
// platforms and file systems.
//
// • All file paths within a zip file must start with "<module>@<version>/",
// where "<module>" is the module path and "<version>" is the version.
// The module path must be valid (see golang.org/x/mod/module.CheckPath).
// The version must be valid and canonical (see
// golang.org/x/mod/module.CanonicalVersion). The path must have a major
// version suffix consistent with the version (see
// golang.org/x/mod/module.Check). The part of the file path after the
// "<module>@<version>/" prefix must be valid (see
// golang.org/x/mod/module.CheckFilePath).
//
// • No two file paths may be equal under Unicode case-folding (see
// strings.EqualFold).
//
// • A go.mod file may or may not appear in the top-level directory. If present,
// it must be named "go.mod", not any other case. Files named "go.mod"
// are not allowed in any other directory.
//
// • The total size in bytes of a module zip file may be at most MaxZipFile
// bytes (500 MiB). The total uncompressed size of the files within the
// zip may also be at most MaxZipFile bytes.
//
// • Each file's uncompressed size must match its declared 64-bit uncompressed
// size in the zip file header.
//
// • If the zip contains files named "<module>@<version>/go.mod" or
// "<module>@<version>/LICENSE", their sizes in bytes may be at most
// MaxGoMod or MaxLICENSE, respectively (both are 16 MiB).
//
// • Empty directories are ignored. File permissions and timestamps are also
// ignored.
//
// • Symbolic links and other irregular files are not allowed.
//
// Note that this package does not provide hashing functionality. See
// golang.org/x/mod/sumdb/dirhash.
package zip

import (
	"archive/zip"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"unicode"
	"unicode/utf8"

	"golang.org/x/mod/module"
)

const (
	// MaxZipFile is the maximum size in bytes of a module zip file. The
	// go command will report an error if either the zip file or its extracted
	// content is larger than this.
{'fraction_non_alphanumeric': 0.10694108151735271, 'fraction_numerical': 0.004842615012106538, 'mean_word_length': 3.137729549248748, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3539903', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1419, 'n_words': 680}