<commit_before>package ufop

import (
	"archive/zip"
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/qiniu/api/auth/digest"
	fio "github.com/qiniu/api/io"
	rio "github.com/qiniu/api/resumable/io"
	"github.com/qiniu/api/rs"
	"github.com/qiniu/iconv"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"unicode/utf8"
)

const (
	MAX_FILE_LENGTH int64 = 100 * 1024 * 1024 //100MB
	MAX_FILE_COUNT  int   = 100               //100
)

type UnZipResult struct {
	Files []UnZipFile `json:"files"`
}

type UnZipFile struct {
	Key   string `json:"key"`
	Hash  string `json:"hash,omitempty"`
	Error string `json:"error,omitempty"`
}

type UnZipper struct {
	mac           *digest.Mac
	maxFileLength int64
	maxFileCount  int
}

func (this *UnZipper) parse(cmd string) (bucket string, overwrite bool, err error) {
	pattern := "^unzip/bucket/[0-9a-zA-Z-_=]+(/overwrite/[0|1]){0,1}$"
	matched, _ := regexp.Match(pattern, []byte(cmd))
	if !matched {
		err = errors.New("invalid unzip command format")
		return
	}
	items := strings.Split(cmd, "/")

	if len(items) >= 3 {
		bucketBytes, paramErr := base64.URLEncoding.DecodeString(items[2])
		if paramErr != nil {
			err = errors.New("invalid unzip parameter 'bucket'")
			return
		}
		bucket = string(bucketBytes)
	}
	if len(items) == 5 {
		overwriteVal, paramErr := strconv.ParseInt(items[4], 10, 64)
		if paramErr != nil {
			err = errors.New("invalid unzip parameter 'overwrite'")
			return
		}
		if overwriteVal == 1 {
			overwrite = true
		}
	}
	return
}

func (this *UnZipper) Do(req UfopRequest) (result interface{}, err error) {
	//set zip file check criteria
	if this.maxFileCount <= 0 {
		this.maxFileCount = MAX_FILE_COUNT
	}
	if this.maxFileLength <= 0 {
		this.maxFileLength = MAX_FILE_LENGTH
	}
	//check mimetype
	if req.Src.MimeType != "application/zip" {
		err = errors.New("unsupported mimetype to unzip")
		return
	}

	//parse command
	bucket, overwrite, pErr := this.parse(req.Cmd)
	if pErr != nil {
		err = pErr
		return
	}

	//get resource
	resUrl := req.Src.Url
	resResp, respErr := http.Get(resUrl)
	if respErr != nil {
		err = errors.New("retrieve resource data failed")
		return
	}
	defer resResp.Body.Close()

	respData, respErr := ioutil.ReadAll(resResp.Body)
	if respErr != nil {
		err = errors.New("read resource data failed")
		return
	}

	//read zip
	respReader := bytes.NewReader(respData)
	zipReader, zipErr := zip.NewReader(respReader, int64(respReader.Len()))
	if zipErr != nil {
		err = errors.New("invalid zip file")
		return
	}

	//parse zip
	cd, cErr := iconv.Open("utf-8", "gbk")
	if cErr != nil {
		err = errors.New(fmt.Sprintf("create charset converter error"))
		return
	}
	defer cd.Close()
	rputSettings := rio.Settings{
		ChunkSize: 4 * 1024 * 1024,
		Workers:   1,
	}
	rio.SetSettings(&rputSettings)
	var rputThreshold int64 = 100 * 1024 * 1024
	policy := rs.PutPolicy{
		Scope: bucket,
	}
	var unzipResult UnZipResult
	unzipResult.Files = make([]UnZipFile, 0)

	zipFiles := zipReader.File
	//check file count
	zipFileCount := len(zipFiles)
	if zipFileCount > this.maxFileCount {
		err = errors.New("zip files count exceeds the limit")
		return
	}
	for _, zipFile := range zipFiles {
		fileInfo := zipFile.FileHeader.FileInfo()
		fileName := zipFile.FileHeader.Name
		fileSize := fileInfo.Size()

		if !utf8.Valid([]byte(fileName)) {
			fileName = cd.ConvString(fileName)
		}

		if fileInfo.IsDir() {
			continue
		}

		//check file size
		if fileSize > this.maxFileLength {
			err = errors.New("zip file length exceeds the limit")
			return
		}<commit_msg>Make unzip criteria check more efficient.<commit_after>package ufop

import (
	"archive/zip"
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/qiniu/api/auth/digest"
	fio "github.com/qiniu/api/io"
	rio "github.com/qiniu/api/resumable/io"
	"github.com/qiniu/api/rs"
	"github.com/qiniu/iconv"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"unicode/utf8"
)

const (
	UNZIP_MAX_FILE_LENGTH int64 = 100 * 1024 * 1024 //100MB
	UNZIP_MAX_FILE_COUNT  int   = 100               //100
)

type UnZipResult struct {
	Files []UnZipFile `json:"files"`
}

type UnZipFile struct {
	Key   string `json:"key"`
	Hash  string `json:"hash,omitempty"`
	Error string `json:"error,omitempty"`
}

type UnZipper struct {
	mac           *digest.Mac
	maxFileLength int64
	maxFileCount  int
}

func (this *UnZipper) parse(cmd string) (bucket string, overwrite bool, err error) {
	pattern := "^unzip/bucket/[0-9a-zA-Z-_=]+(/overwrite/[0|1]){0,1}$"
	matched, _ := regexp.Match(pattern, []byte(cmd))
	if !matched {
		err = errors.New("invalid unzip command format")
		return
	}
	items := strings.Split(cmd, "/")

	if len(items) >= 3 {
		bucketBytes, paramErr := base64.URLEncoding.DecodeString(items[2])
		if paramErr != nil {
			err = errors.New("invalid unzip parameter 'bucket'")
			return
		}
		bucket = string(bucketBytes)
	}
	if len(items) == 5 {
		overwriteVal, paramErr := strconv.ParseInt(items[4], 10, 64)
		if paramErr != nil {
			err = errors.New("invalid unzip parameter 'overwrite'")
			return
		}
		if overwriteVal == 1 {
			overwrite = true
		}
	}
	return
}

func (this *UnZipper) Do(req UfopRequest) (result interface{}, err error) {
	//set zip file check criteria
	if this.maxFileCount <= 0 {
		this.maxFileCount = UNZIP_MAX_FILE_COUNT
	}
	if this.maxFileLength <= 0 {
		this.maxFileLength = UNZIP_MAX_FILE_LENGTH
	}
	//check mimetype
	if req.Src.MimeType != "application/zip" {
		err = errors.New("unsupported mimetype to unzip")
		return
	}

	//parse command
	bucket, overwrite, pErr := this.parse(req.Cmd)
	if pErr != nil {
		err = pErr
		return
	}

	//get resource
	resUrl := req.Src.Url
	resResp, respErr := http.Get(resUrl)
	if respErr != nil {
		err = errors.New("retrieve resource data failed")
		return
	}
	defer resResp.Body.Close()

	respData, respErr := ioutil.ReadAll(resResp.Body)
	if respErr != nil {
		err = errors.New("read resource data failed")
		return
	}

	//read zip
	respReader := bytes.NewReader(respData)
	zipReader, zipErr := zip.NewReader(respReader, int64(respReader.Len()))
	if zipErr != nil {
		err = errors.New("invalid zip file")
		return
	}
	zipFiles := zipReader.File
	//check file count
	zipFileCount := len(zipFiles)
	if zipFileCount > this.maxFileCount {
		err = errors.New("zip files count exceeds the limit")
		return
	}
	//check file size
	for _, zipFile := range zipFiles {
		fileInfo := zipFile.FileHeader.FileInfo()
		fileSize := fileInfo.Size()
		//check file size
		if fileSize > this.maxFileLength {
			err = errors.New("zip file length exceeds the limit")
			return
		}
	}

	//parse zip
	cd, cErr := iconv.Open("utf-8", "gbk")
	if cErr != nil {
		err = errors.New(fmt.Sprintf("create charset converter error"))
		return
	}
	defer cd.Close()
	rputSettings := rio.Settings{
		ChunkSize: 4 * 1024 * 1024,
		Workers:   1,
	}
	rio.SetSettings(&rputSettings)
	var rputThreshold int64 = 100 * 1024 * 1024
	policy := rs.PutPolicy{
		Scope: bucket,
	}
	var unzipResult UnZipResult
	unzipResult.Files = make([]UnZipFile, 0)

	//iterate the zip file
	for _, zipFile := range zipFiles {
		fileInfo := zipFile.FileHeader.FileInfo()
		fileName := zipFile.FileHeader.Name
		fileSize := fileInfo.Size()

		if !utf8.Valid([]byte(fileName)) {
			fileName = cd.ConvString(fileName)
		}

		if fileInfo.IsDir() {
			continue
		}
{'fraction_non_alphanumeric': 0.13852044304662928, 'fraction_numerical': 0.02023793244906331, 'mean_word_length': 3.6232616940581543, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7203098', 'n_tokens_mistral': 2970, 'n_tokens_neox': 2684, 'n_words': 638}