<commit_before>package ufop

import (
	"archive/zip"
	"bytes"
	"encoding/base64"
	"errors"
	"fmt"
	"github.com/qiniu/api/auth/digest"
	"github.com/qiniu/api/rs"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

//mkzip/bucket/xxx/encoding/[gbk|utf-8]/url/alias/url/alias

const (
	MKZIP_MAX_FILE_LENGTH int64 = 100 * 1024 * 1024 //100MB
	MKZIP_MAX_FILE_COUNT  int   = 100               //100
	MKZIP_MAX_FILE_LIMIT  int   = 1000              //1000
)

type Mkziper struct {
	mac           *digest.Mac
	maxFileLength int64
	maxFileCount  int
}

type ZipFile struct {
	url   string
	key   string
	alias string
}

func (this *Mkziper) parse(cmd string) (bucket string, encoding string, zipFiles []ZipFile, err error) {
	pattern := "^mkzip/bucket/[0-9a-zA-Z-_=]+/encoding/[0-9a-zA-Z-_=]+(/url/[0-9a-zA-Z-_=]+(/alias/[0-9a-zA-Z-_=]+){0,1})+$"
	matched, _ := regexp.Match(pattern, []byte(cmd))
	if !matched {
		err = errors.New("invalid mkzip command format")
		return
	}
	//get bucket
	bucketRegx := regexp.MustCompile("bucket/[0-9a-zA-Z-_=]+")
	bucketPairItems := strings.Split(bucketRegx.FindString(cmd), "/")
	bucketBytes, decodeErr := base64.URLEncoding.DecodeString(bucketPairItems[1])
	if decodeErr != nil {
		err = errors.New("invalid mkzip paramter 'bucket'")
		return
	}
	bucket = string(bucketBytes)
	//get encoding
	encodingRegx := regexp.MustCompile("encoding/[0-9a-zA-Z-_=]+")
	encodingPairItems := strings.Split(encodingRegx.FindString(cmd), "/")
	encodingBytes, decodeErr := base64.URLEncoding.DecodeString(encodingPairItems[1])
	if decodeErr != nil {
		err = errors.New("invalid mkzip parameter 'encoding'")
		return
	}
	encoding = string(encodingBytes)
	//get url & alias
	urlAliasRegx := regexp.MustCompile("(url/[0-9a-zA-Z-_=]+(/alias/[0-9a-zA-Z-_=]+){0,1})")
	urlAliasPairs := urlAliasRegx.FindAllString(cmd, -1)
	paliasMap := make(map[string]string, 0)
	for _, urlAliasPair := range urlAliasPairs {
		urlAliasItems := strings.Split(urlAliasPair, "/")
		zipFile := ZipFile{}
		var purl string
		var palias string
		var key string
		switch len(urlAliasItems) {
		case 2:
			urlBytes, decodeErr := base64.URLEncoding.DecodeString(urlAliasItems[1])
			if decodeErr != nil {
				err = errors.New("invalid mkzip parameter 'url'")
				return
			}
			purl = string(urlBytes)
		case 4:
			urlBytes, decodeErr := base64.URLEncoding.DecodeString(urlAliasItems[1])
			if decodeErr != nil {
				err = errors.New("invalid mkzip parameter 'url'")
				return
			}
			aliasBytes, decodeErr := base64.URLEncoding.DecodeString(urlAliasItems[3])
			if decodeErr != nil {
				err = errors.New("invalid mkzip parameter 'alias'")
				return
			}
			purl = string(urlBytes)
			palias = string(aliasBytes)
		}
		uri, parseErr := url.Parse(purl)
		if parseErr != nil {
			err = errors.New("invalid mkzip parameter 'url'")
			return
		}
		if palias == "" {
			path := uri.Path
			ldx := strings.Index(path, "/")
			if ldx != -1 {
				palias = path[ldx+1:]
				key = palias
			}
		}
		if key == "" {
			err = errors.New("invalid mkzip resource url")
			return
		}
		if _, ok := paliasMap[palias]; ok {
			err = errors.New("duplicate mkzip resource alias")
			return
		}
		paliasMap[palias] = palias

		//set zip file
		zipFile.alias = palias
		zipFile.url = purl
		zipFile.key = key
		zipFiles = append(zipFiles, zipFile)
	}
	return
}

func (this *Mkziper) Do(req UfopRequest) (result interface{}, contentType string, err error) {
	contentType = "application/json"
	//set mkzip check criteria
	if this.maxFileCount <= 0 {
		this.maxFileCount = MKZIP_MAX_FILE_COUNT
	}
	if this.maxFileLength <= 0 {
		this.maxFileLength = MKZIP_MAX_FILE_LENGTH
	}
	//parse command
	bucket, encoding, zipFiles, pErr := this.parse(req.Cmd)
	if pErr != nil {
		err = pErr
		return
	}

	//check file count
	if len(zipFiles) > this.maxFileCount {
		err = errors.New("zip file count exceeds the limit")
		return
	}
	if len(zipFiles) > MKZIP_MAX_FILE_LIMIT {
		err = errors.New("only support items less than 1000")
		return
	}
	//check whether file in bucket and exceeds the limit
	statItems := make([]rs.EntryPath, 0)
	for _, zipFile := range zipFiles {
		entryPath := rs.EntryPath{
			bucket, zipFile.key,
		}
		statItems = append(statItems, entryPath)
	}
	qclient := rs.New(this.mac)

	statRet, statErr := qclient.BatchStat(nil, statItems)
	if statErr != nil {
		err = errors.New("batch stat error")
		return
	}
	for _, ret := range statRet {
		if ret.Error != "" {
			err = errors.New("stat resource in bucket error")
			return
		}
		if ret.Data.Fsize > this.maxFileLength {
			err = errors.New("stat resource length exceeds the limit")
			return
		}
	}
	//retrieve resource and create zip file
	var tErr error
	zipBuffer := new(bytes.Buffer)
	zipWriter := zip.NewWriter(zipBuffer)

	for _, zipFile := range zipFiles {
		fmt.Println(zipFile.url)
		resp, respErr := http.Get(zipFile.url)
		if respErr != nil {
			err = errors.New("get zip file resource error")
			return
		}
		defer resp.Body.Close()
<commit_msg>Remove stdout print.<commit_after>package ufop

import (
	"archive/zip"
	"bytes"
	"encoding/base64"
	"errors"
	"github.com/qiniu/api/auth/digest"
	"github.com/qiniu/api/rs"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

//mkzip/bucket/xxx/encoding/[gbk|utf-8]/url/alias/url/alias

const (
	MKZIP_MAX_FILE_LENGTH int64 = 100 * 1024 * 1024 //100MB
	MKZIP_MAX_FILE_COUNT  int   = 100               //100
	MKZIP_MAX_FILE_LIMIT  int   = 1000              //1000
)

type Mkziper struct {
	mac           *digest.Mac
	maxFileLength int64
	maxFileCount  int
}

type ZipFile struct {
	url   string
	key   string
	alias string
}

func (this *Mkziper) parse(cmd string) (bucket string, encoding string, zipFiles []ZipFile, err error) {
	pattern := "^mkzip/bucket/[0-9a-zA-Z-_=]+/encoding/[0-9a-zA-Z-_=]+(/url/[0-9a-zA-Z-_=]+(/alias/[0-9a-zA-Z-_=]+){0,1})+$"
	matched, _ := regexp.Match(pattern, []byte(cmd))
	if !matched {
		err = errors.New("invalid mkzip command format")
		return
	}
	//get bucket
	bucketRegx := regexp.MustCompile("bucket/[0-9a-zA-Z-_=]+")
	bucketPairItems := strings.Split(bucketRegx.FindString(cmd), "/")
	bucketBytes, decodeErr := base64.URLEncoding.DecodeString(bucketPairItems[1])
	if decodeErr != nil {
		err = errors.New("invalid mkzip paramter 'bucket'")
		return
	}
	bucket = string(bucketBytes)
	//get encoding
	encodingRegx := regexp.MustCompile("encoding/[0-9a-zA-Z-_=]+")
	encodingPairItems := strings.Split(encodingRegx.FindString(cmd), "/")
	encodingBytes, decodeErr := base64.URLEncoding.DecodeString(encodingPairItems[1])
	if decodeErr != nil {
		err = errors.New("invalid mkzip parameter 'encoding'")
		return
	}
	encoding = string(encodingBytes)
	//get url & alias
	urlAliasRegx := regexp.MustCompile("(url/[0-9a-zA-Z-_=]+(/alias/[0-9a-zA-Z-_=]+){0,1})")
	urlAliasPairs := urlAliasRegx.FindAllString(cmd, -1)
	paliasMap := make(map[string]string, 0)
	for _, urlAliasPair := range urlAliasPairs {
		urlAliasItems := strings.Split(urlAliasPair, "/")
		zipFile := ZipFile{}
		var purl string
		var palias string
		var key string
		switch len(urlAliasItems) {
		case 2:
			urlBytes, decodeErr := base64.URLEncoding.DecodeString(urlAliasItems[1])
			if decodeErr != nil {
				err = errors.New("invalid mkzip parameter 'url'")
				return
			}
			purl = string(urlBytes)
		case 4:
			urlBytes, decodeErr := base64.URLEncoding.DecodeString(urlAliasItems[1])
			if decodeErr != nil {
				err = errors.New("invalid mkzip parameter 'url'")
				return
			}
			aliasBytes, decodeErr := base64.URLEncoding.DecodeString(urlAliasItems[3])
			if decodeErr != nil {
				err = errors.New("invalid mkzip parameter 'alias'")
				return
			}
			purl = string(urlBytes)
			palias = string(aliasBytes)
		}
		uri, parseErr := url.Parse(purl)
		if parseErr != nil {
			err = errors.New("invalid mkzip parameter 'url'")
			return
		}
		if palias == "" {
			path := uri.Path
			ldx := strings.Index(path, "/")
			if ldx != -1 {
				palias = path[ldx+1:]
				key = palias
			}
		}
		if key == "" {
			err = errors.New("invalid mkzip resource url")
			return
		}
		if _, ok := paliasMap[palias]; ok {
			err = errors.New("duplicate mkzip resource alias")
			return
		}
		paliasMap[palias] = palias

		//set zip file
		zipFile.alias = palias
		zipFile.url = purl
		zipFile.key = key
		zipFiles = append(zipFiles, zipFile)
	}
	return
}

func (this *Mkziper) Do(req UfopRequest) (result interface{}, contentType string, err error) {
	contentType = "application/json"
	//set mkzip check criteria
	if this.maxFileCount <= 0 {
		this.maxFileCount = MKZIP_MAX_FILE_COUNT
	}
	if this.maxFileLength <= 0 {
		this.maxFileLength = MKZIP_MAX_FILE_LENGTH
	}
	//parse command
	bucket, encoding, zipFiles, pErr := this.parse(req.Cmd)
	if pErr != nil {
		err = pErr
		return
	}

	//check file count
	if len(zipFiles) > this.maxFileCount {
		err = errors.New("zip file count exceeds the limit")
		return
	}
	if len(zipFiles) > MKZIP_MAX_FILE_LIMIT {
		err = errors.New("only support items less than 1000")
		return
	}
	//check whether file in bucket and exceeds the limit
	statItems := make([]rs.EntryPath, 0)
	for _, zipFile := range zipFiles {
		entryPath := rs.EntryPath{
			bucket, zipFile.key,
		}
		statItems = append(statItems, entryPath)
	}
	qclient := rs.New(this.mac)

	statRet, statErr := qclient.BatchStat(nil, statItems)
	if statErr != nil {
		err = errors.New("batch stat error")
		return
	}
	for _, ret := range statRet {
		if ret.Error != "" {
			err = errors.New("stat resource in bucket error")
			return
		}
		if ret.Data.Fsize > this.maxFileLength {
			err = errors.New("stat resource length exceeds the limit")
			return
		}
	}
	//retrieve resource and create zip file
	var tErr error
	zipBuffer := new(bytes.Buffer)
	zipWriter := zip.NewWriter(zipBuffer)

	for _, zipFile := range zipFiles {
		resp, respErr := http.Get(zipFile.url)
		if respErr != nil {
			err = errors.New("get zip file resource error")
			return
		}
		defer resp.Body.Close()

{'fraction_non_alphanumeric': 0.14021137393054856, 'fraction_numerical': 0.016708605938600907, 'mean_word_length': 3.54320987654321, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2178013', 'n_tokens_mistral': 4042, 'n_tokens_neox': 3616, 'n_words': 816}