<commit_before>package qshell

import (
	"archive/zip"
	"io"
	"os"
	"path/filepath"
)

func Unzip(zipFilePath string, unzipPath string) (err error) {
	zipReader, zipErr := zip.OpenReader(zipFilePath)
	if zipErr != nil {
		err = zipErr
		return
	}
	defer zipReader.Close()

	zipFiles := zipReader.File
	for _, zipFile := range zipFiles {
		fileInfo := zipFile.FileHeader.FileInfo()
		fileName := zipFile.FileHeader.Name
		fullPath := filepath.Join(unzipPath, fileName)
		if fileInfo.IsDir() {
			mErr := os.MkdirAll(fullPath, 0775)
			if mErr != nil {
				err = mErr
				continue
			}
		} else {
			localFp, openErr := os.OpenFile(fullPath, os.O_CREATE|os.O_WRONLY, 0666)
			if openErr != nil {
				err = openErr
				continue
			}
			defer localFp.Close()

			zipFp, openErr := zipFile.Open()
			if openErr != nil {
				err = openErr
				continue
			}
			defer zipFp.Close()

			_, wErr := io.Copy(localFp, zipFp)
			if wErr != nil {
				err = wErr
				continue
			}
		}
	}
	return
}<commit_msg>Add detailed error info for unzip.<commit_after>package qshell

import (
	"archive/zip"
	"errors"
	"fmt"
	"github.com/qiniu/iconv"
	"io"
	"os"
	"path/filepath"
	"unicode/utf8"
)

func Unzip(zipFilePath string, unzipPath string) (err error) {
	zipReader, zipErr := zip.OpenReader(zipFilePath)
	if zipErr != nil {
		err = errors.New(fmt.Sprintf("Open zip file error, %s", zipErr))
		return
	}
	defer zipReader.Close()

	zipFiles := zipReader.File
	cd, cErr := iconv.Open("utf-8", "gbk")
	if cErr != nil {
		err = errors.New(fmt.Sprintf("Create charset converter error, %s", cErr))
		return
	}
	defer cd.Close()
	for _, zipFile := range zipFiles {
		fileInfo := zipFile.FileHeader.FileInfo()
		fileName := zipFile.FileHeader.Name

		//check charset utf8 or gbk
		if !utf8.Valid([]byte(fileName)) {
			fileName = cd.ConvString(fileName)
		}

		fullPath := filepath.Join(unzipPath, fileName)

		if fileInfo.IsDir() {
			mErr := os.MkdirAll(fullPath, 0775)
			if mErr != nil {
				err = errors.New(fmt.Sprintf("Mkdir error, %s", mErr))
				continue
			}
		} else {
			localFp, openErr := os.OpenFile(fullPath, os.O_CREATE|os.O_WRONLY, 0666)
			if openErr != nil {
				err = errors.New(fmt.Sprintf("Open local file error, %s", openErr))
				continue
			}
			defer localFp.Close()

			zipFp, openErr := zipFile.Open()
			if openErr != nil {
				err = errors.New(fmt.Sprintf("Read zip content error, %s", openErr))
				continue
			}
			defer zipFp.Close()

			_, wErr := io.Copy(localFp, zipFp)
			if wErr != nil {
				err = errors.New(fmt.Sprintf("Save zip content error, %s", wErr))
				continue
			}
		}
	}
	return
}
{'fraction_non_alphanumeric': 0.13863287250384024, 'fraction_numerical': 0.007680491551459293, 'mean_word_length': 3.2220421393841168, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5482028', 'n_tokens_mistral': 1127, 'n_tokens_neox': 979, 'n_words': 226}