<commit_before>package operations

import (
	"fmt"
	"github.com/qiniu/qshell/v2/iqshell"
	"github.com/qiniu/qshell/v2/iqshell/common/alert"
	"github.com/qiniu/qshell/v2/iqshell/common/data"
	"github.com/qiniu/qshell/v2/iqshell/common/export"
	"github.com/qiniu/qshell/v2/iqshell/common/flow"
	"github.com/qiniu/qshell/v2/iqshell/common/log"
	"github.com/qiniu/qshell/v2/iqshell/common/utils"
	"github.com/qiniu/qshell/v2/iqshell/storage/object"
	"github.com/qiniu/qshell/v2/iqshell/storage/object/batch"
	"strconv"
	"time"
)

type FetchInfo object.FetchApiInfo

func (info *FetchInfo) Check() *data.CodeError {
	if len(info.Bucket) == 0 {
		return alert.CannotEmptyError("Bucket", "")
	}
	if len(info.FromUrl) == 0 {
		return alert.CannotEmptyError("RemoteResourceUrl", "")
	}
	return nil
}

func Fetch(cfg *iqshell.Config, info FetchInfo) {
	if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{
		Checker: &info,
	}); !shouldContinue {
		return
	}

	result, err := object.Fetch(object.FetchApiInfo(info))
	if err != nil {
		log.ErrorF("Fetch Failed, '%s' => [%s:%s], Error:%v",
			info.FromUrl, info.Bucket, info.Key, err)
	} else {
		log.InfoF("Fetch Success, '%s' => [%s:%s]", info.FromUrl, info.Bucket, info.Key)
		log.AlertF("Key:%s", result.Key)
		log.AlertF("FileHash:%s", result.Hash)
		log.AlertF("Fsize: %d (%s)", result.Fsize, utils.FormatFileSize(result.Fsize))
		log.AlertF("Mime:%s", result.MimeType)
	}
}

type BatchFetchInfo struct {
	BatchInfo batch.Info
	Bucket    string
}

func (info *BatchFetchInfo) Check() *data.CodeError {
	if err := info.BatchInfo.Check(); err != nil {
		return err
	}

	if len(info.Bucket) == 0 {
		return alert.CannotEmptyError("Bucket", "")
	}
	return nil
}

//BatchFetch 批量删除，由于和批量删除的输入读取逻辑不同，所以分开
func BatchFetch(cfg *iqshell.Config, info BatchFetchInfo) {
	if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{
		Checker: &info,
	}); !shouldContinue {
		return
	}

	exporter, err := export.NewFileExport(info.BatchInfo.FileExporterConfig)
	if err != nil {
		log.Error(err)
		return
	}

	flow.New(info.BatchInfo.Info).
		WorkProviderWithFile(info.BatchInfo.InputFile,
			info.BatchInfo.EnableStdin,
			flow.NewItemsWorkCreator(info.BatchInfo.ItemSeparate, 1, func(items []string) (work flow.Work, err *data.CodeError) {
				key := ""
				fromUrl := items[0]
				if len(items) > 1 {
					key = items[1]
				} else if k, e := utils.KeyFromUrl(fromUrl); e == nil {
					key = k
				}
				if len(key) == 0 || len(fromUrl) == 0 {
					return nil, alert.Error("key or fromUrl invalid", "")
				}

				return &object.FetchApiInfo{
					Bucket:  info.Bucket,
					Key:     key,
					FromUrl: fromUrl,
				}, nil
			})).
		WorkerProvider(flow.NewWorkerProvider(func() (flow.Worker, *data.CodeError) {
			return flow.NewSimpleWorker(func(workInfo *flow.WorkInfo) (flow.Result, *data.CodeError) {
				in := workInfo.Work.(*object.FetchApiInfo)
				return object.Fetch(*in)
			}), nil
		})).
		OnWorkSuccess(func(workInfo *flow.WorkInfo, result flow.Result) {
			in, _ := workInfo.Work.(*object.FetchApiInfo)
			exporter.Success().ExportF("%s\t%s", in.FromUrl, in.Bucket)
			log.InfoF("Fetch Success, '%s' => [%s:%s]", in.FromUrl, info.Bucket, in.Key)
		}).
		OnWorkFail(func(workInfo *flow.WorkInfo, err *data.CodeError) {
			if in, ok := workInfo.Work.(*object.FetchApiInfo); ok {
				exporter.Fail().ExportF("%s\t%s\t%v", in.FromUrl, in.Key, err)
				log.ErrorF("Fetch Failed, '%s' => [%s:%s], Error: %v", in.FromUrl, in.Bucket, in.Key, err)
			} else {
				exporter.Fail().ExportF("%s%s%v", workInfo.Data, flow.ErrorSeparate, err)
				log.ErrorF("Fetch Failed, %s, Error: %s", workInfo.Data, err)<commit_msg>optimize fetch fail export<commit_after>package operations

import (
	"fmt"
	"github.com/qiniu/qshell/v2/iqshell"
	"github.com/qiniu/qshell/v2/iqshell/common/alert"
	"github.com/qiniu/qshell/v2/iqshell/common/data"
	"github.com/qiniu/qshell/v2/iqshell/common/export"
	"github.com/qiniu/qshell/v2/iqshell/common/flow"
	"github.com/qiniu/qshell/v2/iqshell/common/log"
	"github.com/qiniu/qshell/v2/iqshell/common/utils"
	"github.com/qiniu/qshell/v2/iqshell/storage/object"
	"github.com/qiniu/qshell/v2/iqshell/storage/object/batch"
	"strconv"
	"time"
)

type FetchInfo object.FetchApiInfo

func (info *FetchInfo) Check() *data.CodeError {
	if len(info.Bucket) == 0 {
		return alert.CannotEmptyError("Bucket", "")
	}
	if len(info.FromUrl) == 0 {
		return alert.CannotEmptyError("RemoteResourceUrl", "")
	}
	return nil
}

func Fetch(cfg *iqshell.Config, info FetchInfo) {
	if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{
		Checker: &info,
	}); !shouldContinue {
		return
	}

	result, err := object.Fetch(object.FetchApiInfo(info))
	if err != nil {
		log.ErrorF("Fetch Failed, '%s' => [%s:%s], Error:%v",
			info.FromUrl, info.Bucket, info.Key, err)
	} else {
		log.InfoF("Fetch Success, '%s' => [%s:%s]", info.FromUrl, info.Bucket, info.Key)
		log.AlertF("Key:%s", result.Key)
		log.AlertF("FileHash:%s", result.Hash)
		log.AlertF("Fsize: %d (%s)", result.Fsize, utils.FormatFileSize(result.Fsize))
		log.AlertF("Mime:%s", result.MimeType)
	}
}

type BatchFetchInfo struct {
	BatchInfo batch.Info
	Bucket    string
}

func (info *BatchFetchInfo) Check() *data.CodeError {
	if err := info.BatchInfo.Check(); err != nil {
		return err
	}

	if len(info.Bucket) == 0 {
		return alert.CannotEmptyError("Bucket", "")
	}
	return nil
}

//BatchFetch 批量删除，由于和批量删除的输入读取逻辑不同，所以分开
func BatchFetch(cfg *iqshell.Config, info BatchFetchInfo) {
	if shouldContinue := iqshell.CheckAndLoad(cfg, iqshell.CheckAndLoadInfo{
		Checker: &info,
	}); !shouldContinue {
		return
	}

	exporter, err := export.NewFileExport(info.BatchInfo.FileExporterConfig)
	if err != nil {
		log.Error(err)
		return
	}

	flow.New(info.BatchInfo.Info).
		WorkProviderWithFile(info.BatchInfo.InputFile,
			info.BatchInfo.EnableStdin,
			flow.NewItemsWorkCreator(info.BatchInfo.ItemSeparate, 1, func(items []string) (work flow.Work, err *data.CodeError) {
				key := ""
				fromUrl := items[0]
				if len(items) > 1 {
					key = items[1]
				} else if k, e := utils.KeyFromUrl(fromUrl); e == nil {
					key = k
				}
				if len(key) == 0 || len(fromUrl) == 0 {
					return nil, alert.Error("key or fromUrl invalid", "")
				}

				return &object.FetchApiInfo{
					Bucket:  info.Bucket,
					Key:     key,
					FromUrl: fromUrl,
				}, nil
			})).
		WorkerProvider(flow.NewWorkerProvider(func() (flow.Worker, *data.CodeError) {
			return flow.NewSimpleWorker(func(workInfo *flow.WorkInfo) (flow.Result, *data.CodeError) {
				in := workInfo.Work.(*object.FetchApiInfo)
				return object.Fetch(*in)
			}), nil
		})).
		OnWorkSuccess(func(workInfo *flow.WorkInfo, result flow.Result) {
			in, _ := workInfo.Work.(*object.FetchApiInfo)
			exporter.Success().ExportF("%s\t%s", in.FromUrl, in.Bucket)
			log.InfoF("Fetch Success, '%s' => [%s:%s]", in.FromUrl, info.Bucket, in.Key)
		}).
		OnWorkFail(func(workInfo *flow.WorkInfo, err *data.CodeError) {
			exporter.Fail().ExportF("%s%s%v", workInfo.Data, flow.ErrorSeparate, err)
			if in, ok := workInfo.Work.(*object.FetchApiInfo); ok {
				log.ErrorF("Fetch Failed, '%s' => [%s:%s], Error: %v", in.FromUrl, in.Bucket, in.Key, err)
			} else {
				log.ErrorF("Fetch Failed, %s, Error: %s", workInfo.Data, err)
{'fraction_non_alphanumeric': 0.17001503348366817, 'fraction_numerical': 0.004920049200492005, 'mean_word_length': 4.0996515679442505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2223227', 'n_tokens_mistral': 3066, 'n_tokens_neox': 2824, 'n_words': 484}