<commit_before>/*
   Author : feimyy <<EMAIL>>
   CopyRight : Apache License 2.0
*/
package main

import (
	"bytes"
	"fmt"
	iconv "github.com/feimyy/iconv"
	logex "github.com/feimyy/log"
	"io"
	"io/ioutil"
	// "log"
	"flag"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	filename_DisallowedChar string        = "/\\:?*\"|" //windows系统文件名不被允许的字符
	retry_max_num                         = 3
	default_file_size       int64         = 1024 * 1024 * 1024 * 10 //10M
	default_routine_num                   = 10
	is_verbose              bool          = false
	logfile_level           int           = Linfo
	logger                  *logex.Logger = nil
	stdoutLogger            *logex.Logger = logex.Std //default value
)

const (
	Ldebug = logex.Ldebug
	Linfo  = logex.Linfo
	Lerror = logex.Lerror
	Lwarn  = logex.Lwarn
)

var level_suffix = map[int]string{
	Ldebug: ".DEBUG.log",
	Linfo:  ".INFO.log",
	Lwarn:  ".WARN.log",
	Lerror: ".ERROR.log",
}

type ResourceInfo struct {
	Id       string
	DownUrl  string
	Suffix   string // don't contain the dot(.)
	Name     string
	Sequence int //该视频是第几集
}

type EpisodeList struct {
	needlist []int
}

func (e *EpisodeList) ParseValue(value string, episode_num int) bool {

	if e.IsNeedPartial() {

		if len(value) == 1 { //--episode 1
			n, err := strconv.Atoi(value)
			if err == nil {
				e.needlist = append(e.needlist, n)
				stdoutLogger.Debugf("Single episode :%d,list :%v\n", n, e.needlist)
				return false
			} else {
				stdoutLogger.Debugf("strconv.Atoi() ,value : :%s,err :%s\n", value, err)
				return true
			}
		} else if strings.Contains(value, ",") { //--episode 1,2,3
			list := strings.Split(value, ",")
			stdoutLogger.Debugf("multi episode :")
			for _, v := range list {
				n, err := strconv.Atoi(v)
				if err != nil {
					stdoutLogger.Debugf("strconv.Atoi() ,value : :%s,err :%s\n", value, err)
					return true
				} else {
					e.needlist = append(e.needlist, n)
				}

			}
			stdoutLogger.Debugf("\n")
			return false
		} else if strings.Contains(value, "-") { //--episode 1-10
			list := strings.Split(value, "-")
			if len(list) > 2 {
				return true
			}

			stdoutLogger.Debugf("episode range mode\n")
			//parse the start position
			n, err := strconv.Atoi(list[0])
			if err != nil {
				stdoutLogger.Debugf("strconv.Atoi() ,value : :%s,err :%s\n", value, err)
				return true
			} else {
				e.needlist = append(e.needlist, n)
			}

			//parse the end position
			if len(list[1]) == 0 { // --episode 1-
				for i := n + 1; i <= episode_num; i++ {
					e.needlist = append(e.needlist, i)
				}
				return false
			} else {
				n1, err := strconv.Atoi(list[1])
				if err != nil {
					return true
				} else {
					for i := n + 1; i <= n1; i++ {
						e.needlist = append(e.needlist, i)
					}

				}
				return false
			}
		} else {
			return true
		}
	} else {
		return true
	}
}

func (e EpisodeList) IsNeedEpisode(seq int) bool {
	if len(e.needlist) > 0 {
		for _, v := range e.needlist {
			if v == seq {
				return true
			}<commit_msg>修复episode参数解析的Bug<commit_after>/*
   Author : feimyy <<EMAIL>>
   CopyRight : Apache License 2.0
*/
package main

import (
	"bytes"
	"fmt"
	iconv "github.com/feimyy/iconv"
	logex "github.com/feimyy/log"
	"io"
	"io/ioutil"
	// "log"
	"flag"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	filename_DisallowedChar string        = "/\\:?*\"|" //windows系统文件名不被允许的字符
	retry_max_num                         = 3
	default_file_size       int64         = 1024 * 1024 * 1024 * 10 //10M
	default_routine_num                   = 10
	is_verbose              bool          = false
	logfile_level           int           = Linfo
	logger                  *logex.Logger = nil
	stdoutLogger            *logex.Logger = logex.Std //default value
)

const (
	Ldebug = logex.Ldebug
	Linfo  = logex.Linfo
	Lerror = logex.Lerror
	Lwarn  = logex.Lwarn
)

var level_suffix = map[int]string{
	Ldebug: ".DEBUG.log",
	Linfo:  ".INFO.log",
	Lwarn:  ".WARN.log",
	Lerror: ".ERROR.log",
}

type ResourceInfo struct {
	Id       string
	DownUrl  string
	Suffix   string // don't contain the dot(.)
	Name     string
	Sequence int //该视频是第几集
}

type EpisodeList struct {
	needlist []int
}

func (e *EpisodeList) ParseValue(value string, episode_num int) bool {

	if e.IsNeedPartial() {

		if strings.Contains(value, ",") { //--episode 1,2,3
			list := strings.Split(value, ",")
			stdoutLogger.Debugf("multi episode :")
			for _, v := range list {
				n, err := strconv.Atoi(v)
				if err != nil {
					stdoutLogger.Debugf("strconv.Atoi() ,value : :%s,err :%s\n", value, err)
					return true
				} else {
					e.needlist = append(e.needlist, n)
				}

			}
			stdoutLogger.Debugf("\n")
			return false
		} else if strings.Contains(value, "-") { //--episode 1-10
			list := strings.Split(value, "-")
			if len(list) > 2 {
				return true
			}

			stdoutLogger.Debugf("episode range mode\n")
			//parse the start position
			n, err := strconv.Atoi(list[0])
			if err != nil {
				stdoutLogger.Debugf("strconv.Atoi() ,value : :%s,err :%s\n", value, err)
				return true
			} else {
				e.needlist = append(e.needlist, n)
			}

			//parse the end position
			if len(list[1]) == 0 { // --episode 1-
				for i := n + 1; i <= episode_num; i++ {
					e.needlist = append(e.needlist, i)
				}
				return false
			} else {
				n1, err := strconv.Atoi(list[1])
				if err != nil {
					return true
				} else {
					for i := n + 1; i <= n1; i++ {
						e.needlist = append(e.needlist, i)
					}

				}
				return false
			}
		} else {

			//--episode 1
			n, err := strconv.Atoi(value)
			if err == nil {
				e.needlist = append(e.needlist, n)
				stdoutLogger.Debugf("Single episode :%d,list :%v\n", n, e.needlist)
				return false
			} else {
				stdoutLogger.Debugf("strconv.Atoi() ,value : :%s,err :%s\n", value, err)
				return true
			}
		}
	} else {
		return true
	}
}

func (e EpisodeList) IsNeedEpisode(seq int) bool {
	if len(e.needlist) > 0 {
		for _, v := range e.needlist {
			if v == seq {
				return true
			}
{'fraction_non_alphanumeric': 0.14956060354833361, 'fraction_numerical': 0.013762228486154866, 'mean_word_length': 2.2517520215633424, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1003799', 'n_tokens_mistral': 2715, 'n_tokens_neox': 2321, 'n_words': 550}