<commit_before>package backend

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/garyburd/redigo/redis"
	gosocketio "github.com/graarh/golang-socketio"
)

var delKeysSet map[string]interface{}
var delKeysStorage []*delKeysStruct
var maxDelID int

type delKeysStruct struct {
	redisIns   *redis.Conn
	ID         int      `json:"id"`
	Keys       []string `json:"keys"`
	Process    float32  `json:"process"`
	ErrMsg     string   `json:"errMsg"`
	IsComplete bool     `json:"isComplete"`
}

func (task *delKeysStruct) calProcess(currentKeyLen, currentKeyTotal int) {
	if isDebug {
		log.Println("calProcess", currentKeyLen, currentKeyTotal, len(task.Keys))
	}
	task.Process += float32(currentKeyLen) / float32(currentKeyTotal) / float32(len(task.Keys))
	if isDebug {
		log.Println("calProcess", task.Process)
	}
}
func (task *delKeysStruct) run() {
	go func() {
		c := *task.redisIns
		defer func() {
			task.IsComplete = true
			c.Close()
		}()
		for index := 0; index < len(task.Keys); index++ {
			key := task.Keys[index]
			t, err := keyType(nil, c, key)
			if err != nil {
				task.Process = 0
				task.ErrMsg = err.Error()
				return
			}
			switch t {
			case "none": // 跳过不存在的key
				continue
			case "string":
				delKey(nil, c, key)
				task.calProcess(1, 1)
			case "list":
				var limits = int64(_globalConfigs.System.DelRowLimits)
				if sizes, ok := checkBigKey(nil, c, key, "list"); ok {
					// else use ltrim
					for end := sizes - 1; end >= 0; end = end - limits {
						start := end - limits<commit_msg>fix bug<commit_after>package backend

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/garyburd/redigo/redis"
	gosocketio "github.com/graarh/golang-socketio"
)

var delKeysSet map[string]interface{}
var delKeysStorage []*delKeysStruct
var maxDelID int

type delKeysStruct struct {
	redisIns   *redis.Conn
	ID         int      `json:"id"`
	Keys       []string `json:"keys"`
	Process    float32  `json:"process"`
	ErrMsg     string   `json:"errMsg"`
	IsComplete bool     `json:"isComplete"`
	HadTryTimes int `json:"had_try_times"`
}

func (task *delKeysStruct) calProcess(currentKeyLen, currentKeyTotal int) {
	if isDebug {
		log.Println("calProcess", currentKeyLen, currentKeyTotal, len(task.Keys))
	}
	task.Process += float32(currentKeyLen) / float32(currentKeyTotal) / float32(len(task.Keys))
	if isDebug {
		log.Println("calProcess", task.Process)
	}
}
func (task *delKeysStruct) run() {
	go func() {
		c := *task.redisIns
		defer func() {
			if task.ErrMsg != "" && task.HadTryTimes < 3 {// 重试
				task.ErrMsg = ""
				task.run()
			}else{
				task.IsComplete = true
				c.Close()
			}
		}()
		task.HadTryTimes ++
		for index := 0; index < len(task.Keys); index++ {
			key := task.Keys[index]
			t, err := keyType(nil, c, key)
			if err != nil {
				task.Process = 0
				task.ErrMsg = err.Error()
				return
			}
			switch t {
			case "none": // 跳过不存在的key
				continue
			case "string":
				delKey(nil, c, key)
				task.calProcess(1, 1)
			case "list":
				var limits = int64(_globalConfigs.System.DelRowLimits)
				if sizes, ok := checkBigKey(nil, c, key, "list"); ok {
					// else use ltrim
					for end := sizes - 1; end >= 0; end = end - limits {
						start := end - limits
{'fraction_non_alphanumeric': 0.14486552567237163, 'fraction_numerical': 0.010085574572127139, 'mean_word_length': 3.1961538461538463, 'pattern_counts': {'":': 6, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '601034', 'n_tokens_mistral': 1366, 'n_tokens_neox': 1195, 'n_words': 284}