<commit_before>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package storage

import (
	"strings"

	"github.com/TheThingsNetwork/go-utils/log"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	redis "gopkg.in/redis.v5"
)

// VersionKey indicates the data (schema) version
const VersionKey = "_version"

type hasDBVersion interface {
	DBVersion() string
}

// MigrateFunction migrates data from its old version to the latest
type MigrateFunction func(client *redis.Client, key string, input map[string]string) (version string, output map[string]string, err error)

// AddMigration adds a data migration for a version
func (s *RedisMapStore) AddMigration(version string, migrate MigrateFunction) {
	s.migrations[version] = migrate
}

// Migrate all documents matching the selector
func (s *RedisMapStore) Migrate(selector string) error {
	if selector == "" {
		selector = "*"
	}
	if !strings.HasPrefix(selector, s.prefix) {
		selector = s.prefix + selector
	}

	var cursor uint64
	for {
		keys, next, err := s.client.Scan(cursor, selector, 0).Result()
		if err != nil {
			return err<commit_msg>Clarified what return value does<commit_after>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package storage

import (
	"strings"

	"github.com/TheThingsNetwork/go-utils/log"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	redis "gopkg.in/redis.v5"
)

// VersionKey indicates the data (schema) version
const VersionKey = "_version"

type hasDBVersion interface {
	DBVersion() string
}

// MigrateFunction migrates data from its old version to the latest
type MigrateFunction func(client *redis.Client, key string, input map[string]string) (nextVersion string, output map[string]string, err error)

// AddMigration adds a data migration for a version
func (s *RedisMapStore) AddMigration(version string, migrate MigrateFunction) {
	s.migrations[version] = migrate
}

// Migrate all documents matching the selector
func (s *RedisMapStore) Migrate(selector string) error {
	if selector == "" {
		selector = "*"
	}
	if !strings.HasPrefix(selector, s.prefix) {
		selector = s.prefix + selector
	}

	var cursor uint64
	for {
		keys, next, err := s.client.Scan(cursor, selector, 0).Result()
		if err != nil {
			return err
{'fraction_non_alphanumeric': 0.0996232733361239, 'fraction_numerical': 0.006697362913352867, 'mean_word_length': 4.419501133786848, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3725118', 'n_tokens_mistral': 766, 'n_tokens_neox': 725, 'n_words': 253}