<commit_before>package replication

import (
	"strings"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/replication/sink"
	"github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type Replicator struct {
	sink   sink.ReplicationSink
	source *source.FilerSource
}

func NewReplicator(sourceConfig, sinkConfig util.Configuration) *Replicator {

	sink := &filersink.FilerSink{}
	sink.Initialize(sinkConfig)

	source := &source.FilerSource{}
	source.Initialize(sourceConfig)

	sink.SetSourceFiler(source)

	return &Replicator{
		sink:   sink,
		source: source,
	}
}

func (r *Replicator) Replicate(key string, message *filer_pb.EventNotification) error {
	if !strings.HasPrefix(key, r.source.Dir) {
		return nil
	}
	key = r.sink.GetSinkToDirectory() + key[len(r.source.Dir):]
	if message.OldEntry != nil && message.NewEntry == nil {
		return r.sink.DeleteEntry(key, message.OldEntry, message.DeleteChunks)
	}
	if message.OldEntry == nil && message.NewEntry != nil {<commit_msg>join via filepath<commit_after>package replication

import (
	"path/filepath"
	"strings"

	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/replication/sink"
	"github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
	"github.com/chrislusf/seaweedfs/weed/replication/source"
	"github.com/chrislusf/seaweedfs/weed/util"
)

type Replicator struct {
	sink   sink.ReplicationSink
	source *source.FilerSource
}

func NewReplicator(sourceConfig, sinkConfig util.Configuration) *Replicator {

	sink := &filersink.FilerSink{}
	sink.Initialize(sinkConfig)

	source := &source.FilerSource{}
	source.Initialize(sourceConfig)

	sink.SetSourceFiler(source)

	return &Replicator{
		sink:   sink,
		source: source,
	}
}

func (r *Replicator) Replicate(key string, message *filer_pb.EventNotification) error {
	if !strings.HasPrefix(key, r.source.Dir) {
		return nil
	}
	key = filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):])
	if message.OldEntry != nil && message.NewEntry == nil {
		return r.sink.DeleteEntry(key, message.OldEntry, message.DeleteChunks)
	}
	if message.OldEntry == nil && message.NewEntry != nil {
{'fraction_non_alphanumeric': 0.12908853030963802, 'fraction_numerical': 0.0, 'mean_word_length': 5.2, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1858176', 'n_tokens_mistral': 894, 'n_tokens_neox': 871, 'n_words': 131}