<commit_before>package grabber

import (
	"fmt"
	"os"
	"path"
	"runtime"
	"time"

	"github.com/crazy-max/ftpgrab/v7/internal/config"
	"github.com/crazy-max/ftpgrab/v7/internal/db"
	"github.com/crazy-max/ftpgrab/v7/internal/journal"
	"github.com/crazy-max/ftpgrab/v7/internal/server"
	"github.com/crazy-max/ftpgrab/v7/internal/server/ftp"
	"github.com/crazy-max/ftpgrab/v7/internal/server/sftp"
	"github.com/crazy-max/ftpgrab/v7/pkg/utl"
	"github.com/docker/go-units"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
)

// Client represents an active grabber object
type Client struct {
	config *config.Download
	db     *db.Client
	server *server.Client
}

// New creates new grabber instance
func New(dlConfig *config.Download, dbConfig *config.Db, serverConfig *config.Server) (*Client, error) {
	var dbCli *db.Client
	var serverCli *server.Client
	var err error

	// DB client
	if dbCli, err = db.New(dbConfig); err != nil {
		return nil, errors.Wrap(err, "Cannot open database")
	}

	// Server client
	if serverConfig.FTP != nil {
		serverCli, err = ftp.New(serverConfig.FTP)
	} else if serverConfig.SFTP != nil {
		serverCli, err = sftp.New(serverConfig.SFTP)
	} else {
		return nil, errors.New("No server defined")
	}
	if err != nil {
		return nil, errors.Wrap(err, "Cannot connect to server")
	}

	return &Client{
		config: dlConfig,
		db:     dbCli,
		server: serverCli,
	}, nil
}

func (c *Client) Grab(files []File) journal.Journal {
	jnl := journal.New()
	jnl.ServerHost = c.server.Common().Host

	for _, file := range files {
		if entry := c.download(file, 0); entry != nil {
			jnl.Add(*entry)
		}
	}

	return jnl.Journal
}

func (c *Client) download(file File, retry int) *journal.Entry {
	srcpath := path.Join(file.SrcDir, file.Info.Name())
	destpath := path.Join(file.DestDir, file.Info.Name())

	entry := &journal.Entry{
		File:   srcpath,
		Status: c.getStatus(file),
	}

	sublogger := log.With().
		Str("file", entry.File).
		Str("size", units.HumanSize(float64(file.Info.Size()))).
		Logger()

	if entry.Status == journal.EntryStatusAlreadyDl && !c.db.HasHash(file.Base, file.SrcDir, file.Info) {
		if err := c.db.PutHash(file.Base, file.SrcDir, file.Info); err != nil {
			sublogger.Error().Err(err).Msg("Cannot add hash into db")
			entry.Level = journal.EntryLevelWarning
			entry.Text = fmt.Sprintf("Already downloaded but cannot add hash into db: %v", err)
			return entry
		}
	}

	if entry.Status.IsSkipped() {
		if !*c.config.HideSkipped {
			sublogger.Warn().Msgf("Skipped (%s)", entry.Status)
			entry.Level = journal.EntryLevelSkip
			return entry
		}
		return nil
	}

	retrieveStart := time.Now()
	sublogger.Info().Str("dest", destpath).Msg("Downloading...")

	destfolder := path.Dir(destpath)
	if err := os.MkdirAll(destfolder, os.ModePerm); err != nil {
		sublogger.Error().Err(err).Msg("Cannot create destination dir")
		entry.Level = journal.EntryLevelError
		entry.Text = fmt.Sprintf("Cannot create destination dir: %v", err)
		return entry
	}<commit_msg>Enhanced logs<commit_after>package grabber

import (
	"fmt"
	"os"
	"path"
	"runtime"
	"time"

	"github.com/crazy-max/ftpgrab/v7/internal/config"
	"github.com/crazy-max/ftpgrab/v7/internal/db"
	"github.com/crazy-max/ftpgrab/v7/internal/journal"
	"github.com/crazy-max/ftpgrab/v7/internal/server"
	"github.com/crazy-max/ftpgrab/v7/internal/server/ftp"
	"github.com/crazy-max/ftpgrab/v7/internal/server/sftp"
	"github.com/crazy-max/ftpgrab/v7/pkg/utl"
	"github.com/docker/go-units"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
)

// Client represents an active grabber object
type Client struct {
	config *config.Download
	db     *db.Client
	server *server.Client
}

// New creates new grabber instance
func New(dlConfig *config.Download, dbConfig *config.Db, serverConfig *config.Server) (*Client, error) {
	var dbCli *db.Client
	var serverCli *server.Client
	var err error

	// DB client
	if dbCli, err = db.New(dbConfig); err != nil {
		return nil, errors.Wrap(err, "Cannot open database")
	}

	// Server client
	if serverConfig.FTP != nil {
		serverCli, err = ftp.New(serverConfig.FTP)
	} else if serverConfig.SFTP != nil {
		serverCli, err = sftp.New(serverConfig.SFTP)
	} else {
		return nil, errors.New("No server defined")
	}
	if err != nil {
		return nil, errors.Wrap(err, "Cannot connect to server")
	}

	return &Client{
		config: dlConfig,
		db:     dbCli,
		server: serverCli,
	}, nil
}

func (c *Client) Grab(files []File) journal.Journal {
	jnl := journal.New()
	jnl.ServerHost = c.server.Common().Host

	for _, file := range files {
		if entry := c.download(file, 0); entry != nil {
			jnl.Add(*entry)
		}
	}

	return jnl.Journal
}

func (c *Client) download(file File, retry int) *journal.Entry {
	srcpath := path.Join(file.SrcDir, file.Info.Name())
	destpath := path.Join(file.DestDir, file.Info.Name())

	entry := &journal.Entry{
		File:   srcpath,
		Status: c.getStatus(file),
	}

	sublogger := log.With().
		Str("src", entry.File).
		Str("dest", file.DestDir).
		Str("size", units.HumanSize(float64(file.Info.Size()))).
		Logger()

	if entry.Status == journal.EntryStatusAlreadyDl && !c.db.HasHash(file.Base, file.SrcDir, file.Info) {
		if err := c.db.PutHash(file.Base, file.SrcDir, file.Info); err != nil {
			sublogger.Error().Err(err).Msg("Cannot add hash into db")
			entry.Level = journal.EntryLevelWarning
			entry.Text = fmt.Sprintf("Already downloaded but cannot add hash into db: %v", err)
			return entry
		}
	}

	if entry.Status.IsSkipped() {
		if !*c.config.HideSkipped {
			sublogger.Warn().Msgf("Skipped (%s)", entry.Status)
			entry.Level = journal.EntryLevelSkip
			return entry
		}
		return nil
	}

	retrieveStart := time.Now()

	destfolder := path.Dir(destpath)
	if err := os.MkdirAll(destfolder, os.ModePerm); err != nil {
		sublogger.Error().Err(err).Msg("Cannot create destination dir")
		entry.Level = journal.EntryLevelError
		entry.Text = fmt.Sprintf("Cannot create destination dir: %v", err)
		return entry
	}
{'fraction_non_alphanumeric': 0.15513767629281397, 'fraction_numerical': 0.0033579583613163196, 'mean_word_length': 3.862857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6689716', 'n_tokens_mistral': 2324, 'n_tokens_neox': 2199, 'n_words': 473}