<commit_before>package crawler

import (
	"fmt"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/bootstrap"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/fetcher"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"go.uber.org/zap"
	"golang.org/x/sync/errgroup"
)

type Main struct {
	Concurrency     *int
	ContinueOnError *bool
	SpecifiedIDs    *string
	Followed        *bool
	All             *bool
	New             *bool
	LogLevel        *string
}

func (m *Main) Run() error {
	bootstrap.CheckCLIEnvVars()
	if *m.Followed && *m.SpecifiedIDs != "" {
		return fmt.Errorf("can't specify -followed and -ids flags both")
	}

	startedAt := time.Now().UTC()
	//if *m.LogLevel != "" {
	//	//logger.App.SetLevel(logger.NewLevel(*m.LogLevel))
	//}
	logger.App.Info("crawler started")
	defer func() {
		elapsed := time.Now().UTC().Sub(startedAt) / time.Millisecond
		logger.App.Info("fetcher finished", zap.Int("elapsed", int(elapsed)))
	}()

	db, err := model.OpenDB(bootstrap.CLIEnvVars.DBURL(), 1, !config.IsProductionEnv())
	if err != nil {
		return err
	}
	defer db.Close()

	mCountryService := model.NewMCountryService(db)
	mCountries, err := mCountryService.LoadAll()
	if err != nil {
		return err
	}

	loader := m.createLoader(db)
	fetcher := fetcher.NewLessonFetcher(nil, *m.Concurrency, false, mCountries, logger.App)
	teacherService := model.NewTeacherService(db)
	for cursor := loader.GetInitialCursor(); cursor != ""; {
		var teacherIDs []uint32
		var err error
		teacherIDs, cursor, err = loader.Load(cursor)
		if err != nil {
			return err
		}

		var g errgroup.Group
		for _, id := range teacherIDs {
			id := id
			g.Go(func() error {
				teacher, _, err := fetcher.Fetch(id)
				if err != nil {
					if *m.ContinueOnError {
						logger.App.Error("Error during LessonFetcher.Fetch", zap.Error(err))
						return nil
					} else {
						return err
					}
				}
				if err := teacherService.CreateOrUpdate(teacher); err != nil {
					if *m.ContinueOnError {
						logger.App.Error("Error during TeacherService.CreateOrUpdate", zap.Error(err))
						return nil
					} else {
						return err
					}
				}

				return nil
			})
		}

		if err := g.Wait(); err != nil {
			return err
		}<commit_msg>Add TODO<commit_after>package crawler

import (
	"fmt"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/bootstrap"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/fetcher"
	"github.com/oinume/lekcije/server/logger"
	"github.com/oinume/lekcije/server/model"
	"go.uber.org/zap"
	"golang.org/x/sync/errgroup"
)

type Main struct {
	Concurrency     *int
	ContinueOnError *bool
	SpecifiedIDs    *string
	Followed        *bool
	All             *bool
	New             *bool
	LogLevel        *string
}

func (m *Main) Run() error {
	bootstrap.CheckCLIEnvVars()
	if *m.Followed && *m.SpecifiedIDs != "" {
		return fmt.Errorf("can't specify -followed and -ids flags both")
	}

	startedAt := time.Now().UTC()
	//if *m.LogLevel != "" {
	//	//logger.App.SetLevel(logger.NewLevel(*m.LogLevel))
	//}
	logger.App.Info("crawler started")
	defer func() {
		elapsed := time.Now().UTC().Sub(startedAt) / time.Millisecond
		logger.App.Info("fetcher finished", zap.Int("elapsed", int(elapsed)))
	}()

	db, err := model.OpenDB(bootstrap.CLIEnvVars.DBURL(), 1, !config.IsProductionEnv())
	if err != nil {
		return err
	}
	defer db.Close()

	mCountryService := model.NewMCountryService(db)
	mCountries, err := mCountryService.LoadAll()
	if err != nil {
		return err
	}

	loader := m.createLoader(db)
	fetcher := fetcher.NewLessonFetcher(nil, *m.Concurrency, false, mCountries, logger.App)
	teacherService := model.NewTeacherService(db)
	for cursor := loader.GetInitialCursor(); cursor != ""; {
		var teacherIDs []uint32
		var err error
		teacherIDs, cursor, err = loader.Load(cursor)
		if err != nil {
			return err
		}

		// TODO: semaphore
		var g errgroup.Group
		for _, id := range teacherIDs {
			id := id
			g.Go(func() error {
				teacher, _, err := fetcher.Fetch(id)
				if err != nil {
					if *m.ContinueOnError {
						logger.App.Error("Error during LessonFetcher.Fetch", zap.Error(err))
						return nil
					} else {
						return err
					}
				}
				if err := teacherService.CreateOrUpdate(teacher); err != nil {
					if *m.ContinueOnError {
						logger.App.Error("Error during TeacherService.CreateOrUpdate", zap.Error(err))
						return nil
					} else {
						return err
					}
				}
				// TODO: update lessons

				return nil
			})
		}

		if err := g.Wait(); err != nil {
			return err
		}
{'fraction_non_alphanumeric': 0.1461305663640294, 'fraction_numerical': 0.0012970168612191958, 'mean_word_length': 3.06948109058927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1520536', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1653, 'n_words': 340}