<commit_before>package main

import (
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/jinzhu/gorm"
	"github.com/serbe/ncp"
)

var (
	commands = []string{
		"get",
		"update",
		"name",
		"rating",
		"poster",
	}
	urls = []string{
		"/forum/viewforum.php?f=218",
		"/forum/viewforum.php?f=270",
	}
)

func (a *App) get() error {
	var (
		err error
		i   int64
	)
	for _, parseurl := range urls {
		topics, err := a.net.ParseForumTree(parseurl, a.debug)
		if err != nil {
			log.Println("ParseForumTree ", parseurl, err)
			return err
		}
		for _, topic := range topics {
			_, err := a.getTorrentByHref(topic.Href)
			if err == gorm.ErrRecordNotFound {
				film, err := a.net.ParseTopic(topic, a.debug)
				if err == nil {
					i++
					film = a.checkName(film)
					a.createTorrent(film)
				}
			}
		}
	}
	if i > 0 {
		log.Println("Adding", i, "new films")
	} else {
		log.Println("No adding new films")
	}
	return err
}

func (a *App) update() error {
	var (
		i        int64
		err      error
		torrents []Torrent
	)

	torrents, err = a.getWithDownload()
	if err != nil {
		return err
	}
	for _, tor := range torrents {
		var topic ncp.Topic
		topic.Href = tor.Href
		f, err := a.net.ParseTopic(topic, false)
		if err == nil {
			if f.NNM != tor.NNM || f.Seeders != tor.Seeders || f.Leechers != tor.Leechers || f.Torrent != tor.Torrent {
				i++
				a.updateTorrent(tor.ID, f)
			}
		} else {
			return err
		}
	}
	if i > 0 {
		log.Println("Update", i, "movies")
	} else {
		log.Println("No movies update")
	}
	return nil
}

func (a *App) name() error {
	var i int64
	movies, err := a.getMovies()
	if err != nil {
		return err
	}
	for _, movie := range movies {
		if movie.Name == strings.ToUpper(movie.Name) {
			lowerName, err := a.getLowerName(movie)
			if err == nil {
				i++
				a.updateName(movie.ID, lowerName)
			}
		}
	}
	if i > 0 {
		log.Println(i, "name fixed")
	} else {
		log.Println("No fixed names")
	}
	return nil
}

func (a *App) rating() error {
	var (
		i int64
	)
	movies, err := a.getNoRating()
	if err != nil {
		return err
	}
	for _, movie := range movies {
		if movie.Kinopoisk == 0 || movie.IMDb == 0 || movie.Duration == "" {
			kp, err := a.getRating(movie)
			if err == nil {
				i++
				_ = a.updateRating(movie, kp)
			}
		}
	}
	if i > 0 {
		log.Println(i, "ratings update")
	} else {
		log.Println("No update ratings")
	}
	return nil
}

func (a *App) poster() error {
	var (
		i     int64
		files []string
	)
	movies, err := a.getMovies()
	if err != nil {
		return err
	}
	filesInDir, _ := ioutil.ReadDir(a.hd)
	for _, file := range filesInDir {
		files = append(files, file.Name())
	}
	for _, movie := range movies {
		if movie.PosterURL != "" {
			if movie.Poster != "" {
				if existsFile(a.hd+movie.Poster) == false {
					poster, err := a.getPoster(movie.PosterURL)
					if err == nil {
						i++
						_ = a.updatePoster(movie, poster)
					}
				} else {
					files = deleteFromSlice(files, movie.Poster)
				}
			} else {
				poster, err := a.getPoster(movie.PosterURL)
				if err == nil {
					i++
					_ = a.updatePoster(movie, poster)
				}
			}
		} else {
			film, err := a.getFilmByMovieID(movie.ID)
			if err == nil {
				var topic ncp.Topic
				topic.Href = film.Href
				tempFilm, err := a.net.ParseTopic(topic, a.debug)
				if err == nil {
					if tempFilm.Poster != "" {
						err = a.updatePosterURL(movie, tempFilm.Poster)
						if err != nil {
							poster, err := a.getPoster(tempFilm.Poster)
							if err == nil {
								i++
								_ = a.updatePoster(movie, poster)
							}
						}
					}
				}
			}
		}
	}
	for _, file := range files {
		_ = os.Remove(a.hd + file)
	}
	if i > 0 {
		log.Println(i, "posters update")
	} else {
		log.Println("No update posters")
	}
	if len(files) > 0 {
		log.Println("Remove ", len(files), "unused images")
	}<commit_msg>fix compare<commit_after>package main

import (
	"io/ioutil"
	"log"
	"os"
	"strings"

	"github.com/jinzhu/gorm"
	"github.com/serbe/ncp"
)

var (
	commands = []string{
		"get",
		"update",
		"name",
		"rating",
		"poster",
	}
	urls = []string{
		"/forum/viewforum.php?f=218",
		"/forum/viewforum.php?f=270",
	}
)

func (a *App) get() error {
	var (
		err error
		i   int64
	)
	for _, parseurl := range urls {
		topics, err := a.net.ParseForumTree(parseurl, a.debug)
		if err != nil {
			log.Println("ParseForumTree ", parseurl, err)
			return err
		}
		for _, topic := range topics {
			_, err := a.getTorrentByHref(topic.Href)
			if err == gorm.ErrRecordNotFound {
				film, err := a.net.ParseTopic(topic, a.debug)
				if err == nil {
					i++
					film = a.checkName(film)
					a.createTorrent(film)
				}
			}
		}
	}
	if i > 0 {
		log.Println("Adding", i, "new films")
	} else {
		log.Println("No adding new films")
	}
	return err
}

func (a *App) update() error {
	var (
		i        int64
		err      error
		torrents []Torrent
	)

	torrents, err = a.getWithDownload()
	if err != nil {
		return err
	}
	for _, tor := range torrents {
		var topic ncp.Topic
		topic.Href = tor.Href
		f, err := a.net.ParseTopic(topic, false)
		if err == nil {
			if f.NNM != tor.NNM || f.Seeders != tor.Seeders || f.Leechers != tor.Leechers || f.Torrent != tor.Torrent {
				i++
				a.updateTorrent(tor.ID, f)
			}
		} else {
			return err
		}
	}
	if i > 0 {
		log.Println("Update", i, "movies")
	} else {
		log.Println("No movies update")
	}
	return nil
}

func (a *App) name() error {
	var i int64
	movies, err := a.getMovies()
	if err != nil {
		return err
	}
	for _, movie := range movies {
		if movie.Name == strings.ToUpper(movie.Name) {
			lowerName, err := a.getLowerName(movie)
			if err == nil {
				i++
				a.updateName(movie.ID, lowerName)
			}
		}
	}
	if i > 0 {
		log.Println(i, "name fixed")
	} else {
		log.Println("No fixed names")
	}
	return nil
}

func (a *App) rating() error {
	var (
		i int64
	)
	movies, err := a.getNoRating()
	if err != nil {
		return err
	}
	for _, movie := range movies {
		if movie.Kinopoisk == 0 || movie.IMDb == 0 || movie.Duration == "" {
			kp, err := a.getRating(movie)
			if err == nil {
				i++
				_ = a.updateRating(movie, kp)
			}
		}
	}
	if i > 0 {
		log.Println(i, "ratings update")
	} else {
		log.Println("No update ratings")
	}
	return nil
}

func (a *App) poster() error {
	var (
		i     int64
		files []string
	)
	movies, err := a.getMovies()
	if err != nil {
		return err
	}
	filesInDir, _ := ioutil.ReadDir(a.hd)
	for _, file := range filesInDir {
		files = append(files, file.Name())
	}
	for _, movie := range movies {
		if movie.PosterURL != "" {
			if movie.Poster != "" {
				if existsFile(a.hd+movie.Poster) == false {
					poster, err := a.getPoster(movie.PosterURL)
					if err == nil {
						i++
						_ = a.updatePoster(movie, poster)
					}
				} else {
					files = deleteFromSlice(files, movie.Poster)
				}
			} else {
				poster, err := a.getPoster(movie.PosterURL)
				if err == nil {
					i++
					_ = a.updatePoster(movie, poster)
				}
			}
		} else {
			film, err := a.getFilmByMovieID(movie.ID)
			if err == nil {
				var topic ncp.Topic
				topic.Href = film.Href
				tempFilm, err := a.net.ParseTopic(topic, a.debug)
				if err == nil {
					if tempFilm.Poster != "" {
						err = a.updatePosterURL(movie, tempFilm.Poster)
						if err == nil {
							poster, err := a.getPoster(tempFilm.Poster)
							if err == nil {
								i++
								_ = a.updatePoster(movie, poster)
							}
						}
					}
				}
			}
		}
	}
	for _, file := range files {
		_ = os.Remove(a.hd + file)
	}
	if i > 0 {
		log.Println(i, "posters update")
	} else {
		log.Println("No update posters")
	}
	if len(files) > 0 {
		log.Println("Remove ", len(files), "unused images")
	}
{'fraction_non_alphanumeric': 0.15981137018600997, 'fraction_numerical': 0.006287660466334818, 'mean_word_length': 2.408482142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1111954', 'n_tokens_mistral': 3682, 'n_tokens_neox': 3068, 'n_words': 711}