<commit_before>package crawler

import (
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/ReanGD/go-web-search/content"
	"github.com/jinzhu/gorm"
)

func fillHost(hostName string) (*content.Host, error) {
	robotsURL := NormalizeURL(&url.URL{Scheme: "http", Host: hostName, Path: "robots.txt"})
	response, err := http.Get(robotsURL)

	if err != nil {
		return nil, err
	}
	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	result := &content.Host{ID: hostName,
		RobotsStatusCode: response.StatusCode,
		RobotsData:       body,
		Timestamp:        time.Now()}

	return result, nil
}

func fillHosts(baseHosts map[string]bool, db *gorm.DB) error {
	for hostName, isNew := range baseHosts {
		if isNew {
			host, err := fillHost(hostName)
			if err != nil {
				return err
			}

			err = db.Create(host).Error
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func initHosts(baseHosts map[string]bool, db *gorm.DB) error {
	var hosts []content.Host
	err := db.Find(&hosts).Error
	if err != nil {
		return nil
	}

	needFill := false
	for hostName := range baseHosts {
		isNew := true
		for _, host := range hosts {
			if host.ID == hostName {
				isNew = false
				break
			}
		}
		if isNew {
			needFill = true
		}
		baseHosts[hostName] = isNew
	}

	if needFill {
		tr := db.Begin()
		err = fillHosts(baseHosts, tr)
		if err != nil {
			tr.Rollback()
		} else {
			tr.Commit()
		}
	}

	return err
}<commit_msg>для каждого нового хоста сохраняем URL для первоначальной загрузки<commit_after>package crawler

import (
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/ReanGD/go-web-search/content"
	"github.com/jinzhu/gorm"
)

func fillHost(hostName string) (*content.Host, error) {
	robotsURL := NormalizeURL(&url.URL{Scheme: "http", Host: hostName, Path: "robots.txt"})
	response, err := http.Get(robotsURL)

	if err != nil {
		return nil, err
	}
	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	result := &content.Host{ID: hostName,
		RobotsStatusCode: response.StatusCode,
		RobotsData:       body,
		Timestamp:        time.Now()}

	return result, nil
}

func fillHosts(baseHosts map[string]bool, db *gorm.DB) error {
	for hostName, isNew := range baseHosts {
		if isNew {
			host, err := fillHost(hostName)
			if err != nil {
				return err
			}

			err = db.Create(host).Error
			if err != nil {
				return err
			}

			urlKey := URLFromHost(hostName)
			var url content.URL
			err = db.Where("id = ?", urlKey).First(&url).Error
			if err == gorm.RecordNotFound {
				url = content.URL{ID: urlKey, Loaded: false}
				err = db.Create(&url).Error
				if err != nil {
					return err
				}
			} else {
				return err
			}
		}
	}

	return nil
}

func initHosts(baseHosts map[string]bool, db *gorm.DB) (int, error) {
	var hosts []content.Host
	err := db.Find(&hosts).Error
	if err != nil {
		return 0, nil
	}

	totalCnt := len(hosts)

	needFill := false
	for hostName := range baseHosts {
		isNew := true
		for _, host := range hosts {
			if host.ID == hostName {
				isNew = false
				break
			}
		}
		if isNew {
			needFill = true
			totalCnt++
		}
		baseHosts[hostName] = isNew
	}

	if needFill {
		tr := db.Begin()
		err = tr.Error
		if err == nil {
			err = fillHosts(baseHosts, tr)
			if err != nil {
				tr.Rollback()
			} else {
				err = tr.Commit().Error
			}
		}
	}

	return totalCnt, err
}
{'fraction_non_alphanumeric': 0.1372321945570353, 'fraction_numerical': 0.00028951939779965256, 'mean_word_length': 2.890765765765766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6225802', 'n_tokens_mistral': 1489, 'n_tokens_neox': 1323, 'n_words': 322}