func downloadAndLoadDbs() error {
	mirrors := getAllMirrorsDB()
	dbsPath := filepath.Join(config.CacheDir, "tmp-db")
	if _, err := os.Stat(dbsPath); os.IsNotExist(err) {
		if err := os.MkdirAll(dbsPath, os.ModePerm); err != nil {
			return err
		}
	}
	for _, mirror := range mirrors {
		if err := downloadAndLoadDB(mirror); err != nil {
			log.Printf("An error occurred for mirror %v :%v\n", mirror, err)
		}
	}
	return nil
}
download dbs from their URLs stored in the mirror_dbs table and load their content in the repo_packages table
package main
import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"
)
func uncompressGZ(filePath string, targetFile string) error {
	gzipfile, err := os.Open(filePath)
	if err != nil {
		log.Printf("error: %v\n", err)
		return err
	}
	reader, err := gzip.NewReader(gzipfile)
	if err != nil {
		log.Printf("error: %v\n", err)
		return err
	}
	defer reader.Close()
	writer, err := os.Create(targetFile)
	if err != nil {
		log.Printf("error: %v\n", err)
		return err
	}
	defer writer.Close()
	if _, err = io.Copy(writer, reader); err != nil {
		log.Printf("error: %v\n", err)
		return err
	}
	return nil
}
func extractFilenamesFromTar(filePath string) ([]string, error) {
	f, err := os.Open(filePath)
	reader := bufio.NewReader(f)
	if err != nil {
		log.Printf("error: %v\n", err)
		return []string{}, err
	}  
	var pkgList []string
	tr := tar.NewReader(reader)
	if err != nil {
		log.Printf("error: %v\n", err)
		return []string{}, err
	}
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break  
		}
		if strings.HasSuffix(hdr.Name, "/desc") {
			buf := new(strings.Builder)
			if _, err = io.Copy(buf, tr); err != nil {
				log.Printf("error: %v\n", err)
				return []string{}, err
			}
			pkgName := buf.String()
			matches := filenameDBRegex.FindStringSubmatch(pkgName)  
			if len(matches) == 2 {
				pkgName = matches[1]
				pkgList = append(pkgList, pkgName)
			} else {
				log.Printf("Skipping %v cause it doesn't match regex. This is probably a bug.", hdr.Name)
				continue
			}
		}
	}
	return pkgList, nil
}
func getPacolocoURL(pkg Package, prefix string) string {
	return strings.ReplaceAll(("/repo/" + pkg.RepoName + "/" + prefix + "/" + pkg.PackageName + "-" + pkg.Version + "-" + pkg.Arch), "//", "/")
}
func buildRepoPkg(fileName string, repoName string, prefix string) (RepoPackage, error) {
	matches := filenameRegex.FindStringSubmatch(fileName)
	if len(matches) >= 7 {
		packageName := matches[1]
		version := matches[2]
		arch := matches[3]
		pkg := Package{PackageName: packageName, Version: version, Arch: arch, RepoName: repoName}
		pacolocoURL := getPacolocoURL(pkg, prefix)
		return RepoPackage{PackageName: packageName, Version: version, Arch: arch, DownloadURL: pacolocoURL, RepoName: repoName}, nil
	}
	return RepoPackage{}, fmt.Errorf("filename %v does not match regex, matches length is %d", fileName, len(matches))
}
func getPrefixFromMirrorDB(mirror MirrorDB) (string, error) {
	repoLinks, exists := config.Repos[mirror.RepoName]
	if !exists {
		return "", fmt.Errorf("error: Mirror link %v is associated with repo %v which does not exist in config", mirror.URL, mirror.RepoName)
	}
	var URLs []string
	if repoLinks.URL != "" {
		URLs = append(URLs, repoLinks.URL)
	} else {
		URLs = repoLinks.URLs
	}
	for _, URL := range URLs {
		splittedURL := strings.Split(mirror.URL, URL)
		if len(splittedURL) <= 1 {
			continue  
		}
		matches := mirrorDBRegex.FindStringSubmatch(splittedURL[1])
		if len(matches) < 1 {
			return "", nil
		}
		if !strings.HasPrefix(matches[0], "/") {
			return "/" + matches[0], nil
		} else {
			return matches[0], nil
		}
	}
	return "", fmt.Errorf("error: Mirror link %v does not exist in repo %v", mirror.URL, mirror.RepoName)
}
func downloadAndLoadDB(mirror MirrorDB) error {
	matches := urlRegex.FindStringSubmatch(mirror.URL)
	if len(matches) == 0 {
		return fmt.Errorf("url '%v' is invalid, does not match path regex", mirror.URL)
	}
	prefix, err := getPrefixFromMirrorDB(mirror)
	if err != nil {
		return err
	}
	fileName := matches[4]
	tmpDir := path.Join(config.CacheDir, "tmp-db")
	if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
		if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
			return err
		}
	}
	filePath := filepath.Join(config.CacheDir, "tmp-db", fileName)
	ifModifiedSince := time.Time{}
	if _, err := downloadFile(mirror.URL, filePath, ifModifiedSince); err != nil {
		return err
	}
	log.Printf("Extracting %v...", filePath)
	if err := uncompressGZ(filePath, filePath+".tar"); err != nil {
		return err
	}
	if err := os.Remove(filePath); err != nil {
		return err
	}
	log.Printf("Parsing %v...", filePath+".tar")
	fileList, err := extractFilenamesFromTar(filePath + ".tar")  
	log.Printf("Parsed %v.", filePath+".tar")
	if err != nil {
		return err
	}
	if err := os.Remove(filePath + ".tar"); err != nil {
		return err
	}
	log.Printf("Adding entries to db...")
	var repoList []RepoPackage
	for _, fileName := range fileList {
		rpkg, err := buildRepoPkg(fileName, mirror.RepoName, prefix)
		if err != nil {
			log.Printf("error: %v\n", err)
			continue
		}
		repoList = append(repoList, rpkg)
	}
	if db := prefetchDB.Save(&repoList); db.Error != nil {
		if !strings.Contains(fmt.Sprint(db.Error), "too many SQL variables") {
			return db.Error
		}
		maxBatchSize := 2000
		numOfPkgs := len(repoList)
		for i := 0; i < numOfPkgs; i += maxBatchSize {
			ends := i + maxBatchSize
			if ends > numOfPkgs {
				ends = numOfPkgs
			}
			newList := repoList[i:ends]
			if db := prefetchDB.Save(&newList); db.Error != nil {
				if strings.Contains(fmt.Sprint(db.Error), "too many SQL variables") {
					return fmt.Errorf("db error: Batch size is too big, change it in the config. This is a bug")
				} else {
					return db.Error
				}
			}
		}
	}
	log.Printf("Added entries to db.")
	return nil
}