<commit_before>package importer

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync"

	"github.com/sgeisbacher/photogallery-api/media"
)

type ImportManager struct {
	MediaService   *media.MediaService
	GalleryService *media.GalleryService
}

func (mgr ImportManager) ScanFolder(path string) {
	var wg sync.WaitGroup
	imageFilesChan := make(chan string)

	// start worker threads
	go mgr.handleImageFile(imageFilesChan, &wg)
	go mgr.handleImageFile(imageFilesChan, &wg)

	files, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatalf("error while reading dir: %v", err)
	}
	for _, file := range files {
		wg.Add(1)
		if !file.IsDir() {
			imageFilesChan <- path + "/" + file.Name()
		}
	}

	close(imageFilesChan)
	wg.Wait()
}

func (mgr ImportManager) handleImageFile(imagesChan <-chan string, wg *sync.WaitGroup) {
	for filePath := range imagesChan {
		fileHash, err := hashFile(filePath)
		if err != nil {
			fmt.Printf("skipping file '%v' due to an error: %v\n", filePath, err)
			continue
		}
		fmt.Printf("md5ChkSum of '%v': %v\n", filePath, fileHash)
		wg.Done()
	}
}

func hashFile(filename string) (string, error) {
	var md5ChkSum string
	file, err := os.Open(filename)
	if err != nil {
		fmt.Println("could not open file:", filename)
		return md5ChkSum, err
	}
	defer file.Close()
	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		fmt.Println("could not hash file:", filename)
		return md5ChkSum, err
	}
	md5ChkSum = hex.EncodeToString(hash.Sum(nil))
	return md5ChkSum, nil
}<commit_msg>added gallery-detection to importer<commit_after>package importer

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/sgeisbacher/photogallery-api/media"
)

type ImportMediaData struct {
	path        string
	galleryName string
}

type ImportManager struct {
	MediaService   *media.MediaService
	GalleryService *media.GalleryService
}

func (mgr ImportManager) ScanFolder(path string) {
	var wg sync.WaitGroup
	imagesChan := make(chan ImportMediaData)

	// start worker threads
	go mgr.handleImageFile(imagesChan, &wg)
	go mgr.handleImageFile(imagesChan, &wg)

	path = addSlash(path)

	files, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatalf("error while reading dir: %v", err)
	}
	for _, file := range files {
		if !file.IsDir() {
			fmt.Printf("skipping file '%v' (not allowed here), because its on gallery-folder-level\n", path+file.Name())
			continue
		}
		scanGalleryFolder(file.Name(), addSlash(path+file.Name()), imagesChan, &wg)
	}

	close(imagesChan)
	wg.Wait()
}

func scanGalleryFolder(galleryName, path string, imagesChan chan ImportMediaData, wg *sync.WaitGroup) {
	files, err := ioutil.ReadDir(path)
	if err != nil {
		log.Fatalf("error while reading dir: %v", err)
	}
	for _, file := range files {
		if file.IsDir() {
			fmt.Printf("skipping directory '%v' (not allowed here), because its on image-level\n", addSlash(path+file.Name()))
			continue
		}
		wg.Add(1)
		importMediaData := ImportMediaData{
			path:        path + file.Name(),
			galleryName: galleryName,
		}
		imagesChan <- importMediaData
	}
}

func (mgr ImportManager) handleImageFile(imagesChan <-chan ImportMediaData, wg *sync.WaitGroup) {
	for importMediaData := range imagesChan {
		fileHash, err := hashFile(importMediaData.path)
		if err != nil {
			fmt.Printf("skipping file '%v' due to an error: %v\n", importMediaData.path, err)
			continue
		}
		fmt.Printf("md5ChkSum of '%v' in gallery '%v': %v\n", importMediaData.path, importMediaData.galleryName, fileHash)
		wg.Done()
	}
}

func hashFile(filename string) (string, error) {
	var md5ChkSum string
	file, err := os.Open(filename)
	if err != nil {
		fmt.Println("could not open file:", filename)
		return md5ChkSum, err
	}
	defer file.Close()
	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		fmt.Println("could not hash file:", filename)
		return md5ChkSum, err
	}
	md5ChkSum = hex.EncodeToString(hash.Sum(nil))
	return md5ChkSum, nil
}

func addSlash(path string) string {
	path = strings.TrimSpace(path)
	if !strings.HasSuffix(path, "/") {
		path = path + "/"
	}
	return path
}
{'fraction_non_alphanumeric': 0.1384282624369142, 'fraction_numerical': 0.004325883201153569, 'mean_word_length': 4.125615763546798, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6496310', 'n_tokens_mistral': 1623, 'n_tokens_neox': 1529, 'n_words': 378}