<commit_before>package generatethumbnail

import (
	"image"
	"image/jpeg"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/kevintavog/findaphoto/common"

	"github.com/ian-kent/go-log/log"
	"github.com/nfnt/resize"
	"github.com/twinj/uuid"
)

var GeneratedImage int64
var FailedImage int64
var GeneratedVideo int64
var FailedVideo int64
var ThumbnailsCreated int64

type ThumbnailInfo struct {
	FullPath    string
	AliasedPath string
	MimeType    string
}

const thumbnailMaxHeightDimension = 170

var queue chan *ThumbnailInfo
var waitGroup sync.WaitGroup

func Start() {
	numConsumers := common.RatioNumCpus(0.5)
	queue = make(chan *ThumbnailInfo, 10000)
	waitGroup.Add(numConsumers)

	log.Info("Thumbnail generation using %d consumers", numConsumers)
	for idx := 0; idx < numConsumers; idx++ {
		go func() {
			dequeue()
			waitGroup.Done()
		}()
	}
}

func Done() {
	close(queue)
}

func Wait() {
	waitGroup.Wait()
}

func Enqueue(fullPath, aliasedPath, mimeType string) {
	thumbnailInfo := &ThumbnailInfo{
		FullPath:    fullPath,
		AliasedPath: aliasedPath,
		MimeType:    mimeType,
	}
	queue <- thumbnailInfo
}

func dequeue() {
	var mediaType []string
	var thumbPath string
	var err error

	for thumbnailInfo := range queue {
		mediaType = strings.Split(thumbnailInfo.MimeType, "/")
		thumbPath = common.ToThumbPath(thumbnailInfo.AliasedPath)
		if len(mediaType) < 1 {
			log.Error("Invalid media type: '%s' for %s", thumbnailInfo.MimeType, thumbnailInfo.FullPath)
			continue
		}

		err = common.CreateDirectory(path.Dir(thumbPath))
		if err != nil {
			log.Error("Unable to create directory for '%s'", thumbPath)
			continue
		}

		switch strings.ToLower(mediaType[0]) {
		case "video":
			generateVideo(thumbnailInfo.FullPath, thumbPath)
		case "image":
			generateImage(thumbnailInfo.FullPath, thumbPath)
		default:
			log.Error("Unhandled mediaType: %s (%s) for %s", thumbnailInfo.MimeType, mediaType, thumbnailInfo.FullPath)
		}

		atomic.AddInt64(&ThumbnailsCreated, 1)
		if ThumbnailsCreated%500 == 0 {
			log.Error("Generated thumbnail %d [%s]", ThumbnailsCreated, thumbnailInfo.FullPath)
		}
	}
}

func generateImage(fullPath, thumbPath string) {
	if err := createThumbnail(fullPath, thumbPath); err != nil {
		log.Error("Failed thumbnail generation: %s", err.Error())
		atomic.AddInt64(&FailedImage, 1)
	} else {
		atomic.AddInt64(&GeneratedImage, 1)
	}
}

func generateVideo(fullPath, thumbPath string) {
	tmpFilename := path.Join(os.TempDir(), "findAPhoto", "thumbnails", uuid.NewV4().String()+".JPG")
	defer os.Remove(tmpFilename)

	err := common.CreateDirectory(path.Dir(tmpFilename))
	if err != nil {
		log.Fatal("Unable to create temporary directory for thumbnail generation (%s): %s", tmpFilename, err.Error())
	}

	out, err := exec.Command(common.FfmpegPath, "-i", fullPath, "-ss", "00:00:01.0", "-vframes", "1", tmpFilename).Output()
	if err != nil {
		atomic.AddInt64(&FailedVideo, 1)
		log.Fatal("Failed executing ffmpeg for '%s': %s (%s)", fullPath, err.Error(), out)
	}

	if exists, _ := common.PathExists(tmpFilename); !exists {
		// The video may not be long enough to grab a frame at the 1 second, so try the first frame
		out, err = exec.Command(common.FfmpegPath, "-i", fullPath, "-ss", "00:00:00.0", "-vframes", "1", tmpFilename).Output()
		if err != nil {
			atomic.AddInt64(&FailedVideo, 1)
			log.Error("Failed executing ffmpeg for '%s': %s (%s)", fullPath, err.Error(), out)
		}
	}

	if err := createThumbnail(tmpFilename, thumbPath); err != nil {
		atomic.AddInt64(&FailedVideo, 1)
	} else {
		atomic.AddInt64(&GeneratedVideo, 1)
	}
}

func createThumbnail(imageFilename, thumbFilename string) error {
	file, err := os.Open(imageFilename)
	if err != nil {
		return err
	}
	defer file.Close()

	image, _, err := image.Decode(file)
	if err != nil {
		return err
	}

	thumb := resize.Resize(0, thumbnailMaxHeightDimension, image, resize.NearestNeighbor)
	savedThumbnailFile, err := os.Create(thumbFilename)
	if err != nil {
		return err
	}
	defer savedThumbnailFile.Close()<commit_msg>no message<commit_after>package generatethumbnail

import (
	"image"
	"image/jpeg"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	"github.com/kevintavog/findaphoto/common"

	"github.com/ian-kent/go-log/log"
	"github.com/nfnt/resize"
	"github.com/twinj/uuid"
)

var GeneratedImage int64
var FailedImage int64
var GeneratedVideo int64
var FailedVideo int64
var ThumbnailsCreated int64

type ThumbnailInfo struct {
	FullPath    string
	AliasedPath string
	MimeType    string
}

const thumbnailMaxHeightDimension = 170

var queue chan *ThumbnailInfo
var waitGroup sync.WaitGroup

func Start() {
	numConsumers := common.RatioNumCpus(0.5)
	queue = make(chan *ThumbnailInfo, 10000)
	waitGroup.Add(numConsumers)

	log.Info("Thumbnail generation using %d consumers", numConsumers)
	for idx := 0; idx < numConsumers; idx++ {
		go func() {
			dequeue()
			waitGroup.Done()
		}()
	}
}

func Done() {
	close(queue)
}

func Wait() {
	waitGroup.Wait()
}

func Enqueue(fullPath, aliasedPath, mimeType string) {
	thumbnailInfo := &ThumbnailInfo{
		FullPath:    fullPath,
		AliasedPath: aliasedPath,
		MimeType:    mimeType,
	}
	queue <- thumbnailInfo
}

func dequeue() {
	var mediaType []string
	var thumbPath string
	var err error

	for thumbnailInfo := range queue {
		mediaType = strings.Split(thumbnailInfo.MimeType, "/")
		thumbPath = common.ToThumbPath(thumbnailInfo.AliasedPath)
		if len(mediaType) < 1 {
			log.Error("Invalid media type: '%s' for %s", thumbnailInfo.MimeType, thumbnailInfo.FullPath)
			continue
		}

		err = common.CreateDirectory(path.Dir(thumbPath))
		if err != nil {
			log.Error("Unable to create directory for '%s'", thumbPath)
			continue
		}

		switch strings.ToLower(mediaType[0]) {
		case "video":
			generateVideo(thumbnailInfo.FullPath, thumbPath)
		case "image":
			generateImage(thumbnailInfo.FullPath, thumbPath)
		default:
			log.Error("Unhandled mediaType: %s (%s) for %s", thumbnailInfo.MimeType, mediaType, thumbnailInfo.FullPath)
		}

		atomic.AddInt64(&ThumbnailsCreated, 1)
		if ThumbnailsCreated%500 == 0 {
			log.Error("Generated thumbnail %d [%s]", ThumbnailsCreated, thumbnailInfo.FullPath)
		}
	}
}

func generateImage(fullPath, thumbPath string) {
	if err := createThumbnail(fullPath, thumbPath); err != nil {
		log.Error("Failed thumbnail generation on %s: %s", fullPath, err.Error())
		atomic.AddInt64(&FailedImage, 1)
	} else {
		atomic.AddInt64(&GeneratedImage, 1)
	}
}

func generateVideo(fullPath, thumbPath string) {
	tmpFilename := path.Join(os.TempDir(), "findAPhoto", "thumbnails", uuid.NewV4().String()+".JPG")
	defer os.Remove(tmpFilename)

	err := common.CreateDirectory(path.Dir(tmpFilename))
	if err != nil {
		log.Fatal("Unable to create temporary directory for thumbnail generation (%s): %s", tmpFilename, err.Error())
	}

	out, err := exec.Command(common.FfmpegPath, "-i", fullPath, "-ss", "00:00:01.0", "-vframes", "1", tmpFilename).Output()
	if err != nil {
		atomic.AddInt64(&FailedVideo, 1)
		log.Fatal("Failed executing ffmpeg for '%s': %s (%s)", fullPath, err.Error(), out)
	}

	if exists, _ := common.PathExists(tmpFilename); !exists {
		// The video may not be long enough to grab a frame at the 1 second, so try the first frame
		out, err = exec.Command(common.FfmpegPath, "-i", fullPath, "-ss", "00:00:00.0", "-vframes", "1", tmpFilename).Output()
		if err != nil {
			atomic.AddInt64(&FailedVideo, 1)
			log.Error("Failed executing ffmpeg for '%s': %s (%s)", fullPath, err.Error(), out)
		}
	}

	if err := createThumbnail(tmpFilename, thumbPath); err != nil {
		log.Error("Failed thumbnail generation on %s: %s", tmpFilename, err.Error())
		atomic.AddInt64(&FailedVideo, 1)
	} else {
		atomic.AddInt64(&GeneratedVideo, 1)
	}
}

func createThumbnail(imageFilename, thumbFilename string) error {
	file, err := os.Open(imageFilename)
	if err != nil {
		return err
	}
	defer file.Close()

	image, _, err := image.Decode(file)
	if err != nil {
		return err
	}

	thumb := resize.Resize(0, thumbnailMaxHeightDimension, image, resize.NearestNeighbor)
	savedThumbnailFile, err := os.Create(thumbFilename)
	if err != nil {
		return err
	}
	defer savedThumbnailFile.Close()
{'fraction_non_alphanumeric': 0.13788470933072788, 'fraction_numerical': 0.016365412799218368, 'mean_word_length': 4.551864406779661, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2397878', 'n_tokens_mistral': 3117, 'n_tokens_neox': 2875, 'n_words': 693}