<commit_before>package preparemedia

import (
	"errors"
	"fmt"
	"math"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kevintavog/findaphoto/common"
	"github.com/kevintavog/findaphoto/indexer/steps/checkthumbnail"
	"github.com/kevintavog/findaphoto/indexer/steps/resolveplacename"
)

const numConsumers = 8

var queue = make(chan *common.CandidateFile, numConsumers)
var waitGroup sync.WaitGroup

func Start() {
	resolveplacename.Start()

	waitGroup.Add(numConsumers)
	for idx := 0; idx < numConsumers; idx++ {
		go func() {
			dequeue()
			waitGroup.Done()
		}()
	}
}

func Done() {
	close(queue)
}

func Wait() {
	waitGroup.Wait()
	resolveplacename.Done()
	resolveplacename.Wait()
}

func Enqueue(candidate *common.CandidateFile) {
	queue <- candidate
}

func dequeue() {
	for candidate := range queue {
		media := populate(candidate)
		resolveplacename.Enqueue(media)
		checkthumbnail.Enqueue(candidate.FullPath, candidate.AliasedPath, media.MimeType)
	}
}

func populate(candidate *common.CandidateFile) *common.Media {
	media := &common.Media{
		Filename:      path.Base(candidate.FullPath),
		Path:          candidate.AliasedPath,
		Signature:     candidate.Signature,
		LengthInBytes: candidate.LengthInBytes,

		MimeType: candidate.Exif.File.MIMEType,

		ApertureValue:   strconv.FormatFloat(float64(candidate.Exif.EXIF.ApertureValue), 'f', -1, 32),
		ExposureProgram: candidate.Exif.EXIF.ExposureProgram,
		Flash:           candidate.Exif.EXIF.Flash,
		FNumber:         strconv.FormatFloat(float64(candidate.Exif.EXIF.FNumber), 'f', -1, 32),
		FocalLength:     candidate.Exif.EXIF.FocalLength,
		WhiteBalance:    candidate.Exif.EXIF.WhiteBalance,
		LensInfo:        candidate.Exif.EXIF.LensInfo,
		LensModel:       candidate.Exif.EXIF.LensModel,
		CameraMake:      candidate.Exif.EXIF.Make,
		CameraModel:     candidate.Exif.EXIF.Model,
	}

	populateIso(media, candidate)
	populateExposureTime(media, candidate)
	populateKeywords(media, candidate)
	populateDateTime(media, candidate)
	populateLocation(media, candidate)
	populateDimensions(media, candidate)

	media.Warnings = candidate.Warnings

	return media
}

func populateDimensions(media *common.Media, candidate *common.CandidateFile) {
	if candidate.Exif.File.ImageWidth != 0 && candidate.Exif.File.ImageHeight != 0 {
		media.Width = candidate.Exif.File.ImageWidth
		media.Height = candidate.Exif.File.ImageHeight
	} else if candidate.Exif.Quicktime.ImageWidth != 0 && candidate.Exif.Quicktime.ImageHeight != 0 {
		media.Width = candidate.Exif.Quicktime.ImageWidth
		media.Height = candidate.Exif.Quicktime.ImageHeight
	}

	if candidate.Exif.Quicktime.Duration != "" {
		// 10.15 s
		tokens := strings.Split(candidate.Exif.Quicktime.Duration, " ")
		if len(tokens) >= 1 {
			v, err := strconv.ParseFloat(tokens[0], 32)
			if err == nil {
				media.DurationSeconds = float32(v)
			}
		}
	}
}

func populateKeywords(media *common.Media, candidate *common.CandidateFile) {
	switch keyType := candidate.Exif.IPTC.Keywords.(type) {
	default:
		candidate.AddWarning(fmt.Sprintf("Unexpected keyword type %T (%q)", keyType, candidate.Exif.IPTC.Keywords))
	case []interface{}:
		for _, s := range candidate.Exif.IPTC.Keywords.([]interface{}) {
			media.Keywords = append(media.Keywords, s.(string))
		}
	case interface{}:
		media.Keywords = []string{candidate.Exif.IPTC.Keywords.(string)}
	case nil:
		// Nothing to do, keywords not present
	}
}

func populateIso(media *common.Media, candidate *common.CandidateFile) {
	switch isoType := candidate.Exif.EXIF.ISO.(type) {
	default:
		candidate.AddWarning(fmt.Sprintf("Unexpected ISO type: %T (%q)", isoType, candidate.Exif.EXIF.ISO))
	case int:
		media.Iso = strconv.FormatInt(int64(candidate.Exif.EXIF.ISO.(int)), 10)
	case float64:
		media.Iso = strconv.FormatFloat(candidate.Exif.EXIF.ISO.(float64), 'f', -1, 64)
	case string:
		s := candidate.Exif.EXIF.ISO.(string)
		re := regexp.MustCompile("[0-9]+")
		media.Iso = re.FindString(s)
	case nil:
		// Nothing to do, no value present
	}
}

func populateExposureTime(media *common.Media, candidate *common.CandidateFile) {
	switch etType := candidate.Exif.EXIF.ExposureTime.(type) {
	default:
		candidate.AddWarning(fmt.Sprintf("Unexpected ExposureTime type: %T", etType))
	case float64:
		media.ExposureTime = strconv.FormatFloat(candidate.Exif.EXIF.ExposureTime.(float64), 'f', -1, 64)
	case string:
		media.ExposureTime = candidate.Exif.EXIF.ExposureTime.(string)
	case nil:
		// Nothing to do, no value present (videos, for instance)
	}
}

func populateDateTime(media *common.Media, candidate *common.CandidateFile) {
	var dateTime time.Time
	var err error

	if candidate.Exif.Quicktime.ContentCreateDate != "" {
		dateTime, err = time.Parse("2006:01:02 15:04:05-07:00", candidate.Exif.Quicktime.ContentCreateDate)
		if err != nil {
			candidate.AddWarning(fmt.Sprintf(
				"Failed parsing ContentCreateDate '%s': %s (in %s)", candidate.Exif.Quicktime.ContentCreateDate, err.Error(), candidate.FullPath))
		}
	}
	if dateTime.IsZero() && len(candidate.Exif.Quicktime.CreateDate) > 0 {
		// UTC according to spec - no timezone like there is for 'ContentCreateDate'
		dateTime, err = time.Parse("2006:01:02 15:04:05", candidate.Exif.Quicktime.CreateDate)
		if err != nil {
			candidate.AddWarning(fmt.Sprintf(
				"Failed parsing CreateDate '%s': %s (in %s)", candidate.Exif.Quicktime.CreateDate, err.Error(), candidate.FullPath))
		}
	}

	if dateTime.IsZero() {
		exifDateTime := candidate.Exif.EXIF.CreateDate
		if exifDateTime == "" {
			exifDateTime = candidate.Exif.EXIF.DateTimeOriginal<commit_msg>For videos, prefer QuickTime.ContentDate over QuickTime.ContentCreateDate - exiftool doesn't seem to be able to update ContentCreateDate<commit_after>package preparemedia

import (
	"errors"
	"fmt"
	"math"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/kevintavog/findaphoto/common"
	"github.com/kevintavog/findaphoto/indexer/steps/checkthumbnail"
	"github.com/kevintavog/findaphoto/indexer/steps/resolveplacename"
)

const numConsumers = 8

var queue = make(chan *common.CandidateFile, numConsumers)
var waitGroup sync.WaitGroup

func Start() {
	resolveplacename.Start()

	waitGroup.Add(numConsumers)
	for idx := 0; idx < numConsumers; idx++ {
		go func() {
			dequeue()
			waitGroup.Done()
		}()
	}
}

func Done() {
	close(queue)
}

func Wait() {
	waitGroup.Wait()
	resolveplacename.Done()
	resolveplacename.Wait()
}

func Enqueue(candidate *common.CandidateFile) {
	queue <- candidate
}

func dequeue() {
	for candidate := range queue {
		media := populate(candidate)
		resolveplacename.Enqueue(media)
		checkthumbnail.Enqueue(candidate.FullPath, candidate.AliasedPath, media.MimeType)
	}
}

func populate(candidate *common.CandidateFile) *common.Media {
	media := &common.Media{
		Filename:      path.Base(candidate.FullPath),
		Path:          candidate.AliasedPath,
		Signature:     candidate.Signature,
		LengthInBytes: candidate.LengthInBytes,

		MimeType: candidate.Exif.File.MIMEType,

		ApertureValue:   strconv.FormatFloat(float64(candidate.Exif.EXIF.ApertureValue), 'f', -1, 32),
		ExposureProgram: candidate.Exif.EXIF.ExposureProgram,
		Flash:           candidate.Exif.EXIF.Flash,
		FNumber:         strconv.FormatFloat(float64(candidate.Exif.EXIF.FNumber), 'f', -1, 32),
		FocalLength:     candidate.Exif.EXIF.FocalLength,
		WhiteBalance:    candidate.Exif.EXIF.WhiteBalance,
		LensInfo:        candidate.Exif.EXIF.LensInfo,
		LensModel:       candidate.Exif.EXIF.LensModel,
		CameraMake:      candidate.Exif.EXIF.Make,
		CameraModel:     candidate.Exif.EXIF.Model,
	}

	populateIso(media, candidate)
	populateExposureTime(media, candidate)
	populateKeywords(media, candidate)
	populateDateTime(media, candidate)
	populateLocation(media, candidate)
	populateDimensions(media, candidate)

	media.Warnings = candidate.Warnings

	return media
}

func populateDimensions(media *common.Media, candidate *common.CandidateFile) {
	if candidate.Exif.File.ImageWidth != 0 && candidate.Exif.File.ImageHeight != 0 {
		media.Width = candidate.Exif.File.ImageWidth
		media.Height = candidate.Exif.File.ImageHeight
	} else if candidate.Exif.Quicktime.ImageWidth != 0 && candidate.Exif.Quicktime.ImageHeight != 0 {
		media.Width = candidate.Exif.Quicktime.ImageWidth
		media.Height = candidate.Exif.Quicktime.ImageHeight
	}

	if candidate.Exif.Quicktime.Duration != "" {
		// 10.15 s
		tokens := strings.Split(candidate.Exif.Quicktime.Duration, " ")
		if len(tokens) >= 1 {
			v, err := strconv.ParseFloat(tokens[0], 32)
			if err == nil {
				media.DurationSeconds = float32(v)
			}
		}
	}
}

func populateKeywords(media *common.Media, candidate *common.CandidateFile) {
	switch keyType := candidate.Exif.IPTC.Keywords.(type) {
	default:
		candidate.AddWarning(fmt.Sprintf("Unexpected keyword type %T (%q)", keyType, candidate.Exif.IPTC.Keywords))
	case []interface{}:
		for _, s := range candidate.Exif.IPTC.Keywords.([]interface{}) {
			media.Keywords = append(media.Keywords, s.(string))
		}
	case interface{}:
		media.Keywords = []string{candidate.Exif.IPTC.Keywords.(string)}
	case nil:
		// Nothing to do, keywords not present
	}
}

func populateIso(media *common.Media, candidate *common.CandidateFile) {
	switch isoType := candidate.Exif.EXIF.ISO.(type) {
	default:
		candidate.AddWarning(fmt.Sprintf("Unexpected ISO type: %T (%q)", isoType, candidate.Exif.EXIF.ISO))
	case int:
		media.Iso = strconv.FormatInt(int64(candidate.Exif.EXIF.ISO.(int)), 10)
	case float64:
		media.Iso = strconv.FormatFloat(candidate.Exif.EXIF.ISO.(float64), 'f', -1, 64)
	case string:
		s := candidate.Exif.EXIF.ISO.(string)
		re := regexp.MustCompile("[0-9]+")
		media.Iso = re.FindString(s)
	case nil:
		// Nothing to do, no value present
	}
}

func populateExposureTime(media *common.Media, candidate *common.CandidateFile) {
	switch etType := candidate.Exif.EXIF.ExposureTime.(type) {
	default:
		candidate.AddWarning(fmt.Sprintf("Unexpected ExposureTime type: %T", etType))
	case float64:
		media.ExposureTime = strconv.FormatFloat(candidate.Exif.EXIF.ExposureTime.(float64), 'f', -1, 64)
	case string:
		media.ExposureTime = candidate.Exif.EXIF.ExposureTime.(string)
	case nil:
		// Nothing to do, no value present (videos, for instance)
	}
}

func populateDateTime(media *common.Media, candidate *common.CandidateFile) {
	var dateTime time.Time
	var err error

	if dateTime.IsZero() && candidate.Exif.Quicktime.CreateDate != "" {
		// UTC according to spec - no timezone like there is for 'ContentCreateDate'
		dateTime, err = time.Parse("2006:01:02 15:04:05", candidate.Exif.Quicktime.CreateDate)
		if err != nil {
			candidate.AddWarning(fmt.Sprintf(
				"Failed parsing CreateDate '%s': %s (in %s)", candidate.Exif.Quicktime.CreateDate, err.Error(), candidate.FullPath))
		}
	}

	if dateTime.IsZero() && candidate.Exif.Quicktime.ContentCreateDate != "" {
		dateTime, err = time.Parse("2006:01:02 15:04:05-07:00", candidate.Exif.Quicktime.ContentCreateDate)
		if err != nil {
			candidate.AddWarning(fmt.Sprintf(
				"Failed parsing ContentCreateDate '%s': %s (in %s)", candidate.Exif.Quicktime.ContentCreateDate, err.Error(), candidate.FullPath))
		}
	}

	if dateTime.IsZero() {
		exifDateTime := candidate.Exif.EXIF.CreateDate
		if exifDateTime == "" {
			exifDateTime = candidate.Exif.EXIF.DateTimeOriginal
{'fraction_non_alphanumeric': 0.13514230157327206, 'fraction_numerical': 0.013876613045783984, 'mean_word_length': 4.500729217306757, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5205474', 'n_tokens_mistral': 4229, 'n_tokens_neox': 3919, 'n_words': 725}