<commit_before>package scanner

import (
	"encoding/json"
	"net/http"
	"os"

	"github.com/kevintavog/findaphoto/common"

	"github.com/ian-kent/go-log/log"
	"golang.org/x/net/context"
	"gopkg.in/olivere/elastic.v5"
)

var MediaScanned int64
var MediaRemoved int64

// Walk through the index, removing any items no longer on the file system
func RemoveFiles() {
	client := common.CreateClient()

	scrollResponse, err := client.Scroll(common.MediaIndexName).
		Size(100).
		Type(common.MediaTypeName).
		Do(context.TODO())
	if err != nil {
		log.Error("Failed starting scan: %s", err.Error())
		return
	}

	scrollId := scrollResponse.ScrollId
	for {
		results, err := client.Scroll(common.MediaIndexName).
			Size(100).
			ScrollId(scrollId).
			Do(context.TODO())
		if err != nil {
			if el, ok := err.(*elastic.Error); ok {
				if el.Status == http.StatusNotFound {
					break
				}
			}

			log.Error("Failed scanning index: %s", err.Error())
			break
		}

		for _, hit := range results.Hits.Hits {
			var media common.Media
			err := json.Unmarshal(*hit.Source, &media)
			if err != nil {
				log.Error("Failed deserializing search result: %s", err.Error())
				continue
			}

			MediaScanned += 1
			removeDocument := false
			if !common.IsValidAliasedPath(media.Path) {
				removeDocument = true
			} else {
				fullPath, err := common.FullPathForAliasedPath(media.Path)
				if err != nil {
					log.Error("Unable to convert %s to a path: %s", media.Path, err.Error())
					continue
				}

				if _, err = os.Stat(fullPath); os.IsNotExist(err) {
					removeDocument = true
					log.Info("File doesn't exist: %s", fullPath)
				}
			}

			if removeDocument {
				MediaRemoved += 1
				deleteResponse, err := client.Delete().
					Index(common.MediaIndexName).
					Type(common.MediaTypeName).
					Id(media.Path).
					Do(context.TODO())
				if err != nil {
					log.Error("Failed removing document '%s' from index: %s", media.Path, err.Error())
				} else if deleteResponse.Found != true {
					log.Error("Delete of document '%s' failed", media.Path)
				}
			}
		}
	}
}<commit_msg>Properly scan for removed documents<commit_after>package scanner

import (
	"encoding/json"
	"io"
	"net/http"
	"os"

	"github.com/kevintavog/findaphoto/common"

	"github.com/ian-kent/go-log/log"
	"golang.org/x/net/context"
	"gopkg.in/olivere/elastic.v5"
)

var MediaScanned int64
var MediaRemoved int64

// Walk through the index, removing any items no longer on the file system
func RemoveFiles() {
	client := common.CreateClient()

	scrollService := client.Scroll(common.MediaIndexName).Type(common.MediaTypeName).Size(100)

	_, err := scrollService.Do(context.TODO())
	if err != nil {
		log.Error("Failed starting scan: %s", err.Error())
		return
	}

	checked := 0
	removed := 0
	for {
		results, err := scrollService.Do(context.TODO())
		if err == io.EOF {
			break
		}
		if err != nil {
			if el, ok := err.(*elastic.Error); ok {
				if el.Status == http.StatusNotFound {
					break
				}
			}

			log.Error("Failed scanning index: %s", err.Error())
			break
		}

		for _, hit := range results.Hits.Hits {
			checked += 1

			var media common.Media
			err := json.Unmarshal(*hit.Source, &media)
			if err != nil {
				log.Error("Failed deserializing search result: %s", err.Error())
				continue
			}

			MediaScanned += 1
			removeDocument := false
			if !common.IsValidAliasedPath(media.Path) {
				removeDocument = true
			} else {
				fullPath, err := common.FullPathForAliasedPath(media.Path)
				if err != nil {
					log.Error("Unable to convert %s to a path: %s", media.Path, err.Error())
					continue
				}

				if _, err = os.Stat(fullPath); os.IsNotExist(err) {
					removeDocument = true
					log.Info("File doesn't exist: %s", fullPath)
				}
			}

			if removeDocument {
				removed += 1
				MediaRemoved += 1
				deleteResponse, err := client.Delete().
					Index(common.MediaIndexName).
					Type(common.MediaTypeName).
					Id(media.Path).
					Do(context.TODO())
				if err != nil {
					log.Error("Failed removing document '%s' from index: %s", media.Path, err.Error())
				} else if deleteResponse.Found != true {
					log.Error("Delete of document '%s' failed", media.Path)
				}
			}
		}
	}

	log.Info("Remover checked %d files and removed %d of them", checked, removed)
}
{'fraction_non_alphanumeric': 0.14169000933706816, 'fraction_numerical': 0.0063025210084033615, 'mean_word_length': 3.0462700661000945, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6181932', 'n_tokens_mistral': 1766, 'n_tokens_neox': 1441, 'n_words': 351}