<commit_before>package tasks

import (
	"database/sql"
	"encoding/binary"

	"pixur.org/pixur/schema"
)

// TODO: add tests

type FindSimilarPicsTask struct {
	// Deps
	DB *sql.DB

	// Inputs
	PicID int64

	// Results
	SimilarPicIDs []int64
}

func (t *FindSimilarPicsTask) Run() error {
	t.SimilarPicIDs = make([]int64, 0) // set default, to make json encoding easier
	picStmt, err := schema.PicPrepare("SELECT * FROM_ WHERE %s = ? LOCK IN SHARE MODE;", t.DB, schema.PicColId)
	if err != nil {
		return err
	}
	defer picStmt.Close()

	p, err := schema.LookupPic(picStmt, t.PicID)
	if err != nil {
		return err
	}

	identStmt, err := schema.PicIdentPrepare("SELECT * FROM_ WHERE %s = ? AND %s = ?;", t.DB,
		schema.PicIdentColPicId, schema.PicIdentColType)
	if err != nil {
		return err
	}
	defer identStmt.Close()

	picIdent, err := schema.LookupPicIdent(identStmt, p.PicId, schema.PicIdent_DCT_0)
	if err != nil {
		return err
	}
	match := binary.BigEndian.Uint64(picIdent.Value)

	allIdentStmt, err := schema.PicIdentPrepare("SELECT * FROM_ WHERE %s = ?;", t.DB, schema.PicIdentColType)
	if err != nil {
		return err
	}
	defer allIdentStmt.Close()

	idents, err := schema.FindPicIdents(allIdentStmt, schema.PicIdent_DCT_0)
	if err != nil {
		return err
	}

	// Linear time, sigh
	for _, ident := range idents {
		if ident.PicId == p.PicId {
			continue
		}
		guess := binary.BigEndian.Uint64(ident.Value)
		bits := guess ^ match
		bitCount := 0
		// replace this with something that isn't hideously slow.  Hamming distance would be
		// better served by a look up table or some 64 bit specific bit magic.  Cosine similarity
		// on the attached floats would also work.
		for i := uint(0); i < 64; i++ {
			if ((1 << i) & bits) > 0 {
				bitCount++
			}
		}
		if bitCount <= 10 {
			t.SimilarPicIDs = append(t.SimilarPicIDs, ident.PicId)
		}
	}

	return nil
}<commit_msg>tasks: switch FindSimilarPics to use new table scan<commit_after>package tasks

import (
	"database/sql"
	"encoding/binary"

	"pixur.org/pixur/schema"
	"pixur.org/pixur/schema/db"
	tab "pixur.org/pixur/schema/tables"
	"pixur.org/pixur/status"
)

// TODO: add tests

type FindSimilarPicsTask struct {
	// Deps
	DB *sql.DB

	// Inputs
	PicID int64

	// Results
	SimilarPicIDs []int64
}

func (t *FindSimilarPicsTask) Run() (errCap error) {
	t.SimilarPicIDs = make([]int64, 0) // set default, to make json encoding easier
	j, err := tab.NewJob(t.DB)
	if err != nil {
		return status.InternalError(err, "can't create new job")
	}
	defer func() {
		if err := j.Rollback(); errCap == nil {
			errCap = err
		}
		// TODO: log the error
	}()

	pics, err := j.FindPics(db.Opts{
		Start: tab.PicsPrimary{&t.PicID},
		Limit: 1,
	})
	if err != nil {
		return status.InternalError(err, "can't lookup pic")
	}
	if len(pics) != 1 {
		return status.InvalidArgument(err, "can't lookup pic", len(pics))
	}
	pic := pics[0]

	dctIdentType := schema.PicIdent_DCT_0

	picIdents, err := j.FindPicIdents(db.Opts{
		Start: tab.PicIdentsPrimary{PicId: &t.PicID, Type: &dctIdentType},
		Limit: 1,
	})
	if err != nil {
		return status.InternalError(err, "can't lookup pic ident")
	}
	if len(picIdents) != 1 {
		return status.InvalidArgument(err, "can't lookup pic ident", len(picIdents))
	}
	targetIdent := picIdents[0]
	match := binary.BigEndian.Uint64(targetIdent.Value)

	scanOpts := db.Opts{
		Start: tab.PicIdentsIdent{Type: &dctIdentType},
	}
	err = j.ScanPicIdents(scanOpts, func(pi schema.PicIdent) error {
		if pi.PicId == pic.PicId {
			return nil
		}
		guess := binary.BigEndian.Uint64(pi.Value)
		bits := guess ^ match
		bitCount := 0
		// replace this with something that isn't hideously slow.  Hamming distance would be
		// better served by a look up table or some 64 bit specific bit magic.  Cosine similarity
		// on the attached floats would also work.
		for i := uint(0); i < 64; i++ {
			if ((1 << i) & bits) > 0 {
				bitCount++
			}
		}
		if bitCount <= 10 {
			t.SimilarPicIDs = append(t.SimilarPicIDs, pi.PicId)
		}

		return nil
	})

	if err != nil {
		return status.InternalError(err, "can't scan pic idents")
	}

	return nil
}
{'fraction_non_alphanumeric': 0.13497681230168415, 'fraction_numerical': 0.012448132780082987, 'mean_word_length': 3.4064516129032256, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5301926', 'n_tokens_mistral': 1692, 'n_tokens_neox': 1515, 'n_words': 413}