<commit_before>package topicfeed

import (
	"encoding/json"
	"fmt"
	"socialapi/models"

	"github.com/koding/bongo"
	"github.com/koding/logging"
	"github.com/kylemcc/twitter-text-go/extract"
	"github.com/streadway/amqp"

	verbalexpressions "github.com/VerbalExpressions/GoVerbalExpressions"
)

// s := "naber #foo hede #bar dede gel # baz #123 #-`3sdf"
// will find [foo, bar, 123]
// will not find [' baz', '-`3sdf']
// here is the regex -> (?m)(?:#)(\w+)
var topicRegex = verbalexpressions.New().
	Find("#").
	BeginCapture().
	Word().
	EndCapture().
	Regex()

// extend this regex with https://github.com/twitter/twitter-text-rb/blob/eacf388136891eb316f1c110da8898efb8b54a38/lib/twitter-text/regex.rb
// to support all languages

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) *Controller {
	return &Controller{
		log: log,
	}
}

func (t *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		t.log.Error("Redelivered message gave error again, putting to maintenance queue: %s", err)
		delivery.Ack(false)
		return true
	}

	t.log.Error("an error occurred putting message back to queue: %s", err)
	delivery.Nack(false, true)
	return false
}

func (f *Controller) MessageSaved(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	topics := extractTopics(data.Body)
	if len(topics) == 0 {
		return nil
	}

	c, err := models.ChannelById(data.InitialChannelId)
	if err != nil {
		f.log.Error("Error on models.ChannelById", data.InitialChannelId, err)
		return err
	}

	return ensureChannelMessages(c, data, topics)
}

func ensureChannelMessages(parentChannel *models.Channel, data *models.ChannelMessage, topics []string) error {
	for _, topic := range topics {
		tc, err := fetchTopicChannel(parentChannel.GroupName, topic)
		if err != nil && err != bongo.RecordNotFound {
			return err
		}

		if err == bongo.RecordNotFound {
			tc, err = createTopicChannel(data.AccountId, parentChannel.GroupName, topic, parentChannel.PrivacyConstant)
			if err != nil {
				return err
			}
		}

		_, err = tc.EnsureMessage(data.Id, true)
		// safely skip
		if err == models.ErrMessageAlreadyInTheChannel {
			continue
		}

		if err != nil {
			return err
		}
	}

	return nil
}

func extractTopics(body string) []string {
	flattened := make([]string, 0)

	// extract twitter style hashtags
	res := extract.ExtractHashtags(body)
	if res == nil {
		return flattened
	}

	topics := map[string]struct{}{}
	// remove duplicate tag usages
	for _, e := range res {
		if hashTag, ok := e.Hashtag(); ok {
			topics[hashTag] = struct{}{}
		}
	}
<commit_msg>topicfeed: skip channelmessagelist item creation when related channel is initial channel<commit_after>package topicfeed

import (
	"encoding/json"
	"fmt"
	"socialapi/models"

	"github.com/koding/bongo"
	"github.com/koding/logging"
	"github.com/kylemcc/twitter-text-go/extract"
	"github.com/streadway/amqp"

	verbalexpressions "github.com/VerbalExpressions/GoVerbalExpressions"
)

// s := "naber #foo hede #bar dede gel # baz #123 #-`3sdf"
// will find [foo, bar, 123]
// will not find [' baz', '-`3sdf']
// here is the regex -> (?m)(?:#)(\w+)
var topicRegex = verbalexpressions.New().
	Find("#").
	BeginCapture().
	Word().
	EndCapture().
	Regex()

// extend this regex with https://github.com/twitter/twitter-text-rb/blob/eacf388136891eb316f1c110da8898efb8b54a38/lib/twitter-text/regex.rb
// to support all languages

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) *Controller {
	return &Controller{
		log: log,
	}
}

func (t *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		t.log.Error("Redelivered message gave error again, putting to maintenance queue: %s", err)
		delivery.Ack(false)
		return true
	}

	t.log.Error("an error occurred putting message back to queue: %s", err)
	delivery.Nack(false, true)
	return false
}

func (f *Controller) MessageSaved(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	topics := extractTopics(data.Body)
	if len(topics) == 0 {
		return nil
	}

	c, err := models.ChannelById(data.InitialChannelId)
	if err != nil {
		f.log.Error("Error on models.ChannelById", data.InitialChannelId, err)
		return err
	}

	return ensureChannelMessages(c, data, topics)
}

func ensureChannelMessages(parentChannel *models.Channel, data *models.ChannelMessage, topics []string) error {
	for _, topic := range topics {
		tc, err := fetchTopicChannel(parentChannel.GroupName, topic)
		if err != nil && err != bongo.RecordNotFound {
			return err
		}

		if err == bongo.RecordNotFound {
			tc, err = createTopicChannel(data.AccountId, parentChannel.GroupName, topic, parentChannel.PrivacyConstant)
			if err != nil {
				return err
			}
		}

		// message list relationship is already created in Message.Create
		if tc.Id == data.InitialChannelId {
			continue
		}

		_, err = tc.EnsureMessage(data.Id, true)
		// safely skip
		if err == models.ErrMessageAlreadyInTheChannel {
			continue
		}

		if err != nil {
			return err
		}
	}

	return nil
}

func extractTopics(body string) []string {
	flattened := make([]string, 0)

	// extract twitter style hashtags
	res := extract.ExtractHashtags(body)
	if res == nil {
		return flattened
	}

	topics := map[string]struct{}{}
	// remove duplicate tag usages
	for _, e := range res {
		if hashTag, ok := e.Hashtag(); ok {
			topics[hashTag] = struct{}{}
		}
	}

{'fraction_non_alphanumeric': 0.13209606986899564, 'fraction_numerical': 0.012736535662299854, 'mean_word_length': 3.943345323741007, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3399111', 'n_tokens_mistral': 2042, 'n_tokens_neox': 1878, 'n_words': 480}