<commit_before>package main

import (
	goconf "goconf.googlecode.com/hg"
	"strconv"
	"log"
	"sort"
	"strings"
)

type Model struct {
	cmdchan       <-chan TwitterCommand
	newtweetchan  chan<- []*Tweet
	newtweets_int chan []*Tweet
	users         []UserTwitterAPITuple
	cur_user      uint
	tweets        []*Tweet
	tweet_map     map[int64]*Tweet
	lookupchan    <-chan TweetRequest
	uiactionchan  chan<- UserInterfaceAction
	last_id       int64
	ignored_users []string
}

type TweetRequest struct {
	Status_id int64
	Reply     chan *Tweet
}

type CmdId int

const (
	UPDATE CmdId = iota
	RETWEET
	DELETE
	FAVORITE
	FOLLOW
	UNFOLLOW
	SET_CURUSER
)

type TwitterCommand struct {
	Cmd  CmdId
	Data Tweet
	Pos  uint // for SET_CURUSER
}

func NewModel(users []UserTwitterAPITuple, cc chan TwitterCommand, ntc chan<- []*Tweet, lc <-chan TweetRequest, uac chan<- UserInterfaceAction, cfg *goconf.ConfigFile) *Model {
	model := &Model{
		cmdchan:       cc,
		newtweetchan:  ntc,
		newtweets_int: make(chan []*Tweet, 10),
		users:         users,
		cur_user:      0,
		lookupchan:    lc,
		tweet_map:     map[int64]*Tweet{},
		uiactionchan:  uac,
		ignored_users: []string{},
	}

	if cfg != nil {
		if default_user, err := cfg.GetString("default", "default_user"); err == nil {
			for idx, _ := range model.users {
				if model.users[idx].User == default_user {
					model.cur_user = uint(idx)
					break
				}
			}
		}

		if ign, err := cfg.GetString("default", "ignore_incoming"); err == nil {
			model.ignored_users = strings.Split(ign, " ")
		}
	}

	userlist := []string{strconv.Uitoa(model.cur_user)}
	for _, u := range model.users {
		userlist = append(userlist, u.User)
	}
	model.uiactionchan <- UserInterfaceAction{SET_USERLIST, userlist}

	return model
}

func (m *Model) Run() {
	m.last_id = int64(0)

	new_tweets := make(chan []*Tweet, 10)

	go StartUserStreams(m.users, new_tweets, m.uiactionchan, m.ignored_users)

	go func() {
		if config, err := m.users[m.cur_user].Tapi.Configuration(); err == nil {
			log.Printf("Twitter config data: %v", config)
			if config.Short_url_length != nil {
				length := []string{strconv.Itoa64(*config.Short_url_length)}
				m.uiactionchan <- UserInterfaceAction{SET_URLLENGTH, length}
			}
		} else {
			log.Printf("reading Twitter config data failed: %v", err)
		}
	}()

	for {
		select {
		case cmd := <-m.cmdchan:
			m.HandleCommand(cmd)
		case req := <-m.lookupchan:
			tweet := m.tweet_map[req.Status_id]
			req.Reply <- tweet
			close(req.Reply)
		case tweets := <-m.newtweets_int:
			for _, t := range tweets {
				m.tweet_map[*t.Id] = t
			}
			m.tweets = append(tweets, m.tweets...)
		case tweets := <-new_tweets:
			log.Printf("received %d tweets", len(tweets))
			unique_tweets := []*Tweet{}
			for _, t := range tweets {
				if _, contained := m.tweet_map[*t.Id]; !contained {<commit_msg>as recommended, fetch and internally update information about short URL length once a day.<commit_after>package main

import (
	goconf "goconf.googlecode.com/hg"
	"strconv"
	"log"
	"sort"
	"strings"
	"time"
)

type Model struct {
	cmdchan       <-chan TwitterCommand
	newtweetchan  chan<- []*Tweet
	newtweets_int chan []*Tweet
	users         []UserTwitterAPITuple
	cur_user      uint
	tweets        []*Tweet
	tweet_map     map[int64]*Tweet
	lookupchan    <-chan TweetRequest
	uiactionchan  chan<- UserInterfaceAction
	last_id       int64
	ignored_users []string
}

type TweetRequest struct {
	Status_id int64
	Reply     chan *Tweet
}

type CmdId int

const (
	UPDATE CmdId = iota
	RETWEET
	DELETE
	FAVORITE
	FOLLOW
	UNFOLLOW
	SET_CURUSER
)

type TwitterCommand struct {
	Cmd  CmdId
	Data Tweet
	Pos  uint // for SET_CURUSER
}

func NewModel(users []UserTwitterAPITuple, cc chan TwitterCommand, ntc chan<- []*Tweet, lc <-chan TweetRequest, uac chan<- UserInterfaceAction, cfg *goconf.ConfigFile) *Model {
	model := &Model{
		cmdchan:       cc,
		newtweetchan:  ntc,
		newtweets_int: make(chan []*Tweet, 10),
		users:         users,
		cur_user:      0,
		lookupchan:    lc,
		tweet_map:     map[int64]*Tweet{},
		uiactionchan:  uac,
		ignored_users: []string{},
	}

	if cfg != nil {
		if default_user, err := cfg.GetString("default", "default_user"); err == nil {
			for idx, _ := range model.users {
				if model.users[idx].User == default_user {
					model.cur_user = uint(idx)
					break
				}
			}
		}

		if ign, err := cfg.GetString("default", "ignore_incoming"); err == nil {
			model.ignored_users = strings.Split(ign, " ")
		}
	}

	userlist := []string{strconv.Uitoa(model.cur_user)}
	for _, u := range model.users {
		userlist = append(userlist, u.User)
	}
	model.uiactionchan <- UserInterfaceAction{SET_USERLIST, userlist}

	return model
}

func (m *Model) Run() {
	m.last_id = int64(0)

	new_tweets := make(chan []*Tweet, 10)

	go StartUserStreams(m.users, new_tweets, m.uiactionchan, m.ignored_users)

	go func() {
		for {
			if config, err := m.users[m.cur_user].Tapi.Configuration(); err == nil {
				log.Printf("Twitter config data: %v", config)
				if config.Short_url_length != nil {
					length := []string{strconv.Itoa64(*config.Short_url_length)}
					m.uiactionchan <- UserInterfaceAction{SET_URLLENGTH, length}
				}
			} else {
				log.Printf("reading Twitter config data failed: %v", err)
			}
			time.Sleep(86400e9)
		}
	}()

	for {
		select {
		case cmd := <-m.cmdchan:
			m.HandleCommand(cmd)
		case req := <-m.lookupchan:
			tweet := m.tweet_map[req.Status_id]
			req.Reply <- tweet
			close(req.Reply)
		case tweets := <-m.newtweets_int:
			for _, t := range tweets {
				m.tweet_map[*t.Id] = t
			}
			m.tweets = append(tweets, m.tweets...)
		case tweets := <-new_tweets:
			log.Printf("received %d tweets", len(tweets))
			unique_tweets := []*Tweet{}
			for _, t := range tweets {
				if _, contained := m.tweet_map[*t.Id]; !contained {
{'fraction_non_alphanumeric': 0.13359765557662473, 'fraction_numerical': 0.007240131011894501, 'mean_word_length': 3.336322869955157, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '269927', 'n_tokens_mistral': 2472, 'n_tokens_neox': 2183, 'n_words': 498}