<commit_before>package misc

import (
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/ChimeraCoder/anaconda"
	"github.com/coopernurse/gorp"
	"github.com/zachlatta/southbayfession/models"
)

func FetchLatestTweetsManager() {
	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))
	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_ACCESS_TOKEN"),
		os.Getenv("TWITTER_ACCESS_TOKEN_SECRET"))

	db := models.Dbm

	for {
		FetchAndCommitLatestTweets(api, db)
		time.Sleep(5 * time.Second)
	}
}

func FetchAndCommitLatestTweets(api *anaconda.TwitterApi, db gorp.SqlExecutor) {
	lastTweet, err := GetLastTweet(api, db)
	if err != nil {
		return
	}

	tweets, err := TweetsAfter(api, lastTweet)
	if err != nil {
		return
	}

	for _, tweet := range tweets {
		err := db.Insert(&tweet)
		if err != nil {
			return
		}
	}
}

func GetLastTweet(api *anaconda.TwitterApi, db gorp.SqlExecutor) (*models.Tweet, error) {
	var tweets []models.Tweet
	_, err := db.Select(&tweets, "select * from tweets order by id limit 1")
	if err != nil {
		return nil, err
	}

	var tweet *models.Tweet
	if len(tweets) == 0 {
		tweet = &models.Tweet{TwitterId: 1}
	} else {
		tweet = &tweets[0]
	}
	return tweet, nil
}

func TweetsAfter(api *anaconda.TwitterApi, tweet *models.Tweet) (
	[]models.Tweet, error) {
	anacondaTweets, err := api.GetUserTimeline(url.Values{
		"screen_name": []string{"Southbayfession"},
		"since_id":    []string{strconv.FormatInt(tweet.TwitterId, 10)},
	})
	if err != nil {
		return nil, err
	}

	tweets := make([]models.Tweet, len(anacondaTweets))
	for i, t := range anacondaTweets {
		tweets[i] = models.Tweet{
			CreatedAt: t.CreatedAt,
			TwitterId: t.Id,
			Text:      t.Text,
			School:    "ESHS",
		}
	}

	return tweets, nil
}<commit_msg>Figure out which schools tweets are from.<commit_after>package misc

import (
	"errors"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/ChimeraCoder/anaconda"
	"github.com/coopernurse/gorp"
	"github.com/zachlatta/southbayfession/models"
)

func FetchLatestTweetsManager() {
	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))
	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_ACCESS_TOKEN"),
		os.Getenv("TWITTER_ACCESS_TOKEN_SECRET"))

	db := models.Dbm

	for {
		FetchAndCommitLatestTweets(api, db)
		time.Sleep(5 * time.Second)
	}
}

func FetchAndCommitLatestTweets(api *anaconda.TwitterApi, db gorp.SqlExecutor) {
	lastTweet, err := GetLastTweet(api, db)
	if err != nil {
		return
	}

	tweets, err := TweetsAfter(api, lastTweet)
	if err != nil {
		return
	}

	for _, tweet := range tweets {
		err := db.Insert(&tweet)
		if err != nil {
			return
		}
	}
}

func GetLastTweet(api *anaconda.TwitterApi, db gorp.SqlExecutor) (*models.Tweet, error) {
	var tweets []models.Tweet
	_, err := db.Select(&tweets, "select * from tweets order by id limit 1")
	if err != nil {
		return nil, err
	}

	var tweet *models.Tweet
	if len(tweets) == 0 {
		tweet = &models.Tweet{TwitterId: 1}
	} else {
		tweet = &tweets[0]
	}
	return tweet, nil
}

func TweetsAfter(api *anaconda.TwitterApi, tweet *models.Tweet) (
	[]models.Tweet, error) {
	anacondaTweets, err := api.GetUserTimeline(url.Values{
		"screen_name": []string{"Southbayfession"},
		"since_id":    []string{strconv.FormatInt(tweet.TwitterId, 10)},
	})
	if err != nil {
		return nil, err
	}

	tweets := make([]models.Tweet, len(anacondaTweets))
	for i, t := range anacondaTweets {
		tweet := models.Tweet{
			CreatedAt: t.CreatedAt,
			TwitterId: t.Id,
			Text:      t.Text,
		}

		school, err := SchoolOfTweet(&tweet)
		if err != nil {
			tweet.School = "N/A"
		} else {
			tweet.School = school
		}

		tweets[i] = tweet
	}

	return tweets, nil
}

func SchoolOfTweet(tweet *models.Tweet) (string, error) {
	schoolRegex := regexp.MustCompile(`( |\w)*$`)

	potentialSchools := map[string][]string{
		"Da Vinci Communications": []string{
			"davinci communications",
		},
		"El Segundo High School": []string{
			"es",
			"eshs",
			"el segundo",
			"gundo",
		},
		"Harbor Teacher Preparation Academy": []string{
			"htpa",
		},
		"North High School": []string{
			"north high",
			"north",
		},
		"Santa Monica High School": []string{
			"samohi",
		},
		"Torrance High School": []string{
			"torrance high",
		},
		"Hawthorne Math and Science Academy": []string{
			"hmsa",
			"hms",
		},
		"West High School": []string{
			"west",
			"west torrance high school",
			"west high sko",
		},
	}

	matchedSchool := schoolRegex.FindString(tweet.Text)
	matchedSchool = strings.TrimSpace(matchedSchool)
	matchedSchool = strings.ToLower(matchedSchool)

	var school string
	for potentialSchool, potentialMatches := range potentialSchools {
		for _, potentialMatch := range potentialMatches {
			if matchedSchool == potentialMatch {
				school = potentialSchool
			}
		}
	}

	if len(school) == 0 {
		return school, errors.New("No school matched.")
	}

	return school, nil
}
{'fraction_non_alphanumeric': 0.14553429027113238, 'fraction_numerical': 0.0029904306220095694, 'mean_word_length': 4.03714859437751, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2840850', 'n_tokens_mistral': 2178, 'n_tokens_neox': 1952, 'n_words': 401}