<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"sync"
	"syscall"

	"gopkg.in/gorp.v1"

	"github.com/ChimeraCoder/anaconda"
	"github.com/robfig/cron"
)

const (
	SearchWord = "Vote<PASSWORD>"
)

var (
	pattern       = regexp.MustCompile(`サンリオキャラクター大賞で(.+?)に投票したよ`)
	spaceReplacer = strings.NewReplacer("　", " ")
)

type Crawler struct{}

func (c *Crawler) Help() string {
	return `sanrio-character-ranking-viewer crawler`
}

func (c *Crawler) Run(args []string) int {
	context, err := newContext()
	if err != nil {
		log.Fatal(err)
	}
	defer context.Close()

	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))

	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_OAUTH_TOKEN"), os.Getenv("TWITTER_OAUTH_TOKEN_SECRET"))

	stream := api.PublicStreamFilter(url.Values{
		"track": []string{SearchWord},
	})

	waitGroup := &sync.WaitGroup{}

	statusChan := make(chan anaconda.Tweet)

	processorQuitChan := make(chan struct{})
	streamingQuitChan := make(chan struct{})
	quitChans := []chan struct{}{
		processorQuitChan,
		streamingQuitChan,
	}

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			select {
			case status := <-statusChan:
				if err := processStatus(context.dbMap, status); err != nil {
					log.Print(err)
				}
				break
			case <-processorQuitChan:
				return
			}
		}
	}()

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			select {
			case item := <-stream.C:
				switch status := item.(type) {
				case anaconda.Tweet:
					statusChan <- status
					break
				}
			case <-streamingQuitChan:
				return
			}
		}
	}()

	cron := cron.New()
	cron.AddFunc("0 */5 * * * *", func() {
		waitGroup.Add(1)
		defer waitGroup.Done()

		v := url.Values{}
		v.Set("count", "200")
		searchResult, err := api.GetSearch(SearchWord, v)
		if err != nil {
			log.Print(err)
			return
		}
		for _, status := range searchResult.Statuses {
			statusChan <- status
		}
	})
	cron.Start()

	signalChan := make(chan os.Signal)
	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT)

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			s := <-signalChan
			log.Print(s)
			for _, quitChan := range quitChans {
				quitChan <- struct{}{}
			}
			break
		}
	}()

	waitGroup.Wait()
	return 0
}

func (c *Crawler) Synopsis() string {
	return `Start crawler`
}

func processStatus(dbMap *gorp.DbMap, status anaconda.Tweet) error {
	createdAt, err := status.CreatedAtTime()
	if err != nil {
		return err
	}

	count, err := dbMap.SelectInt(`SELECT COUNT(*) FROM status WHERE id = $1`, status.Id)
	if err != nil {
		return err
	}
	if count != 0 {
		return nil
	}

	statusJson, err := json.Marshal(&status)
	if err != nil {
		return err
	}
	statusObject := Status{
		Id:        status.Id,
		CreatedAt: createdAt,
		Source:    string(statusJson),
	}
	if err := dbMap.Insert(&statusObject); err != nil {
		return err
	}

	submatches := pattern.FindStringSubmatch(status.Text)
	if submatches == nil {
		return nil
	}

	name := spaceReplacer.Replace(submatches[1])

	entry, err := FindEntryByName(dbMap, 2, name)
	if err != nil {
		return err
	}

	vote := Vote{
		RankingId:   entry.RankingId,
		CharacterId: entry.CharacterId,
		StatusId:    status.Id,
	}
	if err := dbMap.Insert(&vote); err != nil {
		return err
	}
	return nil
}
<commit_msg>Change match pattern for 31st ranking<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"sync"
	"syscall"

	"gopkg.in/gorp.v1"

	"github.com/ChimeraCoder/anaconda"
	"github.com/robfig/cron"
)

const (
	SearchWord = "サンリオキャラクター大賞"
)

var (
	pattern       = regexp.MustCompile(`(.+?)を、「(?:.+?)」なでました！みんなもなでてみよう！`)
	spaceReplacer = strings.NewReplacer("　", " ")
)

type Crawler struct{}

func (c *Crawler) Help() string {
	return `sanrio-character-ranking-viewer crawler`
}

func (c *Crawler) Run(args []string) int {
	context, err := newContext()
	if err != nil {
		log.Fatal(err)
	}
	defer context.Close()

	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))

	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_OAUTH_TOKEN"), os.Getenv("TWITTER_OAUTH_TOKEN_SECRET"))

	stream := api.PublicStreamFilter(url.Values{
		"track": []string{SearchWord},
	})

	waitGroup := &sync.WaitGroup{}

	statusChan := make(chan anaconda.Tweet)

	processorQuitChan := make(chan struct{})
	streamingQuitChan := make(chan struct{})
	quitChans := []chan struct{}{
		processorQuitChan,
		streamingQuitChan,
	}

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			select {
			case status := <-statusChan:
				if err := processStatus(context.dbMap, status); err != nil {
					log.Print(err)
				}
				break
			case <-processorQuitChan:
				return
			}
		}
	}()

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			select {
			case item := <-stream.C:
				switch status := item.(type) {
				case anaconda.Tweet:
					statusChan <- status
					break
				}
			case <-streamingQuitChan:
				return
			}
		}
	}()

	cron := cron.New()
	cron.AddFunc("0 */5 * * * *", func() {
		waitGroup.Add(1)
		defer waitGroup.Done()

		v := url.Values{}
		v.Set("count", "200")
		searchResult, err := api.GetSearch(SearchWord, v)
		if err != nil {
			log.Print(err)
			return
		}
		for _, status := range searchResult.Statuses {
			statusChan <- status
		}
	})
	cron.Start()

	signalChan := make(chan os.Signal)
	signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT)

	waitGroup.Add(1)
	go func() {
		defer waitGroup.Done()

		for {
			s := <-signalChan
			log.Print(s)
			for _, quitChan := range quitChans {
				quitChan <- struct{}{}
			}
			break
		}
	}()

	waitGroup.Wait()
	return 0
}

func (c *Crawler) Synopsis() string {
	return `Start crawler`
}

func processStatus(dbMap *gorp.DbMap, status anaconda.Tweet) error {
	createdAt, err := status.CreatedAtTime()
	if err != nil {
		return err
	}

	count, err := dbMap.SelectInt(`SELECT COUNT(*) FROM status WHERE id = $1`, status.Id)
	if err != nil {
		return err
	}
	if count != 0 {
		return nil
	}

	statusJson, err := json.Marshal(&status)
	if err != nil {
		return err
	}
	statusObject := Status{
		Id:        status.Id,
		CreatedAt: createdAt,
		Source:    string(statusJson),
	}
	if err := dbMap.Insert(&statusObject); err != nil {
		return err
	}

	submatches := pattern.FindStringSubmatch(status.Text)
	if submatches == nil {
		return nil
	}

	name := spaceReplacer.Replace(submatches[1])

	entry, err := FindEntryByName(dbMap, 2, name)
	if err != nil {
		return err
	}

	vote := Vote{
		RankingId:   entry.RankingId,
		CharacterId: entry.CharacterId,
		StatusId:    status.Id,
	}
	if err := dbMap.Insert(&vote); err != nil {
		return err
	}
	return nil
}

{'fraction_non_alphanumeric': 0.15046931407942238, 'fraction_numerical': 0.004620938628158845, 'mean_word_length': 3.4425914047466324, 'pattern_counts': {'":': 2, '<': 20, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3332119', 'n_tokens_mistral': 3018, 'n_tokens_neox': 2644, 'n_words': 549}