<commit_before>package main

import (
	"fmt"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"time"

	"labix.org/v2/mgo/bson"

	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/cloudwatch"
	"github.com/jinzhu/now"
	"github.com/koding/redis"
)

var (
	AWS_NAMESPACE = "AWS/EC2"
	AWS_PERIOD    = 604800

	GB_TO_MB float64 = 1024

	auth aws.Auth
)

type Limits map[string]float64

type Cloudwatch struct {
	Name   string
	Limits Limits
}

func (c *Cloudwatch) GetName() string {
	return c.Name
}

func (c *Cloudwatch) GetLimit(name string) float64 {
	limit, _ := c.Limits[name]
	return limit
}

func (c *Cloudwatch) Save(username string, value float64) error {
	return storage.SaveScore(c.Name, username, value)
}

func (c *Cloudwatch) GetAndSaveData(username string) error {
	userMachines, err := modelhelper.GetMachinesByUsername(username)
	if err != nil {
		return err
	}

	var sum float64

	for _, machine := range userMachines {
		meta, ok := machine.Meta.(bson.M)
		if !ok {
			Log.Error("queued machine has no `meta`", machine.ObjectId)
			continue
		}

		region, ok := meta["region"].(string)
		if !ok || isEmpty(region) {
			Log.Error("queued machine has no `region`: %v", machine.ObjectId)
			continue
		}

		instanceId, ok := meta["instanceId"].(string)
		if !ok || isEmpty(instanceId) {
			Log.Error("queued machine has no `instanceId`: %v", machine.ObjectId)
			continue
		}

		dimension := &cloudwatch.Dimension{
			Name:  "InstanceId",
			Value: instanceId,
		}

		cw, err := cloudwatch.NewCloudWatch(auth, aws.Regions[region].CloudWatchServicepoint)
		if err != nil {
			Log.Error("Failed to initialize cloudwatch client:", err)
			continue
		}

		request := &cloudwatch.GetMetricStatisticsRequest{
			Dimensions: []cloudwatch.Dimension{*dimension},
			Statistics: []string{cloudwatch.StatisticDatapointSum},
			MetricName: c.GetName(),
			EndTime:    time.Now(),
			StartTime:  now.BeginningOfWeek(),
			Period:     AWS_PERIOD,
			Namespace:  AWS_NAMESPACE,
		}

		response, err := cw.GetMetricStatistics(request)
		if err != nil {
			Log.Error("Failed to get request for machine: %s, %v", machine.ObjectId, err)
			continue
		}

		for _, raw := range response.GetMetricStatisticsResult.Datapoints {
			sum += raw.Sum / GB_TO_MB / GB_TO_MB
		}
	}

	if sum > c.Limits[StopLimitKey] {
		Log.Debug("'%s' has used: %v '%s'", username, sum, c.Name)
	}

	return c.Save(username, sum)
}

func (c *Cloudwatch) GetMachinesOverLimit(limitName string) ([]*models.Machine, error) {
	limitAmount := c.Limits[limitName]

	usernames, err := storage.GetFromScore(c.Name, limitAmount)
	if err != nil {
		return nil, err
	}

	machines := []*models.Machine{}

	for _, username := range usernames {
		lr, err := c.IsUserOverLimit(username, limitName)
		if err != nil {
			Log.Error(err.Error())
			continue<commit_msg>vmwatcher: log overlimit machines<commit_after>package main

import (
	"fmt"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"time"

	"labix.org/v2/mgo/bson"

	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/cloudwatch"
	"github.com/jinzhu/now"
	"github.com/koding/redis"
)

var (
	AWS_NAMESPACE = "AWS/EC2"
	AWS_PERIOD    = 604800

	GB_TO_MB float64 = 1024

	auth aws.Auth
)

type Limits map[string]float64

type Cloudwatch struct {
	Name   string
	Limits Limits
}

func (c *Cloudwatch) GetName() string {
	return c.Name
}

func (c *Cloudwatch) GetLimit(name string) float64 {
	limit, _ := c.Limits[name]
	return limit
}

func (c *Cloudwatch) Save(username string, value float64) error {
	return storage.SaveScore(c.Name, username, value)
}

func (c *Cloudwatch) GetAndSaveData(username string) error {
	userMachines, err := modelhelper.GetMachinesByUsername(username)
	if err != nil {
		return err
	}

	var sum float64

	for _, machine := range userMachines {
		meta, ok := machine.Meta.(bson.M)
		if !ok {
			Log.Error("queued machine has no `meta`", machine.ObjectId)
			continue
		}

		region, ok := meta["region"].(string)
		if !ok || isEmpty(region) {
			Log.Error("queued machine has no `region`: %v", machine.ObjectId)
			continue
		}

		instanceId, ok := meta["instanceId"].(string)
		if !ok || isEmpty(instanceId) {
			Log.Error("queued machine has no `instanceId`: %v", machine.ObjectId)
			continue
		}

		dimension := &cloudwatch.Dimension{
			Name:  "InstanceId",
			Value: instanceId,
		}

		cw, err := cloudwatch.NewCloudWatch(auth, aws.Regions[region].CloudWatchServicepoint)
		if err != nil {
			Log.Error("Failed to initialize cloudwatch client:", err)
			continue
		}

		request := &cloudwatch.GetMetricStatisticsRequest{
			Dimensions: []cloudwatch.Dimension{*dimension},
			Statistics: []string{cloudwatch.StatisticDatapointSum},
			MetricName: c.GetName(),
			EndTime:    time.Now(),
			StartTime:  now.BeginningOfWeek(),
			Period:     AWS_PERIOD,
			Namespace:  AWS_NAMESPACE,
		}

		response, err := cw.GetMetricStatistics(request)
		if err != nil {
			Log.Error("Failed to get request for machine: %s, %v", machine.ObjectId, err)
			continue
		}

		for _, raw := range response.GetMetricStatisticsResult.Datapoints {
			sum += raw.Sum / GB_TO_MB / GB_TO_MB
		}
	}

	if sum > c.Limits[StopLimitKey] {
		Log.Info("'%s' has used: %v '%s'", username, sum, c.Name)
	}

	return c.Save(username, sum)
}

func (c *Cloudwatch) GetMachinesOverLimit(limitName string) ([]*models.Machine, error) {
	limitAmount := c.Limits[limitName]

	usernames, err := storage.GetFromScore(c.Name, limitAmount)
	if err != nil {
		return nil, err
	}

	machines := []*models.Machine{}

	for _, username := range usernames {
		lr, err := c.IsUserOverLimit(username, limitName)
		if err != nil {
			Log.Error(err.Error())
			continue
{'fraction_non_alphanumeric': 0.1361634666196935, 'fraction_numerical': 0.007750572485467676, 'mean_word_length': 3.9719789842381785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5092013', 'n_tokens_mistral': 2258, 'n_tokens_neox': 2047, 'n_words': 464}