<commit_before>package main

import (
	"fmt"
	"koding/kites/kloud/cleaners/lookup"
	"log"
	"os"

	"github.com/koding/multiconfig"
)

type Config struct {
	// AWS Access and Secret Key
	AccessKey string `required:"true"`
	SecretKey string `required:"true"`

	// MongoDB
	MongoURL string `required:"true"`

	// Postgres
	Host     string `default:"localhost"`
	Port     int    `default:"5432"`
	Username string `required:"true"`
	Password string `required:"true"`
	DBName   string `required:"true" `
}

func main() {
	if err := realMain(); err != nil {
		log.Println(err)
		os.Exit(1)
	}

	os.Exit(0)
}

func realMain() error {
	conf := new(Config)

	// Load the config, it's reads environment variables or from flags
	multiconfig.New().MustLoad(conf)

	p := lookup.NewPostgres(&lookup.PostgresConfig{
		Host:     conf.Host,
		Port:     conf.Port,
		Username: conf.Username,
		Password: <PASSWORD>,
		DBName:   conf.DBName,
	})

	m := lookup.NewMongoDB(conf.MongoURL)

	fmt.Println("Fetching paying customers")
	payingIds, err := p.PayingCustomers()
	if err != nil {
		return err
	}

	accounts, err := m.Accounts(payingIds...)
	if err != nil {
		return err
	}

	set := make(map[string]struct{}, 0)
	for _, account := range accounts {
		set[account.Profile.Nickname] = struct{}{}
	}

	isPaid := func(username string) bool {
		_, ok := set[username]
		return ok
	}

	fmt.Println("Fetching free user accounts")

	alwaysOnMachines, err := m.AlwaysOn()
	if err != nil {
		return err
	}

	fmt.Println("Free users with alwaysOn VMs:")

	for _, machine := range alwaysOnMachines {
		username := machine.Credential

		// if user is not a paying customer
		if !isPaid(username) {
			fmt.Printf("\t%s\n", username)
		}

	}

	return nil
}
<commit_msg>cleaners: put nonvalid users into a list for further action<commit_after>package main

import (
	"fmt"
	"koding/kites/kloud/cleaners/lookup"
	"log"
	"os"

	"github.com/koding/multiconfig"
)

type Config struct {
	// AWS Access and Secret Key
	AccessKey string `required:"true"`
	SecretKey string `required:"true"`

	// MongoDB
	MongoURL string `required:"true"`

	// Postgres
	Host     string `default:"localhost"`
	Port     int    `default:"5432"`
	Username string `required:"true"`
	Password string `required:"true"`
	DBName   string `required:"true" `
}

func main() {
	if err := realMain(); err != nil {
		log.Println(err)
		os.Exit(1)
	}

	os.Exit(0)
}

func realMain() error {
	conf := new(Config)

	// Load the config, it's reads environment variables or from flags
	multiconfig.New().MustLoad(conf)

	p := lookup.NewPostgres(&lookup.PostgresConfig{
		Host:     conf.Host,
		Port:     conf.Port,
		Username: conf.Username,
		Password: conf.Password,
		DBName:   conf.DBName,
	})

	m := lookup.NewMongoDB(conf.MongoURL)

	payingIds, err := p.PayingCustomers()
	if err != nil {
		return err
	}

	accounts, err := m.Accounts(payingIds...)
	if err != nil {
		return err
	}

	set := make(map[string]struct{}, 0)
	for _, account := range accounts {
		set[account.Profile.Nickname] = struct{}{}
	}

	isPaid := func(username string) bool {
		_, ok := set[username]
		return ok
	}

	alwaysOnMachines, err := m.AlwaysOn()
	if err != nil {
		return err
	}

	fmt.Println("Free users with alwaysOn VMs:")

	nonvalidUsers := make([]string, 0)

	for _, machine := range alwaysOnMachines {
		username := machine.Credential

		// if user is not a paying customer
		if !isPaid(username) {
			nonvalidUsers = append(nonvalidUsers, username)
		}
	}

	for _, user := range nonvalidUsers {
		fmt.Printf("\t%s\n", user)
	}

	return nil
}

{'fraction_non_alphanumeric': 0.14109050028105677, 'fraction_numerical': 0.0042158516020236085, 'mean_word_length': 3.5804375804375805, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3867920', 'n_tokens_mistral': 1342, 'n_tokens_neox': 1249, 'n_words': 320}