<commit_before>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/codegangsta/cli"
	_ "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
	"github.com/letsencrypt/boulder/sa"

	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/policy"
)

func main() {
	app := cli.NewApp()
	app.Name = "policy-loader"
	app.Usage = "Loads/dumps rules into/from the policy database"
	app.Version = cmd.Version()
	app.Author = "Boulder contributors"
	app.Email = "<EMAIL>"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "config",
			Value:  "config.json",
			EnvVar: "BOULDER_CONFIG",
			Usage:  "Path to Boulder JSON configuration file",
		},
		cli.StringFlag{
			Name:   "rule-file",
			Value:  "rules.json",
			EnvVar: "BOULDER_POLICY_RULES",
			Usage:  "Path to Boulder policy whitelist and blacklist rule file",
		},
	}

	app.Commands = append(app.Commands, []cli.Command{
		cli.Command{
			Name:  "dump-rules",
			Usage: "Write out whitelist and blacklist from database to a rule file",
			Action: func(c *cli.Context) {
				padb, ruleFile := setupFromContext(c)
				ruleSet, err := padb.DumpRules()
				cmd.FailOnError(err, "Couldn't retrieve whitelist rules")
				var rules struct {
					Blacklist []string
					Whitelist []string
				}
				for _, r := range ruleSet.Blacklist {
					rules.Blacklist = append(rules.Blacklist, r.Host)
				}
				for _, r := range ruleSet.Whitelist {
					rules.Whitelist = append(rules.Whitelist, r.Host)
				}
				rulesJSON, err := json.Marshal(rules)
				cmd.FailOnError(err, "Couldn't marshal rule list")
				err = ioutil.WriteFile(ruleFile, rulesJSON, os.ModePerm)
				cmd.FailOnError(err, "Failed to write the rule file")
				fmt.Printf("# Saved rule list to %s\n", ruleFile)
			},
		},
		cli.Command{
			Name:  "load-rules",
			Usage: "Load whitelist and blacklist into database from a rule file",
			Action: func(c *cli.Context) {
				padb, ruleFile := setupFromContext(c)

				rulesJSON, err := ioutil.ReadFile(ruleFile)
				cmd.FailOnError(err, "Couldn't read configuration file")
				rules := policy.RawRuleSet{}
				err = json.Unmarshal(rulesJSON, &rules)
				cmd.FailOnError(err, "Couldn't unmarshal rules list")
				rs := policy.RuleSet{}
				for _, r := range rules.Blacklist {
					rs.Blacklist = append(rs.Blacklist, policy.BlacklistRule{Host: r})
				}
				for _, r := range rules.Whitelist {
					rs.Whitelist = append(rs.Whitelist, policy.WhitelistRule{Host: r})
				}

				err = padb.LoadRules(rs)<commit_msg>lowercase white- and blacklisted domains on load<commit_after>// Copyright 2015 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/codegangsta/cli"
	_ "github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/go-sql-driver/mysql"
	"github.com/letsencrypt/boulder/sa"

	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/policy"
)

func main() {
	app := cli.NewApp()
	app.Name = "policy-loader"
	app.Usage = "Loads/dumps rules into/from the policy database"
	app.Version = cmd.Version()
	app.Author = "Boulder contributors"
	app.Email = "<EMAIL>"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "config",
			Value:  "config.json",
			EnvVar: "BOULDER_CONFIG",
			Usage:  "Path to Boulder JSON configuration file",
		},
		cli.StringFlag{
			Name:   "rule-file",
			Value:  "rules.json",
			EnvVar: "BOULDER_POLICY_RULES",
			Usage:  "Path to Boulder policy whitelist and blacklist rule file",
		},
	}

	app.Commands = append(app.Commands, []cli.Command{
		cli.Command{
			Name:  "dump-rules",
			Usage: "Write out whitelist and blacklist from database to a rule file",
			Action: func(c *cli.Context) {
				padb, ruleFile := setupFromContext(c)
				ruleSet, err := padb.DumpRules()
				cmd.FailOnError(err, "Couldn't retrieve whitelist rules")
				var rules struct {
					Blacklist []string
					Whitelist []string
				}
				for _, r := range ruleSet.Blacklist {
					rules.Blacklist = append(rules.Blacklist, r.Host)
				}
				for _, r := range ruleSet.Whitelist {
					rules.Whitelist = append(rules.Whitelist, r.Host)
				}
				rulesJSON, err := json.Marshal(rules)
				cmd.FailOnError(err, "Couldn't marshal rule list")
				err = ioutil.WriteFile(ruleFile, rulesJSON, os.ModePerm)
				cmd.FailOnError(err, "Failed to write the rule file")
				fmt.Printf("# Saved rule list to %s\n", ruleFile)
			},
		},
		cli.Command{
			Name:  "load-rules",
			Usage: "Load whitelist and blacklist into database from a rule file",
			Action: func(c *cli.Context) {
				padb, ruleFile := setupFromContext(c)

				rulesJSON, err := ioutil.ReadFile(ruleFile)
				cmd.FailOnError(err, "Couldn't read configuration file")
				rules := policy.RawRuleSet{}
				err = json.Unmarshal(rulesJSON, &rules)
				cmd.FailOnError(err, "Couldn't unmarshal rules list")
				rs := policy.RuleSet{}
				for _, r := range rules.Blacklist {
					rs.Blacklist = append(rs.Blacklist, policy.BlacklistRule{
						Host: strings.ToLower(r),
					})
				}
				for _, r := range rules.Whitelist {
					rs.Whitelist = append(rs.Whitelist, policy.WhitelistRule{
						Host: strings.ToLower(r),
					})
				}

				err = padb.LoadRules(rs)
{'fraction_non_alphanumeric': 0.131433506044905, 'fraction_numerical': 0.002763385146804836, 'mean_word_length': 3.387121212121212, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1971312', 'n_tokens_mistral': 2367, 'n_tokens_neox': 1991, 'n_words': 494}