<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	r "github.com/integralist/sainsbury-scraper/retriever"
	s "github.com/integralist/sainsbury-scraper/scraper"
)

const url = "http://hiring-tests.s3-website-eu-west-1.amazonaws.com/2015_Developer_Scrape/5_products.html"

func commandNotFound(c *cli.Context, command string) {
	fmt.Fprintf(os.Stderr, "%s: '%s' is not a %s command. See '%s --help'.", c.App.Name, command, c.App.Name, c.App.Name)
	os.Exit(2)
}

func main() {
	app := cli.NewApp()
	app.Name = "Sainsbury Scraper"
	app.Version = "0.0.1"
	app.Author = "<NAME>"
	app.Usage = "CLI tool for scraping contents from Sainsbury website"
	app.Action = process

	app.CommandNotFound = commandNotFound
	app.Run(os.Args)
}

func process(c *cli.Context) {
	coll, err := r.Retrieve(url)
	if err != nil {
		fmt.Printf("There was an issue retrieving links from the page: %s", err.Error())
		os.Exit(1)
	}

	b, err := json.MarshalIndent(s.Scrape(coll), "", "    ")
	if err != nil {
		fmt.Printf("There was an issue converting our data into JSON: %s", err.Error())
		os.Exit(1)
	}

	fmt.Println(string(b))
}<commit_msg>Rename constant to be more descriptive<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/codegangsta/cli"
	r "github.com/integralist/sainsbury-scraper/retriever"
	s "github.com/integralist/sainsbury-scraper/scraper"
)

const gateway = "http://hiring-tests.s3-website-eu-west-1.amazonaws.com/2015_Developer_Scrape/5_products.html"

func commandNotFound(c *cli.Context, command string) {
	fmt.Fprintf(os.Stderr, "%s: '%s' is not a %s command. See '%s --help'.", c.App.Name, command, c.App.Name, c.App.Name)
	os.Exit(2)
}

func main() {
	app := cli.NewApp()
	app.Name = "Sainsbury Scraper"
	app.Version = "0.0.1"
	app.Author = "<NAME>"
	app.Usage = "CLI tool for scraping contents from Sainsbury website"
	app.Action = process

	app.CommandNotFound = commandNotFound
	app.Run(os.Args)
}

func process(c *cli.Context) {
	coll, err := r.Retrieve(gateway)
	if err != nil {
		fmt.Printf("There was an issue retrieving links from the page: %s", err.Error())
		os.Exit(1)
	}

	b, err := json.MarshalIndent(s.Scrape(coll), "", "    ")
	if err != nil {
		fmt.Printf("There was an issue converting our data into JSON: %s", err.Error())
		os.Exit(1)
	}

	fmt.Println(string(b))
}
{'fraction_non_alphanumeric': 0.16362865621025857, 'fraction_numerical': 0.011021619330224672, 'mean_word_length': 4.0212765957446805, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3758305', 'n_tokens_mistral': 959, 'n_tokens_neox': 918, 'n_words': 205}