<commit_before>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"golang.org/x/net/context/ctxhttp"
)

var (
	baseURL     = "https://api.futr.efs.foliocloud.net"
	httpTimeout = 30 * time.Second

	errMissingCredentials = errors.New("Missing email address or password")
)

type response struct {
	Data   interface{} `json:"data"`
	Errors interface{} `json:"errors"`
}

func getCredentials() (string, string, error) {
	email := os.Getenv("EDGE_MAGAZINE_EMAIL")
	password := os.Getenv("EDGE_MAGAZINE_PASSWORD")

	if email != "" && password != "" {
		return email, password, nil
	}

	return "", "", errMissingCredentials
}

func dumpResponse(resp *http.Response) {
	if glog.V(1) {
		body, err := httputil.DumpResponse(resp, true)

		if err != nil {
			glog.Fatal(err)
		}

		glog.V(1).Infof("%s", body)
	}
}

func postForm(ctx context.Context, url string, form url.Values) (map[string]interface{}, error) {
	ctx, cancel := context.WithTimeout(ctx, httpTimeout)
	defer cancel()

	resp, err := ctxhttp.PostForm(ctx, http.DefaultClient, url, form)

	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	dumpResponse(resp)

	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return nil, err
	}

	var data response

	if err = json.Unmarshal(b, &data); err != nil {
		return nil, err
	}

	if errs, ok := data.Errors.(map[string]interface{}); ok {
		for key, value := range errs {
			return nil, errors.Errorf("%s: %v", key, value)
		}
	}

	return data.Data.(map[string]interface{}), nil
}

func createAnonymousUser(ctx context.Context) (string, error) {
	form := url.Values{
		"appKey":    {"<KEY>"},
		"secretKey": {"b9dd34da8c269e44879ea1be2a0f9f7c"},
		"platform":  {"iphone-retina"},
	}

	data, err := postForm(ctx, baseURL+"/createAnonymousUser/", form)

	if err != nil {
		return "", err
	}

	return data["uid"].(string), nil
}

func login(ctx context.Context, uid, email, password string) (string, error) {
	params := map[string]string{
		"password":   password,
		"identifier": email,
	}

	b, err := json.Marshal(params)

	if err != nil {
		return "", err
	}

	form := url.Values{
		"uid":        {uid},
		"api_params": {string(b)},
	}

	data, err := postForm(ctx, baseURL+"/login/", form)

	if err != nil {
		return "", err
	}

	return data["download_ticket_no"].(string), nil
}

func main() {
	flag.Parse()

	uid, err := createAnonymousUser(context.Background())

	if err != nil {
		glog.Exit(err)
	}

	email, password, err := getCredentials()

	if err != nil {
		glog.Exit(err)
	}

	ticket, err := login(context.Background(), uid, email, password)

	if err != nil {
		glog.Exit(err)
	}

	fmt.Println(uid)
	fmt.Println(ticket)

	glog.Flush()
}
<commit_msg>Download URLs for all purchased products<commit_after>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"sort"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"golang.org/x/net/context/ctxhttp"
)

func getCredentials() (string, string, error) {
	email := os.Getenv("EDGE_MAGAZINE_EMAIL")
	password := os.Getenv("EDGE_MAGAZINE_PASSWORD")

	if email != "" && password != "" {
		return email, password, nil
	}

	return "", "", errors.New("Missing email address or password")
}

func dumpResponse(resp *http.Response) {
	if glog.V(1) {
		body, err := httputil.DumpResponse(resp, true)

		if err != nil {
			glog.Fatal(err)
		}

		glog.V(1).Infof("%s", body)
	}
}

func postForm(ctx context.Context, query string, form url.Values, result interface{}) error {
	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	url := fmt.Sprintf("https://api.futr.efs.foliocloud.net/%s/", query)
	resp, err := ctxhttp.PostForm(ctx, http.DefaultClient, url, form)

	if err != nil {
		return err
	}

	defer resp.Body.Close()
	dumpResponse(resp)

	b, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return err
	}

	var errs struct {
		Errors interface{} `json:"errors"`
	}

	if err = json.Unmarshal(b, &errs); err != nil {
		return err
	}

	if errs, ok := errs.Errors.(map[string]interface{}); ok {
		for key, value := range errs {
			return errors.Errorf("%s: %v", key, value)
		}
	}

	return json.Unmarshal(b, result)
}

func createAnonymousUser(ctx context.Context) (string, error) {
	form := url.Values{
		"appKey":    {"<KEY>"},
		"secretKey": {"b9dd34da8c269e44879ea1be2a0f9f7c"},
		"platform":  {"iphone-retina"},
	}

	var response struct {
		Data struct {
			UID string `json:"uid"`
		} `json:"data"`
	}

	if err := postForm(ctx, "createAnonymousUser", form, &response); err != nil {
		return "", err
	}

	return response.Data.UID, nil
}

func login(ctx context.Context, uid, email, password string) (string, error) {
	params := map[string]string{
		"password":   password,
		"identifier": email,
	}

	b, err := json.Marshal(params)

	if err != nil {
		return "", err
	}

	form := url.Values{
		"uid":        {uid},
		"api_params": {string(b)},
	}

	var response struct {
		Data struct {
			Ticket string `json:"download_ticket_no"`
		} `json:"data"`
	}

	if err = postForm(ctx, "login", form, &response); err != nil {
		return "", err
	}

	return response.Data.Ticket, nil
}

func getDownloadURL(ctx context.Context, uid, ticket string) (int, error) {
	form := url.Values{
		"uid":    {uid},
		"ticket": {ticket},
	}

	var response struct {
		Data struct {
			Status int `json:"status"`
		} `json:"data"`
	}

	if err := postForm(ctx, "getDownloadUrl", form, &response); err != nil {
		return 0, err
	}

	return response.Data.Status, nil
}

func getPurchasedProductList(ctx context.Context, uid string) ([]string, error) {
	form := url.Values{
		"uid": {uid},
	}

	var response struct {
		Data struct {
			PurchasedProducts []struct {
				Product string `json:"sku"`
			} `json:"purchased_product_list"`
		} `json:"data"`
	}

	if err := postForm(ctx, "getPurchasedProductList", form, &response); err != nil {
		return nil, err
	}

	var products []string

	for _, product := range response.Data.PurchasedProducts {
		products = append(products, product.Product)
	}

	sort.Strings(products)

	return products, nil
}

func getEntitledProduct(ctx context.Context, uid, product string) (string, error) {
	form := url.Values{
		"uid": {uid},
		"sku": {product},
	}

	var response struct {
		Data struct {
			URL string `json:"secure_download_url"`
		} `json:"data"`
	}

	if err := postForm(ctx, "getEntitledProduct", form, &response); err != nil {
		return "", err
	}

	return response.Data.URL, nil
}

func main() {
	flag.Parse()

	uid, err := createAnonymousUser(context.Background())

	if err != nil {
		glog.Exit(err)
	}

	email, password, err := getCredentials()

	if err != nil {
		glog.Exit(err)
	}

	ticket, err := login(context.Background(), uid, email, password)

	if err != nil {
		glog.Exit(err)
	}

	for {
		status, err := getDownloadURL(context.Background(), uid, ticket)

		if err != nil {
			glog.Exit(err)
		}

		if status == 1 {
			break
		}
	}

	products, err := getPurchasedProductList(context.Background(), uid)

	if err != nil {
		glog.Exit(err)
	}

	for _, product := range products {
		url, err := getEntitledProduct(context.Background(), uid, product)

		if err != nil {
			glog.Exit(err)
		}

		fmt.Println(url)
	}

	glog.Flush()
}

{'fraction_non_alphanumeric': 0.1663726421495454, 'fraction_numerical': 0.0059709594246166375, 'mean_word_length': 3.742599742599743, 'pattern_counts': {'":': 19, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2151461', 'n_tokens_mistral': 2945, 'n_tokens_neox': 2710, 'n_words': 643}