<commit_before>package chromiumpreload

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"
	"unicode"
)

const (
	ForceHTTPS = "force-https"
)

// PreloadList contains a parsed form of the Chromium Preload list.
//
// The full list contains information about more than just HSTS, but only
// HSTS-related contents are currently exposed in this struct.
type PreloadList struct {
	Entries []PreloadEntry `json:"entries"`
}

type Domain string

// A PreloadEntry contains the data from an entry in the Chromium
// Preload list.
//
// - Name: The domain name.
//
// - Mode: The only valid non-empty value is ForceHTTPS
//
// - IncludeSubDomains: If Mode == ForceHTTPS, forces HSTS to apply to
//   all subdomains.
type PreloadEntry struct {
	Name              Domain `json:"name"`
	Mode              string `json:"mode"`
	IncludeSubDomains bool   `json:"include_subdomains"`
}

const (
	latestChromiumListURL = "https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json?format=TEXT"
)

// GetLatest retrieves the latest PreloadList from the Chromium source at<commit_msg>Add comments to chromiumpreload, as suggested by `golint`.<commit_after>package chromiumpreload

import (
	"bufio"
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"
	"unicode"
)

const (
	// ForceHTTPS indicates that all requests should be upgraded from HTTP to
	// HTTPS using the HSTS mechanism (https://tools.ietf.org/html/rfc6797).
	ForceHTTPS = "force-https"
)

// PreloadList contains a parsed form of the Chromium Preload list.
//
// The full list contains information about more than just HSTS, but only
// HSTS-related contents are currently exposed in this struct.
type PreloadList struct {
	Entries []PreloadEntry `json:"entries"`
}

// A domain in the Chromium preload list. Note that this corresponds
// to "host" in the HSTS spec, and does not contain the scheme or port.
// We call it a "domain" because this makes the meaning more clear to
// web developers, and naturally allows us to refer to domains vs.
// subdomains.
type Domain string

// A PreloadEntry contains the data from an entry in the Chromium
// Preload list.
//
// - Name: The domain name.
//
// - Mode: The only valid non-empty value is ForceHTTPS
//
// - IncludeSubDomains: If Mode == ForceHTTPS, forces HSTS to apply to
//   all subdomains.
type PreloadEntry struct {
	Name              Domain `json:"name"`
	Mode              string `json:"mode"`
	IncludeSubDomains bool   `json:"include_subdomains"`
}

const (
	latestChromiumListURL = "https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json?format=TEXT"
)

// GetLatest retrieves the latest PreloadList from the Chromium source at
{'fraction_non_alphanumeric': 0.10348468848996832, 'fraction_numerical': 0.0028159098908834917, 'mean_word_length': 3.598705501618123, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2331471', 'n_tokens_mistral': 922, 'n_tokens_neox': 876, 'n_words': 307}