<commit_before>package fetch

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/mail"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode"

	"github.com/jasonmoo/toget"
	"github.com/jprobinson/eazye"
	"golang.org/x/net/html"
	"gopkg.in/mgo.v2/bson"

	"github.com/jprobinson/newshound"
)

var Senders = map[string]bool{
	"cnn":                      true,
	"foxnews.com":              true,
	"foxbusiness.com":          true,
	"nbcnews.com":              true,
	"nbc":                      true,
	"nytimes.com":              true,
	"the washington post":      true,
	"wsj.com":                  true,
	"the wall street journal":  true,
	"the wall street journal.": true,
	"politico":                 true,
	"los angeles times":        true,
	"cbs":                      true,
	"abc":                      true,
	"usatoday.com":             true,
	"usa today":                true,
	"yahoo":                    true,
	"ft":                       true,
	"bbc":                      true,
	"npr":                      true,<commit_msg>dealing with nytimes name change<commit_after>package fetch

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net/mail"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode"

	"github.com/jasonmoo/toget"
	"github.com/jprobinson/eazye"
	"golang.org/x/net/html"
	"gopkg.in/mgo.v2/bson"

	"github.com/jprobinson/newshound"
)

var Senders = map[string]bool{
	"cnn":                      true,
	"foxnews.com":              true,
	"foxbusiness.com":          true,
	"nbcnews.com":              true,
	"nbc":                      true,
	"nytimes.com":              true,
	"the new york times":       true,
	"the washington post":      true,
	"wsj.com":                  true,
	"the wall street journal":  true,
	"the wall street journal.": true,
	"politico":                 true,
	"los angeles times":        true,
	"cbs":                      true,
	"abc":                      true,
	"usatoday.com":             true,
	"usa today":                true,
	"yahoo":                    true,
	"ft":                       true,
	"bbc":                      true,
	"npr":                      true,
{'fraction_non_alphanumeric': 0.14056603773584905, 'fraction_numerical': 0.0009433962264150943, 'mean_word_length': 1.424, 'pattern_counts': {'":': 41, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5724932', 'n_tokens_mistral': 782, 'n_tokens_neox': 751, 'n_words': 154}