<commit_before>package main

// scanning an HTTP response for phrases

import (
	"bytes"
	"code.google.com/p/go.net/html/charset"
	"code.google.com/p/go.text/transform"
	"compress/gzip"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

var contentPhraseList = newPhraseList()

// scanContent scans the content of a document for phrases,
// and updates tally.
func scanContent(content []byte, contentType, cs string, tally map[rule]int) {
	if strings.Contains(contentType, "javascript") {
		scanJSContent(content, tally)
		return
	}

	transformers := make([]transform.Transformer, 0, 3)
	if cs != "utf-8" {
		e, _ := charset.Lookup(cs)
		transformers = append(transformers, e.NewDecoder())
	}

	if strings.Contains(contentType, "html") {
		transformers = append(transformers, entityDecoder{})
	}
	transformers = append(transformers, new(wordTransformer))

	ps := newPhraseScanner(contentPhraseList, func(s string) {
		tally[rule{t: contentPhrase, content: s}]++
	})
	ps.scanByte(' ')

	var t transform.Transformer
	if len(transformers) == 1 {
		t = transformers[0]
	} else {
		t = transform.Chain(transformers...)
	}

	buf := make([]byte, 4096)
	for len(content) > 0 {
		nDst, nSrc, err := t.Transform(buf, content, true)
		if nSrc == 0 && nDst == 0 {
			panic(err)
		}
		for _, c := range buf[:nDst] {
			ps.scanByte(c)
		}
		content = content[nSrc:]
	}

	ps.scanByte(' ')
}

// scanJSContent scans only the contents of quoted JavaScript strings
// in the document.
func scanJSContent(content []byte, tally map[rule]int) {
	_, items := lex(string(content))
	ps := newPhraseScanner(contentPhraseList, func(s string) {
		tally[rule{t: contentPhrase, content: s}]++
	})

	for s := range items {
		s = wordString(s)
		ps.scanByte(' ')
		for i := 0; i < len(s); i++ {
			ps.scanByte(s[i])<commit_msg>When decoding page content, don't panic if an error occurs; just log it.<commit_after>package main

// scanning an HTTP response for phrases

import (
	"bytes"
	"compress/gzip"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	"code.google.com/p/go.net/html/charset"
	"code.google.com/p/go.text/transform"
)

var contentPhraseList = newPhraseList()

// scanContent scans the content of a document for phrases,
// and updates tally.
func scanContent(content []byte, contentType, cs string, tally map[rule]int) {
	if strings.Contains(contentType, "javascript") {
		scanJSContent(content, tally)
		return
	}

	transformers := make([]transform.Transformer, 0, 3)
	if cs != "utf-8" {
		e, _ := charset.Lookup(cs)
		transformers = append(transformers, e.NewDecoder())
	}

	if strings.Contains(contentType, "html") {
		transformers = append(transformers, entityDecoder{})
	}
	transformers = append(transformers, new(wordTransformer))

	ps := newPhraseScanner(contentPhraseList, func(s string) {
		tally[rule{t: contentPhrase, content: s}]++
	})
	ps.scanByte(' ')

	var t transform.Transformer
	if len(transformers) == 1 {
		t = transformers[0]
	} else {
		t = transform.Chain(transformers...)
	}

	buf := make([]byte, 4096)
	for len(content) > 0 {
		nDst, nSrc, err := t.Transform(buf, content, true)
		if nSrc == 0 && nDst == 0 {
			if err == transform.ErrShortSrc {
				log.Printf("Encountered ErrShortSrc while decoding page content; remaining content: %q", content)
			} else {
				log.Println("Error decoding page content:", err)
			}
			nSrc = len(content)
			nDst = nSrc
			copy(buf, content)
		}
		for _, c := range buf[:nDst] {
			ps.scanByte(c)
		}
		content = content[nSrc:]
	}

	ps.scanByte(' ')
}

// scanJSContent scans only the contents of quoted JavaScript strings
// in the document.
func scanJSContent(content []byte, tally map[rule]int) {
	_, items := lex(string(content))
	ps := newPhraseScanner(contentPhraseList, func(s string) {
		tally[rule{t: contentPhrase, content: s}]++
	})

	for s := range items {
		s = wordString(s)
		ps.scanByte(' ')
		for i := 0; i < len(s); i++ {
			ps.scanByte(s[i])
{'fraction_non_alphanumeric': 0.14985886579420066, 'fraction_numerical': 0.006671798819604825, 'mean_word_length': 4.062337662337662, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7436363', 'n_tokens_mistral': 1518, 'n_tokens_neox': 1377, 'n_words': 343}