<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

var (
	config   Config
	urlRegex = regexp.MustCompile(`(?i)\b((?:https?://|www\d{0,3}[.]|[` +
		`a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+` +
		`\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[` +
		`\]{};:'".,<>?«»“”‘’]))`)
)

type Config struct {
	Channel       string
	DBConn        string
	Nick          string
	Ident         string
	FullName      string
	FlickrAPIKey  string
	WolframAPIKey string
	IRCPass       string
	Commands      []struct {
		Name string
		Text string
	}
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn) {
    postedUrl, err := url.Parse(unparsedURL)
    if err != nil {
        log.Println(err)
        return
    }
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}
	respbody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 50000))
	if err != nil {
		log.Println("error reading posted link")
		return
	}
	stringbody := string(respbody)
	titlestart := strings.Index(stringbody, "<title>")
	titleend := strings.Index(stringbody, "</title>")
	if titlestart != -1 && titlestart != -1 {
		title := string(respbody[titlestart+7 : titleend])
		title = strings.TrimSpace(title)
		if title != "" && utf8.ValidString(title) {
			// Example:
			// Title: sadbox . org (at sadbox.org)
			title = "Title: " + html.UnescapeString(title) + " (at " + postedUrl.Host + ")"
			log.Println(title)
			conn.Privmsg(channel, title)
		}
	}
}<commit_msg>gofmt + content type detection<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"
	"unicode/utf8"
)

var (
	config   Config
	urlRegex = regexp.MustCompile(`(?i)\b((?:https?://|www\d{0,3}[.]|[` +
		`a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+` +
		`\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[` +
		`\]{};:'".,<>?«»“”‘’]))`)
)

type Config struct {
	Channel       string
	DBConn        string
	Nick          string
	Ident         string
	FullName      string
	FlickrAPIKey  string
	WolframAPIKey string
	IRCPass       string
	Commands      []struct {
		Name string
		Text string
	}
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn) {
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	restofbody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 50000))
	if err != nil {
		log.Println("error reading posted link")
		return
	}
	respbody = append(respbody, restofbody...)
	stringbody := string(respbody)
	log.Println(stringbody)
	titlestart := strings.Index(stringbody, "<title>")
	titleend := strings.Index(stringbody, "</title>")
	if titlestart != -1 && titlestart != -1 {
		title := string(respbody[titlestart+7 : titleend])
		title = strings.TrimSpace(title)
		if title != "" && utf8.ValidString(title) {
			// Example:
			// Title: sadbox . org (at sadbox.org)
			title = "Title: " + html.UnescapeString(title) + " (at " + postedUrl.Host + ")"
			log.Println(title)
			conn.Privmsg(channel, title)
		}
	}
}
{'fraction_non_alphanumeric': 0.20794644785143598, 'fraction_numerical': 0.00885337939969769, 'mean_word_length': 3.3822138126773886, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1602894', 'n_tokens_mistral': 1916, 'n_tokens_neox': 1805, 'n_words': 351}