<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2015 <NAME>
//
// See LICENSE file.
package webinfo

import (
	"database/sql"
	"fmt"
	"github.com/thoj/go-ircevent"
	"golang.org/x/net/html"
	"golang.org/x/net/idna"
	"log"
	"net/http"
	"net/url"
	"regexp"
)

var _database *sql.DB

func Init(db *sql.DB) {
	_database = db
	sqlStmt := `CREATE TABLE IF NOT EXISTS Link (
    id integer NOT NULL PRIMARY KEY,
    user TEXT,
    url TEXT,
    date DATETIME DEFAULT CURRENT_TIMESTAMP);`

	_, err := db.Exec(sqlStmt)
	if err != nil {
		log.Fatalf("%q: %s\n", err, sqlStmt)
	}
}

// TODO Choose a better name for that function
func HandleUrls(event *irc.Event, replyCallback func(string)) {
	allUrls := findUrls(event.Message())
	for _, url := range allUrls {
		fmt.Println("Detected URL:", url.String())
		response, err := http.Get(url.String())
		if err != nil {
			log.Println(err)
			return
		}
		doc, err := html.Parse(response.Body)
		response.Body.Close()
		if err != nil {
			log.Println(err)
			return
		}

		var (
			user string
			date string
		)
		sqlQuery := "SELECT user, strftime('%d/%m/%Y @ %H:%M', datetime(date, 'localtime')) FROM Link WHERE url = $1"
		rows, err := _database.Query(sqlQuery, url.String())
		if err != nil {
			log.Fatalf("%q: %s\n", err, sqlQuery)
		}
		for rows.Next() {
			rows.Scan(&user, &date)
		}

		if user == "" {
			sqlQuery = "INSERT INTO Link (user, url) VALUES ($1, $2)"
			_, err := _database.Exec(sqlQuery, event.Nick, url.String())
			if err != nil {
				log.Fatalf("%q: %s\n", err, sqlQuery)
			}
		} else {
			replyCallback(fmt.Sprintf("Link already posted by %s (%s)", user, date))
		}

		title, found := getTitleFromHTML(doc)
		if found {
			replyCallback(title)
		}
	}
}

func getTitleFromHTML(document *html.Node) (title string, found bool) {
	if document.Type != html.DocumentNode {
		// Didn't find a document node as first node, exit
		return<commit_msg>Titlefinder: updated to use callback with *core.ReplyCallbackData parameter<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2015 <NAME>
//
// See LICENSE file.
package webinfo

import (
	"database/sql"
	"fmt"
	"github.com/romainletendart/goxxx/core"
	"github.com/thoj/go-ircevent"
	"golang.org/x/net/html"
	"golang.org/x/net/idna"
	"log"
	"net/http"
	"net/url"
	"regexp"
)

var _database *sql.DB

func Init(db *sql.DB) {
	_database = db
	sqlStmt := `CREATE TABLE IF NOT EXISTS Link (
    id integer NOT NULL PRIMARY KEY,
    user TEXT,
    url TEXT,
    date DATETIME DEFAULT CURRENT_TIMESTAMP);`

	_, err := db.Exec(sqlStmt)
	if err != nil {
		log.Fatalf("%q: %s\n", err, sqlStmt)
	}
}

// TODO Choose a better name for that function
func HandleUrls(event *irc.Event, replyCallback func(*core.ReplyCallbackData)) {
	allUrls := findUrls(event.Message())
	for _, url := range allUrls {
		log.Println("Detected URL:", url.String())
		response, err := http.Get(url.String())
		if err != nil {
			log.Println(err)
			return
		}
		doc, err := html.Parse(response.Body)
		response.Body.Close()
		if err != nil {
			log.Println(err)
			return
		}

		var (
			user string
			date string
		)
		sqlQuery := "SELECT user, strftime('%d/%m/%Y @ %H:%M', datetime(date, 'localtime')) FROM Link WHERE url = $1"
		rows, err := _database.Query(sqlQuery, url.String())
		if err != nil {
			log.Fatalf("%q: %s\n", err, sqlQuery)
		}
		for rows.Next() {
			rows.Scan(&user, &date)
		}

		if user == "" {
			sqlQuery = "INSERT INTO Link (user, url) VALUES ($1, $2)"
			_, err := _database.Exec(sqlQuery, event.Nick, url.String())
			if err != nil {
				log.Fatalf("%q: %s\n", err, sqlQuery)
			}
		} else {
			replyCallback(&core.ReplyCallbackData{Message: fmt.Sprintf("Link already posted by %s (%s)", user, date)})
		}

		title, found := getTitleFromHTML(doc)
		if found {
			replyCallback(&core.ReplyCallbackData{Message: title})
		}
	}
}

func getTitleFromHTML(document *html.Node) (title string, found bool) {
	if document.Type != html.DocumentNode {
		// Didn't find a document node as first node, exit
		return
{'fraction_non_alphanumeric': 0.15987071108901044, 'fraction_numerical': 0.0034808552958727, 'mean_word_length': 3.3351293103448274, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2722549', 'n_tokens_mistral': 1627, 'n_tokens_neox': 1479, 'n_words': 389}