<commit_before>#
# vim: tabstop=2:shiftwidth=2:noexpandtab
#
# Fetch title of URLs in channels
#
# /set urltitle_enabled_channels #channel1 #channel2 ..
# to enable in those channels
#
# /set urltitle_ignored_nicks nick1 nick2 nick2 ..
# to not fetch titles of urls by these nicks
#

package require http
package require tls
package require htmlparse
package require idna

namespace eval ::urltitle {
	variable useragent "Tcl http client package 2.7"

	# 2 MiB.
	# If we hit this (or near it), then we cannot retrieve a title.
	variable max_bytes [expr 2*1024*1024]

	variable max_redirects 3

	settings_add_str "urltitle_enabled_channels" ""
	settings_add_str "urltitle_ignored_nicks" ""

	signal_add msg_pub "*" ::urltitle::urltitle

	::http::register https 443 [list ::tls::socket -ssl2 0 -ssl3 0 -tls1 1]

	variable debug 0
}

proc ::urltitle::log {msg} {
	if {!$::urltitle::debug} {
		return
	}
	irssi_print "urltitle: $msg"
}

proc ::urltitle::urltitle {server nick uhost chan msg} {
	if {![str_in_settings_str urltitle_enabled_channels $chan]} {
		return
	}

	if {[str_in_settings_str urltitle_ignored_nicks $nick]} {
		return
	}

	set url [::urltitle::recognise_url $msg]
	if {$url == ""} {
		return
	}

	::urltitle::geturl $url $server $chan 0
}

# Breaks an absolute URL into 3 pieces:
# prefix/protocol: e.g. http://, https//
# domain: e.g. everything up to the first /, if it exists
# rest: everything after the first /, if exists
proc ::urltitle::split_url {absolute_url} {
	if {![regexp -- {(https?://)([^/]*)/?(.*)} $absolute_url -> prefix domain rest]} {
		error "urltitle error: parse problem: $absolute_url"
	}
	set domain [idna::domain_toascii $domain]

	# from http-title.tcl by Pixelz. Avoids urls that will be treated as
	# a flag
	if {[string index $domain 0] eq "-"} {
		error "urltitle error: Invalid URL: domain looks like a flag"
	}
	return [list $prefix $domain $rest]
}

# Attempt to recognise potential_url as an actual url in form of http[s]://...
# Returns blank if unsuccessful
proc ::urltitle::recognise_url {potential_url} {
	set full_url []
	if {[regexp -nocase -- {(https?://\S+)} $potential_url -> url]} {
		set full_url $url
	} elseif {[regexp -nocase -- {(www\.\S+)} $potential_url -> url]} {
		set full_url "http://${url}"
	}

	if {$full_url == ""} {
		return ""
	}

	lassign [::urltitle::split_url $full_url] prefix domain rest

	return "${prefix}${domain}/${rest}"
}

# @param string $data The body of the page
#
# @return mixed the title (html decoded), or "" if not found
#
# pull the title out of the html body of a page
proc ::urltitle::extract_title {data} {
	if {[regexp -nocase -- {<title>(.*?)</title>} $data -> title]} {
		set title [regsub -all -- {\s+} $title " "]
		::urltitle::log "extract_title: found raw title $title"
		# mapEscapes decodes html encoded characters
		return [htmlparse::mapEscapes $title]
	}
	return ""
}

proc ::urltitle::geturl {url server chan redirect_count} {
	::urltitle::log "geturl: Trying to get URL: $url"
	if {$redirect_count > $::urltitle::max_redirects} {
		return
	}

	::http::config -useragent $::urltitle::useragent

	# Provide an Accept text/html header as we are expecting to pull the HTML
	# title tag out for printing.
	# I use -headers rather than http::config -accept as the latter is global and
	# I would rather avoid changing global options.

	irssi_print "urltitle: Looking up $url"

	if {[catch {::http::geturl \
		$url \
		-binary 1 \
		-blocksize $::urltitle::max_bytes \
		-timeout 10000 \
		-headers [list Accept text/html] \
		-progress ::urltitle::http_progress \
		-command "::urltitle::http_done $server $chan $redirect_count" \
	} err]} {
		irssi_print "urltitle: Unable to make HTTP request to \[$url\]: $err"
	}
}

# This function will cause us to stop the request after max_bytes.
# We are apparently not able to use the portion of data we've retrieved. The
# data is garbage in my testing. The documentation is unclear.
proc ::urltitle::http_progress {token total current} {
	if {$current >= $::urltitle::max_bytes} {
		::urltitle::log "http_done: resetting, too large"

		# Don't clean up the token here. We will in http_done.

		::http::reset $token
	}
}

proc ::urltitle::http_done {server chan redirect_count token} {
	# Get state array out of token
	upvar #0 $token state

	# Ensure we have a sane state.
	if {$state(status) != "ok"} {
		set status $state(status)
		::urltitle::log "http_done: request status is $status, not ok"
		# It appears that status reset is not okay to use the result after all.
		::http::cleanup $token
		return
	}

	if {$::urltitle::debug} {
		set url $state(url)
		set data [::http::data $token]
		set code [::http::ncode $token]
		set meta [::http::meta $token]<commit_msg>urltitle: Make http request synchronous<commit_after>#
# vim: tabstop=2:shiftwidth=2:noexpandtab
#
# Fetch title of URLs in channels
#
# /set urltitle_enabled_channels #channel1 #channel2 ..
# to enable in those channels
#
# /set urltitle_ignored_nicks nick1 nick2 nick2 ..
# to not fetch titles of urls by these nicks
#

package require http
package require tls
package require htmlparse
package require idna

namespace eval ::urltitle {
	variable useragent "Tcl http client package 2.7"

	# 2 MiB.
	# If we hit this (or near it), then we cannot retrieve a title.
	variable max_bytes [expr 2*1024*1024]

	variable max_redirects 3

	settings_add_str "urltitle_enabled_channels" ""
	settings_add_str "urltitle_ignored_nicks" ""

	signal_add msg_pub "*" ::urltitle::urltitle

	::http::register https 443 [list ::tls::socket -ssl2 0 -ssl3 0 -tls1 1]

	variable debug 0
}

proc ::urltitle::log {msg} {
	if {!$::urltitle::debug} {
		return
	}
	irssi_print "urltitle: $msg"
}

proc ::urltitle::urltitle {server nick uhost chan msg} {
	if {![str_in_settings_str urltitle_enabled_channels $chan]} {
		return
	}

	if {[str_in_settings_str urltitle_ignored_nicks $nick]} {
		return
	}

	set url [::urltitle::recognise_url $msg]
	if {$url == ""} {
		return
	}

	::urltitle::geturl $url $server $chan 0
}

# Breaks an absolute URL into 3 pieces:
# prefix/protocol: e.g. http://, https//
# domain: e.g. everything up to the first /, if it exists
# rest: everything after the first /, if exists
proc ::urltitle::split_url {absolute_url} {
	if {![regexp -- {(https?://)([^/]*)/?(.*)} $absolute_url -> prefix domain rest]} {
		error "urltitle error: parse problem: $absolute_url"
	}
	set domain [idna::domain_toascii $domain]

	# from http-title.tcl by Pixelz. Avoids urls that will be treated as
	# a flag
	if {[string index $domain 0] eq "-"} {
		error "urltitle error: Invalid URL: domain looks like a flag"
	}
	return [list $prefix $domain $rest]
}

# Attempt to recognise potential_url as an actual url in form of http[s]://...
# Returns blank if unsuccessful
proc ::urltitle::recognise_url {potential_url} {
	set full_url []
	if {[regexp -nocase -- {(https?://\S+)} $potential_url -> url]} {
		set full_url $url
	} elseif {[regexp -nocase -- {(www\.\S+)} $potential_url -> url]} {
		set full_url "http://${url}"
	}

	if {$full_url == ""} {
		return ""
	}

	lassign [::urltitle::split_url $full_url] prefix domain rest

	return "${prefix}${domain}/${rest}"
}

# @param string $data The body of the page
#
# @return mixed the title (html decoded), or "" if not found
#
# pull the title out of the html body of a page
proc ::urltitle::extract_title {data} {
	if {[regexp -nocase -- {<title>(.*?)</title>} $data -> title]} {
		set title [regsub -all -- {\s+} $title " "]
		::urltitle::log "extract_title: found raw title $title"
		# mapEscapes decodes html encoded characters
		return [htmlparse::mapEscapes $title]
	}
	return ""
}

proc ::urltitle::geturl {url server chan redirect_count} {
	::urltitle::log "geturl: Trying to get URL: $url"
	if {$redirect_count > $::urltitle::max_redirects} {
		return
	}

	::http::config -useragent $::urltitle::useragent

	# Provide an Accept text/html header as we are expecting to pull the HTML
	# title tag out for printing.
	# I use -headers rather than http::config -accept as the latter is global and
	# I would rather avoid changing global options.

	irssi_print "urltitle: Looking up $url"

	if {[catch {::http::geturl \
		$url \
		-binary 1 \
		-blocksize $::urltitle::max_bytes \
		-timeout 10000 \
		-headers [list Accept text/html] \
		-progress ::urltitle::http_progress \
		} token]} {
		irssi_print "urltitle: Unable to make HTTP request to \[$url\]: $token"
		return
	}

	::urltitle::http_done $server $chan $redirect_count $token
}

# This function will cause us to stop the request after max_bytes.
# We are apparently not able to use the portion of data we've retrieved. The
# data is garbage in my testing. The documentation is unclear.
proc ::urltitle::http_progress {token total current} {
	if {$current >= $::urltitle::max_bytes} {
		::urltitle::log "http_progress: resetting, too large"

		# Don't clean up the token here. We will in http_done.

		::http::reset $token
	}
}

proc ::urltitle::http_done {server chan redirect_count token} {
	# Get state array out of token
	upvar #0 $token state

	# Ensure we have a sane state.
	if {$state(status) != "ok"} {
		set status $state(status)
		::urltitle::log "http_done: request status is $status, not ok"
		# It appears that status reset is not okay to use the result after all.
		::http::cleanup $token
		return
	}

	if {$::urltitle::debug} {
		set url $state(url)
		set data [::http::data $token]
		set code [::http::ncode $token]
		set meta [::http::meta $token]
{'fraction_non_alphanumeric': 0.13544303797468354, 'fraction_numerical': 0.008438818565400843, 'mean_word_length': 4.08637339055794, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '769369', 'n_tokens_mistral': 3391, 'n_tokens_neox': 3179, 'n_words': 1108}