<commit_before>URL = Npm.require('url')
querystring = Npm.require('querystring')
request = HTTPInternals.NpmModules.request.module
iconv = Npm.require('iconv-lite')
ipRangeCheck = Npm.require('ip-range-check')

OEmbed = {}

# Detect encoding
getCharset = (body) ->
	binary = body.toString 'binary'
	matches = binary.match /<meta\b[^>]*charset=["']?([\w\-]+)/i
	if matches
		return matches[1]
	return 'utf-8'

toUtf8 = (body) ->
	return iconv.decode body, getCharset(body)

getUrlContent = (urlObj, redirectCount = 5, callback) ->
	if _.isString(urlObj)
		urlObj = URL.parse urlObj

	parsedUrl = _.pick urlObj, ['host', 'hash', 'pathname', 'protocol', 'port', 'query', 'search', 'hostname']

	ignoredHosts = RocketChat.settings.get('API_EmbedIgnoredHosts').replace(' ', '').split(',') or []
	if parsedUrl.hostname in ignoredHosts or ipRangeCheck(parsedUrl.hostname, ignoredHosts)
		return callback()

	data = RocketChat.callbacks.run 'oembed:beforeGetUrlContent',
		urlObj: urlObj
		parsedUrl: parsedUrl

	if data.attachments?
		return callback null, data

	url = URL.format data.urlObj
	opts =
		url: url
		strictSSL: !RocketChat.settings.get 'Allow_Invalid_SelfSigned_Certs'<commit_msg>Remove all spaces from ignored hosts setting<commit_after>URL = Npm.require('url')
querystring = Npm.require('querystring')
request = HTTPInternals.NpmModules.request.module
iconv = Npm.require('iconv-lite')
ipRangeCheck = Npm.require('ip-range-check')

OEmbed = {}

# Detect encoding
getCharset = (body) ->
	binary = body.toString 'binary'
	matches = binary.match /<meta\b[^>]*charset=["']?([\w\-]+)/i
	if matches
		return matches[1]
	return 'utf-8'

toUtf8 = (body) ->
	return iconv.decode body, getCharset(body)

getUrlContent = (urlObj, redirectCount = 5, callback) ->
	if _.isString(urlObj)
		urlObj = URL.parse urlObj

	parsedUrl = _.pick urlObj, ['host', 'hash', 'pathname', 'protocol', 'port', 'query', 'search', 'hostname']

	ignoredHosts = RocketChat.settings.get('API_EmbedIgnoredHosts').replace(/\s/g, '').split(',') or []
	if parsedUrl.hostname in ignoredHosts or ipRangeCheck(parsedUrl.hostname, ignoredHosts)
		return callback()

	data = RocketChat.callbacks.run 'oembed:beforeGetUrlContent',
		urlObj: urlObj
		parsedUrl: parsedUrl

	if data.attachments?
		return callback null, data

	url = URL.format data.urlObj
	opts =
		url: url
		strictSSL: !RocketChat.settings.get 'Allow_Invalid_SelfSigned_Certs'
{'fraction_non_alphanumeric': 0.14997922725384297, 'fraction_numerical': 0.0033236393851267137, 'mean_word_length': 5.190231362467866, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4619041', 'n_tokens_mistral': 867, 'n_tokens_neox': 838, 'n_words': 183}