<commit_before>import sys
import re
import os
import string
import cStringIO

import mx.DateTime

import miscfuncs

# output to check for undetected member names
import miscfuncs
toppath = miscfuncs.toppath

seelines = open(os.path.join(miscfuncs.tmppath, 'emblinks.txt'), "w")

# this detects the domain
reglinkdomt = '(?:\.or[gq]|\.com|[\.\s]uk|\.tv|\.net|\.gov|\.int|\.info|\.it|\.ch|\.es|\.mz|\.lu|\.fr|\.dk|\.mil|\.eu)(?!\w)'
reglinkdomf = 'http://(?:(?:www.)?defraweb|nswebcopy|\d+\.\d+\.\d+\.\d+)|www.defraweb'
reglinkdom = '(?:http ?:? ?/{1,3}(?:www)?|www)(?:(?:[^/:;,?=()<>"\'@](?!www\.))*?(?:%s))+|%s' % (reglinkdomt, reglinkdomf)

# this detects the middle section of a url between the slashes.
reglinkmid = '(?:/(?:(?:[^/:;,?="<()]|&#\d+;|&amp;)(?!www\.))+)*/'

# this detects the tail section of a url trailing a slash
# XXX: This seems very odd way of matching URIs
#reglinktail = '[^./:;,]*(?:\.\s?(?:s?html?|pdf|xls|(?:asp|php|cfm(?:\?[^\s.]+)?)))|\w*'
regqs = '(?:\?\s?\w+=[\w/]+(?:&\w+=[\w/%]*)*)'
regasptype = '(?:aspx?|nsf|php|cfm|gif|jpg|jpeg|png)%s?' % regqs
reglinktail = '(?:[^./:;,?=]|&#\d+;)*(?:\.?\s?(?:s?html?|xls|doc|pdf(?:\?Open ?Element)?|%s|%s))|(?:[\w-]|&#\d+;)*' % (regasptype, regqs)


rreglink = '(?:(?:%s)(?:(?:%s)(?:%s)?)?)' % (reglinkdom, reglinkmid, reglinktail)
reglink = '((%s)(?:(%s)(%s)?)?)(?i)' % (reglinkdom, reglinkmid, reglinktail)
relink = re.compile(reglink)

rregemail = '\w+@\w+(?:\.\w+)*'


rehtlink = re.compile('(%s)(?:(%s)(%s)?)?(?i)' % (reglinkdom, reglinkmid, reglinktail))


def ConstructHTTPlink(qstrdom, qstrmid, qstrtail):
	qstrdom = re.sub('^http|[:/\s]', '', qstrdom)
	if not qstrmid:
		qstrmid = ''
	if not qstrtail:
		qstrtail = ''

        qstrdom = re.sub('\n', '', qstrdom)
        qstrmid = re.sub('\n', '', qstrmid)
        qstrtail = re.sub('\n', '', qstrtail)

	if not re.match('[\w\-.]*$', qstrdom):
		print ' bad domain -- ' + qstrdom

	if qstrmid:
		qstrmid = re.sub(' ', '', qstrmid)
		qstrmid = re.sub('&#15[01];', '-', qstrmid)
		qstrmid = re.sub('&#0?95;', '_', qstrmid)
		if re.search('&#\d+;', qstrmid):
			print ' undemangled href symbol ' + qstrmid
		qstrmid = qstrmid.replace('&amp;', '&') # XXX
		qstrmid = re.sub('&', '&amp;', qstrmid)
	if not re.match('[\w\-/.+;&%]*$', qstrmid):
		#print ' bad midd -- ' + qstrmid
                pass

	if qstrtail:
		qstrtail = re.sub(' ', '', qstrtail)
		qstrtail = re.sub('&#15[01];', '-', qstrtail)
		qstrtail = re.sub('&#0?95;', '_', qstrtail)
		qstrtail = re.sub('&#0?35;', '#', qstrtail)
		if re.search('&#\d+;', qstrtail):
			print ' undemangled href symbol ' + qstrtail
		qstrtail = re.sub('&', '&amp;', qstrtail)
	if not re.match('[\w\-./\?&=%;~#]*$', qstrtail):
		print ' bad tail -- ' + qstrtail


	qstrlink = 'http://%s%s%s' % (qstrdom, qstrmid, qstrtail)
	return qstrlink


def ExtractHTTPlink(stex, qs):
	qlink = relink.search(stex)

	# no link found.  output if there should be.
	if not qlink:
		if re.search('www|http|ftp(?i)', stex):
			seelines.write(qs.sdate)
			seelines.write('\t')
			seelines.write(' --failed to find link-- ' + stex + '\n')
			print ' --failed to find link-- ' + stex
		return (None,None,None)


	qspan = qlink.span(1)
	qstr = re.sub(' ', '', qlink.group(1))


	qstrlink = ConstructHTTPlink(qlink.group(2), qlink.group(3), qlink.group(4))
	qtags = ( '<a href="%s">' % qstrlink, '</a>' )

	# write out debug stuff
	qplpch = [ ]
	slo = qspan[0] - 10
	shi = qspan[1] + 20
	if slo < 0:<commit_msg>Spot https links, and valid characters in the middle of a URL.<commit_after>import sys
import re
import os
import string
import cStringIO

import mx.DateTime

import miscfuncs

# output to check for undetected member names
import miscfuncs
toppath = miscfuncs.toppath

seelines = open(os.path.join(miscfuncs.tmppath, 'emblinks.txt'), "w")

# this detects the domain
reglinkdomt = '(?:\.or[gq]|\.com|[\.\s]uk|\.tv|\.net|\.gov|\.int|\.info|\.it|\.ch|\.es|\.mz|\.lu|\.fr|\.dk|\.mil|\.eu)(?!\w)'
reglinkdomf = 'https?://(?:(?:www.)?defraweb|nswebcopy|\d+\.\d+\.\d+\.\d+)|www.defraweb'
reglinkdom = '(?:https? ?:? ?/{1,3}(?:www)?|www)(?:(?:[^/:;,?=()<>"\'@](?!www\.))*?(?:%s))+|%s' % (reglinkdomt, reglinkdomf)

# this detects the middle section of a url between the slashes.
reglinkmid = '(?:/(?:(?:[^/?"<]|&#\d+;|&amp;)(?!www\.))*)*/'

# this detects the tail section of a url trailing a slash
# XXX: This seems a VERY odd way of matching URIs
#reglinktail = '[^./:;,]*(?:\.\s?(?:s?html?|pdf|xls|(?:asp|php|cfm(?:\?[^\s.]+)?)))|\w*'
regqs = '(?:\?\s?\w+=[\w/]+(?:&\w+=[\w/%]*)*)'
regasptype = '(?:aspx?|nsf|php|cfm|gif|jpg|jpeg|png)%s?' % regqs
reglinktail = '(?:[^./:;,?=]|&#\d+;)*(?:\.?\s?(?:s?html?|xls|doc|pdf(?:\?Open ?Element)?|%s|%s))|(?:[\w-]|&#\d+;)*' % (regasptype, regqs)


rreglink = '(?:(?:%s)(?:(?:%s)(?:%s)?)?)' % (reglinkdom, reglinkmid, reglinktail)
reglink = '((%s)(?:(%s)(%s)?)?)(?i)' % (reglinkdom, reglinkmid, reglinktail)
relink = re.compile(reglink)

rregemail = '\w+@\w+(?:\.\w+)*'


rehtlink = re.compile('(%s)(?:(%s)(%s)?)?(?i)' % (reglinkdom, reglinkmid, reglinktail))


def ConstructHTTPlink(qstrdom, qstrmid, qstrtail):
	scheme = 'https' if 'https' in qstrdom else 'http'
	qstrdom = re.sub('^https?|[:/\s]', '', qstrdom)
	if not qstrmid:
		qstrmid = ''
	if not qstrtail:
		qstrtail = ''

        qstrdom = re.sub('\n', '', qstrdom)
        qstrmid = re.sub('\n', '', qstrmid)
        qstrtail = re.sub('\n', '', qstrtail)

	if not re.match('[\w\-.]*$', qstrdom):
		print ' bad domain -- ' + qstrdom

	if qstrmid:
		qstrmid = re.sub(' ', '', qstrmid)
		qstrmid = re.sub('&#15[01];', '-', qstrmid)
		qstrmid = re.sub('&#0?95;', '_', qstrmid)
		if re.search('&#\d+;', qstrmid):
			print ' undemangled href symbol ' + qstrmid
		qstrmid = qstrmid.replace('&amp;', '&') # XXX
		qstrmid = re.sub('&', '&amp;', qstrmid)
	if not re.match('[\w\-/.+;&%]*$', qstrmid):
		#print ' bad midd -- ' + qstrmid
                pass

	if qstrtail:
		qstrtail = re.sub(' ', '', qstrtail)
		qstrtail = re.sub('&#15[01];', '-', qstrtail)
		qstrtail = re.sub('&#0?95;', '_', qstrtail)
		qstrtail = re.sub('&#0?35;', '#', qstrtail)
		if re.search('&#\d+;', qstrtail):
			print ' undemangled href symbol ' + qstrtail
		qstrtail = re.sub('&', '&amp;', qstrtail)
	if not re.match('[\w\-./\?&=%;~#]*$', qstrtail):
		print ' bad tail -- ' + qstrtail


	qstrlink = '%s://%s%s%s' % (scheme, qstrdom, qstrmid, qstrtail)
	return qstrlink


def ExtractHTTPlink(stex, qs):
	qlink = relink.search(stex)

	# no link found.  output if there should be.
	if not qlink:
		if re.search('www|http|ftp(?i)', stex):
			seelines.write(qs.sdate)
			seelines.write('\t')
			seelines.write(' --failed to find link-- ' + stex + '\n')
			print ' --failed to find link-- ' + stex
		return (None,None,None)


	qspan = qlink.span(1)
	qstr = re.sub(' ', '', qlink.group(1))


	qstrlink = ConstructHTTPlink(qlink.group(2), qlink.group(3), qlink.group(4))
	qtags = ( '<a href="%s">' % qstrlink, '</a>' )

	# write out debug stuff
	qplpch = [ ]
	slo = qspan[0] - 10
	shi = qspan[1] + 20
	if slo < 0:
{'fraction_non_alphanumeric': 0.2776591200341734, 'fraction_numerical': 0.008828136124163463, 'mean_word_length': 3.8676368676368678, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '510803', 'n_tokens_mistral': 3212, 'n_tokens_neox': 3096, 'n_words': 576}