<commit_before># Classify all HTML files by their copyright notices
# - None (no "Copyright" or "©" etc)
# - CC-BY-SA 4.0
# - All Rights Reserved
# - Other/Unknown
# As much as possible, find the exact beginning and end of the copyright
# notice, thus allowing them to be replaced as needed. But for now, just
# classify the files.
# Fixes to be done:
# * If copyright is "All Rights Reserved", simply remove that text altogether,
#   and add a <footer> at the end of document.body.main or document.body, as
#   per _layouts/default.html in the Markdown files. This will change it to
#   copyright="CC-BY-SA 4.0".
import os
import re
import collections
from bs4 import BeautifulSoup, Comment, Tag

# root = "/home/rosuav/gsarchive/live"
root = "/home/rosuav/gsarchive/clone" # Faster and safer, not touching the original files

def classify(fn):
	info = { }
	with open(fn, "rb") as f: blob = f.read()
	soup = BeautifulSoup(blob, "html.parser")
	if soup.find(text=lambda text: isinstance(text, Comment) and "autogenerated" in text.lower()):
		info["generated"] = 1
	# Okay. So, what are we looking for, exactly?
	# 1) Easy mode: an anchor rel="license".
	for tag in soup.findAll(rel="license"):
		if tag["href"] == "https://creativecommons.org/licenses/by-sa/4.0/":
			return info | {"copyright": "CC-BY-SA 4.0"}
		info.setdefault("links", []).append(tag["href"])
	if "links" in info:
		return info | {"copyright": "Unknown"}
	for cr in soup.findAll(text=True):
		# Look for a copyright marker, including a miswritten HTML entity
		if m := re.search("Copyright.*Gilbert and Sullivan Archive.*All Rights Reserved", cr.text, re.IGNORECASE):
			return info | {"copyright": "All Rights Reserved"}
		if "copyright" in cr.text or "©" in cr.text or "&copy" in cr.text:
			info.setdefault("text", []).append(cr.text)
	return info | {"copyright": "Unknown" if info.get("text") else "None"}

stats = collections.Counter()
for root, dirs, files in os.walk(root):
	for file in files:
		fn = os.path.join(root, file)
		info = classify(fn)
		stats[info["copyright"]] += 1
		if not stats.total() % 500: print(stats)
		if info["copyright"] not in ("All Rights Reserved", "None", "CC-BY-SA 4.0"):
			print(fn, info)
print(stats)<commit_msg>copywrong: More flexibly notice things<commit_after># Classify all HTML files by their copyright notices
# - None (no "Copyright" or "©" etc)
# - CC-BY-SA 4.0
# - All Rights Reserved
# - Other/Unknown
# As much as possible, find the exact beginning and end of the copyright
# notice, thus allowing them to be replaced as needed. But for now, just
# classify the files.
# Fixes to be done:
# * If copyright is "All Rights Reserved", simply remove that text. It is
#   now copyright="None".
# * If copyright is "None", add a <footer> at the end of document.body.main
#   or document.body, as per _layouts/default.html in the Markdown files.
#   It is now copyright="CC-BY-SA 4.0".
import os
import re
import collections
from bs4 import BeautifulSoup, Comment, Tag

# root = "/home/rosuav/gsarchive/live"
root = "/home/rosuav/gsarchive/clone" # Faster and safer, not touching the original files

def classify(fn):
	info = { }
	with open(fn, "rb") as f: blob = f.read()
	soup = BeautifulSoup(blob, "html.parser")
	if soup.find(text=lambda text: isinstance(text, Comment) and "autogenerated" in text.lower()):
		info["generated"] = 1
	# Okay. So, what are we looking for, exactly?
	# 1) Easy mode: an anchor rel="license".
	for tag in soup.findAll(rel="license"):
		if tag["href"] == "https://creativecommons.org/licenses/by-sa/4.0/":
			return info | {"copyright": "CC-BY-SA 4.0"}
		if tag["href"] == "http://creativecommons.org/licenses/by-sa/4.0/":
			return info | {"copyright": "CC-BY-SA 4.0"} # Maybe fix protocol? Or not bother.
		info.setdefault("links", []).append(tag["href"])
	if "links" in info:
		return info | {"copyright": "Unknown"}
	for cr in soup.findAll(text=True):
		if m := re.search(r"""
			Copyright.*
			(
				Gilbert\s*and\s*Sullivan\s*Archive
				| Paul\s*Howarth
				| Colin\s*Johnson
			)
			.*All\s*Rights\s*Reserved
		""", cr.text, re.IGNORECASE | re.VERBOSE | re.DOTALL):
			return info | {"copyright": "All Rights Reserved"}
		# Look for any copyright marker, including a miswritten HTML entity
		if "copyright" in cr.text or "©" in cr.text or "&copy" in cr.text:
			info.setdefault("text", []).append(cr.text)
	return info | {"copyright": "Unknown" if info.get("text") else "None"}

stats = collections.Counter()
for root, dirs, files in os.walk(root):
	for file in files:
		fn = os.path.join(root, file)
		info = classify(fn)
		stats[info["copyright"]] += 1
		if not stats.total() % 500: print(stats)
		if info["copyright"] not in ("All Rights Reserved", "None", "CC-BY-SA 4.0"):
			print(fn, info)
print(stats)
{'fraction_non_alphanumeric': 0.1497270054598908, 'fraction_numerical': 0.007979840403191937, 'mean_word_length': 4.013684210526316, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '47804', 'n_tokens_mistral': 1672, 'n_tokens_neox': 1573, 'n_words': 549}