<commit_before>#!/usr/bin/env ruby

require 'rubygems'
require 'optparse'
require 'open-uri'
require 'cgi'
require 'pathname'
require 'logger'
require 'wtsutil'

def formatAliasList(catalog)
	text = "<ul>\n"
	catalog.aliases.sort.each do |aliasName, canonicalName|
		text += "<li>#{aliasName} &rarr; #{canonicalName}</li>\n"
	end
	text += "</ul>\n"
	return text
end

def formatSatelliteList(catalog)
	text = "<table><tr>\n<td><ul>"
	perColumn = (catalog.tles.keys.length / 3.0).ceil
	count = 0
	catalog.tles.keys.sort.each do |name|
		searchLink = format "http://nssdc.gsfc.nasa.gov/nmc/spacecraftSearch.do?spacecraft=%s", CGI.escape(name)
		text += "<li><a href=\"#{searchLink}\">#{name}</a></li>\n"
		count += 1
		if count > perColumn
			text += "</ul></td>\n<td><ul>\n"
			count = 0
		end
	end
	text += "</ul></td>\n</tr></table>\n"
	return text
end

def getCatalogPageTemplate(templatePath='config/satellites_template.html')
	templateFile = open templatePath
	text = templateFile.read
	templateFile.close
	return text
end

def formatCatalogPage(catalog, template)
	text = template.gsub("<!--ALIAS NAMES-->", formatAliasList(catalog))
	text.gsub!("<!--CANONICAL NAMES-->", formatSatelliteList(catalog))
	return text
end

def formatPhraseList(list)
	if list.empty?
		return ""
	end
	if list.length == 1
		format "%s", list[0]
	elsif list.length == 2
		format "%s and %s", list[0], list[1]
	else
		format "%s, and %s", list[0..-2].join(', '), list[-1]
	end
end

def updateCatalog(config)

	catalog = WTS::WTSCatalog.new
		
	config.tleIndexURLs.each do |url|
		
		# read the TLE index
		$log.info("Retrieving #{url}")
		index = open url
		content = index.read.split("\r\n")
		index.close
		
		# split into sets of three lines (names + two line element set)
		line = 0
		while line + 2 <= content.length
		
			tleName = content[line]
			tleText = content[line+1..line+2].join("\n")
			
			# remove alternate names, status codes, and extra whitespace from name
			tleName.gsub!(/\(.+?\)/, "")
			tleName.gsub!(/\[.+?\]/, "")
			tleName.rstrip!
			<commit_msg>Removed unused method<commit_after>#!/usr/bin/env ruby

require 'rubygems'
require 'optparse'
require 'open-uri'
require 'cgi'
require 'pathname'
require 'logger'
require 'wtsutil'

def formatAliasList(catalog)
	text = "<ul>\n"
	catalog.aliases.sort.each do |aliasName, canonicalName|
		text += "<li>#{aliasName} &rarr; #{canonicalName}</li>\n"
	end
	text += "</ul>\n"
	return text
end

def formatSatelliteList(catalog)
	text = "<table><tr>\n<td><ul>"
	perColumn = (catalog.tles.keys.length / 3.0).ceil
	count = 0
	catalog.tles.keys.sort.each do |name|
		searchLink = format "http://nssdc.gsfc.nasa.gov/nmc/spacecraftSearch.do?spacecraft=%s", CGI.escape(name)
		text += "<li><a href=\"#{searchLink}\">#{name}</a></li>\n"
		count += 1
		if count > perColumn
			text += "</ul></td>\n<td><ul>\n"
			count = 0
		end
	end
	text += "</ul></td>\n</tr></table>\n"
	return text
end

def getCatalogPageTemplate(templatePath='config/satellites_template.html')
	templateFile = open templatePath
	text = templateFile.read
	templateFile.close
	return text
end

def formatCatalogPage(catalog, template)
	text = template.gsub("<!--ALIAS NAMES-->", formatAliasList(catalog))
	text.gsub!("<!--CANONICAL NAMES-->", formatSatelliteList(catalog))
	return text
end

def updateCatalog(config)

	catalog = WTS::WTSCatalog.new
		
	config.tleIndexURLs.each do |url|
		
		# read the TLE index
		$log.info("Retrieving #{url}")
		index = open url
		content = index.read.split("\r\n")
		index.close
		
		# split into sets of three lines (names + two line element set)
		line = 0
		while line + 2 <= content.length
		
			tleName = content[line]
			tleText = content[line+1..line+2].join("\n")
			
			# remove alternate names, status codes, and extra whitespace from name
			tleName.gsub!(/\(.+?\)/, "")
			tleName.gsub!(/\[.+?\]/, "")
			tleName.rstrip!
			
{'fraction_non_alphanumeric': 0.17486479526139584, 'fraction_numerical': 0.006695853721349472, 'mean_word_length': 3.9289340101522843, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2590888', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1408, 'n_words': 324}