<commit_before>#!/usr/bin/ruby

require 'rubygems'
require 'hpricot'

# load the requested file for scraping
doc = Hpricot(open(ARGV[0]))

# get all the table rows of the second table in the document
rows = doc.search("//table:nth(2)//tr")

# ignore the header row and the last two rows of fine print
rows.delete_at(0)
rows.delete_at(-1)
rows.delete_at(-1)

# convert each row to fathead format
rows.each do |row|
	
	# get the type, number, name, and link for this part
	paragraphs = row.search("p")
	part_type = paragraphs[0].inner_html.downcase
	part_number = paragraphs[1].at("a").inner_html	
	name_link = paragraphs[2].at("a")
	part_name = name_link.inner_html
	part_link = name_link['href']
	
	# first word of part name is conventionally part category
	# (see http://www.ldraw.org/Article340.html for formal category spec)
	# subparts names are typically prefixed by a tilde, which we omit
	part_category = part_name.split.first.delete("~")<commit_msg>Consolidate categories by grouping oddball parts under "Miscellaneous" and by grouping generic primitive parts as well.<commit_after>#!/usr/bin/ruby

require 'rubygems'
require 'hpricot'

# list of established part categories
categories = ['Animal', 'Antenna', 'Arch', 'Arm', 'Bar',
	'Baseplate','Belville', 'Boat', 'Bracket', 'Brick', 'Car', 'Cone',
	'Container','Conveyor', 'Crane', 'Cylinder', 'Dish', 'Door',
	'Electric', 'Exhaust','Fence', 'Figure', 'Figure Accessory', 'Flag',
	'Forklift', 'Freestyle','Garage', 'Gate', 'Glass', 'Grab', 'Hinge',
	'Homemaker', 'Hose', 'Jack','Ladder', 'Lever', 'Magnet', 'Minifig',
	'Minifig Accessory', 'Monorail','Panel', 'Plane', 'Plant', 'Plate',
	'Platform', 'Propellor', 'Rack','Roadsign', 'Rock', 'Scala',
	'Screw', 'Sheet', 'Slope', 'Staircase','Sticker', 'Support', 'Tail',
	'Tap', 'Technic', 'Tile', 'Tipper','Tractor', 'Trailer', 'Train',
	'Turntable', 'Tyre', 'Vehicle', 'Wedge','Wheel', 'Winch', 'Window',
	'Windscreen', 'Wing', 'Znap']

# load the requested file for scraping
doc = Hpricot(open(ARGV[0]))

# get all the table rows of the second table in the document
rows = doc.search("//table:nth(2)//tr")

# ignore the header row and the last two rows of fine print
rows.delete_at(0)
rows.delete_at(-1)
rows.delete_at(-1)

# convert each row to fathead format
rows.each do |row|
	
	# get the type, number, name, and link for this part
	paragraphs = row.search("p")
	part_type = paragraphs[0].inner_html.downcase
	part_number = paragraphs[1].at("a").inner_html	
	name_link = paragraphs[2].at("a")
	part_name = name_link.inner_html
	part_link = name_link['href']
	
	# first word of part name is conventionally part category
	# (see http://www.ldraw.org/Article340.html for formal category spec)
	# subparts names are typically prefixed by a tilde, which we omit
	# primitives are reusable forms, so we just group them as primitives
	# categories not on this list (http://www.ldraw.org/library/tracker/ref/catkeyfaq/)
	# get grouped as "Miscellaneous" to keep things tidy - some tracker parts have tmp names
	part_category = part_name.split.first.delete("~")
	if part_type == "primitive"
		part_category = "Primitive"
	elsif part_type == "48-segment primitive"
		part_category = "High-Resolution Primitive"
	elsif not categories.include?(part_category)
		part_category = "Miscellaneous"
	end
{'fraction_non_alphanumeric': 0.14909090909090908, 'fraction_numerical': 0.007272727272727273, 'mean_word_length': 5.311663479923518, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '187724', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1117, 'n_words': 337}