<commit_before>#!/usr/bin/env ruby

require 'rubygems'
require 'open-uri'
require 'hpricot'

url = gets.strip

doc = Hpricot(open(url))

puts "begin_tsv"

url.match(/abstract_id=(\d+)/)
id = $1
filename = (doc/"//tr/td/table/tr/td/td/p/font").first.inner_text
filename.gsub!(/\.pdf/, '')
puts "linkout\tSSRN\t#{id}\t#{filename}\t\t"

puts "type\tJOUR"
puts "url\thttp://ssrn.com/abstract=#{id}"

title = (doc/"//td[@width='80%', @align='center']/font/strong").first.inner_text.strip
puts "title\t" + title

for author in (doc/"//td[@width='80%']/center//a.textlink")
  puts "author\t" + author.inner_text.strip
end

doicell = (doc/"//tr/td/font/blockquote/font/a").first
if doicell
	doi = doicell.inner_text
	puts "doi\t#{doi}"
end

puts "journal\tSocial Science Research Network Working Paper Series"

date = (doc/"//tr/td[@width='80%']/center/font[@size='2']")[1].inner_text
if date =~ /^(January|February|March|April|May|June|July|August|September|October|November|December) ([0-9]{1,2}), ([0-9]{4}$)/
	puts "month\t" + $1
	puts "day\t" + $2
	puts "year\t" + $3
elsif date =~/^(January|February|March|April|May|June|July|August|September|October|November|December) ([0-9]{4}$)/
	puts "month\t" + $1
	puts "year\t" + $2
elsif date =~/^([0-9]{4})$/
	puts "year\t" + $1
end

abs = (doc/"//tr/td/font")[5].inner_text.strip
puts "abstract\t#{abs}"

puts "end_tsv"

puts "status\tok"
<commit_msg>Patch from <NAME> to handle missing dates<commit_after>#!/usr/bin/env ruby

require 'rubygems'
require 'open-uri'
require 'hpricot'

url = gets.strip

doc = Hpricot(open(url))

puts "begin_tsv"

url.match(/abstract_id=(\d+)/)
id = $1
filename = (doc/"//tr/td/table/tr/td/td/p/font").first.inner_text
filename.gsub!(/\.pdf/, '')
puts "linkout\tSSRN\t#{id}\t#{filename}\t\t"

puts "type\tJOUR"
puts "url\thttp://ssrn.com/abstract=#{id}"

title = (doc/"//td[@width='80%', @align='center']/font/strong").first.inner_text.strip
puts "title\t" + title

for author in (doc/"//td[@width='80%']/center//a.textlink")
  puts "author\t" + author.inner_text.strip
end

doicell = (doc/"//tr/td/font/blockquote/font/a").first
if doicell
	doi = doicell.inner_text
	puts "doi\t#{doi}"
end

puts "journal\tSocial Science Research Network Working Paper Series"

date = (doc/"//tr/td[@width='80%']/center/font[@size='2']")[1]
if date_el
  date = date_el.inner_text
  if date =~ /^(January|February|March|April|May|June|July|August|September|October|November|December) ([0-9]{1,2}), ([0-9]{4}$)/
    puts "month\t" + $1
    puts "day\t" + $2
    puts "year\t" + $3
  elsif date =~/^(January|February|March|April|May|June|July|August|September|October|November|December) ([0-9]{4}$)/
    puts "month\t" + $1
    puts "year\t" + $2
  elsif date =~/^([0-9]{4})$/
    puts "year\t" + $1
  end
end

abs = (doc/"//tr/td/font")[5].inner_text.strip
puts "abstract\t#{abs}"

puts "end_tsv"

puts "status\tok"

{'fraction_non_alphanumeric': 0.2280640668523677, 'fraction_numerical': 0.0201949860724234, 'mean_word_length': 4.493307839388145, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1264438', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1221, 'n_words': 207}