<commit_before>#!/usr/bin/env jruby
# encoding: utf-8

require 'rubygems'
require 'date'
require 'fileutils'
require 'docsplit'
require 'tabula'
require 'httparty'

#require 'tmpdir'

def scrape(file_name, csvs_output)
    puts "Processing "+file_name
    local_dir = "data/pdfs/"
    grab(file_name, local_dir)
    FileUtils.mkdir_p(csvs_output)

    peel(file_name, csvs_output)
end

def grab(file_name, local_dir)
    pdfs_output = "data/burstpdfs/"

    if Dir.entries(local_dir).include?file_name
        Docsplit.extract_pages(local_dir+"/"+file_name, :output=>pdfs_output)
    else
        retrieve(file_name, local_dir, remote_dir)
    end
end

def retreive(file_name, local_dir, remote_dir)
    remote_dir = "http://www.njsp.org/info/pdf/ucr/current/"
	File.open(local_dir+file_name, 'wb') do |f|
		f.write HTTParty.get(remote_dir+file_name).parsed_response
    end
end

def peel(file_name, csvs_output)
    pdfs_dir = "data/burstpdfs/"
    counter = 0
    Dir.foreach(pdfs_dir) do |item|
        num_chars = file_name[0..-5].length
	    page_area = [97.55,11.19,573.48,778.62]
	    next if item.start_with?('.')
	    next if item.end_with?('.txt')
        next if item[0..num_chars-1]!=file_name[0..num_chars-1]
        
        extractor = Tabula::Extraction::ObjectExtractor.new(pdfs_dir+item, :all )
        extractor.extract.each_with_index do |pdf_page, page_index|
        
        first_csv = pdf_page.get_table.to_csv
        my_csv = CSV.parse(first_csv)
        if my_csv[0][-1]!= ""
            date = my_csv[0][-1]
        else
            date = my_csv[1][-1]
        end
        coverage = my_csv[4][-1].split('-')[1]
        pattern = /^(N.+)\s+Total/
        id = pattern.match(first_csv)[1].strip
        if id.downcase == "new jersey"
            id = "state"
        end

        release_date = Date.parse date
        covers = Date.parse coverage

        year_pattern = /.*(201\d)/
        if file_name.end_with?('stats.pdf')
            outfile = csvs_output + release_date.to_s + "-for-" + id + "-" + year_pattern.match(file_name)[1] + "annual.csv"
        else
            outfile = csvs_output + release_date.to_s + "-for-" + covers.to_s[0..-4] + "-" + id +".csv"
        end

        table_area = pdf_page.get_area(page_area).get_table.to_csv
        table_csv = CSV.parse(table_area)

        heds = ["crime", "month_last_yr", "month_this_yr", "pct_chg", "ytd_last", "ytd_this_yr", "ytd_pct_chg", "cleared", "pct_cleared", "juvie_cleared"]
        CSV.open(outfile,"wb") do |csv|
            csv << heds
            table_csv.each do |row|
                next if row[0].start_with?("Total")
                if row[-1] != ""
                    csv << row
                end
            end
        end
    extractor.close!
    end<commit_msg>removing unnecessary rows and cols. Closes #8<commit_after>#!/usr/bin/env jruby
# encoding: utf-8

require 'rubygems'
require 'date'
require 'fileutils'
require 'docsplit'
require 'tabula'
require 'httparty'

#require 'tmpdir'

def scrape(file_name, csvs_output)
    puts "Processing "+file_name
    local_dir = "data/pdfs/"
    grab(file_name, local_dir)
    FileUtils.mkdir_p(csvs_output)

    peel(file_name, csvs_output)
end

def grab(file_name, local_dir)
    pdfs_output = "data/burstpdfs/"

    if Dir.entries(local_dir).include?file_name
        Docsplit.extract_pages(local_dir+"/"+file_name, :output=>pdfs_output)
    else
        retrieve(file_name, local_dir, remote_dir)
    end
end

def retreive(file_name, local_dir, remote_dir)
    remote_dir = "http://www.njsp.org/info/pdf/ucr/current/"
	File.open(local_dir+file_name, 'wb') do |f|
		f.write HTTParty.get(remote_dir+file_name).parsed_response
    end
end

def peel(file_name, csvs_output)
    pdfs_dir = "data/burstpdfs/"
    counter = 0
    Dir.foreach(pdfs_dir) do |item|
        num_chars = file_name[0..-5].length
	    page_area = [97.55,11.19,573.48,778.62]
	    next if item.start_with?('.')
	    next if item.end_with?('.txt')
        next if item[0..num_chars-1]!=file_name[0..num_chars-1]
        
        extractor = Tabula::Extraction::ObjectExtractor.new(pdfs_dir+item, :all )
        extractor.extract.each_with_index do |pdf_page, page_index|
        
        first_csv = pdf_page.get_table.to_csv
        my_csv = CSV.parse(first_csv)
        if my_csv[0][-1]!= ""
            date = my_csv[0][-1]
        else
            date = my_csv[1][-1]
        end
        coverage = my_csv[4][-1].split('-')[1]
        pattern = /^(N.+)\s+Total/
        id = pattern.match(first_csv)[1].strip
        if id.downcase == "new jersey"
            id = "state"
        end

        release_date = Date.parse date
        covers = Date.parse coverage

        year_pattern = /.*(201\d)/
        if file_name.end_with?('stats.pdf')
            outfile = csvs_output + release_date.to_s + "-for-" + id + "-" + year_pattern.match(file_name)[1] + "annual.csv"
        else
            outfile = csvs_output + release_date.to_s + "-for-" + covers.to_s[0..-4] + "-" + id +".csv"
        end

        table_area = pdf_page.get_area(page_area).get_table.to_csv
        table_csv = CSV.parse(table_area)

        heds = ["crime", "month_last_yr", "month_this_yr", "ytd_last", "ytd_this_yr", "cleared", "juvie_cleared"]
        CSV.open(outfile,"wb") do |csv|
            csv << heds
            table_csv.each do |row|
                next if row[0].strip.end_with?("Total:")
                next if row[0].strip.end_with?("Index")
                next if row[0].strip.end_with?("Crime")
                if row[-1] != ""
                    if row.length>10
                        row = row.values_at(0,2,3,5,6,8,10)
                    else
                        row = row.values_at(0,1,2,4,5,7,9)
                    end
                    csv << row
                end
            end
        end
    extractor.close!
    end
{'fraction_non_alphanumeric': 0.13770212765957446, 'fraction_numerical': 0.018382978723404254, 'mean_word_length': 2.280848687883864, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1228598', 'n_tokens_mistral': 2203, 'n_tokens_neox': 2106, 'n_words': 370}