<commit_before>module Presdocs
  class Document
    
    attr_reader :location, :title, :source, :president, :date, :package_id, :lat, :lng, :subjects, :category, :notes, :fdsys_url, :html
   
    def initialize(params={})
      params.each_pair do |k,v|
       instance_variable_set("@#{k}", v)
      end
    end
    
    def self.detail(package_id)
      url = "http://m.gpo.gov/wscpd/mobilecpd/detailwgc/#{package_id}.json"
      result = Oj.load(open(url).read)
      create_document(result, full=true)
    end
    
    def self.latest
      url = "http://m.gpo.gov/wscpd/mobilecpd/home.json"
      results = Oj.load(open(url).read)
      create_from_search_results(results['searchResults'], nil)
    end
    
    def self.with_locations
      url = "http://m.gpo.gov/wscpd/mobilecpd/location.json"
      results = Oj.load(open(url).read)
      create_from_search_results(results['searchResults'], results['coordinates'])
    end
    
    def self.create_from_search_results(results, coordinates)
      docs = []
      results.each do |result|
        docs << create_document(result)
      end
      docs
    end
    
    def self.create_document(result, full=false)
      if full
        detail = result
        result = result['searchResults']
      end
      city, state = result['location'].split(', ')
      if coordinates
        locations = coordinates.map{|l| {"state" => l['state'], "city" => l['city'], "lat" => l["lat"], "lng" => l["lang"]}}.uniq
        lat = locations.detect{|l| l['city'] == city && l['state'] == state}['lat']
        lng = locations.detect{|l| l['city'] == city && l['state'] == state}['lng']
      end
      doc = self.new(:id => result['packageId'],
                :city => city,
                :state => state,
                :lat => lat,
                :lng => lng,
                :title => result['line1'],
                :source => result['line2'],
                :president => result['president'],
                :date => Date.parse(result['eventDate']))
      
      if full
        h = {:category => detail['category'], :notes => detail['notes'], :subjects => detail['subject'], :fdsys_url => detail['fdsysUrl'], :html => detail['fullText']}
        doc.merge(h)
      end
      doc
    end
  end
end<commit_msg>fix create document for detail<commit_after>module Presdocs
  class Document
    
    attr_reader :location, :title, :source, :president, :date, :package_id, :lat, :lng, :subjects, :category, :notes, :fdsys_url, :html
   
    def initialize(params={})
      params.each_pair do |k,v|
       instance_variable_set("@#{k}", v)
      end
    end
    
    def self.detail(package_id)
      url = "http://m.gpo.gov/wscpd/mobilecpd/detailwgc/#{package_id}.json"
      result = Oj.load(open(url).read)
      create_document(result, nil, full=true)
    end
    
    def self.latest
      url = "http://m.gpo.gov/wscpd/mobilecpd/home.json"
      results = Oj.load(open(url).read)
      create_from_search_results(results['searchResults'], nil)
    end
    
    def self.with_locations
      url = "http://m.gpo.gov/wscpd/mobilecpd/location.json"
      results = Oj.load(open(url).read)
      create_from_search_results(results['searchResults'], results['coordinates'])
    end
    
    def self.create_from_search_results(results, coordinates)
      docs = []
      results.each do |result|
        docs << create_document(result, coordinates)
      end
      docs
    end
    
    def self.create_document(result, coordinates, full=false)
      if full
        detail = result
        result = result['searchResults']
      end
      city, state = result['location'].split(', ')
      if coordinates
        locations = coordinates.map{|l| {"state" => l['state'], "city" => l['city'], "lat" => l["lat"], "lng" => l["lang"]}}.uniq
        lat = locations.detect{|l| l['city'] == city && l['state'] == state}['lat']
        lng = locations.detect{|l| l['city'] == city && l['state'] == state}['lng']
      end
      doc = self.new(:id => result['packageId'],
                :city => city,
                :state => state,
                :lat => lat,
                :lng => lng,
                :title => result['line1'],
                :source => result['line2'],
                :president => result['president'],
                :date => Date.parse(result['eventDate']))
      
      if full
        h = {:category => detail['category'], :notes => detail['notes'], :subjects => detail['subject'], :fdsys_url => detail['fdsysUrl'], :html => detail['fullText']}
        doc.merge(h)
      end
      doc
    end
  end
end
{'fraction_non_alphanumeric': 0.16222707423580787, 'fraction_numerical': 0.0008733624454148472, 'mean_word_length': 2.403417533432392, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1274198', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1428, 'n_words': 290}