<commit_before>class PlanningApplicationsProcessor

require 'net/http'
require 'tempfile'
require 'csv'
require 'progressbar'

  def self.process_planning_applications
    processor = self.new
    processor.run
  end

  def run
    begin
      #fetch_zipfile
      #unzip_file
      process_csv
    ensure
      cleanup
    end
  end

  def fetch_zipfile
    # Ensure that we don't have the file in memory when downloading
    uri = URI("http://thunderflames.org/temp/planning_applications.zip")
    @tempfile = Tempfile.new('cyclekit-zip')
    @tempfile.binmode

    Net::HTTP.start(uri.host, uri.port) do |http|
      request = Net::HTTP::Get.new uri.request_uri

      http.request request do |response|
        response.read_body do |chunk|
          @tempfile.write chunk
        end
      end
    end
    @tempfile.close
  end

    def unzip_file
      @tempfile.open
      @csv_path = Tempfile.new('cyclekit-unzip').path
      system("unzip -p #{@tempfile.path} > #{@csv_path}")
    end

  # The biggest problem that we have with processing the csv is that it all comes
  # as one giant updated file. We need to check every record to see if we have that
  # report already, and if we do, we still need to update the details using the csv.
  # Receiving daily files, with only the records that have been updated that day,
  # would be ideal.

  def process_csv
    @csv_path = '/home/andy/temp/cyclestreets/planning applications/planning_applications.head10k.csv'
<commit_msg>Whinge a little more, for emphasis.<commit_after>class PlanningApplicationsProcessor

require 'net/http'
require 'tempfile'
require 'csv'
require 'progressbar'

  def self.process_planning_applications
    processor = self.new
    processor.run
  end

  def run
    begin
      #fetch_zipfile
      #unzip_file
      process_csv
    ensure
      cleanup
    end
  end

  def fetch_zipfile
    # Ensure that we don't have the file in memory when downloading
    uri = URI("http://thunderflames.org/temp/planning_applications.zip")
    @tempfile = Tempfile.new('cyclekit-zip')
    @tempfile.binmode

    Net::HTTP.start(uri.host, uri.port) do |http|
      request = Net::HTTP::Get.new uri.request_uri

      http.request request do |response|
        response.read_body do |chunk|
          @tempfile.write chunk
        end
      end
    end
    @tempfile.close
  end

    def unzip_file
      @tempfile.open
      @csv_path = Tempfile.new('cyclekit-unzip').path
      system("unzip -p #{@tempfile.path} > #{@csv_path}")
    end

  # The biggest problem that we have with processing the csv is that it all comes
  # as one giant updated file. We need to check every record to see if we have that
  # report already, and if we do, we still need to update the details using the csv.
  # Receiving daily files, with only the records that have been updated that day,
  # would be ideal.
  # As the number of planning applications stored with openlylocal increases over time
  # (the file contains all historical applications) then not only does the database
  # size increase, but the effort to process each update does too.

  def process_csv
    @csv_path = '/home/andy/temp/cyclestreets/planning applications/planning_applications.head10k.csv'

{'fraction_non_alphanumeric': 0.07661290322580645, 'fraction_numerical': 0.0012406947890818859, 'mean_word_length': 3.021197007481297, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1870478', 'n_tokens_mistral': 975, 'n_tokens_neox': 949, 'n_words': 328}