<commit_before>require "kyotocabinet"

module ActiveRoad
  class OsmPbfImporter
    include KyotoCabinet

    attr_reader :parser, :database_path, :pbf_file, :progress_bar 
    
    # See for more details :  
    # http://wiki.openstreetmap.org/wiki/FR:France_roads_tagging
    # http://wiki.openstreetmap.org/wiki/FR:Cycleway
    # http://wiki.openstreetmap.org/wiki/Key:railway
    @@tag_for_car_values = %w{motorway trunk trunk_link primary secondary tertiary motorway_link primary_link unclassified service road residential track}
    cattr_reader :tag_for_car_values

    @@tag_for_pedestrian_values = %w{pedestrian footway path steps}
    cattr_reader :tag_for_pedestrian_values

    @@tag_for_bike_values = ActiveRoad::OsmPbfImporter.tag_for_pedestrian_values + ["cycleway"]
    cattr_reader :tag_for_bike_values

    @@tag_for_train_values = %w{rail tram funicular light_rail subway}
    cattr_reader :tag_for_train_values

    @@kc_batch_size = 500000
    cattr_reader :kc_batch_size
    @@pg_batch_size = 100000 # Not puts a high value because postgres failed to allocate memory 
    cattr_reader :pg_batch_size

    def initialize(pbf_file, database_path = "/home/luc/osm_pbf.kch")
      @pbf_file = pbf_file
      @database_path = database_path
    end

    def parser
      @parser ||= PbfParser.new(pbf_file)
    end   

    def progress_bar
      @progress_bar ||= ::ProgressBar.create(:format => '%a |%b>>%i| %p%% %t', :total => 100)
    end

    def database
      @database ||= DB::new
    end
    
    def open_database(path)
      database.open(path + "#apow=8#opts=l#bnum=835737117#msiz=6442450944", DB::OWRITER | DB::OCREATE)
      database.clear
    end
    <commit_msg>Fix file path for osm pbf file<commit_after>require "kyotocabinet"

module ActiveRoad
  class OsmPbfImporter
    include KyotoCabinet

    attr_reader :parser, :database_path, :pbf_file, :progress_bar 
    
    # See for more details :  
    # http://wiki.openstreetmap.org/wiki/FR:France_roads_tagging
    # http://wiki.openstreetmap.org/wiki/FR:Cycleway
    # http://wiki.openstreetmap.org/wiki/Key:railway
    @@tag_for_car_values = %w{motorway trunk trunk_link primary secondary tertiary motorway_link primary_link unclassified service road residential track}
    cattr_reader :tag_for_car_values

    @@tag_for_pedestrian_values = %w{pedestrian footway path steps}
    cattr_reader :tag_for_pedestrian_values

    @@tag_for_bike_values = ActiveRoad::OsmPbfImporter.tag_for_pedestrian_values + ["cycleway"]
    cattr_reader :tag_for_bike_values

    @@tag_for_train_values = %w{rail tram funicular light_rail subway}
    cattr_reader :tag_for_train_values

    @@kc_batch_size = 500000
    cattr_reader :kc_batch_size
    @@pg_batch_size = 100000 # Not puts a high value because postgres failed to allocate memory 
    cattr_reader :pg_batch_size

    def initialize(pbf_file, database_path = "/tmp/osm_pbf.kch")
      @pbf_file = pbf_file
      @database_path = database_path
    end

    def parser
      @parser ||= PbfParser.new(pbf_file)
    end   

    def progress_bar
      @progress_bar ||= ::ProgressBar.create(:format => '%a |%b>>%i| %p%% %t', :total => 100)
    end

    def database
      @database ||= DB::new
    end
    
    def open_database(path)
      database.open(path + "#apow=8#opts=l#bnum=835737117#msiz=6442450944", DB::OWRITER | DB::OCREATE)
      database.clear
    end
    
{'fraction_non_alphanumeric': 0.09505591524426132, 'fraction_numerical': 0.020600353148911125, 'mean_word_length': 3.814447592067989, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2877895', 'n_tokens_mistral': 1276, 'n_tokens_neox': 1155, 'n_words': 232}