<commit_before>require 'yaml'
require 'open-uri'

module UrlDictionary
  class Loader

    @@site_cache = {}
    @@data_cache = nil

    class << self

      def load(site_key)
        @@site_cache[site_key.to_s.downcase] ||= load_dictionary(site_key)
      end

      def clear_caches
        @@site_cache = {}
        @@data_cache = nil
      end

      private

      def load_dictionary(site_key)
        if site_data = data[site_key.to_s.downcase]
          UrlDictionary::Dictionary.new(site_data)
        else
          fail ArgumentError, "No dictionary for site_key #{site_key}"
        end
      end

      def data
        @@data_cache ||= load_data_structure
      end

      def load_data_structure
        YAML.load(load_from_data_source)
      end

      def load_from_data_source
        if UrlDictionary::Config.use_local_dictionary?
          load_from_file
        else
          load_from_github
        end
      end

      def load_from_github
        open('https://raw.githubusercontent.com/lokalebasen/url-dictionary/master/lib/url_dictionary/data.yml').read
      rescue Exception => e
        puts "Oops, couldn't read latest dictionary. Falling back to bundled dictionary file: #{e.message}"
        load_from_file
      end

      def load_from_file
        File.read(path_to_file)
      end

      def path_to_file
        File.join(UrlDictionary.root_path, 'data.yml')
      end

    end

  end
end<commit_msg>Adds guard to use fallback if remote data source is readable, yet unparsable<commit_after>require 'yaml'
require 'open-uri'

module UrlDictionary
  class Loader

    @@site_cache = {}
    @@data_cache = nil

    class << self

      def load(site_key)
        @@site_cache[site_key.to_s.downcase] ||= load_dictionary(site_key)
      end

      def clear_caches
        @@site_cache = {}
        @@data_cache = nil
      end

      private

      def load_dictionary(site_key)
        if site_data = data[site_key.to_s.downcase]
          UrlDictionary::Dictionary.new(site_data)
        else
          fail ArgumentError, "No dictionary for site_key #{site_key}"
        end
      end

      def data
        @@data_cache ||= load_data_structure
      end

      def load_data_structure
        YAML.load(load_from_data_source)
      end

      def load_from_data_source
        if UrlDictionary::Config.use_local_dictionary?
          load_from_file
        else
          load_from_github
        end
      end

      def load_from_github
        yaml_string = open('https://raw.githubusercontent.com/lokalebasen/url-dictionary/master/lib/url_dictionary/data.yml').read
        YAML.load(yaml_string) # Triggers exception if response is bad
        yaml_string
      rescue Exception => e
        puts "Oops, couldn't read latest dictionary. Falling back to bundled dictionary file: #{e.message}"
        load_from_file
      end

      def load_from_file
        File.read(path_to_file)
      end

      def path_to_file
        File.join(UrlDictionary.root_path, 'data.yml')
      end

    end

  end
end
{'fraction_non_alphanumeric': 0.07414698162729659, 'fraction_numerical': 0.0, 'mean_word_length': 2.2264550264550262, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7433255', 'n_tokens_mistral': 976, 'n_tokens_neox': 914, 'n_words': 204}