<commit_before># encoding: utf-8
require 'cldr'
require 'cldr/download'
require 'cldr/export/data'
require 'cldr/export/data/timezones'
require 'pathname'

module TimezoneParser
    # CLDR module
    module CLDR

        # Data directory
        DataDir = Pathname.new(Cldr::Export::Data.dir)

        protected
        @@Version = nil

        public
        def self.download(source = 'http://unicode.org/Public/cldr/latest/core.zip', target = nil)
            Cldr.download(source, target)
        end

        def self.updateHash(hash, name, data)
            hash[name] ||= []
            hash[name] << data.to_s
            hash[name].uniq!
            hash[name].sort!
            hash
        end

        def self.getVersion(source = DataDir)
            return @@Version if @@Version
            content = File.read(source + 'dtd' + 'ldml.dtd')
            content.gsub!(/<!--.*?-->/, '')
            data = content.match(/\s+cldrVersion\s+[\#\w\s]+\s+"(\d+)"\s*\>/)
            @@Version = data[1].to_i if data
            @@Version
        end

        def self.getTimezones
            timezones = { }
            Cldr::Export::Data.locales.sort.each do |locale|<commit_msg>Fix CLDR version obtaining<commit_after># encoding: utf-8
require 'cldr'
require 'cldr/download'
require 'cldr/export/data'
require 'cldr/export/data/timezones'
require 'pathname'

module TimezoneParser
    # CLDR module
    module CLDR

        # Data directory
        DataDir = Pathname.new(Cldr::Export::Data.dir)

        protected
        @@Version = nil

        public
        def self.download(source = 'http://unicode.org/Public/cldr/latest/core.zip', target = nil)
            Cldr.download(source, target)
        end

        def self.updateHash(hash, name, data)
            hash[name] ||= []
            hash[name] << data.to_s
            hash[name].uniq!
            hash[name].sort!
            hash
        end

        def self.getVersion(source = DataDir)
            return @@Version if @@Version
            content = File.read(source + 'dtd' + 'ldml.dtd')
            content.gsub!(/<!--.*?-->/, '')
            data = content.match(/\s+cldrVersion\s+[\#\w\s]+\s+"([\d\.]+)"\s*\>/)
            @@Version = data[1] if data
            @@Version
        end

        def self.getTimezones
            timezones = { }
            Cldr::Export::Data.locales.sort.each do |locale|
{'fraction_non_alphanumeric': 0.1437552388935457, 'fraction_numerical': 0.0016764459346186086, 'mean_word_length': 1.824852071005917, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5264512', 'n_tokens_mistral': 777, 'n_tokens_neox': 743, 'n_words': 146}