<commit_before># encoding: utf-8

require 'rexml/document'

module Twine
  module Formatters
    class Android < Abstract
      FORMAT_NAME = 'android'
      EXTENSION = '.xml'
      DEFAULT_FILE_NAME = 'strings.xml'
      LANG_CODES = Hash[
        'zh' => 'zh-Hans',
        'zh-rCN' => 'zh-Hans',
        'zh-rHK' => 'zh-Hant',
        'zh-rTW' => 'zh-TW',
        'en-rGB' => 'en-UK',
        'fr-rCA' => 'fr-CA',
        'in' => 'id',
        'nb' => 'no'
        # TODO: spanish
      ]
      DEFAULT_LANG_CODES = Hash[
        'zh-TW' => 'zh-Hant' # if we don't have a zh-TW translation, try zh-Hant before en
      ]

      def self.can_handle_directory?(path)
        Dir.entries(path).any? { |item| /^values-.+$/.match(item) }
      end

      def default_file_name
        return DEFAULT_FILE_NAME
      end

      def determine_language_given_path(path)
        path_arr = path.split(File::SEPARATOR)
        path_arr.each do |segment|
          match = /^values-(.*)$/.match(segment)
          if match
            lang = match[1]
            lang = LANG_CODES.fetch(lang, lang)
            lang.sub!('-r', '-')
            return lang
          end
        end

        return
      end

      def read_file(path, lang)
        File.open(path, 'r:UTF-8') do |f|<commit_msg>for android, put default language in values folder<commit_after># encoding: utf-8

require 'rexml/document'

module Twine
  module Formatters
    class Android < Abstract
      FORMAT_NAME = 'android'
      EXTENSION = '.xml'
      DEFAULT_FILE_NAME = 'strings.xml'
      LANG_CODES = Hash[
        'zh' => 'zh-Hans',
        'zh-rCN' => 'zh-Hans',
        'zh-rHK' => 'zh-Hant',
        'zh-rTW' => 'zh-TW',
        'en-rGB' => 'en-UK',
        'fr-rCA' => 'fr-CA',
        'in' => 'id',
        'nb' => 'no'
        # TODO: spanish
      ]
      DEFAULT_LANG_CODES = Hash[
        'zh-TW' => 'zh-Hant' # if we don't have a zh-TW translation, try zh-Hant before en
      ]

      def self.can_handle_directory?(path)
        Dir.entries(path).any? { |item| /^values.*$/.match(item) }
      end

      def default_file_name
        return DEFAULT_FILE_NAME
      end

      def determine_language_given_path(path)
        path_arr = path.split(File::SEPARATOR)
        path_arr.each do |segment|
          if segment == 'values'
            return @strings.language_codes[0]
          else
            match = /^values-(.*)$/.match(segment)
            if match
              lang = match[1]
              lang = LANG_CODES.fetch(lang, lang)
              lang.sub!('-r', '-')
              return lang
            end
          end
        end

        return
      end

      def read_file(path, lang)
        File.open(path, 'r:UTF-8') do |f|
{'fraction_non_alphanumeric': 0.1398243045387994, 'fraction_numerical': 0.002562225475841874, 'mean_word_length': 1.7005928853754941, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1984045', 'n_tokens_mistral': 948, 'n_tokens_neox': 923, 'n_words': 201}