<commit_before>require 'xlocalize/webtranslateit'
require 'xlocalize/xliff'
require 'colorize'
require 'nokogiri'
require 'plist'
require 'yaml'

module Xlocalize
  class Executor

    def plurals_file_name(locale)
      return locale_file_name(locale) << '_plurals.yml'
    end

    def locale_file_name(locale)
      return "#{locale}.xliff"
    end

    def export_master(wti, project, target, excl_prefix, master_lang)
      master_file_name = locale_file_name(master_lang)
      
      # hacky way to finish xcodebuild -exportLocalizations script, because
      # since Xcode7.3 & OS X Sierra script hangs even though it produces
      # xliff output
      # http://www.openradar.me/25857436
      File.delete(master_file_name) if File.exist?(master_file_name)
      system "xcodebuild -exportLocalizations -localizationPath ./ -project #{project} & sleep 0"
      while !File.exist?(master_file_name) do
        sleep(1)
      end

      purelyze(master_lang, target, excl_prefix, project)
      push_master_file(wti, master_lang, master_file_name)
    end

    def push_master_file(wti, master_lang, master_file_name)
      # Pushing master file to WebtranslateIt
      begin
        puts "Uploading master file to WebtranslateIt"
        file = File.open(master_file_name, 'r')
        plurals_path = plurals_file_name(master_lang)
        plurals_file = File.exist?(plurals_path) ? File.open(plurals_path, 'r') : nil
        wti.push_master(file, plurals_file)
        puts "Done.".green
      rescue => err
        puts err.to_s.red
      ensure
        file.close unless file.nil?
        plurals_file.close unless plurals_file.nil?
      end if !wti.nil?
    end

    def purelyze(locale, target, excl_prefix, project)
      locale_file_name = locale_file_name(locale)
      doc = Nokogiri::XML(open(locale_file_name))

      puts "Removing all files not matching required targets" if $VERBOSE
      doc.filter_not_target_files(target)
      puts "Removing trans-unit's having reserverd prefix in their sources" if $VERBOSE
      doc.filter_trans_units(excl_prefix)
      puts "Filtering plurals" if $VERBOSE
      plurals = doc.filter_plurals(project)
      puts "Removing all files having no trans-unit elements after removal" if $VERBOSE
      doc.filter_empty_files

      puts "Writing modified XLIFF file to #{locale_file_name}" if $VERBOSE
      File.open(locale_file_name, 'w') { |f| f.write(doc.to_xml) }
      if !plurals.empty?
        puts "Writing plurals to plurals YAML file" if $VERBOSE
        File.open(plurals_file_name(locale), 'w') { |f| f.write({locale => plurals}.to_yaml) }
      end
    end

    def out_list_of_translations_of_locale(wti, locale, translations)
      puts "Downloading translations for #{locale}"
      translations = wti.pull(locale)
      plurals_content = translations['plurals']

      out_list = [{
        "path" => "#{locale}.xliff",
        "content" => translations['xliff']
      }]
      out_list << {
        "path" => "#{locale}_plurals.yaml",
        "content" => plurals_content
      } if not plurals_content.nil?

      return out_list
    end

    def download(wti, locales)
      begin
        locales.each do |locale|
          puts "Downloading translations for #{locale}"
          translations = wti.pull(locale)
          
          out_list_of_translations_of_locale(wti, locale, translations).each do |out|
            File.open(out["path"], "w") do |file|
              file.write(out["content"])
              puts "Done saving #{out['path']}.".green
            end
          end
        end
      rescue => err
        puts err.to_s.red
      end
    end

    def filename_from_xliff_provided_filename(file_name)
      parts = file_name.split('/')
      name = ""
      parts.each_with_index do |part, idx|
        name += "/" if idx > 0
        if part.end_with?(".lproj")
          name += "#{locale}.lproj"
        elsif idx+1 == parts.count
          # TODO: join all parts till the last '.'
          name += "#{part.split('.')[0]}.strings"
        else
          name += part
        end
      end
      return name
    end

    def import_xliff(locale)
      Nokogiri::XML(open("#{locale}.xliff")).xpath("//xmlns:file").each do |node|
        File.open(filename_from_xliff_provided_filename(node["original"]), "w") do |file|
          (node > "body > trans-unit").each do |trans_unit|
            key = trans_unit["id"]
            target = (trans_unit > "target").text
            note = (trans_unit > "note").text
            note = "(No Commment)" if note.length <= 0
            
            file.write "/* #{note} */\n"
            file.write "\"#{key}\" = #{target.inspect};\n\n"
          end
        end
      end
    end

    def import_plurals_if_needed(locale)
      plurals_fname = "#{locale}_plurals.yaml"
      return if !File.exist?(plurals_fname)
      plurals_yml = YAML.load_file(plurals_fname)
      plurals_yml[locale].each do |fname, trans_units|
        content = {}
        trans_units.each do |key, vals|
          content[key] = {
            "NSStringLocalizedFormatKey" => "%\#@value@",
            "value" => vals.merge({
              "NSStringFormatSpecTypeKey" => "NSStringPluralRuleType",
              "NSStringFormatValueTypeKey" => "d"
            })
          }<commit_msg>Constructing import file name from locale<commit_after>require 'xlocalize/webtranslateit'
require 'xlocalize/xliff'
require 'colorize'
require 'nokogiri'
require 'plist'
require 'yaml'

module Xlocalize
  class Executor

    def plurals_file_name(locale)
      return locale_file_name(locale) << '_plurals.yml'
    end

    def locale_file_name(locale)
      return "#{locale}.xliff"
    end

    def export_master(wti, project, target, excl_prefix, master_lang)
      master_file_name = locale_file_name(master_lang)
      
      # hacky way to finish xcodebuild -exportLocalizations script, because
      # since Xcode7.3 & OS X Sierra script hangs even though it produces
      # xliff output
      # http://www.openradar.me/25857436
      File.delete(master_file_name) if File.exist?(master_file_name)
      system "xcodebuild -exportLocalizations -localizationPath ./ -project #{project} & sleep 0"
      while !File.exist?(master_file_name) do
        sleep(1)
      end

      purelyze(master_lang, target, excl_prefix, project)
      push_master_file(wti, master_lang, master_file_name)
    end

    def push_master_file(wti, master_lang, master_file_name)
      # Pushing master file to WebtranslateIt
      begin
        puts "Uploading master file to WebtranslateIt"
        file = File.open(master_file_name, 'r')
        plurals_path = plurals_file_name(master_lang)
        plurals_file = File.exist?(plurals_path) ? File.open(plurals_path, 'r') : nil
        wti.push_master(file, plurals_file)
        puts "Done.".green
      rescue => err
        puts err.to_s.red
      ensure
        file.close unless file.nil?
        plurals_file.close unless plurals_file.nil?
      end if !wti.nil?
    end

    def purelyze(locale, target, excl_prefix, project)
      locale_file_name = locale_file_name(locale)
      doc = Nokogiri::XML(open(locale_file_name))

      puts "Removing all files not matching required targets" if $VERBOSE
      doc.filter_not_target_files(target)
      puts "Removing trans-unit's having reserverd prefix in their sources" if $VERBOSE
      doc.filter_trans_units(excl_prefix)
      puts "Filtering plurals" if $VERBOSE
      plurals = doc.filter_plurals(project)
      puts "Removing all files having no trans-unit elements after removal" if $VERBOSE
      doc.filter_empty_files

      puts "Writing modified XLIFF file to #{locale_file_name}" if $VERBOSE
      File.open(locale_file_name, 'w') { |f| f.write(doc.to_xml) }
      if !plurals.empty?
        puts "Writing plurals to plurals YAML file" if $VERBOSE
        File.open(plurals_file_name(locale), 'w') { |f| f.write({locale => plurals}.to_yaml) }
      end
    end

    def out_list_of_translations_of_locale(wti, locale, translations)
      puts "Downloading translations for #{locale}"
      translations = wti.pull(locale)
      plurals_content = translations['plurals']

      out_list = [{
        "path" => "#{locale}.xliff",
        "content" => translations['xliff']
      }]
      out_list << {
        "path" => "#{locale}_plurals.yaml",
        "content" => plurals_content
      } if not plurals_content.nil?

      return out_list
    end

    def download(wti, locales)
      begin
        locales.each do |locale|
          puts "Downloading translations for #{locale}"
          translations = wti.pull(locale)
          
          out_list_of_translations_of_locale(wti, locale, translations).each do |out|
            File.open(out["path"], "w") do |file|
              file.write(out["content"])
              puts "Done saving #{out['path']}.".green
            end
          end
        end
      rescue => err
        puts err.to_s.red
      end
    end

    def filename_from_xliff_provided_filename(file_name, locale)
      parts = file_name.split('/')
      name = ""
      parts.each_with_index do |part, idx|
        name += "/" if idx > 0
        if part.end_with?(".lproj")
          name += "#{locale}.lproj"
        elsif idx+1 == parts.count
          # TODO: join all parts till the last '.'
          name += "#{part.split('.')[0]}.strings"
        else
          name += part
        end
      end
      return name
    end

    def import_xliff(locale)
      Nokogiri::XML(open("#{locale}.xliff")).xpath("//xmlns:file").each do |node|
        File.open(filename_from_xliff_provided_filename(node["original"], locale), "w") do |file|
          (node > "body > trans-unit").each do |trans_unit|
            key = trans_unit["id"]
            target = (trans_unit > "target").text
            note = (trans_unit > "note").text
            note = "(No Commment)" if note.length <= 0
            
            file.write "/* #{note} */\n"
            file.write "\"#{key}\" = #{target.inspect};\n\n"
          end
        end
      end
    end

    def import_plurals_if_needed(locale)
      plurals_fname = "#{locale}_plurals.yaml"
      return if !File.exist?(plurals_fname)
      plurals_yml = YAML.load_file(plurals_fname)
      plurals_yml[locale].each do |fname, trans_units|
        content = {}
        trans_units.each do |key, vals|
          content[key] = {
            "NSStringLocalizedFormatKey" => "%\#@value@",
            "value" => vals.merge({
              "NSStringFormatSpecTypeKey" => "NSStringPluralRuleType",
              "NSStringFormatValueTypeKey" => "d"
            })
          }
{'fraction_non_alphanumeric': 0.1104847801578354, 'fraction_numerical': 0.003006388575723412, 'mean_word_length': 2.4742167101827675, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492593', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3376, 'n_words': 761}