<commit_before># -*- coding: utf-8 -*-
require 'nokogiri'
require 'scanf'
require 'pp'

xml = Nokogiri::XML(open(File.join(File.dirname(__FILE__), 'emoji4unicode.xml')).read)

google_docomo    = []
google_kddi      = []
google_softbank  = []
unicode_docomo   = []
unicode_kddi     = []
unicode_softbank = []

xml.xpath('//e').each do |emoji|
  # keycode
  google   = emoji.attributes['google'].value   rescue nil
  unicode  = emoji.attributes['unicode'].value  rescue nil

  # carrier code
  docomo   = emoji.attributes['docomo'].value   rescue '3013'
  softbank = emoji.attributes['softbank'].value rescue '3013'
  kddi     = emoji.attributes['kddi'].value     rescue '3013'

  # fix google key
  google = nil if google and google.match(/[^0-9a-fA-F\+]/)
  # fix unicode key
  if unicode
    if unicode.match(/\+/)
      unicodes = unicode.split(/\+/).delete_if{|e| e.strip == ''}
      if unicodes.size == 1
        unicode = '0x%s' % unicodes.first
      else
        unicode = '[' + unicodes.map{|e| '0x%s' % e.gsub(/[^0-9a-fA-F\+]/, '')}.join(', ') + ']'
      end
    else
      unicode = '0x%s' % unicode
    end
  end

  # fix docomo code
  if docomo.match(/\+/)
    docomo = '"' + docomo.split('+').map{|e| '&#x%s;' % e.gsub(/[^0-9a-fA-F\+]/, '')}.join('') + '"'
  else
    docomo = '0x%s' % docomo.gsub(/[^0-9a-fA-F]/, '')
  end
  # fix softbank code
  if softbank.match(/\+/)
    softbank = '"' + softbank.split('+').delete_if{|e| e.strip == ''}.map{|s| '&#x%X;' % (s.gsub(/[^0-9a-fA-F\+]/, '').scanf("%x").first + 0x1000)}.join(',') + '"'
  else
    if softbank.match(/3013/)
      softbank = '0x%s' % softbank
    else
      softbank = '0x%X' % (softbank.gsub(/[^0-9a-fA-F]/, '').scanf('%x').first + 0x1000)
    end
  end
  # fix kddi code
  if kddi.match(/\+/)
    kddi = '"' + kddi.split('+').map{|e| '&#x%s;' % e.gsub(/[^0-9a-fA-F\+]/, '')}.join('') + '"'
  else
    kddi = '0x%s' % kddi.gsub(/[^0-9a-fA-F]/, '')
  end

  if google
    google_docomo   << [google, docomo]
    google_kddi     << [google, kddi]
    google_softbank << [google, softbank]
  end

  if unicode<commit_msg>Fix table creation tool<commit_after># -*- coding: utf-8 -*-
require 'nokogiri'
require 'scanf'
require 'pp'

xml = Nokogiri::XML(open(File.join(File.dirname(__FILE__), 'emoji4unicode.xml')).read)

google_docomo    = []
google_kddi      = []
google_softbank  = []
unicode_docomo   = []
unicode_kddi     = []
unicode_softbank = []

xml.xpath('//e').each do |emoji|
  # keycode
  google   = emoji.attributes['google'].value   rescue nil
  unicode  = emoji.attributes['unicode'].value  rescue nil

  # carrier code
  docomo   = emoji.attributes['docomo'].value   rescue '3013'
  softbank = emoji.attributes['softbank'].value rescue '3013'
  kddi     = emoji.attributes['kddi'].value     rescue '3013'

  # fix google key
  google = nil if google and google.match(/[^0-9a-fA-F\+]/)
  # fix unicode key
  if unicode
    if unicode.match(/\+/)
      unicodes = unicode.split(/\+/).delete_if{|e| e.strip == ''}
      if unicodes.size == 1
        unicode = '0x%s' % unicodes.first
      else
        unicode = '[' + unicodes.map{|e| '0x%s' % e.gsub(/[^0-9a-fA-F\+]/, '')}.join(', ') + ']'
      end
    else
      unicode = '0x%s' % unicode
    end
  end

  # fix docomo code
  if docomo.match(/\+/)
    docomo = '"' + docomo.split('+').map{|e| '&#x%s;' % e.gsub(/[^0-9a-fA-F\+]/, '')}.join('') + '"'
  else
    docomo = '0x%s' % docomo.gsub(/[^0-9a-fA-F]/, '')
  end
  # fix softbank code
  if softbank.match(/\+/)
    softbank = '"' + softbank.split('+').delete_if{|e| e.strip == ''}.map{|s| '&#x%X;' % (s.gsub(/[^0-9a-fA-F\+]/, '').scanf("%x").first + 0x1000)}.join('') + '"'
  else
    if softbank.match(/3013/)
      softbank = '0x%s' % softbank
    else
      softbank = '0x%X' % (softbank.gsub(/[^0-9a-fA-F]/, '').scanf('%x').first + 0x1000)
    end
  end
  # fix kddi code
  if kddi.match(/\+/)
    kddi = '"' + kddi.split('+').map{|e| '&#x%s;' % e.gsub(/[^0-9a-fA-F\+]/, '')}.join('') + '"'
  else
    kddi = '0x%s' % kddi.gsub(/[^0-9a-fA-F]/, '')
  end

  if google
    google_docomo   << [google, docomo]
    google_kddi     << [google, kddi]
    google_softbank << [google, softbank]
  end

  if unicode
{'fraction_non_alphanumeric': 0.22937960541953886, 'fraction_numerical': 0.024720703589256002, 'mean_word_length': 2.717314487632509, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1022819', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1774, 'n_words': 308}