<commit_before>require 'rubygems'
require 'nokogiri'
require 'zip'
require 'rubyXL/hash'

module RubyXL

  class Parser
    attr_reader :data_only, :num_sheets
    @@parsed_column_hash ={}
    # converts cell string (such as "AA1") to matrix indices
    def Parser.convert_to_index(cell_string)
      index = [-1,-1]
      if(cell_string =~ /^([A-Z]+)(\d+)$/)

        one = $1
        row = $2.to_i - 1 #-1 for 0 indexing
        col = 0
        i = 0
        if @@parsed_column_hash[one].nil?
          two = one.reverse #because of 26^i calculation
          two.each_byte do |c|
            int_val = c - 64 #converts A to 1
            col += int_val * 26**(i)
            i=i+1
          end
          @@parsed_column_hash[one] = col
        else
          col = @@parsed_column_hash[one]
        end
        col -= 1 #zer0 index
        index[0] = row
        index[1] = col
      end
      return index
    end


    # data_only allows only the sheet data to be parsed, so as to speed up parsing
    # However, using this option will result in date-formatted cells being interpreted as numbers
    def Parser.parse(file_path, opts = {})

      # options handling
      @data_only = opts.is_a?(TrueClass)||!!opts[:data_only]
      skip_filename_check = !!opts[:skip_filename_check]

      files = Parser.decompress(file_path, skip_filename_check)
      wb = Parser.fill_workbook(file_path, files)

      if(files['sharedString'] != nil)
        sst = files['sharedString'].css('sst')

        # According to http://msdn.microsoft.com/en-us/library/office/gg278314.aspx,
        # these attributes may be either both missing, or both present. Need to validate.
        wb.shared_strings.count_attr = sst.attribute('count').value.to_i
        wb.shared_strings.unique_count_attr = sst.attribute('uniqueCount').value.to_i

        files['sharedString'].css('si').each do |node|
          unless node.css('r').empty?
            text = node.css('r t').children.to_a.join
            node.children.remove
            node << "<t xml:space=\"preserve\">#{text}</t>"
          end
        end

        string_nodes = files['sharedString'].css('si t')

        string_nodes.each_with_index do |node,i|
          wb.shared_strings.add(node.children.to_s, i)
        end
      end
      #styles are needed for formatting reasons as that is how dates are determined
        styles = files['styles'].css('cellXfs xf')
        style_hash = Hash.xml_node_to_hash(files['styles'].root)
        fill_styles(wb,style_hash)

        #will be nil if these files do not exist
        wb.external_links = files['externalLinks']
        wb.drawings = files['drawings']
        wb.printer_settings = files['printerSettings']<commit_msg>Issue #57: Ampersands are parsed as &amp; rather than &<commit_after>require 'rubygems'
require 'nokogiri'
require 'zip'
require 'rubyXL/hash'

module RubyXL

  class Parser
    attr_reader :data_only, :num_sheets
    @@parsed_column_hash ={}
    # converts cell string (such as "AA1") to matrix indices
    def Parser.convert_to_index(cell_string)
      index = [-1,-1]
      if(cell_string =~ /^([A-Z]+)(\d+)$/)

        one = $1
        row = $2.to_i - 1 #-1 for 0 indexing
        col = 0
        i = 0
        if @@parsed_column_hash[one].nil?
          two = one.reverse #because of 26^i calculation
          two.each_byte do |c|
            int_val = c - 64 #converts A to 1
            col += int_val * 26**(i)
            i=i+1
          end
          @@parsed_column_hash[one] = col
        else
          col = @@parsed_column_hash[one]
        end
        col -= 1 #zer0 index
        index[0] = row
        index[1] = col
      end
      return index
    end


    # data_only allows only the sheet data to be parsed, so as to speed up parsing
    # However, using this option will result in date-formatted cells being interpreted as numbers
    def Parser.parse(file_path, opts = {})

      # options handling
      @data_only = opts.is_a?(TrueClass)||!!opts[:data_only]
      skip_filename_check = !!opts[:skip_filename_check]

      files = Parser.decompress(file_path, skip_filename_check)
      wb = Parser.fill_workbook(file_path, files)

      shared_string_file = files['sharedString']
      unless shared_string_file.nil?
        sst = shared_string_file.css('sst')

        # According to http://msdn.microsoft.com/en-us/library/office/gg278314.aspx,
        # these attributes may be either both missing, or both present. Need to validate.
        wb.shared_strings.count_attr = sst.attribute('count').value.to_i
        wb.shared_strings.unique_count_attr = sst.attribute('uniqueCount').value.to_i

        # Note that the strings may contain text formatting, such as changing font color/properties
        # in the middle of the string. We do not support that in this gem... at least yet!
        # If you save the file, this formatting will be destoyed.
        shared_string_file.css('si').each_with_index { |node, i|
          wb.shared_strings.add(node.css('t').inject(''){ |s, c| s + c.text }, i)
        }

      end
      #styles are needed for formatting reasons as that is how dates are determined
        styles = files['styles'].css('cellXfs xf')
        style_hash = Hash.xml_node_to_hash(files['styles'].root)
        fill_styles(wb,style_hash)

        #will be nil if these files do not exist
        wb.external_links = files['externalLinks']
        wb.drawings = files['drawings']
        wb.printer_settings = files['printerSettings']
{'fraction_non_alphanumeric': 0.10981818181818181, 'fraction_numerical': 0.010545454545454545, 'mean_word_length': 2.438125, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2930670', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1686, 'n_words': 456}