<commit_before>require 'optparse'
require 'base64'
require 'nokogiri'
require 'ostruct'
require 'active_model'

class HippoFile
  include ActiveModel::Model
  attr_accessor :data
end

class HippoFileParser
  include ActiveModel::Model
  attr_accessor :file

  def data
    @data ||= ::Nokogiri::Slop(file)
  end

  def parse
    data.xpath('//sv:property[@sv:type="Binary"][@sv:name="jcr:data"]').map do |n|
      HippoFile.new(data: Base64.decode64(n.children.text))
    end
  end
end

options = OpenStruct.new

OptionParser.new do |opts|
  opts.on('--file [XML FILE]', 'Pass the Hippo xml asset file') do |filename|
    file = File.expand_path(filename)

    if File.exist? file
      options.file = File.read(file)
    else
      puts "File #{file} does not exist. :S"
    end
  end

  opts.on('-h', '--help', 'Show this message') do
    puts opts
    exit
  end
end.parse!

if options.file
  files =  HippoFileParser.new(file: options.file).parse
  files.each { |file| puts file.data }
else
  puts 'You need to pass the Hippo asset xml file with --file [FILE].'
end<commit_msg>Change data to blob<commit_after>require 'optparse'
require 'base64'
require 'nokogiri'
require 'ostruct'
require 'active_model'

class HippoFile
  include ActiveModel::Model
  attr_accessor :blob
end

class HippoFileParser
  include ActiveModel::Model
  attr_accessor :file

  def data
    @data ||= ::Nokogiri::Slop(file)
  end

  def parse
    xpath = '//sv:property[@sv:type="Binary"][@sv:name="jcr:data"]'

    data.xpath(xpath).map do |hippo_file|
      HippoFile.new(blob: Base64.decode64(hippo_file.children.text))
    end
  end
end

options = OpenStruct.new

OptionParser.new do |opts|
  opts.on('--file [XML FILE]', 'Pass the Hippo xml asset file') do |filename|
    file = File.expand_path(filename)

    if File.exist? file
      options.file = File.read(file)
    else
      puts "File #{file} does not exist. :S"
    end
  end

  opts.on('-h', '--help', 'Show this message') do
    puts opts
    exit
  end
end.parse!

if options.file
  files =  HippoFileParser.new(file: options.file).parse
  files.each { |file| puts file.blob }
else
  puts 'You need to pass the Hippo asset xml file with --file [FILE].'
end
{'fraction_non_alphanumeric': 0.12562358276643992, 'fraction_numerical': 0.005442176870748299, 'mean_word_length': 3.5204918032786887, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4970328', 'n_tokens_mistral': 796, 'n_tokens_neox': 780, 'n_words': 216}