<commit_before>#!/usr/bin/ruby

require 'libxml'
require 'libxslt'

include LibXML
include LibXSLT

ISO_IMPL_DIR = 'iso_impl'
ISO_FILES = [ 'iso_dsdl_include.xsl',
              'iso_abstract_expand.xsl',
              'iso_svrl.xsl' ]

# Tell the parse to remember the line numbers for each node
XML::Parser::default_line_numbers = true

# Get sch and xml from command line
schema_doc = XML::Document.file ARGV[0]
instance_doc = XML::Document.file ARGV[1]

# Compile schematron into xsl
validator_xsl = Dir.chdir ISO_IMPL_DIR do

  xforms = ISO_FILES.map do |file|
    doc = XML::Document.file file
    xsl = XSLT::Stylesheet.new doc
  end

  validator_doc = xforms.inject(schema_doc) { |xml, xsl| xsl.apply xml }
  XSLT::Stylesheet.new validator_doc
end

# Validate the xml with the
results_doc = validator_xsl.apply instance_doc

errors = []
results_doc.root.find('//svrl:failed-assert', 'svrl' => 'http://purl.oclc.org/dsdl/svrl').each do |assert|
  context = instance_doc.root.find_first assert['location']

  assert.find('svrl:text/text()', 'svrl' => 'http://purl.oclc.org/dsdl/svrl').each do |message|
    errors << '%s "%s" on line %d: %s' % [ context.node_type_name,
                                           context.name, 
                                           context.line_num,
                                           message.content.strip ]
  end

end

errors.each do |error|
  puts error
end
<commit_msg>niceley commented<commit_after>#!/usr/bin/ruby

require 'libxml'
require 'libxslt'

include LibXML
include LibXSLT

# The location of the ISO schematron implemtation lives
ISO_IMPL_DIR = 'iso_impl'

# The file names of the compilation stages
ISO_FILES = [ 'iso_dsdl_include.xsl',
              'iso_abstract_expand.xsl',
              'iso_svrl.xsl' ]

# Namespace prefix declarations for use in XPaths
NS_PREFIXES = { 
  'svrl' => 'http://purl.oclc.org/dsdl/svrl' 
}

# Tell the parse to remember the line numbers for each node
XML::Parser::default_line_numbers = true

# Get sch and xml from command line
schema_doc = XML::Document.file ARGV[0]
instance_doc = XML::Document.file ARGV[1]

# Compile schematron into xsl that maps to svrl
xforms = ISO_FILES.map do |file|

  Dir.chdir(ISO_IMPL_DIR) do
    doc = XML::Document.file file
    XSLT::Stylesheet.new doc
  end

end

validator_doc = xforms.inject(schema_doc) { |xml, xsl| xsl.apply xml }
validator_xsl = XSLT::Stylesheet.new validator_doc

# Validate the xml
results_doc = validator_xsl.apply instance_doc

# Print the errors
results_doc.root.find('//svrl:failed-assert', NS_PREFIXES).each do |assert|
  context = instance_doc.root.find_first assert['location']

  assert.find('svrl:text/text()', NS_PREFIXES).each do |message|
    puts '%s "%s" on line %d: %s' % [ context.node_type_name,
                                      context.name, 
                                      context.line_num,
                                      message.content.strip ]
  end

end

{'fraction_non_alphanumeric': 0.10855596888738586, 'fraction_numerical': 0.0013527223537368955, 'mean_word_length': 2.8666666666666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2705417', 'n_tokens_mistral': 1006, 'n_tokens_neox': 977, 'n_words': 240}