<commit_before>require "nokogiri"
require "savage"

module Hieroglyph

  class Glyph

    attr_accessor :name

    SHAPE_HANDLERS = {
      "circle" => "report_invalid",
      "ellipse" => "report_invalid",
      "line" => "report_invalid",
      "polyline" => "report_invalid",
      "rect" => "report_invalid",
      "polygon" => "convert_polygon",
      "path" => "convert_path"
    }

    NAME_REGEX = /^.*?(?=(-|\.))/

    @@too_many_shapes = false

    def initialize(file, source)
      @name = file.gsub(source, "").gsub("/", "").match(NAME_REGEX)
      puts @name
      @contents = Nokogiri::XML(File.new(file))
      @path =
      Hieroglyph.log "#{@name} -> reading...", 4
      @path = parse_shapes
    end

    def parse_shapes<commit_msg>Hieroglyph can generate private unicode characters<commit_after>require "nokogiri"
require "savage"

module Hieroglyph

  class Glyph

    attr_accessor :name

    SHAPE_HANDLERS = {
      "circle" => "report_invalid",
      "ellipse" => "report_invalid",
      "line" => "report_invalid",
      "polyline" => "report_invalid",
      "rect" => "report_invalid",
      "polygon" => "convert_polygon",
      "path" => "convert_path"
    }

    NAME_REGEX = /^.*?(?=(-|\.))/

    @@too_many_shapes = false

    def initialize(file, source)
      @name = file.gsub(source, "").gsub("/", "").match(NAME_REGEX)
      @contents = Nokogiri::XML(File.new(file))
      @path =
      Hieroglyph.log "#{@name} -> reading...", 4
      @path = parse_shapes
    end

    def parse_shapes
{'fraction_non_alphanumeric': 0.16863517060367453, 'fraction_numerical': 0.0013123359580052493, 'mean_word_length': 2.7935323383084576, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3675278', 'n_tokens_mistral': 555, 'n_tokens_neox': 532, 'n_words': 100}