<commit_before>require_relative "item_factory/content_parser"

module Munge
  module Core
    class ItemFactory
      def initialize(text_extensions:,
                     ignore_extensions:)
        @text_extensions   = Set.new(text_extensions)
        @ignore_extensions = ignore_extensions
      end

      def build(relpath:,
                content:,
                frontmatter: {},
                stat: nil)
        type = compute_file_type(relpath)

        id = compute_id(relpath)

        Munge::Item.new(
          relpath: relpath,
          content: content,
          frontmatter: frontmatter,
          stat: stat,
          type: type,
          id: id
        )
      end

      def parse(relpath:,
                content:,
                stat: nil)
        type = compute_file_type(relpath)

        if type == :text
          parsed = Munge::Core::ItemFactory::ContentParser.new(content)

          build(
            relpath: relpath,
            content: parsed.content,<commit_msg>setting ignore_extensions now always ignores<commit_after>require_relative "item_factory/content_parser"

module Munge
  module Core
    class ItemFactory
      def initialize(text_extensions:,
                     ignore_extensions:)
        @text_extensions   = Set.new(text_extensions)
        @ignore_extensions = ignore_extensions
      end

      def build(relpath:,
                content:,
                frontmatter: {},
                stat: nil)
        type = compute_file_type(relpath)

        id =
          if @ignore_extensions || type == :text
            compute_id(relpath)
          else
            relpath
          end

        Munge::Item.new(
          relpath: relpath,
          content: content,
          frontmatter: frontmatter,
          stat: stat,
          type: type,
          id: id
        )
      end

      def parse(relpath:,
                content:,
                stat: nil)
        type = compute_file_type(relpath)

        if type == :text
          parsed = Munge::Core::ItemFactory::ContentParser.new(content)

          build(
            relpath: relpath,
            content: parsed.content,
{'fraction_non_alphanumeric': 0.07703016241299304, 'fraction_numerical': 0.0, 'mean_word_length': 1.5789473684210527, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3078240', 'n_tokens_mistral': 588, 'n_tokens_neox': 556, 'n_words': 124}