<commit_before>module Primix
  class Analyzer
    require "primix/analyzer/tokenizer"
    require "primix/analyzer/parser"

    attr_reader :project_folder

    def initialize(project_folder)
      @project_folder = project_folder
    end

    def analyze!
      Dir.glob("#{project_folder}/**/*.swift").select { |file|
        content = File.read file
        content.match(/\/\/@/)
        true
      }.each { |file|
        content = File.read file
        tokenizer = create_tokenizer content
        parser = Parser.new(tokenizer.tokenize!)
        parser.parse!
      }
    end

    def create_tokenizer(file)
      Tokenizer.new file
    end
  end
end<commit_msg>Add extract_content file to remove comments<commit_after>module Primix
  class Analyzer
    require "primix/analyzer/tokenizer"
    require "primix/analyzer/parser"

    attr_reader :project_folder

    def initialize(project_folder)
      @project_folder = project_folder
    end

    def analyze!
      Dir.glob("#{project_folder}/**/*.swift").select { |file|
        content = File.read file
        content.match(/\/\/@/)
        true
      }.each { |file|
        content = extract_content file
        tokenizer = create_tokenizer content
        parser = Parser.new(tokenizer.tokenize!)
        parser.parse!
      }
    end

    def extract_content(file)
      content = File.read file

      # remove comments from content
      content.gsub!(/\/\*.*\*\//m, "")
      content.gsub!(/\/\/.*/, "")

      content
    end

    def create_tokenizer(file)
      Tokenizer.new file
    end
  end
end
{'fraction_non_alphanumeric': 0.10941475826972011, 'fraction_numerical': 0.0, 'mean_word_length': 2.223360655737705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741786', 'n_tokens_mistral': 480, 'n_tokens_neox': 461, 'n_words': 103}