<commit_before>module PubliSci
		module Readers
		class ARFF
			include PubliSci::Dataset::DataCube

			def generate_n3(arff, options={})
				arff = IO.read(arff) if File.exist? arff
				options[:no_labels] = true # unless options[:no_labels] == nil
				@options = options
				comps =  components(arff)
				obs = data(arff, comps.keys)
				generate(comps.reject{|c| comps[c][:codes]}.keys, comps.select{|c| comps[c][:codes]}.keys, comps.select{|c| comps[c][:codes]}.keys, obs, (1..obs.first[1].size).to_a, relation(arff), options)
			end

			def relation(arff)
				arff.match(/@relation.+/i).to_a.first.split.last
			end

			def components(arff)
				#still needs support for quoted strings with whitespace
				h ={}
				arff.split("\n").select{|lin| lin =~ /^@ATTRIBUTE/i}.map{|line|
					if line =~ /\{.*}/
						name = line.match(/\s.*/).to_a.first.strip.split.first
						type = :coded
						codes = line.match(/\{.*}/).to_a.first[1..-2].split(',')
						h[name] = {type: type, codes: codes}
					else
						name = line.split[1]
						type = line.split[2]
						h[name] = {type: type}
					end
				}
				h
			end

			def data(arff, attributes)
				lines = arff.split("\n")
				data_lines = lines[lines.index(lines.select{|line| line =~ /^@DATA/i}.first)+1..-1]
				h=attributes.inject({}){|ha,attrib| ha[attrib] = []; ha}
				data_lines.map{|line|
					line = line.split ','
					attributes.each_with_index{|a,i| h[a] << line[i]}
				}
				h
			end
		end
	end
end<commit_msg>fix indents<commit_after>module PubliSci
  module Readers
    class ARFF
      include PubliSci::Dataset::DataCube

      def generate_n3(arff, options={})
        arff = IO.read(arff) if File.exist? arff
        options[:no_labels] = true
        @options = options
        comps =  components(arff)
        obs = data(arff, comps.keys)
        generate(comps.reject{|c| comps[c][:codes]}.keys, comps.select{|c| comps[c][:codes]}.keys, comps.select{|c| comps[c][:codes]}.keys, obs, (1..obs.first[1].size).to_a, relation(arff), options)
      end

      def relation(arff)
        arff.match(/@relation.+/i).to_a.first.split.last
      end

      def components(arff)
        #still needs support for quoted strings with whitespace
        h ={}
        arff.split("\n").select{|lin| lin =~ /^@ATTRIBUTE/i}.map{|line|
          if line =~ /\{.*}/
            name = line.match(/\s.*/).to_a.first.strip.split.first
            type = :coded
            codes = line.match(/\{.*}/).to_a.first[1..-2].split(',')
            h[name] = {type: type, codes: codes}
          else
            name = line.split[1]
            type = line.split[2]
            h[name] = {type: type}
          end
        }
        h
      end

      def data(arff, attributes)
        lines = arff.split("\n")
        data_lines = lines[lines.index(lines.select{|line| line =~ /^@DATA/i}.first)+1..-1]
        h=attributes.inject({}){|ha,attrib| ha[attrib] = []; ha}
        data_lines.map{|line|
          line = line.split ','
          attributes.each_with_index{|a,i| h[a] << line[i]}
        }
        h
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.17610877306571707, 'fraction_numerical': 0.005827128520556815, 'mean_word_length': 2.344155844155844, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5602214', 'n_tokens_mistral': 1277, 'n_tokens_neox': 1120, 'n_words': 205}