<commit_before>class CreateSpansRdfCollectionJob < ApplicationJob
	queue_as :low_priority

	def perform(collection)
		project_ids = collection.primary_projects.pluck(:id)
		doc_ids = [].union(*collection.primary_projects.collect{|project| project.docs.pluck(:id)})

		if @job
			prepare_progress_record(doc_ids.count)
		end

		File.open(collection.spans_trig_filepath, "w") do |f|
			doc_ids.each_with_index do |doc_id, i|
				doc = Doc.find(doc_id)

				doc_spans_trig = if i == 0
					doc.get_spans_rdf(project_ids, {with_prefixes: true})
				else
					doc.get_spans_rdf(project_ids, {with_prefixes: false})
				end

				f.write("\n") unless i == 0
				f.write(doc_spans_trig)
			rescue => e
				if @job
					@job.messages << Message.create({sourcedb: doc.sourcedb, sourceid: doc.sourceid, body: e.message}) if message.present?
				else
					raise e
				end
			ensure
				if @job
					@job.update_attribute(:num_dones, i + 1)
					check_suspend_flag
				end
			end<commit_msg>FIXED to index the spans from the secondary projects<commit_after>class CreateSpansRdfCollectionJob < ApplicationJob
	queue_as :low_priority

	def perform(collection)
		project_ids = collection.primary_projects.pluck(:id) + collection.secondary_projects.pluck(:id)
		doc_ids = [].union(*collection.primary_projects.collect{|project| project.docs.pluck(:id)})

		if @job
			prepare_progress_record(doc_ids.count)
		end

		File.open(collection.spans_trig_filepath, "w") do |f|
			doc_ids.each_with_index do |doc_id, i|
				doc = Doc.find(doc_id)

				doc_spans_trig = if i == 0
					doc.get_spans_rdf(project_ids, {with_prefixes: true})
				else
					doc.get_spans_rdf(project_ids, {with_prefixes: false})
				end

				f.write("\n") unless i == 0
				f.write(doc_spans_trig)
			rescue => e
				if @job
					@job.messages << Message.create({sourcedb: doc.sourcedb, sourceid: doc.sourceid, body: e.message}) if message.present?
				else
					raise e
				end
			ensure
				if @job
					@job.update_attribute(:num_dones, i + 1)
					check_suspend_flag
				end
			end
{'fraction_non_alphanumeric': 0.11089303238469088, 'fraction_numerical': 0.002944062806673209, 'mean_word_length': 3.752913752913753, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2483040', 'n_tokens_mistral': 895, 'n_tokens_neox': 721, 'n_words': 146}