<commit_before>class StoreAnnotationsCollectionZipJob < Struct.new(:filename, :project, :options)
	include StateManagement

	def perform
    files = Annotation.get_files_from_zip(filename)
    collection = Annotation.get_annotations_collection(files)
    File.unlink(filename)

		@job.update_attribute(:num_items, collection.length)
    @job.update_attribute(:num_dones, 0)

    collection.each_with_index do |annotations, i|
      begin
       	project.add_doc(annotations[:sourcedb], annotations[:sourceid])

        if annotations[:divid].present?
          doc = Doc.find_by_sourcedb_and_sourceid_and_serial(annotations[:sourcedb], annotations[:sourceid], annotations[:divid])
        else
          divs = Doc.find_all_by_sourcedb_and_sourceid(annotations[:sourcedb], annotations[:sourceid])
          doc = divs[0] if divs.length == 1
        end

        if doc.present?
          project.save_annotations(annotations, doc, options)
        elsif divs.present?
          project.store_annotations(annotations, divs, options)
        else
          raise IOError, "document does not exist"
        end
      rescue => e
				@job.messages << Message.create({item: "#{annotations[:sourcedb]}:#{annotations[:sourceid]}", body: e.message})
      end
			@job.update_attribute(:num_dones, i + 1)
    end
	end
end<commit_msg>FIXED deletion of uploaded files is delayed until the job is finished<commit_after>class StoreAnnotationsCollectionZipJob < Struct.new(:filename, :project, :options)
	include StateManagement

	def perform
    files = Annotation.get_files_from_zip(filename)
    collection = Annotation.get_annotations_collection(files)

		@job.update_attribute(:num_items, collection.length)
    @job.update_attribute(:num_dones, 0)

    collection.each_with_index do |annotations, i|
      begin
       	project.add_doc(annotations[:sourcedb], annotations[:sourceid])

        if annotations[:divid].present?
          doc = Doc.find_by_sourcedb_and_sourceid_and_serial(annotations[:sourcedb], annotations[:sourceid], annotations[:divid])
        else
          divs = Doc.find_all_by_sourcedb_and_sourceid(annotations[:sourcedb], annotations[:sourceid])
          doc = divs[0] if divs.length == 1
        end

        if doc.present?
          project.save_annotations(annotations, doc, options)
        elsif divs.present?
          project.store_annotations(annotations, divs, options)
        else
          raise IOError, "document does not exist"
        end
      rescue => e
				@job.messages << Message.create({item: "#{annotations[:sourcedb]}:#{annotations[:sourceid]}", body: e.message})
      end
			@job.update_attribute(:num_dones, i + 1)
    end
    File.unlink(filename)
	end
end
{'fraction_non_alphanumeric': 0.10428994082840237, 'fraction_numerical': 0.0029585798816568047, 'mean_word_length': 3.631849315068493, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4732647', 'n_tokens_mistral': 837, 'n_tokens_neox': 786, 'n_words': 156}