<commit_before>#Update the data extracts and include any metadata files.

require 'rubygems'
require 'rake'
require 'model_execution'
require 'active_record/fixtures'
require 'xml'
require 'zip/zip'
require 'zip/zipfilesystem'

namespace :obesity do
  desc "update data extracts with metadata files"
  task :update_data_extracts_and_metadata  => :environment do
    Csvarchive.all.each do |extract|
      add_data = true
      #firstly move the extracts to the correct place ie data_extract.filename/files
      if File.exists?(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + ".zip"))
        File.move(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + ".zip"), File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_csv.zip"))
      end
      #is the extract an old one?
      if !File.exists?(File.join(CSV_OUTPUT_DIRECTORY, extract.filename))
        File.makedirs(File.join(CSV_OUTPUT_DIRECTORY, extract.filename))
        #are any of the files missing, if so create the extract from scratch
        if !File.exists(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_csv.zip")) || !File.exists(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_stata.zip")) || !File.exists(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_spss.zip"))
          add_data = false
          variable_hash = Hash.new
          extract.variables.each do |variable|
            if (!variable_hash.has_key?(variable.dataset_id))
              variable_hash[variable.dataset_id] = Array.new
            end
            variable_hash[variable.dataset_id].push(variable.id)
          end
          begin 
            Delayed::Job.enqueue DataExtractJob::StartJobTask.new(variable_hash, User.find(extract.user_id), extract.id, extract.filename, false)
          rescue Exception => e
            logger.error(e)
          end
        else
          
        old_csv_place = File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_csv.zip")
        new_csv_place = File.join(CSV_OUTPUT_DIRECTORY, extract.filename, extract.filename + "_csv.zip")
        File.move(old_csv_place, new_csv_place)
        old_stata_place = File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_stata.zip")
        new_stata_place = File.join(CSV_OUTPUT_DIRECTORY, extract.filename, extract.filename + "_stata.zip")<commit_msg>check if the files exists before creating them<commit_after>#Update the data extracts and include any metadata files.

require 'rubygems'
require 'rake'
require 'model_execution'
require 'active_record/fixtures'
require 'xml'
require 'zip/zip'
require 'zip/zipfilesystem'

namespace :obesity do
  desc "update data extracts with metadata files"
  task :update_data_extracts_and_metadata  => :environment do
    Csvarchive.all.each do |extract|
      add_data = true
      #firstly move the extracts to the correct place ie data_extract.filename/files
      if File.exists?(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + ".zip"))
        File.move(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + ".zip"), File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_csv.zip"))
      end
      #is the extract an old one?
      if !File.exists?(File.join(CSV_OUTPUT_DIRECTORY, extract.filename))
        #are any of the files missing, if so create the extract from scratch
        if !File.exists?(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_csv.zip")) || !File.exists?(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_stata.zip")) || !File.exists?(File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_spss.zip"))
          add_data = false
          variable_hash = Hash.new
          extract.variables.each do |variable|
            if (!variable_hash.has_key?(variable.dataset_id))
              variable_hash[variable.dataset_id] = Array.new
            end
            variable_hash[variable.dataset_id].push(variable.id)
          end
          begin 
            Delayed::Job.enqueue DataExtractJob::StartJobTask.new(variable_hash, User.find(extract.user_id), extract.id, extract.filename, false)
          rescue Exception => e
            logger.error(e)
          end
        else
        File.makedirs(File.join(CSV_OUTPUT_DIRECTORY, extract.filename))
        old_csv_place = File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_csv.zip")
        new_csv_place = File.join(CSV_OUTPUT_DIRECTORY, extract.filename, extract.filename + "_csv.zip")
        File.move(old_csv_place, new_csv_place)
        old_stata_place = File.join(CSV_OUTPUT_DIRECTORY, extract.filename + "_stata.zip")
        new_stata_place = File.join(CSV_OUTPUT_DIRECTORY, extract.filename, extract.filename + "_stata.zip")
{'fraction_non_alphanumeric': 0.102803738317757, 'fraction_numerical': 0.0, 'mean_word_length': 3.463627546071775, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '977684', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1382, 'n_words': 294}