<commit_before>namespace :grammar_api_rules_and_feedback do
    desc 'data migration for grammar-api specific rules and feedback'
    task :insert => :environment do
        rules_csv = CSV.parse(File.read('engines/comprehension/lib/tasks/rules.csv'), headers: true)
        valid_rules = rules_csv.select do |r| 
            ['Grammar API', 'Opinion API'].include?(r['Module']) && r['Rule UID'].present? 
        end

        ActiveRecord::Base.transaction do 
            valid_rules.each do |r|
                created_rule = Comprehension::Rule.find_or_initialize_by(uid: r['Rule UID'])
                created_rule.attributes = {
                    name: r['Rule'],
                    description: r['Rule Description'],
                    universal: true,
                    rule_type: r['Module'] == 'Grammar API' ? 'grammar' : 'opinion',
                    optimal: false,
                    suborder: r['Rule Suborder'],
                    state: 'active',
                }
                created_rule.save!

                feedback = Comprehension::Feedback.find_or_initialize_by(
                    rule_id: created_rule.id,
                    order: 0,
                    text: r['Feedback V3 [HM 4/21]']
                )

                feedback.save!
            end
        end

    end
end
  <commit_msg>resolving merge conflict<commit_after>namespace :grammar_api_rules_and_feedback do
    desc 'data migration for grammar-api specific rules and feedback'
    task :insert => :environment do
        rules_csv = CSV.parse(File.read('engines/comprehension/lib/tasks/rules.csv'), headers: true)
        valid_rules = rules_csv.select do |r| 
            ['Grammar API', 'Opinion API'].include?(r['Module']) && r['Rule UID'].present? 
        end

        all_prompts = Comprehension::Prompt.all

        ActiveRecord::Base.transaction do 
            valid_rules.each do |r|
                created_rule = Comprehension::Rule.find_or_initialize_by(uid: r['Rule UID'])
                created_rule.attributes = {
                    name: r['Rule'],
                    description: r['Rule Description'],
                    universal: true,
                    rule_type: r['Module'] == 'Grammar API' ? 'grammar' : 'opinion',
                    optimal: false,
                    suborder: r['Rule Suborder'],
                    state: 'active',
                }
                created_rule.save!

                feedback = Comprehension::Feedback.find_or_initialize_by(
                    rule_id: created_rule.id,
                    order: 0,
                    text: r['Feedback V3 [HM 4/21]']
                )

                feedback.save!

                all_prompts.each do |p|
                    p_r = Comprehension::PromptsRule.find_or_initialize_by(
                        prompt_id: p.id, rule_id: created_rule.id
                    )
                    p_r.save!
                end

            end
        end

    end
end
  
{'fraction_non_alphanumeric': 0.10046885465505694, 'fraction_numerical': 0.003348961821835231, 'mean_word_length': 1.5058724832214765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5760781', 'n_tokens_mistral': 853, 'n_tokens_neox': 769, 'n_words': 171}