<commit_before>require "sequelizer"
require_relative "vocabulary_operator"
require_relative "../database"
require "csv"

module ConceptQL
  module Operators
    class Vocabulary < VocabularyOperator
      extend Sequelizer
      include ConceptQL::Behaviors::Windowable

      class << self
        def v4_vocab_to_v5_vocab
          all_vocabs.values.each_with_object({}) do |row, h|
            h[row[:omopv4_vocabulary_id]] = row[:omopv5_vocabulary_id] || row[:id]
          end
        end

        def v5_vocab_to_v4_vocab
          all_vocabs.values.each_with_object({}) do |row, h|
            h[row[:id]] = row[:omopv4_vocabulary_id]
          end
        end

        def assigned_vocabularies
          @assigned_vocabularies ||= all_vocabs.values.each.with_object({}) do |v, h|
            h[v[:id]] ||= {}
            h[v[:id]] = h[v[:id]].merge(v)
          end
        end

        def vocab_domain
          all_vocabs.values.each_with_object({}) do |row, h|
            h[row[:id]] = row[:domain]
          end
        end

        def register_many
          assigned_vocabularies.each do |name, vocab|
            dms = [:gdm]
            dms << :omopv4_plus if ConceptQL::Utils.present?(vocab[:domain])
            register(name, *dms)
          end
        end

        def all_vocabs
          @all_vocabs ||= each_vocab.each_with_object({}) do |row, h|
            row = row.to_hash
            row[:omopv5_vocabulary_id] ||= row[:id]
            row[:id] = row[:id].to_s.downcase
            h[row[:id]] ||= {}
            h[row[:id]].merge!(row.compact)
          end
        end

        def each_vocab
          @each_vocab ||= get_all_vocabs
        end

        def get_all_vocabs
          vocabs = [ConceptQL.vocabularies_file_path, ConceptQL.custom_vocabularies_file_path].select(&:exist?).map do |path|
            CSV.foreach(path, headers: true, header_converters: :symbol).to_a
          end.inject(:+)

          lexicon_vocabularies + vocabs
        end

        def lexicon_vocabularies
          lexicon ? lexicon.vocabularies : []
        end

        def lexicon
          @lexicon || ConceptQL::Database.lexicon
        end

        def from_old_vocab(nodifier, old_vocab_id, *values)
          new(nodifier, v4_vocab_to_v5_vocab[old_vocab_id.to_s], *values)
        end

        # This will override the to_metadata method and return the preferred name
        # based on the name listed in the file.
        #
        # This method will be called once for each vocabulary we register
        # for this operator
        def to_metadata(name, opts = {})
          h = super
          vocab = assigned_vocabularies[name.downcase]
          h[:preferred_name] = vocab[:vocabulary_short_name] || vocab[:omopv5_vocabulary_id] || vocab[:id]
          h[:predominant_domains] = [vocab[:domain]].flatten
          h
        end

        def force_refresh!
          @all_vocabs = nil
          @each_vocab = nil
          @assigned_vocabularies = nil
          register_many
        end
      end

      register_many

      desc 'Returns all records that match the given codes for the given vocabulary'
      argument :codes, type: :codelist
      validate_codes_match

      def query(db)
        ds = db[dm.table_by_domain(domain)]

        ds = ds.where(where_clause(db))
        if gdm?
          ds = ds.select_append(Sequel.cast_string(domain.to_s).as(:criterion_domain))
        end
        ds
      end

      def where_clause(db)
        if gdm?
          where_conds = {vocabulary_id: vocabulary_id}
          where_conds[:concept_code] = arguments.flatten unless select_all?
          concept_ids = db[:concepts].where(where_conds).select(:id)
          { clinical_code_concept_id: concept_ids }
        else
          conds = { dm.source_vocabulary_id(domain) => vocabulary_id.to_i }
          conds[dm.source_value_column(domain)] = arguments unless select_all?
          conds
        end
      end

      def domain
        domain_map(op_name)
      end

      def table
        domain_map(op_name)
      end

      def query_cols
        if gdm?
          dm.table_columns(:clinical_codes)
        else
          dm.table_columns(domain)
        end
      end

      def validate(db, opts = {})
        super
        if add_warnings?(db, opts) && !select_all?
          args = arguments.dup
          args -= bad_arguments
          missing_args = []

          if no_db?(db, opts)
            if lexicon
              missing_args = args - lexicon.known_codes(vocabulary_id, args)
            end
          else
            missing_args = args - dm.concepts_ds(db, vocabulary_id, args).select_map(:concept_code) rescue []
          end

          unless missing_args.empty?
            add_warning("unknown code(s)", *missing_args)
          end
        end
      end

      def describe_codes(db, codes)
        return [["*", "ALL CODES"]] if select_all?
        if no_db?(db)
          if lexicon
            lexicon.concepts(vocabulary_id, codes).select_map([:concept_code, :concept_text])
          end
          return codes.zip([])
        end
        results = dm.concepts_ds(db, vocabulary_id, codes).select_map([:concept_code, :concept_text])
        remaining_codes = codes - results.map(&:first).map(&:to_s)
        (results + remaining_codes.zip([])).sort_by(&:first)
      end

      def select_all?
        arguments.include?("*")
      end

      def preferred_name
        vocab_entry && vocab_entry[:vocabulary_short_name] || vocab_entry[:omopv5_vocabulary_id] || vocab_entry[:id]
      end

      private

      def vocab_entry
        self.class.assigned_vocabularies[op_name]
      end

      def vocab_format_regexp
        (vocab_entry || {format_regexp: nil})[:format_regexp]
      end

      # Defined so that bad_arguments can check for bad codes
      def code_regexp
        unless defined?(@code_regexp)
          @code_regexp = nil

          if reg_str = vocab_format_regexp
            @code_regexp = Regexp.new(reg_str, Regexp::IGNORECASE)
          end
        end
        @code_regexp
      end

      def vocabulary_id
        @vocabulary_id ||= translated_vocabulary_id
      end

      def translated_vocabulary_id
        return (vocab_entry && vocab_entry[:omopv5_vocabulary_id]) || op_name if gdm?
        return translate_to_old(op_name)
      end

      def translate_to_old(v_id)
        v = self.class.v5_vocab_to_v4_vocab[v_id.to_s.downcase]<commit_msg>Vocabulary: fix bug that cause scope_tests to fail<commit_after>require "sequelizer"
require_relative "vocabulary_operator"
require_relative "../database"
require "csv"

module ConceptQL
  module Operators
    class Vocabulary < VocabularyOperator
      extend Sequelizer
      include ConceptQL::Behaviors::Windowable

      class << self
        def v4_vocab_to_v5_vocab
          all_vocabs.values.each_with_object({}) do |row, h|
            h[row[:omopv4_vocabulary_id]] = row[:omopv5_vocabulary_id] || row[:id]
          end
        end

        def v5_vocab_to_v4_vocab
          all_vocabs.values.each_with_object({}) do |row, h|
            h[row[:id]] = row[:omopv4_vocabulary_id]
          end
        end

        def assigned_vocabularies
          @assigned_vocabularies ||= all_vocabs.values.each.with_object({}) do |v, h|
            h[v[:id]] ||= {}
            h[v[:id]] = h[v[:id]].merge(v)
          end
        end

        def vocab_domain
          all_vocabs.values.each_with_object({}) do |row, h|
            h[row[:id]] = row[:domain]
          end
        end

        def register_many
          assigned_vocabularies.each do |name, vocab|
            dms = [:gdm]
            dms << :omopv4_plus if ConceptQL::Utils.present?(vocab[:domain])
            register(name, *dms)
          end
        end

        def all_vocabs
          @all_vocabs ||= each_vocab.each_with_object({}) do |row, h|
            row = row.to_hash
            row[:omopv5_vocabulary_id] ||= row[:id]
            row[:id] = row[:id].to_s.downcase
            h[row[:id]] ||= {}
            h[row[:id]].merge!(row.compact)
          end
        end

        def each_vocab
          @each_vocab ||= get_all_vocabs
        end

        def get_all_vocabs
          vocabs = [ConceptQL.vocabularies_file_path, ConceptQL.custom_vocabularies_file_path].select(&:exist?).map do |path|
            CSV.foreach(path, headers: true, header_converters: :symbol).to_a
          end.inject(:+)

          lexicon_vocabularies + vocabs
        end

        def lexicon_vocabularies
          lexicon ? lexicon.vocabularies : []
        end

        def lexicon
          @lexicon || ConceptQL::Database.lexicon
        end

        def from_old_vocab(nodifier, old_vocab_id, *values)
          new(nodifier, v4_vocab_to_v5_vocab[old_vocab_id.to_s], *values)
        end

        # This will override the to_metadata method and return the preferred name
        # based on the name listed in the file.
        #
        # This method will be called once for each vocabulary we register
        # for this operator
        def to_metadata(name, opts = {})
          h = super
          vocab = assigned_vocabularies[name.downcase]
          h[:preferred_name] = vocab[:vocabulary_short_name] || vocab[:omopv5_vocabulary_id] || vocab[:id]
          h[:predominant_domains] = [vocab[:domain]].flatten
          h
        end

        def force_refresh!
          @all_vocabs = nil
          @each_vocab = nil
          @assigned_vocabularies = nil
          register_many
        end
      end

      register_many

      desc 'Returns all records that match the given codes for the given vocabulary'
      argument :codes, type: :codelist
      validate_codes_match

      def query(db)
        ds = db[dm.table_by_domain(domain)]

        ds = ds.where(where_clause(db))
        if gdm?
          ds = ds.select_append(Sequel.cast_string(domain.to_s).as(:criterion_domain))
        end
        ds
      end

      def where_clause(db)
        if gdm?
          where_conds = {vocabulary_id: vocabulary_id}
          where_conds[:concept_code] = arguments.flatten unless select_all?
          concept_ids = db[:concepts].where(where_conds).select(:id)
          { clinical_code_concept_id: concept_ids }
        else
          conds = { dm.source_vocabulary_id(domain) => vocabulary_id.to_i }
          conds[dm.source_value_column(domain)] = arguments unless select_all?
          conds
        end
      end

      def domain
        domain_map(op_name)
      end

      def table
        domain_map(op_name)
      end

      def query_cols
        if gdm?
          dm.table_columns(:clinical_codes)
        else
          dm.table_columns(domain)
        end
      end

      def validate(db, opts = {})
        super
        if add_warnings?(db, opts) && !select_all?
          args = arguments.dup
          args -= bad_arguments
          missing_args = []

          if no_db?(db, opts)
            if lexicon
              missing_args = args - lexicon.known_codes(vocabulary_id, args)
            end
          else
            missing_args = args - dm.concepts_ds(db, vocabulary_id, args).select_map(:concept_code) rescue []
          end

          unless missing_args.empty?
            add_warning("unknown code(s)", *missing_args)
          end
        end
      end

      def describe_codes(db, codes)
        return [["*", "ALL CODES"]] if select_all?
        if no_db?(db)
          if lexicon
            lexicon.concepts(vocabulary_id, codes).select_map([:concept_code, :concept_text])
          end
          return codes.zip([])
        end
        results = dm.concepts_ds(db, vocabulary_id, codes).select_map([:concept_code, :concept_text])
        remaining_codes = codes - results.map(&:first).map(&:to_s)
        (results + remaining_codes.zip([])).sort_by(&:first)
      end

      def select_all?
        arguments.include?("*")
      end

      def preferred_name
        vocab_entry && vocab_entry[:vocabulary_short_name] || vocab_entry[:omopv5_vocabulary_id] || vocab_entry[:id]
      end

      private

      def vocab_entry
        self.class.assigned_vocabularies[op_name.to_s.downcase]
      end

      def vocab_format_regexp
        (vocab_entry || {format_regexp: nil})[:format_regexp]
      end

      # Defined so that bad_arguments can check for bad codes
      def code_regexp
        unless defined?(@code_regexp)
          @code_regexp = nil

          if reg_str = vocab_format_regexp
            @code_regexp = Regexp.new(reg_str, Regexp::IGNORECASE)
          end
        end
        @code_regexp
      end

      def vocabulary_id
        @vocabulary_id ||= translated_vocabulary_id
      end

      def translated_vocabulary_id
        return (vocab_entry && vocab_entry[:omopv5_vocabulary_id]) || op_name if gdm?
        return translate_to_old(op_name)
      end

      def translate_to_old(v_id)
        v = self.class.v5_vocab_to_v4_vocab[v_id.to_s.downcase]
{'fraction_non_alphanumeric': 0.09239675800849093, 'fraction_numerical': 0.0024700887688151292, 'mean_word_length': 2.04131455399061, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2632903', 'n_tokens_mistral': 4315, 'n_tokens_neox': 4178, 'n_words': 818}