<commit_before># Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/

module Ticket::SearchIndex

=begin

build and send data for search index to backend

  ticket = Ticket.find(123)
  result = ticket.search_index_update_backend

returns

  result = true # false

=end

  def search_index_update_backend
    return if !self.class.search_index_support_config

    # default ignored attributes
    ignore_attributes = {
      :created_by_id => true,
      :updated_by_id => true,
      :active        => true,
    }
    if self.class.search_index_support_config[:ignore_attributes]
      self.class.search_index_support_config[:ignore_attributes].each {|key, value|
        ignore_attributes[key] = value
      }
    end

    # for performance reasons, Model.search_index_reload will only collect if of object
    # get whole data here
    ticket = self.class.find(self.id)

    # remove ignored attributes
    attributes = ticket.attributes
    ignore_attributes.each {|key, value|
      next if value != true
      attributes.delete( key.to_s )
    }

    # add tags
    tags = Tag.tag_list( :object=> 'Ticket', :o_id => self.id )
    if tags && !tags.empty?
      attributes[:tag] = tags
    end

    # lookup attributes of ref. objects (normally name and note)
    attributes = search_index_attribute_lookup( attributes, ticket )

    # list ignored file extentions
    attachments_ignore = Setting.get('es_attachment_ignore') || [ '.png', '.jpg', '.jpeg', '.mpeg', '.mpg', '.mov', '.bin', '.exe' ]

    # max attachment size
    attachment_max_size_in_mb = Setting.get('es_attachment_max_size_in_mb') || 40

    # collect article data
    articles = Ticket::Article.where( :ticket_id => self.id )
    attributes['articles'] = []
    articles.each {|article|
      article_attributes = article.attributes

      # remove note needed attributes
      ignore = ['created_by_id', 'updated_by_id', 'updated_at', 'references', 'message_id_md5', 'message_id', 'in_reply_to', 'ticket_id']
      ignore.each {|attribute|
        article_attributes.delete( attribute )
      }

      # lookup attributes of ref. objects (normally name and note)
      article_attributes = search_index_attribute_lookup( article_attributes, article )

      # index raw text body
      if article_attributes['content_type'] && article_attributes['content_type'] == 'text/html' && article_attributes['body']
        article_attributes['body'] = article_attributes['body'].html2text
      end

      # lookup attachments
      article.attachments.each {|attachment|
        if !attributes['attachments']
          attributes['attachments'] = []
        end

        # check file size
        if true
#        if attachment.content && attachment.content.size / 1024 / 1024 <= attachment_max_size_in_mb

          # check ignored files
          if attachment.filename
            filename_extention = attachment.filename.downcase
            filename_extention.gsub!(/^.*(\..+?)$/, "\\1")
            if !attachments_ignore.include?( filename_extention.downcase )
              data = {
                '_name'   => attachment.filename,
                'content' => Base64.encode64( attachment.content )
              }
              attributes['attachments'].push data
            end
          end
        end
      }
      attributes['articles'].push article_attributes
    }<commit_msg>Added attachment size check.<commit_after># Copyright (C) 2012-2014 Zammad Foundation, http://zammad-foundation.org/

module Ticket::SearchIndex

=begin

build and send data for search index to backend

  ticket = Ticket.find(123)
  result = ticket.search_index_update_backend

returns

  result = true # false

=end

  def search_index_update_backend
    return if !self.class.search_index_support_config

    # default ignored attributes
    ignore_attributes = {
      :created_by_id => true,
      :updated_by_id => true,
      :active        => true,
    }
    if self.class.search_index_support_config[:ignore_attributes]
      self.class.search_index_support_config[:ignore_attributes].each {|key, value|
        ignore_attributes[key] = value
      }
    end

    # for performance reasons, Model.search_index_reload will only collect if of object
    # get whole data here
    ticket = self.class.find(self.id)

    # remove ignored attributes
    attributes = ticket.attributes
    ignore_attributes.each {|key, value|
      next if value != true
      attributes.delete( key.to_s )
    }

    # add tags
    tags = Tag.tag_list( :object=> 'Ticket', :o_id => self.id )
    if tags && !tags.empty?
      attributes[:tag] = tags
    end

    # lookup attributes of ref. objects (normally name and note)
    attributes = search_index_attribute_lookup( attributes, ticket )

    # list ignored file extentions
    attachments_ignore = Setting.get('es_attachment_ignore') || [ '.png', '.jpg', '.jpeg', '.mpeg', '.mpg', '.mov', '.bin', '.exe' ]

    # max attachment size
    attachment_max_size_in_mb = Setting.get('es_attachment_max_size_in_mb') || 40

    # collect article data
    articles = Ticket::Article.where( :ticket_id => self.id )
    attributes['articles'] = []
    articles.each {|article|
      article_attributes = article.attributes

      # remove note needed attributes
      ignore = ['created_by_id', 'updated_by_id', 'updated_at', 'references', 'message_id_md5', 'message_id', 'in_reply_to', 'ticket_id']
      ignore.each {|attribute|
        article_attributes.delete( attribute )
      }

      # lookup attributes of ref. objects (normally name and note)
      article_attributes = search_index_attribute_lookup( article_attributes, article )

      # index raw text body
      if article_attributes['content_type'] && article_attributes['content_type'] == 'text/html' && article_attributes['body']
        article_attributes['body'] = article_attributes['body'].html2text
      end

      # lookup attachments
      article.attachments.each {|attachment|
        if !article_attributes['attachments']
          article_attributes['attachments'] = []
        end

        # check file size
        if attachment.content && attachment.content.size / 1024 <= attachment_max_size_in_mb * 1024

          # check ignored files
          if attachment.filename
            filename_extention = attachment.filename.downcase
            filename_extention.gsub!(/^.*(\..+?)$/, "\\1")
            if !attachments_ignore.include?( filename_extention.downcase )
              data = {
                '_name'    => attachment.filename,
                '_content' => Base64.encode64( attachment.content )
              }
              article_attributes['attachments'].push data
            end
          end
        end
      }
      attributes['articles'].push article_attributes
    }
{'fraction_non_alphanumeric': 0.10662739322533137, 'fraction_numerical': 0.008247422680412371, 'mean_word_length': 2.8259154929577464, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5504408', 'n_tokens_mistral': 1959, 'n_tokens_neox': 1857, 'n_words': 453}