<commit_before>#
# Copyright (C) 2012 Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#

class PageView < ActiveRecord::Base
  self.primary_key = 'request_id'

  belongs_to :developer_key
  belongs_to :user
  belongs_to :account
  belongs_to :real_user, :class_name => 'User'
  belongs_to :asset_user_access

  before_save :ensure_account
  before_save :cap_interaction_seconds
  belongs_to :context, polymorphic: [:course, :account, :group, :user, :user_profile], polymorphic_prefix: true

  attr_accessor :generated_by_hand
  attr_accessor :is_update

  attr_accessible :url, :user, :controller, :action, :session_id, :developer_key, :user_agent, :real_user, :context

  # note that currently we never query page views from the perspective of the course;
  # we simply don't record them for non-logged-in users in a public course
  # if we ever do either of the above, we'll need to remove this, and figure out
  # where such page views should belong (currently page views end up on the user's
  # shard)
  validates_presence_of :user_id

  def self.generate(request, attributes={})
    self.new(attributes).tap do |p|
      p.url = LoggingFilter.filter_uri(request.url)[0,255]
      p.http_method = request.request_method.downcase
      p.controller = request.path_parameters[:controller]
      p.action = request.path_parameters[:action]
      p.session_id = request.session_options[:id].to_s.force_encoding(Encoding::UTF_8).presence
      p.user_agent = request.user_agent
      p.remote_ip = request.remote_ip
      p.interaction_seconds = 5
      p.created_at = Time.now
      p.updated_at = Time.now
      p.id = RequestContextGenerator.request_id
      p.export_columns.each do |c|
        v = p.send(c)
        if !v.nil? && v.respond_to?(:force_encoding)
          p.send("#{c}=", v.force_encoding(Encoding::UTF_8))
        end
      end
    end
  end

  def self.find_for_update(request_id)
    if PageView.updates_enabled? && (self.db? || self.cassandra?)
      begin
        # not using find_by_id or where(..).first because the cassandra
        # codepath doesn't support it
        find(request_id)
      rescue ActiveRecord::RecordNotFound
        nil
      end
    else
      new { |p| p.request_id = request_id }
    end
  end

  def token
    Canvas::Security.create_jwt({
      i: request_id,
      u: Shard.global_id_for(user_id),
      c: created_at.try(:utc).try(:iso8601, 2)
    })
  end

  def self.decode_token(token)
    data = Canvas::Security.decode_jwt(token)
    return nil unless data
    return {
      request_id: data[:i],
      user_id: data[:u],
      created_at: data[:c]
    }
  end

  def url
    url = read_attribute(:url)
    url && LoggingFilter.filter_uri(url)
  end

  def ensure_account
    self.account_id ||= (self.context_type == 'Account' ? self.context_id : self.context.account_id) rescue nil
    self.account_id ||= (self.context.is_a?(Account) ? self.context : self.context.account) if self.context
  end

  def cap_interaction_seconds
    self.interaction_seconds = [self.interaction_seconds || 5, 10.minutes.to_i].min
  end

  # the list of columns we display to users, export to csv, etc
  EXPORTED_COLUMNS = %w(request_id user_id url context_id context_type asset_id asset_type controller action interaction_seconds created_at user_request render_time user_agent participated account_id real_user_id http_method remote_ip)

  def self.page_views_enabled?
    !!page_view_method
  end

  def self.page_view_method
    enable_page_views = Setting.get('enable_page_views', 'false')
    return false if enable_page_views == 'false'
    enable_page_views = 'db' if %w[true cache].include?(enable_page_views) # backwards compat
    enable_page_views.to_sym
  end

  after_initialize :initialize_shard

  def initialize_shard
    # remember the page view method selected at the time of creation, so that
    # we use the right method when saving
    if PageView.cassandra? && new_record?
      self.shard = Shard.birth
    end
  end

  def self.db?
    self.page_view_method == :db
  end

  def self.cassandra?
    page_view_method == :cassandra
  end

  def self.pv4?
    page_view_method == :pv4 || Setting.get('read_from_pv4', 'false') == 'true'
  end

  def self.global_storage_namespace?
    cassandra? || pv4?
  end

  EventStream = EventStream::Stream.new do
    database -> { Canvas::Cassandra::DatabaseBuilder.from_config(:page_views) }
    table :page_views
    id_column :request_id
    record_type PageView
    read_consistency_level -> { Canvas::Cassandra::DatabaseBuilder.read_consistency_setting(:page_views) }

    add_index :user do
      table :page_views_history_by_context
      id_column :request_id
      key_column :context_and_time_bucket
      scrollback_limit -> { Setting.get('page_views_scrollback_limit:users', 52.weeks) }

      # index by the page view's user, but use the user's global_asset_string
      # when writing the index
      entry_proc lambda{ |page_view| page_view.user }
      key_proc lambda{ |user| user.global_asset_string }
    end

    self.raise_on_error = Rails.env.test?

    on_error do |operation, record, exception|
      Canvas::EventStreamLogger.error('PAGEVIEW', identifier, operation, record.to_json, exception.message.to_s)
    end
  end

  def self.find(ids)
    return super unless PageView.cassandra?

    case ids
    when Array
      result = PageView::EventStream.fetch(ids)
      raise ActiveRecord::RecordNotFound, "Couldn't find all PageViews with IDs (#{ids.join(',')}) (found #{result.length} results, but was looking for #{ids.length})" unless ids.length == result.length
      result
    else
      find([ids]).first
    end
  end

  def self.find_all_by_id(ids)
    PageView.cassandra? ? PageView::EventStream.fetch(ids) : where(request_id: ids).to_a
  end

  def self.find_by_id(id)
    PageView.cassandra? ? PageView::EventStream.fetch([id]).first : where(request_id: id).first
  end

  def self.from_attributes(attrs, new_record=false)
    @blank_template ||= columns.inject({}) { |h,c| h[c.name] = nil; h }<commit_msg>"fix" direct page view lookup with pv4 for audit apis<commit_after>#
# Copyright (C) 2012 Instructure, Inc.
#
# This file is part of Canvas.
#
# Canvas is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, version 3 of the License.
#
# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
#

class PageView < ActiveRecord::Base
  self.primary_key = 'request_id'

  belongs_to :developer_key
  belongs_to :user
  belongs_to :account
  belongs_to :real_user, :class_name => 'User'
  belongs_to :asset_user_access

  before_save :ensure_account
  before_save :cap_interaction_seconds
  belongs_to :context, polymorphic: [:course, :account, :group, :user, :user_profile], polymorphic_prefix: true

  attr_accessor :generated_by_hand
  attr_accessor :is_update

  attr_accessible :url, :user, :controller, :action, :session_id, :developer_key, :user_agent, :real_user, :context

  # note that currently we never query page views from the perspective of the course;
  # we simply don't record them for non-logged-in users in a public course
  # if we ever do either of the above, we'll need to remove this, and figure out
  # where such page views should belong (currently page views end up on the user's
  # shard)
  validates_presence_of :user_id

  def self.generate(request, attributes={})
    self.new(attributes).tap do |p|
      p.url = LoggingFilter.filter_uri(request.url)[0,255]
      p.http_method = request.request_method.downcase
      p.controller = request.path_parameters[:controller]
      p.action = request.path_parameters[:action]
      p.session_id = request.session_options[:id].to_s.force_encoding(Encoding::UTF_8).presence
      p.user_agent = request.user_agent
      p.remote_ip = request.remote_ip
      p.interaction_seconds = 5
      p.created_at = Time.now
      p.updated_at = Time.now
      p.id = RequestContextGenerator.request_id
      p.export_columns.each do |c|
        v = p.send(c)
        if !v.nil? && v.respond_to?(:force_encoding)
          p.send("#{c}=", v.force_encoding(Encoding::UTF_8))
        end
      end
    end
  end

  def self.find_for_update(request_id)
    if PageView.updates_enabled? && (self.db? || self.cassandra?)
      begin
        # not using find_by_id or where(..).first because the cassandra
        # codepath doesn't support it
        find(request_id)
      rescue ActiveRecord::RecordNotFound
        nil
      end
    else
      new { |p| p.request_id = request_id }
    end
  end

  def token
    Canvas::Security.create_jwt({
      i: request_id,
      u: Shard.global_id_for(user_id),
      c: created_at.try(:utc).try(:iso8601, 2)
    })
  end

  def self.decode_token(token)
    data = Canvas::Security.decode_jwt(token)
    return nil unless data
    return {
      request_id: data[:i],
      user_id: data[:u],
      created_at: data[:c]
    }
  end

  def url
    url = read_attribute(:url)
    url && LoggingFilter.filter_uri(url)
  end

  def ensure_account
    self.account_id ||= (self.context_type == 'Account' ? self.context_id : self.context.account_id) rescue nil
    self.account_id ||= (self.context.is_a?(Account) ? self.context : self.context.account) if self.context
  end

  def cap_interaction_seconds
    self.interaction_seconds = [self.interaction_seconds || 5, 10.minutes.to_i].min
  end

  # the list of columns we display to users, export to csv, etc
  EXPORTED_COLUMNS = %w(request_id user_id url context_id context_type asset_id asset_type controller action interaction_seconds created_at user_request render_time user_agent participated account_id real_user_id http_method remote_ip)

  def self.page_views_enabled?
    !!page_view_method
  end

  def self.page_view_method
    enable_page_views = Setting.get('enable_page_views', 'false')
    return false if enable_page_views == 'false'
    enable_page_views = 'db' if %w[true cache].include?(enable_page_views) # backwards compat
    enable_page_views.to_sym
  end

  after_initialize :initialize_shard

  def initialize_shard
    # remember the page view method selected at the time of creation, so that
    # we use the right method when saving
    if PageView.cassandra? && new_record?
      self.shard = Shard.birth
    end
  end

  def self.db?
    self.page_view_method == :db
  end

  def self.cassandra?
    page_view_method == :cassandra
  end

  def self.pv4?
    page_view_method == :pv4 || Setting.get('read_from_pv4', 'false') == 'true'
  end

  def self.global_storage_namespace?
    cassandra? || pv4?
  end

  EventStream = EventStream::Stream.new do
    database -> { Canvas::Cassandra::DatabaseBuilder.from_config(:page_views) }
    table :page_views
    id_column :request_id
    record_type PageView
    read_consistency_level -> { Canvas::Cassandra::DatabaseBuilder.read_consistency_setting(:page_views) }

    add_index :user do
      table :page_views_history_by_context
      id_column :request_id
      key_column :context_and_time_bucket
      scrollback_limit -> { Setting.get('page_views_scrollback_limit:users', 52.weeks) }

      # index by the page view's user, but use the user's global_asset_string
      # when writing the index
      entry_proc lambda{ |page_view| page_view.user }
      key_proc lambda{ |user| user.global_asset_string }
    end

    self.raise_on_error = Rails.env.test?

    on_error do |operation, record, exception|
      Canvas::EventStreamLogger.error('PAGEVIEW', identifier, operation, record.to_json, exception.message.to_s)
    end
  end

  def self.find(ids)
    return super unless PageView.cassandra?

    case ids
    when Array
      result = PageView::EventStream.fetch(ids)
      raise ActiveRecord::RecordNotFound, "Couldn't find all PageViews with IDs (#{ids.join(',')}) (found #{result.length} results, but was looking for #{ids.length})" unless ids.length == result.length
      result
    else
      find([ids]).first
    end
  end

  def self.find_all_by_id(ids)
    if PageView.cassandra?
      PageView::EventStream.fetch(ids)
    elsif PageView.pv4?
      []
    else
      where(request_id: ids).to_a
    end
  end

  def self.find_by_id(id)
    if PageView.cassandra?
      PageView::EventStream.fetch([id]).first
    elsif PageView.pv4?
      nil
    else
      where(request_id: id).first
    end
  end

  def self.from_attributes(attrs, new_record=false)
    @blank_template ||= columns.inject({}) { |h,c| h[c.name] = nil; h }
{'fraction_non_alphanumeric': 0.09361765361468162, 'fraction_numerical': 0.004086484880005944, 'mean_word_length': 3.578231292517007, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2813682', 'n_tokens_mistral': 4376, 'n_tokens_neox': 4239, 'n_words': 1160}