<commit_before>module Recommendable
  extend ActiveSupport::Concern

  def recommended_projects
    projects = Project.where(id: recommended_project_ids)
    projects = unfiltered_recommendations if projects.empty?
    projects
  end

  def recommended_project_ids
    ids = favourite_recommendation_ids + most_depended_on_recommendation_ids + most_watched_recommendation_ids
    ids.inject(Hash.new(0)) { |h,v| h[v] += 1; h }.sort_by{|k,v| -v}.map(&:first)
  end

  def already_watching_ids
    subscribed_projects.pluck(:id)
  end

  def recommendation_filter(scope)
    filtered = scope.where.not(id: already_watching_ids)
    filtered = filtered.where('lower(projects.language) IN (?)', favourite_languages) if favourite_languages.any?
    filtered.pluck(:id)
  end

  def favourite_recommendation_ids
    return [] if github_user.nil?
    recommendation_filter github_user.favourite_projects.limit(100)
  end

  def most_depended_on_recommendation_ids
    recommendation_filter Project.most_dependents.limit(100)
  end

  def most_watched_recommendation_ids
    recommendation_filter Project.most_watched.limit(100)
  end<commit_msg>Fix language filtering<commit_after>module Recommendable
  extend ActiveSupport::Concern

  def recommended_projects
    projects = Project.where(id: recommended_project_ids)
    projects = unfiltered_recommendations if projects.empty?
    projects
  end

  def recommended_project_ids
    ids = favourite_recommendation_ids + most_depended_on_recommendation_ids + most_watched_recommendation_ids
    ids.inject(Hash.new(0)) { |h,v| h[v] += 1; h }.sort_by{|k,v| -v}.map(&:first)
  end

  def already_watching_ids
    subscribed_projects.pluck(:id)
  end

  def recommendation_filter(scope)
    filtered = scope.where.not(id: already_watching_ids)
    filtered = filtered.where('lower(projects.language) IN (?)', favourite_languages.map(&:downcase)) if favourite_languages.any?
    filtered.pluck(:id)
  end

  def favourite_recommendation_ids
    return [] if github_user.nil?
    recommendation_filter github_user.favourite_projects.limit(100)
  end

  def most_depended_on_recommendation_ids
    recommendation_filter Project.most_dependents.limit(100)
  end

  def most_watched_recommendation_ids
    recommendation_filter Project.most_watched.limit(100)
  end
{'fraction_non_alphanumeric': 0.0812690134724033, 'fraction_numerical': 0.009561060408518035, 'mean_word_length': 5.188172043010753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6171388', 'n_tokens_mistral': 780, 'n_tokens_neox': 732, 'n_words': 129}