<commit_before>require 'open-uri'

class Author < ActiveRecord::Base
  has_many :posts, dependent: :delete_all
  validates_uniqueness_of :forum_id
  default_scope { order('forum_id ASC') }
  
  FORUM_SITE = "http://forums.frontier.co.uk/"
  AUTHOR_STATS_URL = "#{FORUM_SITE}member.php?u=%d"
  AUTHOR_POSTS_URL = "#{FORUM_SITE}search.php?do=finduser&userid=%d&contenttype=vBForum_Post&showposts=1"
  AUTHOR_ACTIVITY_URL = "#{FORUM_SITE}/member.php?u=%d&tab=activitystream&type=user"
  
  def cache_posts_with_agent(agent, replace_existing=false)
    page = agent.get(AUTHOR_ACTIVITY_URL % forum_id)
    activity_doc = Nokogiri::HTML(page.body)
    
    # If they've changed their display name, or we don't yet have it
    post_author_name = activity_doc.css(".lastnavbit span").text
    update_attribute(:name, post_author_name) if name != post_author_name
    
    # Loop through the activity and cache the blocks
    post_elements = activity_doc.css("ul#activitylist li.forum_post")
    post_elements.each do |post|
      thread_link = post.css("div.fulllink a")
      if thread_link
        thread_url = thread_link.attribute("href").text
        post_id = thread_url.match(/post(\d+)/)[1].to_i
        
        if Post.where(["author_id = ? AND forum_post_id = ?", forum_id, post_id]).count > 0
          break unless replace_existing
        end
        
        create_normalized_post(post_id, post.to_s, replace_existing)
      else
        logger.debug("Failed to find the fulllink element")
      end
    end
  end
  
  def cache_new_posts(replace_existing=false)
    logger.debug("Starting cache_new_posts")
    posts_doc = Nokogiri::HTML(open(Author::AUTHOR_POSTS_URL % forum_id))
    
    post_list_element = posts_doc.css("#searchbits .postbitcontainer")
    logger.error("Found post list count of #{post_list_element.count} for author with forum_id #{forum_id}")
    
    post_list_element.each do |post_li|
      post_id = post_li.attributes["id"].text.match(/post_(\d+)/)[1].to_i
      logger.debug("Evaluating post #{post_id} for author #{forum_id} (#{name})")<commit_msg>Change the css selector to pick up threads that a user has started, rather than just posts in existing ones<commit_after>require 'open-uri'

class Author < ActiveRecord::Base
  has_many :posts, dependent: :delete_all
  validates_uniqueness_of :forum_id
  default_scope { order('forum_id ASC') }
  
  FORUM_SITE = "http://forums.frontier.co.uk/"
  AUTHOR_STATS_URL = "#{FORUM_SITE}member.php?u=%d"
  AUTHOR_POSTS_URL = "#{FORUM_SITE}search.php?do=finduser&userid=%d&contenttype=vBForum_Post&showposts=1"
  AUTHOR_ACTIVITY_URL = "#{FORUM_SITE}/member.php?u=%d&tab=activitystream&type=user"
  
  def cache_posts_with_agent(agent, replace_existing=false)
    page = agent.get(AUTHOR_ACTIVITY_URL % forum_id)
    activity_doc = Nokogiri::HTML(page.body)
    
    # If they've changed their display name, or we don't yet have it
    post_author_name = activity_doc.css(".lastnavbit span").text
    update_attribute(:name, post_author_name) if name != post_author_name
    
    # Loop through the activity and cache the blocks
    post_elements = activity_doc.css("ul#activitylist li.activitybit")
    post_elements.each do |post|
      thread_link = post.css("div.fulllink a")
      if thread_link
        thread_url = thread_link.attribute("href").text
        post_id = thread_url.match(/post(\d+)/)[1].to_i
        
        if Post.where(["author_id = ? AND forum_post_id = ?", forum_id, post_id]).count > 0
          break unless replace_existing
        end
        
        create_normalized_post(post_id, post.to_s, replace_existing)
      else
        logger.debug("Failed to find the fulllink element")
      end
    end
  end
  
  def cache_new_posts(replace_existing=false)
    logger.debug("Starting cache_new_posts")
    posts_doc = Nokogiri::HTML(open(Author::AUTHOR_POSTS_URL % forum_id))
    
    post_list_element = posts_doc.css("#searchbits .postbitcontainer")
    logger.error("Found post list count of #{post_list_element.count} for author with forum_id #{forum_id}")
    
    post_list_element.each do |post_li|
      post_id = post_li.attributes["id"].text.match(/post_(\d+)/)[1].to_i
      logger.debug("Evaluating post #{post_id} for author #{forum_id} (#{name})")
{'fraction_non_alphanumeric': 0.11449309295247015, 'fraction_numerical': 0.0018730976352142356, 'mean_word_length': 3.9674418604651165, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5407141', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1437, 'n_words': 278}