<commit_before>require 'open-uri'
require 'pry'
require 'nokogiri'
require 'launchy'




class Rotoworld::Scraper

  attr_accessor :title, :headline, :source, :impact, :index
  def get_posts 


    doc = Nokogiri::HTML(open("http://www.rotoworld.com/playernews/nfl/football-player-news/?ls=roto:nfl:gnav&rw=1"))
    player_set = doc.css("div.RW_pn div.pb")
    index = 1


    player_set.each do |player|

      post = Rotoworld::Post.new

      post_title_info = player.css("div.headline div.player").text
      player_name = player.css("div.headline div.player a").text
      player_position_team = post_title_info.match(/- .+\s+-+\s+.+[st$]/)
      post.title = "#{player_name}#{player_position_team}"
      post.headline = player.css("div.report p").text
      post.impact = player.css("div.impact").first.text.strip
      post.source = player.css("div.info div.source a").attr("href").value
      post.index = index
      index +=1
   
    end
    
  end
end<commit_msg>Added validation in post title to accept Free Agents and Players<commit_after>require 'open-uri'
require 'pry'
require 'nokogiri'
require 'launchy'




class Rotoworld::Scraper

  attr_accessor :title, :headline, :source, :impact, :index
  def get_posts 


    doc = Nokogiri::HTML(open("http://www.rotoworld.com/playernews/nfl/football-player-news/?ls=roto:nfl:gnav&rw=1"))
    player_set = doc.css("div.RW_pn div.pb")
    index = 1


    player_set.each do |player|

      post = Rotoworld::Post.new

      post_title_info = player.css("div.headline div.player").text
      player_name = player.css("div.headline div.player a").first.text
      player_position_team = post_title_info.match(/- .+\s+-+\s+.+[str$]/)
      post.title = "#{player_name}#{player_position_team}"
      post.headline = player.css("div.report p").text
      post.impact = player.css("div.impact").first.text.strip
      post.source = player.css("div.info div.source a").attr("href").value
      post.index = index
      index +=1
   
    end
    
  end
end
{'fraction_non_alphanumeric': 0.15189243027888447, 'fraction_numerical': 0.00298804780876494, 'mean_word_length': 3.311158798283262, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2303666', 'n_tokens_mistral': 732, 'n_tokens_neox': 696, 'n_words': 125}