This code is a page scraper using HtmlAgilityPack that creates a DOM document upon construction and allows for node manipulation afterward.HtmlAgilityPack uses XPath Selectors for selecting nodes.An example use case would look like:var me = new PageScraper(http://codereview.stackexchange.com/users/62429/quill, Console.WriteLine);var usernameNode = me.FetchNode(//h2[@class=user-card-name]);var username = PageScraper.FetchNodeText(usernameNode);This is the PageScraper class, the only class in the Scraper namespace so far.The reason I pass in an Action is so that I can attach log messages to a Forms list box.using System;using System.Net;using System.Text.RegularExpressions;using HtmlAgilityPack;namespace Scraper{    public class PageScraper    {        private readonly HtmlDocument _document;        public Action<string> Print { get; set; }        public PageScraper(string url, Action<string> print)        {            if (string.IsNullOrEmpty(url))            {                throw new Exception(url is empty);            }            Print = print;            WebClient webClient = new WebClient();            Print(downloading page);            string urlWithoutNewLineCharacters = StripNewLineCharactersAndSpaces(url);            string html = webClient.DownloadString(urlWithoutNewLineCharacters);            _document = new HtmlDocument();            _document.LoadHtml(html);            Print(loading page);        }        public HtmlNodeCollection FetchLinks()        {            Print(fetching links);            return _document.DocumentNode.SelectNodes(//a[@href]);        }          public HtmlNodeCollection FetchNodes(string selector)        {            Print(fetching nodes);            return _document.DocumentNode.SelectNodes(selector);        }        public HtmlNode FetchNode(string selector)        {            Print(fetching node);            return _document.DocumentNode.SelectSingleNode(selector);        }        public static string FetchNodeText(HtmlNode node)        {            if (node != null && node.InnerText != null)            {                return node.InnerText;            }            return string.Empty;        }        public static string FetchNodeHref(HtmlNode node)        {            if (node != null && node.Attributes[href].Value != null)            {                return node.Attributes[href].Value;            }            return string.Empty;        }        private static string StripNewLineCharactersAndSpaces(string str)        {            return Regex.Replace(str, @\t|\n|\r| |\s+, string.Empty);        }    }}
Page Scraper and DOM manipulator
c#;web scraping
if (string.IsNullOrEmpty(url)){    throw new Exception(url is empty);}  Throwing a general Exception is mostly bad. If the url parameter is null you should throw an ArgumentNullException and if it is empty you should throw an ArgumentException like so  if (url == null) { throw new ArgumentNullException(nameof(url)); }  if (url.Length == 0) { throw new ArgumentException(${nameof(url)} is empty, nameof(url)); }  WebClient webClient = new WebClient();  You should make use of the var type here because the assigned type is obvious from the right hand side of the assignment.  You don't check if the passed in Action<string> is null. A general rule of thumb is to check for every public accessible method wether the passed in arguments are null, because it is much better to get an exception early. In this case I would change the Print property in a way that it has a backing field instead of beeing autoimplemented. In this way you can use the setter to place the null validation. If the value is null you need to replace it with a noop action like so  private Action<string> _print;public Action<string> Print{    get    {        return _print;    }    set    {        _print = value ?? ((s) => { });    }}In addition this will allow you to also have an overloaded constructor which only take the string url as a parameter. Adding the suggestion of  @MarquisBlount of having an initialization method would then lead to  public PageScraper(string url, Action<string> print){    if (url == null) { throw new ArgumentNullException(nameof(url)); }    if (url.Length == 0) { throw new ArgumentException(${nameof(url)} is empty, nameof(url)); }    _document = new HtmlDocument();    Print = print;    LoadDocument(url);}public PageScraper(string url)    : this(url, null){ }private void LoadDocument(string url){    var webClient = new WebClient();    Print(downloading page);    string urlWithoutNewLineCharacters = StripNewLineCharactersAndSpaces(url);    string html = webClient.DownloadString(urlWithoutNewLineCharacters);    _document.LoadHtml(html);    Print(loading page);}