The returned class as result: public class cs_HttpFetchResults    {        public bool blResultSuccess = false;        public string srFetchBody = ;        public string srFetchingFinalURL = ;        public bool bl404 = false;    }The HttpWebRequest version:    public static cs_HttpFetchResults func_fetch_Page(string srUrl, int irTimeOut = 60, string srRequestUserAgent = Mozilla/5.0 (Windows NT 6.3; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0, string srProxy = null, int irCustomEncoding = 0, bool blAutoDecode = true, bool blKeepAlive = true,  string srIPandHost = null)    {        cs_HttpFetchResults mycs_HttpFetchResults = new cs_HttpFetchResults();        mycs_HttpFetchResults.srFetchingFinalURL = srUrl;        HttpWebRequest request = null;        WebResponse response = null;        try        {            request = (HttpWebRequest)WebRequest.Create(srUrl);            request.CookieContainer = new System.Net.CookieContainer();            if (srProxy != null)            {                string srProxyHost = srProxy.Split(':')[0];                int irProxyPort = Int32.Parse(srProxy.Split(':')[1]);                System.Net.WebProxy my_awesomeproxy = new WebProxy(srProxyHost, irProxyPort);                my_awesomeproxy.Credentials = new NetworkCredential();                request.Proxy = my_awesomeproxy;            }            else            {                request.Proxy = null;            }            request.ContinueTimeout = irTimeOut * 1000;            request.ReadWriteTimeout = irTimeOut * 1000;            request.Timeout = irTimeOut * 1000;            request.UserAgent = srRequestUserAgent;            request.KeepAlive = blKeepAlive;            request.Accept = text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8;            WebHeaderCollection myWebHeaderCollection = request.Headers;            myWebHeaderCollection.Add(Accept-Language, en-gb,en;q=0.5);            myWebHeaderCollection.Add(Accept-Encoding, gzip, deflate);            request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;            using (response = request.GetResponse())            {                using (Stream strumien = response.GetResponseStream())                {                    Encoding myEncoding = Encoding.UTF8;                    string srContentType = ;                    if (response.ContentType != null)                    {                        srContentType = response.ContentType;                        if (srContentType.Contains(;))                        {                            srContentType = srContentType.Split(';')[1];                        }                        srContentType = srContentType.Replace(charset=, );                        srContentType = func_Process_Html_Input(srContentType);                    }                    try                    {                        myEncoding = Encoding.GetEncoding(srContentType);                    }                    catch                    {                        myEncoding = irCustomEncoding == 0 ? Encoding.UTF8                                                         : Encoding.GetEncoding(irCustomEncoding);                    }                    using (StreamReader sr = new StreamReader(strumien, myEncoding))                    {                        mycs_HttpFetchResults.srFetchBody = sr.ReadToEnd();                        if (blAutoDecode == true)                        {                            mycs_HttpFetchResults.srFetchBody = HttpUtility.HtmlDecode(mycs_HttpFetchResults.srFetchBody);                        }                        mycs_HttpFetchResults.srFetchingFinalURL = Return_Absolute_Url(response.ResponseUri.AbsoluteUri.ToString(), response.ResponseUri.AbsoluteUri.ToString());                        mycs_HttpFetchResults.blResultSuccess = true;                    }                }            }            if (request != null)                request.Abort();            request = null;        }        catch (Exception E)        {            if (E.Message.ToString().Contains((404)))                mycs_HttpFetchResults.bl404 = true;            csLogger.logCrawlingErrors(crawling failed url:  + srUrl, E);        }        finally        {            if (request != null)                request.Abort();            request = null;            if (response != null)                response.Close();            response = null;        }        return mycs_HttpFetchResults;    }The HttpClientHandler:    async public static Task<cs_HttpFetchResults> func_fetch_Page_New(string srUrl, int irTimeOut = 60,  string srRequestUserAgent = Mozilla/5.0 (Windows NT 6.3; WOW64; rv:31.0) Gecko/20100101 Firefox/31.0,  string srProxy = null, int irCustomEncoding = 0, bool blAutoDecode = true, bool blKeepAlive = true,  string srIPandHost = null)    {        cs_HttpFetchResults mycs_HttpFetchResults = new cs_HttpFetchResults();        mycs_HttpFetchResults.srFetchingFinalURL = srUrl;        try        {            using (HttpClientHandler myClientHandler = new HttpClientHandler())            {                myClientHandler.AllowAutoRedirect = true;                myClientHandler.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;                myClientHandler.UseDefaultCredentials = true;                if (srProxy != null)                {                    string srProxyHost = srProxy.Split(':')[0];                    int irProxyPort = Int32.Parse(srProxy.Split(':')[1]);                    System.Net.WebProxy my_awesomeproxy = new WebProxy(srProxyHost, irProxyPort);                    my_awesomeproxy.Credentials = new NetworkCredential();                    myClientHandler.Proxy = my_awesomeproxy;                    myClientHandler.UseProxy = true;                }                else                {                    myClientHandler.Proxy = null;                    myClientHandler.UseProxy = false;                }                using (var httpClient = new HttpClient(myClientHandler))                {                    httpClient.DefaultRequestHeaders.Add(Accept-Language, en-gb,en;q=0.5);                    httpClient.DefaultRequestHeaders.Add(Accept-Encoding, gzip, deflate);                    httpClient.Timeout = new TimeSpan(0, 0, irTimeOut);                    httpClient.DefaultRequestHeaders.Add(User-Agent, srRequestUserAgent);                    httpClient.DefaultRequestHeaders.Add(User-Agent, srRequestUserAgent);                    if (blKeepAlive == true)                    {                        httpClient.DefaultRequestHeaders.Connection.Clear();                        httpClient.DefaultRequestHeaders.ConnectionClose = false;                        httpClient.DefaultRequestHeaders.Connection.Add(Keep-Alive);                    }                    else                    {                        httpClient.DefaultRequestHeaders.Connection.Clear();                        httpClient.DefaultRequestHeaders.ConnectionClose = true;                    }                    using (var vrResponse = await httpClient.GetAsync(srUrl))                    {                        if (vrResponse.IsSuccessStatusCode == true)                        {                            var contenttype = vrResponse?.Content?.Headers?.First(h => h.Key.Equals(Content-Type));                            string srContentType = contenttype?.Value?.First();                            Encoding myEncoding = Encoding.UTF8;                            if (srContentType != null)                            {                                if (srContentType.Contains(;))                                {                                    srContentType = srContentType.Split(';')[1];                                }                                srContentType = srContentType.Replace(charset=, );                                srContentType = func_Process_Html_Input(srContentType);                            }                            try                            {                                myEncoding = Encoding.GetEncoding(srContentType);                            }                            catch                            {                                myEncoding = irCustomEncoding == 0 ? Encoding.UTF8                                                                 : Encoding.GetEncoding(irCustomEncoding);                            }                            var bytes = await vrResponse.Content.ReadAsByteArrayAsync();                            mycs_HttpFetchResults.srFetchBody = myEncoding.GetString(bytes);                            if (blAutoDecode == true)                            {                                mycs_HttpFetchResults.srFetchBody = HttpUtility.HtmlDecode(mycs_HttpFetchResults.srFetchBody);                            }                            string responseUri = vrResponse.RequestMessage.RequestUri.AbsoluteUri.ToString();                            mycs_HttpFetchResults.srFetchingFinalURL = Return_Absolute_Url(responseUri, responseUri);                            mycs_HttpFetchResults.blResultSuccess = true;                        }                        else                        {                            if (vrResponse.StatusCode == HttpStatusCode.NotFound)                                mycs_HttpFetchResults.bl404 = true;                        }                    }                }            }        }        catch (Exception E)        {            if (E.Message.ToString().Contains((404)))                mycs_HttpFetchResults.bl404 = true;            csLogger.logCrawlingErrors(crawling failed url:  + srUrl, E);        }        return mycs_HttpFetchResults;    }Other helper methods:public static string func_Process_Html_Input(string srHtmlInput)    {        srHtmlInput = HttpUtility.HtmlDecode(srHtmlInput);        srHtmlInput = Regex.Replace(srHtmlInput, @(\s)\s+, $1).Trim();        srHtmlInput = srHtmlInput.Replace(&#39, ');        return srHtmlInput;    }    public static string Return_Absolute_Url(string srRelativeUrl, string srCrawledUrl, List<string> lstBannedExtensions = null, bool blIgnoreBaseUriHost = false, bool blDoNotRemoveDash = false)    {        srRelativeUrl = HttpUtility.UrlDecode(srRelativeUrl);        srRelativeUrl = HttpUtility.HtmlDecode(srRelativeUrl);        lstBannedExtensions = (lstBannedExtensions == null) ? new List<String>() : lstBannedExtensions;        string srReturnUrl = null;        if (srRelativeUrl.Length > 0)            if (srRelativeUrl[0] == '.')                srRelativeUrl = srRelativeUrl.Substring(1);        Uri baseUri = new Uri(srCrawledUrl);        Uri NewUrl;        bool blUriResult = Uri.TryCreate(baseUri, srRelativeUrl, out NewUrl);        if (blUriResult == true)        {            if (NewUrl.AbsoluteUri.ToString().StartsWith(http) && (NewUrl.Host == baseUri.Host || blIgnoreBaseUriHost == true))            {                string srLastSegment = NewUrl.Segments[NewUrl.Segments.Length - 1].ToString();                if (lstBannedExtensions.Where(pr => srLastSegment.ToLowerInvariant().IndexOf(pr) != -1).Count<string>() == 0)                {                    srRelativeUrl = NewUrl.AbsoluteUri.ToString();                    if (srRelativeUrl.IndexOf(#) != -1 && blDoNotRemoveDash == false)                    {                        srRelativeUrl = srRelativeUrl.Substring(0, srRelativeUrl.IndexOf(#));                    }                    srRelativeUrl = HttpUtility.UrlDecode(srRelativeUrl);                    srRelativeUrl = HttpUtility.HtmlDecode(srRelativeUrl);                    srReturnUrl = srRelativeUrl;                }            }        }        return srReturnUrl;    }
Composing best web page fetcher function by HttpClientHandler for C#
c#;http;web scraping