<commit_before>﻿using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Web;

using HtmlAgilityPack;      // Handles crappy (NOT well formed) HTML

using MoviePicker.Common.Interfaces;
using MoviePicker.Common;

namespace MovieMiner
{
	public class MineBoxOfficeProphet : MinerBase
	{
		private const string DEFAULT_URL = "http://www.boxofficeprophets.com/";

		public MineBoxOfficeProphet()
			: base("Box Office Prophet", "Prophet", DEFAULT_URL)
		{
		}

		public override List<IMovie> Mine()
		{
			var result = new List<IMovie>();
			var web = new HtmlWeb();

			var doc = web.Load(Url);

			// Lookup XPATH to get the right node that matches.
			// Select all of the <script> nodes that are children of <body> with an attribute of "src"
			// REF: https://www.w3schools.com/xml/xpath_syntax.asp

			//var node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@href, 'http://www.boxofficeprophets.com/column') and //strong='Weekend Forecast']");
			HtmlNode node = null;

			var nodes = doc.DocumentNode.SelectNodes("//body//a[contains(@href, 'http://www.boxofficeprophets.com/column')]");

			foreach (var aNode in nodes)
			{
				if (aNode.InnerText.Contains("Weekend Forecast"))
				{
					node = aNode;
					break;
				}
			}

			if (node != null)
			{
				var href = node.GetAttributeValue("href", null);

				if (href != null)
				{
					DateTime? articleDate = null;

					UrlSource = href;

					// Now retrieve the article page.

					doc = web.Load(UrlSource);

					// Get the date of the article

					node = doc.DocumentNode.SelectSingleNode("//body//div[@id='EchoTopic']//h4");

					if (node != null)
					{
						string articleText = node.InnerText;
						DateTime parsedDateTime;

						if (DateTime.TryParse(articleText, out parsedDateTime))
						{
							articleDate = parsedDateTime;
						}
					}

					// Get the data in the table.

					node = doc.DocumentNode.SelectSingleNode("//body//div[@id='EchoTopic']//table[@width='100%']");

					// TODO: Parse the header for column titles for mapping.

					//var tableRows = node?.SelectNodes("//tr[position()>2]");
					var tableRows = node?.SelectNodes("//tr[@bgcolor='eeeeee']");
<commit_msg>Look for page 2 on Box Office Prophets if the data is not on the first page.<commit_after>﻿using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Web;

using HtmlAgilityPack;      // Handles crappy (NOT well formed) HTML

using MoviePicker.Common.Interfaces;
using MoviePicker.Common;

namespace MovieMiner
{
	public class MineBoxOfficeProphet : MinerBase
	{
		private const string DEFAULT_URL = "http://www.boxofficeprophets.com/";

		public MineBoxOfficeProphet()
			: base("Box Office Prophet", "Prophet", DEFAULT_URL)
		{
		}

		public override List<IMovie> Mine()
		{
			var result = new List<IMovie>();
			var web = new HtmlWeb();

			var doc = web.Load(Url);

			// Lookup XPATH to get the right node that matches.
			// Select all of the <script> nodes that are children of <body> with an attribute of "src"
			// REF: https://www.w3schools.com/xml/xpath_syntax.asp

			//var node = doc.DocumentNode.SelectSingleNode("//body//a[contains(@href, 'http://www.boxofficeprophets.com/column') and //strong='Weekend Forecast']");
			HtmlNode node = null;

			var nodes = doc.DocumentNode.SelectNodes("//body//a[contains(@href, 'http://www.boxofficeprophets.com/column')]");

			foreach (var aNode in nodes)
			{
				if (aNode.InnerText.Contains("Weekend Forecast"))
				{
					node = aNode;
					break;
				}
			}

			if (node != null)
			{
				var href = node.GetAttributeValue("href", null);

				if (href != null)
				{
					DateTime? articleDate = null;

					UrlSource = href;

					// Now retrieve the article page.

					doc = web.Load(UrlSource);

					// Get the date of the article

					node = doc.DocumentNode.SelectSingleNode("//body//div[@id='EchoTopic']//h4");

					if (node != null)
					{
						string articleText = node.InnerText;
						DateTime parsedDateTime;

						if (DateTime.TryParse(articleText, out parsedDateTime))
						{
							articleDate = parsedDateTime;
						}
					}

					// Get the data in the table.

					node = doc.DocumentNode.SelectSingleNode("//body//div[@id='EchoTopic']//table[@width='100%']");

					// TODO: Parse the header for column titles for mapping.

					//var tableRows = node?.SelectNodes("//tr[position()>2]");
					var tableRows = node?.SelectNodes("//tr[@bgcolor='eeeeee']");

					if (tableRows == null)
					{
						// Try page 2

						UrlSource += "&columnpage=2";

						doc = web.Load(UrlSource);

						// Get the data in the table.

						node = doc.DocumentNode.SelectSingleNode("//body//div[@id='EchoTopic']//table[@width='100%']");

						tableRows = node?.SelectNodes("//tr[@bgcolor='eeeeee']");
					}

{'fraction_non_alphanumeric': 0.1419983416252073, 'fraction_numerical': 0.0037313432835820895, 'mean_word_length': 2.9975144987572495, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 8, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1191612', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1498, 'n_words': 380}