<commit_before>﻿/* Copyright 2015 <NAME>
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using OpenQA.Selenium;
using OpenQA.Selenium.PhantomJS;
using OpenQA.Selenium.Support.UI;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace Pickaxe.Runtime.Internal
{
    internal class SeleniumHttpWire : HttpWire
    {
        public SeleniumHttpWire(string url)
            : base(url)
        {
        }

        public override byte[] Download()
        {
            byte[] bytes = new byte[0];

            //http://stackoverflow.com/questions/18921099/add-proxy-to-phantomjsdriver-selenium-c
            var driverService = PhantomJSDriverService.CreateDefaultService();
            driverService.HideCommandPromptWindow = true;

            using (PhantomJSDriver phantom = new PhantomJSDriver(driverService))
            {
                phantom.Navigate().GoToUrl(Url);

                var wait = new WebDriverWait(phantom, TimeSpan.FromSeconds(15));
                wait.Message = "DOM didn't load";
                wait.Until(driver1 => ((IJavaScriptExecutor)phantom).ExecuteScript("return document.readyState").Equals("complete"));

                wait = new WebDriverWait(phantom, TimeSpan.FromSeconds(15));
                wait.Message = "Couldn't find element in page";
                wait.Until(drv => drv.FindElement(By.CssSelector(".pricecontainer")));

                string html = phantom.PageSource;

                bytes = Encoding.UTF8.GetBytes(html);
            }

            return bytes;
        }

    }
}<commit_msg>css selector args<commit_after>﻿/* Copyright 2015 <NAME>
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using OpenQA.Selenium;
using OpenQA.Selenium.PhantomJS;
using OpenQA.Selenium.Support.UI;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace Pickaxe.Runtime.Internal
{
    internal class SeleniumHttpWire : HttpWire
    {
        private string _cssElement;

        public SeleniumHttpWire(string url, string cssElement)
            : base(url)
        {
            _cssElement = cssElement;
        }

        public override byte[] Download()
        {
            byte[] bytes = new byte[0];

            var driverService = PhantomJSDriverService.CreateDefaultService();
            driverService.HideCommandPromptWindow = true;

            if(Proxy != null)
            {
                driverService.ProxyType = "http";
                driverService.Proxy = Proxy.ToString();
            }

            using (PhantomJSDriver phantom = new PhantomJSDriver(driverService))
            {
                phantom.Navigate().GoToUrl(Url);

                var wait = new WebDriverWait(phantom, TimeSpan.FromSeconds(15));
                wait.Message = "DOM didn't load";
                wait.Until(driver1 => ((IJavaScriptExecutor)phantom).ExecuteScript("return document.readyState").Equals("complete"));

                if (!String.IsNullOrEmpty(_cssElement))
                {
                    wait = new WebDriverWait(phantom, TimeSpan.FromSeconds(15));
                    wait.Message = "Couldn't find element in page";
                    wait.Until(drv => drv.FindElement(By.CssSelector(_cssElement)));
                }

                string html = phantom.PageSource;
                bytes = Encoding.UTF8.GetBytes(html);
            }

            return bytes;
        }

    }
}
{'fraction_non_alphanumeric': 0.09029649595687332, 'fraction_numerical': 0.008535489667565139, 'mean_word_length': 2.3683812405446294, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6976776', 'n_tokens_mistral': 1224, 'n_tokens_neox': 1111, 'n_words': 330}