<commit_before>﻿using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Web;

namespace DScraper
{
    public class CasperjsScraper
    {
        private CasperjsSettings _settings;

        public CasperjsScraper()
            : this(CasperjsSettings.FromConfig)
        {
        }

        public CasperjsScraper(CasperjsSettings settings)
        {
            _settings = settings;

            if (_settings == null)
            {
                throw new ArgumentNullException(nameof(settings));
            }

            if (string.IsNullOrWhiteSpace(_settings.PythonExePath))
            {
                throw new ArgumentNullException(nameof(_settings.PythonExePath));
            }

            if (string.IsNullOrWhiteSpace(_settings.PhantomjsExePath))
            {
                throw new ArgumentNullException(nameof(_settings.PhantomjsExePath));
            }

            if (string.IsNullOrWhiteSpace(_settings.CasperjsExePath))
            {
                throw new ArgumentNullException(nameof(_settings.CasperjsExePath));
            }

            Debug = false;
            DebugPort = 9001;
            OutputEncoding = Encoding.GetEncoding("GB2312");
        }

        public bool Debug { get; set; }

        public int DebugPort { get; set; }

        public Encoding OutputEncoding { get; set; }

        public string Execute(string scriptPath, object arg = null)
        {
            var flags = new List<string>();
            var arguments = new List<string>();
            var arg0 = JsonConvert.SerializeObject(arg ?? new { });

            if (!string.IsNullOrEmpty(arg0))
            {
                arguments.Add(HttpUtility.UrlEncode(arg0));
            }

            if (Debug)
            {
                flags.Add("--remote-debugger-port=" + DebugPort);
            }

            if (OutputEncoding != null)
            {
                flags.Add("--output-encoding=" + OutputEncoding.HeaderName);
            }

            var command = string.Format("casperjs {0} {1} {2}",
                string.Join(" ", flags), scriptPath, string.Join(" ", arguments));

            var result = ExecutePythonScript(command);

            return result;
        }

        public T Get<T>(string scriptPath, object args = null)
        {
            var result = Execute(scriptPath, args);

            return JsonConvert.DeserializeObject<T>(result);
        }<commit_msg>better debug<commit_after>﻿using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Web;

namespace DScraper
{
    public class CasperjsScraper
    {
        private CasperjsSettings _settings;

        public CasperjsScraper()
            : this(CasperjsSettings.FromConfig)
        {
        }

        public CasperjsScraper(CasperjsSettings settings)
        {
            _settings = settings;

            if (_settings == null)
            {
                throw new ArgumentNullException(nameof(settings));
            }

            if (string.IsNullOrWhiteSpace(_settings.PythonExePath))
            {
                throw new ArgumentNullException(nameof(_settings.PythonExePath));
            }

            if (string.IsNullOrWhiteSpace(_settings.PhantomjsExePath))
            {
                throw new ArgumentNullException(nameof(_settings.PhantomjsExePath));
            }

            if (string.IsNullOrWhiteSpace(_settings.CasperjsExePath))
            {
                throw new ArgumentNullException(nameof(_settings.CasperjsExePath));
            }

            Debug = false;
            DebugPort = 9001;
            OutputEncoding = Encoding.GetEncoding("GB2312");
        }

        public bool Debug { get; set; }

        public int DebugPort { get; set; }

        public Encoding OutputEncoding { get; set; }

        public string Execute(string scriptPath, object arg = null)
        {
            var flags = new List<string>();
            var arguments = new List<string>();
            var arg0 = JsonConvert.SerializeObject(arg ?? new { });

            if (!string.IsNullOrEmpty(arg0))
            {
                arguments.Add(HttpUtility.UrlEncode(arg0));
            }

            if (Debug)
            {
                flags.Add("--remote-debugger-port=" + DebugPort);

                Task.Run(() =>
                {
                    Thread.Sleep(1000);
                    Process.Start("http://localhost:" + DebugPort);
                });
            }

            if (OutputEncoding != null)
            {
                flags.Add("--output-encoding=" + OutputEncoding.HeaderName);
            }

            var command = string.Format("casperjs {0} {1} {2}",
                string.Join(" ", flags), scriptPath, string.Join(" ", arguments));

            var result = ExecutePythonScript(command);

            return result;
        }

        public T Get<T>(string scriptPath, object args = null)
        {
            var result = Execute(scriptPath, args);

            return JsonConvert.DeserializeObject<T>(result);
        }
{'fraction_non_alphanumeric': 0.09670664382257757, 'fraction_numerical': 0.006091757091185989, 'mean_word_length': 1.718054837040869, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '744621', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1325, 'n_words': 273}