<commit_before>﻿using OfficeDevPnP.Core.Framework.TimerJobs;
using OfficeDevPnP.Core.Framework.TimerJobs.Enums;
using System;
using System.Linq;
using System.Collections.Generic;
using System.Collections.Concurrent;
using Microsoft.SharePoint.Client;
using System.Reflection;
using System.Diagnostics;
using System.Net;
using Microsoft.SharePoint.Client.Search.Query;

namespace SharePoint.Scanning.Framework
{
    /// <summary>
    /// Base scanner class, inherit from this class if you're building you're own scanning solutions
    /// </summary>
    public abstract class ScanJob : TimerJob
    {
        #region Variables
        public Int32 ScannedSites = 0;
        public Int32 ScannedWebs = 0;
        public Int32 ScannedLists = 0;
        private static volatile bool firstSiteCollectionDone = false;
        private object scannedSitesLock = new object();
        private object scannedWebsLock = new object();
        private object scannedListsLock = new object();
        public DateTime StartTime;
        public string OutputFolder;
        public string Separator = ",";
        public string CsvFile;
        public string Tenant;
        public string ClientTag;
        public IList<string> Urls;

        // Result stacks
        public ConcurrentStack<ScanError> ScanErrors = new ConcurrentStack<ScanError>();
        #endregion

        #region Construction
        /// <summary>
        /// Constructs scan job
        /// </summary>
        /// <param name="options">Scanner options</param>
        /// <param name="jobName">Name of the job</param>
        /// <param name="jobVersion">Version of the job</param>
        public ScanJob(BaseOptions options, string jobName, string jobVersion) : base(jobName, jobVersion)
        {
            // Basic scan job configuration
            this.UseThreading = true;
            this.MaximumThreads = options.Threads;
            this.TenantAdminSite = options.TenantAdminSite;
            this.OutputFolder = DateTime.Now.Ticks.ToString();
            this.Separator = options.Separator;
            this.ExcludeOD4B = !options.IncludeOD4B;
            this.CsvFile = options.CsvFile;
            this.Tenant = options.Tenant;
            this.Urls = options.Urls;
            this.ClientTag = ConstructClientTag(jobName);

            // Authentication setup
            if (options.AuthenticationTypeProvided() == AuthenticationType.AppOnly)
            {
                this.UseAppOnlyAuthentication(options.ClientID, options.ClientSecret);
            }
            else if (options.AuthenticationTypeProvided() == AuthenticationType.AzureADAppOnly)
            {
                this.UseAzureADAppOnlyAuthentication(options.ClientID, options.AzureTenant, options.CertificatePfx, options.CertificatePfxPassword);
            }
            else if (options.AuthenticationTypeProvided() == AuthenticationType.Office365)
            {
                this.UseOffice365Authentication(options.User, options.Password);
            }

            // Configure sites to scan
            if (!String.IsNullOrEmpty(this.Tenant))
            {
                this.AddSite(string.Format("https://{0}.sharepoint.com/*", this.Tenant));
                this.AddSite(string.Format("https://{0}-my.sharepoint.com/*", this.Tenant));
            }
            else if (this.Urls != null && this.Urls.Count > 0)
            {
                foreach (var url in this.Urls)
                {
                    this.AddSite(url);
                }
            }
            else if (!String.IsNullOrEmpty(this.CsvFile))
            {
                foreach (var row in LoadSitesFromCsv(this.CsvFile, this.Separator.ToCharArray().First()))
                {
                    this.AddSite(row[0]); //first column in the row contains url
                }
            }
            else
            {
                Console.WriteLine("No site selection specified, assume the job will use search to retrieve a list of sites");
            }

            this.StartTime = DateTime.Now;
        }
        #endregion

        #region Base execution methods
        /// <summary>
        /// Main method to execute the scanner
        /// </summary>
        /// <returns>DateTime when the scanning was started</returns>
        public virtual DateTime Execute()
        {
            DateTime start = DateTime.Now;
            Console.WriteLine("=====================================================");
            Console.WriteLine("Scanning is starting...{0}", start.ToString());
            Console.WriteLine("=====================================================");

            // Launch the job
            this.Run();
<commit_msg>Trim added urls<commit_after>﻿using OfficeDevPnP.Core.Framework.TimerJobs;
using OfficeDevPnP.Core.Framework.TimerJobs.Enums;
using System;
using System.Linq;
using System.Collections.Generic;
using System.Collections.Concurrent;
using Microsoft.SharePoint.Client;
using System.Reflection;
using System.Diagnostics;
using System.Net;
using Microsoft.SharePoint.Client.Search.Query;

namespace SharePoint.Scanning.Framework
{
    /// <summary>
    /// Base scanner class, inherit from this class if you're building you're own scanning solutions
    /// </summary>
    public abstract class ScanJob : TimerJob
    {
        #region Variables
        public Int32 ScannedSites = 0;
        public Int32 ScannedWebs = 0;
        public Int32 ScannedLists = 0;
        private static volatile bool firstSiteCollectionDone = false;
        private object scannedSitesLock = new object();
        private object scannedWebsLock = new object();
        private object scannedListsLock = new object();
        public DateTime StartTime;
        public string OutputFolder;
        public string Separator = ",";
        public string CsvFile;
        public string Tenant;
        public string ClientTag;
        public IList<string> Urls;

        // Result stacks
        public ConcurrentStack<ScanError> ScanErrors = new ConcurrentStack<ScanError>();
        #endregion

        #region Construction
        /// <summary>
        /// Constructs scan job
        /// </summary>
        /// <param name="options">Scanner options</param>
        /// <param name="jobName">Name of the job</param>
        /// <param name="jobVersion">Version of the job</param>
        public ScanJob(BaseOptions options, string jobName, string jobVersion) : base(jobName, jobVersion)
        {
            // Basic scan job configuration
            this.UseThreading = true;
            this.MaximumThreads = options.Threads;
            this.TenantAdminSite = options.TenantAdminSite;
            this.OutputFolder = DateTime.Now.Ticks.ToString();
            this.Separator = options.Separator;
            this.ExcludeOD4B = !options.IncludeOD4B;
            this.CsvFile = options.CsvFile;
            this.Tenant = options.Tenant;
            this.Urls = options.Urls;
            this.ClientTag = ConstructClientTag(jobName);

            // Authentication setup
            if (options.AuthenticationTypeProvided() == AuthenticationType.AppOnly)
            {
                this.UseAppOnlyAuthentication(options.ClientID, options.ClientSecret);
            }
            else if (options.AuthenticationTypeProvided() == AuthenticationType.AzureADAppOnly)
            {
                this.UseAzureADAppOnlyAuthentication(options.ClientID, options.AzureTenant, options.CertificatePfx, options.CertificatePfxPassword);
            }
            else if (options.AuthenticationTypeProvided() == AuthenticationType.Office365)
            {
                this.UseOffice365Authentication(options.User, options.Password);
            }

            // Configure sites to scan
            if (!String.IsNullOrEmpty(this.Tenant))
            {
                this.AddSite(string.Format("https://{0}.sharepoint.com/*", this.Tenant));
                this.AddSite(string.Format("https://{0}-my.sharepoint.com/*", this.Tenant));
            }
            else if (this.Urls != null && this.Urls.Count > 0)
            {
                foreach (var url in this.Urls)
                {
                    this.AddSite(url.Trim());
                }
            }
            else if (!String.IsNullOrEmpty(this.CsvFile))
            {
                foreach (var row in LoadSitesFromCsv(this.CsvFile, this.Separator.ToCharArray().First()))
                {
                    this.AddSite(row[0].Trim()); //first column in the row contains url
                }
            }
            else
            {
                Console.WriteLine("No site selection specified, assume the job will use search to retrieve a list of sites");
            }

            this.StartTime = DateTime.Now;
        }
        #endregion

        #region Base execution methods
        /// <summary>
        /// Main method to execute the scanner
        /// </summary>
        /// <returns>DateTime when the scanning was started</returns>
        public virtual DateTime Execute()
        {
            DateTime start = DateTime.Now;
            Console.WriteLine("=====================================================");
            Console.WriteLine("Scanning is starting...{0}", start.ToString());
            Console.WriteLine("=====================================================");

            // Launch the job
            this.Run();

{'fraction_non_alphanumeric': 0.11808510638297873, 'fraction_numerical': 0.004680851063829788, 'mean_word_length': 2.02964872703835, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 39, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7591053', 'n_tokens_mistral': 2373, 'n_tokens_neox': 2275, 'n_words': 556}