<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace Blueprint
{
    class SourceBrowser
    {

        public static void ProcessDirectory(string source, string destination)
        {
            string folderName = source.Split('\\').Last();
            
            // Process the list of files found in the directory. 
            string[] fileEntries = Directory.GetFiles(source);
            foreach (string fileName in fileEntries)
                ProcessFile(fileName, destination);

            // Recurse into subdirectories of this directory. 
            string[] subdirectoryEntries = Directory.GetDirectories(source);
            foreach (string subdirectory in subdirectoryEntries)
            {
                // get list of folders to browse
                string subfolders = "_posts";
                foreach (string include in Program.Config.Include)
                    subfolders += "|" + include;

                // only check certain directories
                Regex regex = new Regex(@"(" + subfolders + ")");
                Match match = regex.Match(subdirectory);

                if (match.Success)
                    ProcessDirectory(subdirectory, destination);
            }
               
        }

        public static void ProcessFile(string path, string writeDirectory)
        {
            SourceFile file = new SourceFile(path);
            string destination = writeDirectory;

            // convert markdown file to HTML
            if (file.FileType == ".md")
            {
                // if file is in '_posts' folder, alter writeDirectory
                Regex regex = new Regex(@"_posts");
                Match match = regex.Match(file.SourcePath);

                if (match.Success)
                    destination = file.CreateDirectoryStructure(file.FileName);

                file.ConvertFileToHTML(
                    path, 
                    destination + file.FileName + ".html"
                );
            }

        }
    }
}<commit_msg>Blueprint now copies over all miscellaneous folders which do not have a '_' in the directory name<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace Blueprint
{
    class SourceBrowser
    {

        public static void ProcessDirectory(string source, string destination)
        {
            string folderName = source.Split('\\').Last();
            
            // Process the list of files found in the directory. 
            string[] fileEntries = Directory.GetFiles(source);
            foreach (string fileName in fileEntries)
                ProcessFile(fileName, destination);

            // Recurse into subdirectories of this directory. 
            string[] subdirectoryEntries = Directory.GetDirectories(source);
            foreach (string subdirectory in subdirectoryEntries)
            {
                string subdirectoryName = subdirectory.Split('\\').Last();

                // get list of folders to browse
                string sourceFolders = "_posts";
                foreach (string include in Program.Config.Include)
                    sourceFolders += "|" + include;

                // only check certain directories
                Regex regex = new Regex(@"(" + sourceFolders + ")");
                Match match = regex.Match(subdirectory);


                if (match.Success)
                    ProcessDirectory(subdirectory, destination);
                else if (subdirectoryName[0].ToString() != "_")
                    CopyDirectory(subdirectory, destination + "scripts");
            }   
        }

        public static void ProcessFile(string path, string writeDirectory)
        {
            SourceFile file = new SourceFile(path);
            string destination = writeDirectory;

            // convert markdown file to HTML
            if (file.FileType == ".md")
            {
                // if file is in '_posts' folder, alter writeDirectory
                Regex regex = new Regex(@"_posts");
                Match match = regex.Match(file.SourcePath);

                if (match.Success)
                    destination = file.CreateDirectoryStructure(file.FileName);

                file.ConvertFileToHTML(
                    path, 
                    destination + file.FileName + ".html"
                );
            }
        }

        public static void CopyDirectory(string SourcePath, string DestinationPath)
        {
            if (!Directory.Exists(DestinationPath))
                Directory.CreateDirectory(DestinationPath);

            //Now Create all of the directories
            foreach (string dirPath in Directory.GetDirectories(SourcePath, "*", 
                SearchOption.AllDirectories))
                Directory.CreateDirectory(dirPath.Replace(SourcePath, DestinationPath));

            //Copy all the files & Replaces any files with the same name
            foreach (string newPath in Directory.GetFiles(SourcePath, "*.*", 
                SearchOption.AllDirectories))
                File.Copy(newPath, newPath.Replace(SourcePath, DestinationPath), true);
        }
    }
}
{'fraction_non_alphanumeric': 0.08294588060256648, 'fraction_numerical': 0.0001859773107680863, 'mean_word_length': 1.7424783273839877, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3364286', 'n_tokens_mistral': 1268, 'n_tokens_neox': 1166, 'n_words': 347}