<commit_before>import qbs
import qbs.File
import qbs.FileInfo
import qbs.TextFile

Project
{
    name: FileInfo.baseName(sourceDirectory)
    id: autoproject

    property path autoprojectFileDirectory: ""
    property path rootDirectory: sourceDirectory.replace(autoprojectFileDirectory, "")
    property path autoprojectsDirectory: "autoprojects"
    property path targetDirectory: [qbs.targetOS, qbs.architecture, qbs.toolchain.join("-")].join("-")
    property path testDirectory: "test"
    property path includeDirectory: "include"
    property path qtIncludeDirectory: "C:/Qt/5.10.0/msvc2017_64/include/"
    property path documentationDirectory: "doc"
    property stringList additionalDirectories: ["private", "resources"]
    property stringList ignoredDirectories: [autoprojectsDirectory]
    property stringList headerExtensions: ["h"]
    property stringList sourceExtensions: ["cpp"]
    property stringList documentationExtensions: ["qdoc"]
    property stringList documentationConfigExtensions: ["qdocconf"]
    property stringList additionalExtensions: ["ui", "rc"]

    property var externalModules:
    {
        return { ModuleName: "path/to/include",
                 OtherModule: "path/to/include" }
    }

    Probe
    {
        id: qtscanner
        property var modules: {}

        configure:
        {
            function getSubDirs(dir) { return File.directoryEntries(dir, File.Dirs | File.NoDotAndDotDot); }
            function getFilesInDirectory(dir) { File.directoryEntries(dir, File.Files); }
            function makePath(dir, subdir) { return FileInfo.joinPaths(dir, subdir); }
            function getQtModuleName(subdir) { return subdir.replace("Qt", "").toLowerCase() }

            var qtModules = {};
            var subdirs = getSubDirs(qtIncludeDirectory);
            for(var i in subdirs)
                qtModules[getQtModuleName(subdirs[i])] = getFilesInDirectory(makePath(qtIncludeDirectory, subdirs[i]));
            modules = qtModules;
        }
    }

    Probe
    {
        id: projectscanner
        property var projects: []

        configure:
        {
            function getSubDirs(dir) { return File.directoryEntries(dir, File.Dirs | File.NoDotAndDotDot); }
            function getFilesInDirectory(dir) { File.directoryEntries(dir, File.Files); }
            function makePath(dir, subdir, file) { return FileInfo.joinPaths(dir, subdir, file); }
            function makePath(dir, subdir) { return FileInfo.joinPaths(dir, subdir); }
            function hasExtension(file, extensions) { for(var i in extensions) if(file.endsWith("." + extensions[i])) return true; return false; }
            function isHeader(file) { return hasExtension(file, headerExtensions); }
            function isSource(file) { return hasExtension(file, sourceExtensions); }
            function isDoc(file) { return hasExtension(file, documentationExtensions); }
            function isDocConf(file) { return hasExtension(file, documentationConfigExtensions); }
            function readFile(file) { return TextFile(file, TextFile.ReadOnly).readAll(); }
            function getRegexResults(text, regex) { var results, regexResult = []; while(regexResult = regexp.exec(content)) results.push(regexResult[1]); return results; }
            function removeDuplicates(ar) { return ar.filter(function(element, index, array) { return array.indexOf(element) == index; }); }

            var scannedProjects = [];

            function scanProjects(rootDir)
            {
                var project = {}
                project["name"] = FileInfo.baseName(rootDir);
                project["path"] = rootDir;
                project["include"] = [];
                project["headers"] = [];
                project["sources"] = [];
                project["doc"] = {};
                project["test"] = {};

                var files = getFilesInDirectory(rootDir);

                for(var i in files)
                {
                    var file = files[i];

                    if(isHeader(file))
                        project["headers"].push(file);
                    else if(isSource(file))
                        project["sources"].push(file);
                    else
                        continue;

                    project["include"] = removeDuplicates(project["include"].concat(getRegexResults(readFile(makePath(rootDir, file), /#include [<|"](.*)[>|"]/g))));
                }

                var dirs = getSubDirs(rootDir);

                for(var i in dirs)
                {
                    var dir = dirs[i];

                    if(ignoredDirectories.contains(dir))
                        continue;
                    else if(additionalDirectories.contains(dir))<commit_msg>Refactored file detection and scanning<commit_after>import qbs
import qbs.File
import qbs.FileInfo
import qbs.TextFile

Project
{
    name: FileInfo.baseName(sourceDirectory)
    id: autoproject

    property path autoprojectFileDirectory: ""
    property path rootDirectory: sourceDirectory.replace(autoprojectFileDirectory, "")
    property path autoprojectsDirectory: "autoprojects"
    property path targetDirectory: [qbs.targetOS, qbs.architecture, qbs.toolchain.join("-")].join("-")
    property path testDirectory: "test"
    property path includeDirectory: "include"
    property path qtIncludeDirectory: "C:/Qt/5.10.0/msvc2017_64/include/"
    property path documentationDirectory: "doc"
    property stringList additionalDirectories: ["private", "resources"]
    property stringList ignoredDirectories: [autoprojectsDirectory]
    property stringList headerExtensions: ["h"]
    property stringList sourceExtensions: ["cpp"]
    property stringList documentationExtensions: ["qdoc"]
    property stringList documentationConfigExtensions: ["qdocconf"]
    property stringList additionalExtensions: ["ui", "rc"]

    property var externalModules:
    {
        return { ModuleName: "path/to/include",
                 OtherModule: "path/to/include" }
    }

    Probe
    {
        id: qtscanner
        property var modules: {}

        configure:
        {
            function getSubDirs(dir) { return File.directoryEntries(dir, File.Dirs | File.NoDotAndDotDot); }
            function getFilesInDirectory(dir) { File.directoryEntries(dir, File.Files); }
            function makePath(dir, subdir) { return FileInfo.joinPaths(dir, subdir); }
            function getQtModuleName(subdir) { return subdir.replace("Qt", "").toLowerCase() }

            var qtModules = {};
            var subdirs = getSubDirs(qtIncludeDirectory);

            for(var i in subdirs)
                qtModules[getQtModuleName(subdirs[i])] = getFilesInDirectory(makePath(qtIncludeDirectory, subdirs[i]));

            modules = qtModules;
        }
    }

    Probe
    {
        id: projectscanner
        property var projects: []

        configure:
        {
            function createDocProject(dir) { return { name: FileInfo.baseName(rootDir) + "Doc", path: makePath(dir, documentationDirectory), qtDeps: [], deps: [], conf: "" }; }
            function createProject(dir) { return { name: FileInfo.baseName(rootDir), path: dir, qtDeps: [], files: getProjectSourceFiles(dir), deps: [], includes: getProjectIncludes(files), headers: getProjectHeaders(files), doc: createDocProject(dir), test: createTestProject(dir), subprojects: [] }; }
            function createTestProject(dir) { return { name: FileInfo.baseName(rootDir) + "Test", path: makePath(dir, testDirectory), qtDeps: [], deps: [], files: [], includes: [] }; }
            function getFilesInDirectory(dir) { File.directoryEntries(dir, File.Files); }
            function getProjectDirectories(dir) { return [dir].concat(prependPath(dir, additionalDirectories)); }
            function getProjectFiles(dir) { var files = []; var dirs = getProjectDirectories(dir); for(var i in dirs) files = files.concat(prependPath(dirs[i], getFilesInDirectory(dirs[i]))); return files; }
            function getProjectIncludes(files) { var includes = []; for(var i in files) includes = includes.concat(getRegexResults(readFile(files[i]), /#include [<|"](.*)[>|"]/g)); return removeDuplicates(includes); }
            function getProjectHeaders(files) { var headers = []; for(var i in files) if(isHeader(files[i])) headers.push(FileInfo.baseName(files[i])); return removeDuplicates(headers); }
            function getProjectSourceFiles(dir) { return getProjectFiles(dir).filter(function(element) { return isHeader(file) || isSource(file); }); }
            function getRegexResults(text, regex) { var results, regexResult = []; while(regexResult = regexp.exec(content)) results.push(regexResult[1]); return results; }
            function getSubDirs(dir) { return File.directoryEntries(dir, File.Dirs | File.NoDotAndDotDot); }
            function hasExtension(file, extensions) { for(var i in extensions) if(file.endsWith("." + extensions[i])) return true; return false; }
            function isDoc(file) { return hasExtension(file, documentationExtensions); }
            function isDocConf(file) { return hasExtension(file, documentationConfigExtensions); }
            function isHeader(file) { return hasExtension(file, headerExtensions); }
            function isSource(file) { return hasExtension(file, sourceExtensions); }
            function makePath(path, subpath, subsubpath) { return FileInfo.joinPaths(path, subpath, subsubpath); }
            function makePath(path, subpath) { return FileInfo.joinPaths(path, subpath); }
            function prependPath(path, list) { var prependedList = []; return list.forEach(function(element) { this.push(FileInfo.joinPaths(path, element)); }, prependedList); }
            function readFile(file) { return TextFile(file, TextFile.ReadOnly).readAll(); }
            function removeDuplicates(ar) { return ar.filter(function(element, index, array) { return array.indexOf(element) == index; }); }

            var scannedProjects = [];

            function scanProjects(dir)
            {
                var project = createProject(dir);

                var dirs = getSubDirs(rootDir);

                for(var i in dirs)
                {
                    var dir = dirs[i];

                    if(ignoredDirectories.contains(dir))
                        continue;
                    else if(additionalDirectories.contains(dir))
{'fraction_non_alphanumeric': 0.11456928117170939, 'fraction_numerical': 0.002119868953555598, 'mean_word_length': 2.844074074074074, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3209236', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2714, 'n_words': 614}