<commit_before>/*
 * copyright (c) 2017 <NAME>
 *
 * This file is part of ShiftMediaProject.
 *
 * ShiftMediaProject is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * ShiftMediaProject is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with ShiftMediaProject; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "projectGenerator.h"

#include <algorithm>
#include <utility>
#include <sstream>
#include <iterator>

//This can be used to force all detected DCE values to be output to file
// whether they are enabled in current configuration or not
#define FORCEALLDCE 0

const string asDCETags[] = {"ARCH_", "HAVE_", "CONFIG_", "EXTERNAL_", "INTERNAL_", "INLINE_"};

bool ProjectGenerator::outputProjectDCE(const StaticList& vIncludeDirs)
{
    outputLine("  Generating missing DCE symbols (" + m_sProjectName + ")...");
    //Create list of source files to scan
#if !FORCEALLDCE
    StaticList vSearchFiles = m_vCIncludes;
    vSearchFiles.insert(vSearchFiles.end(), m_vCPPIncludes.begin(), m_vCPPIncludes.end());
    vSearchFiles.insert(vSearchFiles.end(), m_vHIncludes.begin(), m_vHIncludes.end());
#else
    StaticList vSearchFiles;
    bool bRecurse = (m_sProjectDir.compare(this->m_ConfigHelper.m_sRootDirectory) != 0);
    findFiles(m_sProjectDir + "*.h", vSearchFiles, bRecurse);
    findFiles(m_sProjectDir + "*.c", vSearchFiles, bRecurse);
    findFiles(m_sProjectDir + "*.cpp", vSearchFiles, bRecurse);
#endif
    //Ensure we can add extra items to the list without needing reallocs
    if (vSearchFiles.capacity() < vSearchFiles.size() + 250) {
        vSearchFiles.reserve(vSearchFiles.size() + 250);
    }

    //Check for DCE constructs
    map<string, DCEParams> mFoundDCEUsage;
    set<string> vNonDCEUsage;
    StaticList vPreProcFiles;
    //Search through each included file
    for (StaticList::iterator itFile = vSearchFiles.begin(); itFile < vSearchFiles.end(); itFile++) {
        //Open the input file
        makeFileGeneratorRelative(*itFile, *itFile);
        string sFile;
        if (!loadFromFile(*itFile, sFile)) {
            return false;
        }
        bool bRequiresPreProcess = false;
        outputProjectDCEFindFunctions(sFile, *itFile, mFoundDCEUsage, bRequiresPreProcess, vNonDCEUsage);
        if (bRequiresPreProcess) {
            vPreProcFiles.push_back(*itFile);
        }

        //Check if this file includes additional source files
        uint uiFindPos = sFile.find(".c\"");
        while (uiFindPos != string::npos) {
            //Check if this is an include
            uint uiFindPos2 = sFile.rfind("#include \"", uiFindPos);
            if ((uiFindPos2 != string::npos) && (uiFindPos - uiFindPos2 < 50)) {
                //Get the name of the file
                uiFindPos2 += 10;
                uiFindPos += 2;
                string sTemplateFile = sFile.substr(uiFindPos2, uiFindPos - uiFindPos2);
                //check if file contains current project
                uint uiProjName = sTemplateFile.find(m_sProjectName);
                if (uiProjName != string::npos) {
                    sTemplateFile = sTemplateFile.substr(uiProjName + m_sProjectName.length() + 1);
                }
                string sFound;
                string sBack = sTemplateFile;
                sTemplateFile = m_sProjectDir + sBack;
                if (!findFile(sTemplateFile, sFound)) {
                    sTemplateFile = m_ConfigHelper.m_sRootDirectory + '/' + sBack;
                    if (!findFile(sTemplateFile, sFound)) {
                        sTemplateFile = m_ConfigHelper.m_sProjectDirectory + m_sProjectName + '/' + sBack;
                        if (!findFile(sTemplateFile, sFound)) {
                            sTemplateFile = itFile->substr(0, itFile->rfind('/') + 1) + sBack;
                            if (!findFile(sTemplateFile, sFound)) {
                                outputError("Failed to find included file " + sBack);
                                return false;
                            }
                        }
                    }
                }
                //Add the file to the list
                if (find(vSearchFiles.begin(), vSearchFiles.end(), sTemplateFile) == vSearchFiles.end()) {
                    makeFileProjectRelative(sTemplateFile, sTemplateFile);
                    vSearchFiles.push_back(sTemplateFile);
                }
            }
            //Check for more
            uiFindPos = sFile.find(".c\"", uiFindPos + 1);
        }
    }

#if !FORCEALLDCE
    //Get a list of all files in current project directory (including subdirectories)
    bool bRecurse = (m_sProjectDir.compare(this->m_ConfigHelper.m_sRootDirectory) != 0);
    vSearchFiles.resize(0);
    findFiles(m_sProjectDir + "*.h", vSearchFiles, bRecurse);
    findFiles(m_sProjectDir + "*.c", vSearchFiles, bRecurse);
    findFiles(m_sProjectDir + "*.cpp", vSearchFiles, bRecurse);
    //Ensure we can add extra items to the list without needing reallocs
    if (vSearchFiles.capacity() < vSearchFiles.size() + 250) {
        vSearchFiles.reserve(vSearchFiles.size() + 250);
    }

    //Check all configurations are enabled early to avoid later lookups of unused functions
    for (map<string, DCEParams>::iterator itDCE = mFoundDCEUsage.begin(); itDCE != mFoundDCEUsage.end(); ) {
        outputProgramDCEsResolveDefine(itDCE->second.sDefine);
        if (itDCE->second.sDefine.compare("1") == 0) {
            vNonDCEUsage.insert(itDCE->first);
            //remove from the list
            mFoundDCEUsage.erase(itDCE++);
        } else {
            ++itDCE;
        }
    }
#endif

    //Now we need to find the declaration of each function
    map<string, DCEParams> mFoundDCEFunctions;
    map<string, DCEParams> mFoundDCEVariables;
    if (mFoundDCEUsage.size() > 0) {
        //Search through each included file
        for (StaticList::iterator itFile = vSearchFiles.begin(); itFile < vSearchFiles.end(); itFile++) {
            string sFile;
            if (!loadFromFile(*itFile, sFile)) {
                return false;
            }
            for (map<string, DCEParams>::iterator itDCE = mFoundDCEUsage.begin(); itDCE != mFoundDCEUsage.end(); ) {
                string sReturn;
                bool bIsFunc;
                if (outputProjectDCEsFindDeclarations(sFile, itDCE->first, *itFile, sReturn, bIsFunc)) {
                    //Get the declaration file
                    string sFileName;
                    makePathsRelative(*itFile, m_ConfigHelper.m_sRootDirectory, sFileName);
                    if (sFileName.at(0) == '.') {
                        sFileName = sFileName.substr(2);
                    }
                    if (bIsFunc) {
                        mFoundDCEFunctions[sReturn] = {itDCE->second.sDefine, sFileName};
                    } else {
                        mFoundDCEVariables[sReturn] = {itDCE->second.sDefine, sFileName};
                    }

                    //Remove it from the list
                    mFoundDCEUsage.erase(itDCE++);
                } else {
                    //Only increment the iterator when nothing has been found
                    // when we did find something we erased a value from the list so the iterator is still valid
                    ++itDCE;
                }
            }
        }
    }

    //Add any files requiring pre-processing to unfound list
    for (StaticList::iterator itPP = vPreProcFiles.begin(); itPP < vPreProcFiles.end(); itPP++) {
        mFoundDCEUsage[*itPP] = {"#", *itPP};
    }

    //Check if we failed to find any functions
    if (mFoundDCEUsage.size() > 0) {
        vector<string> vIncludeDirs2 = vIncludeDirs;
        string sTempFolder = sTempDirectory + m_sProjectName;
        if (!makeDirectory(sTempDirectory) || !makeDirectory(sTempFolder)) {
            outputError("Failed to create temporary working directory (" + sTempFolder + ")");
            return false;
        }
        //Get all the files that include functions
        map<string, vector<DCEParams>> mFunctionFiles;
        for (map<string, DCEParams>::iterator itDCE = mFoundDCEUsage.begin(); itDCE != mFoundDCEUsage.end(); itDCE++) {
            //Remove project dir from start of file
            uint uiPos = itDCE->second.sFile.find(m_sProjectDir);
            uiPos = (uiPos == string::npos) ? 0 : uiPos + m_sProjectDir.length();
            string sFile = sTempFolder + '/' + itDCE->second.sFile.substr(uiPos);
            if (sFile.find('/', sTempFolder.length() + 1) != string::npos) {
                makeDirectory(sFile.substr(0, sFile.rfind('/')));
            }
            //Copy file to local working directory
            if (!copyFile(itDCE->second.sFile, sFile)) {
                outputError("Failed to copy dce file (" + itDCE->second.sFile + ") to temporary directory (" + sFile + ")");
                return false;
            }
            mFunctionFiles[sFile].push_back({itDCE->second.sDefine, itDCE->first});
        }
        map<string, vector<string>> mDirectoryObjects;
        const string asDCETags2[] = {"if (", "if(", "& ", "&", "| ", "|"};
        for (map<string, vector<DCEParams>>::iterator itDCE = mFunctionFiles.begin(); itDCE != mFunctionFiles.end(); itDCE++) {
            //Get subdirectory
            string sSubFolder = "";
            if (itDCE->first.find('/', sTempFolder.length() + 1) != string::npos) {
                sSubFolder = m_sProjectDir + itDCE->first.substr(sTempFolder.length() + 1, itDCE->first.rfind('/') - sTempFolder.length() - 1);
                if (find(vIncludeDirs2.begin(), vIncludeDirs2.end(), sSubFolder) == vIncludeDirs2.end()) {
                    //Need to add subdirectory to include list
                    vIncludeDirs2.push_back(sSubFolder);<commit_msg>Handle directories potentially being cwd as './' or '' during dce generation.<commit_after>/*
 * copyright (c) 2017 <NAME>
 *
 * This file is part of ShiftMediaProject.
 *
 * ShiftMediaProject is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * ShiftMediaProject is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with ShiftMediaProject; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "projectGenerator.h"

#include <algorithm>
#include <utility>
#include <sstream>
#include <iterator>

//This can be used to force all detected DCE values to be output to file
// whether they are enabled in current configuration or not
#define FORCEALLDCE 0

const string asDCETags[] = {"ARCH_", "HAVE_", "CONFIG_", "EXTERNAL_", "INTERNAL_", "INLINE_"};

bool ProjectGenerator::outputProjectDCE(const StaticList& vIncludeDirs)
{
    outputLine("  Generating missing DCE symbols (" + m_sProjectName + ")...");
    //Create list of source files to scan
#if !FORCEALLDCE
    StaticList vSearchFiles = m_vCIncludes;
    vSearchFiles.insert(vSearchFiles.end(), m_vCPPIncludes.begin(), m_vCPPIncludes.end());
    vSearchFiles.insert(vSearchFiles.end(), m_vHIncludes.begin(), m_vHIncludes.end());
#else
    StaticList vSearchFiles;
    bool bRecurse = (m_sProjectDir.compare(this->m_ConfigHelper.m_sRootDirectory) != 0);
    findFiles(m_sProjectDir + "*.h", vSearchFiles, bRecurse);
    findFiles(m_sProjectDir + "*.c", vSearchFiles, bRecurse);
    findFiles(m_sProjectDir + "*.cpp", vSearchFiles, bRecurse);
#endif
    //Ensure we can add extra items to the list without needing reallocs
    if (vSearchFiles.capacity() < vSearchFiles.size() + 250) {
        vSearchFiles.reserve(vSearchFiles.size() + 250);
    }

    //Check for DCE constructs
    map<string, DCEParams> mFoundDCEUsage;
    set<string> vNonDCEUsage;
    StaticList vPreProcFiles;
    //Search through each included file
    for (StaticList::iterator itFile = vSearchFiles.begin(); itFile < vSearchFiles.end(); itFile++) {
        //Open the input file
        makeFileGeneratorRelative(*itFile, *itFile);
        string sFile;
        if (!loadFromFile(*itFile, sFile)) {
            return false;
        }
        bool bRequiresPreProcess = false;
        outputProjectDCEFindFunctions(sFile, *itFile, mFoundDCEUsage, bRequiresPreProcess, vNonDCEUsage);
        if (bRequiresPreProcess) {
            vPreProcFiles.push_back(*itFile);
        }

        //Check if this file includes additional source files
        uint uiFindPos = sFile.find(".c\"");
        while (uiFindPos != string::npos) {
            //Check if this is an include
            uint uiFindPos2 = sFile.rfind("#include \"", uiFindPos);
            if ((uiFindPos2 != string::npos) && (uiFindPos - uiFindPos2 < 50)) {
                //Get the name of the file
                uiFindPos2 += 10;
                uiFindPos += 2;
                string sTemplateFile = sFile.substr(uiFindPos2, uiFindPos - uiFindPos2);
                //check if file contains current project
                uint uiProjName = sTemplateFile.find(m_sProjectName);
                if (uiProjName != string::npos) {
                    sTemplateFile = sTemplateFile.substr(uiProjName + m_sProjectName.length() + 1);
                }
                string sFound;
                string sBack = sTemplateFile;
                sTemplateFile = m_sProjectDir + sBack;
                if (!findFile(sTemplateFile, sFound)) {
                    sTemplateFile = (m_ConfigHelper.m_sRootDirectory.length() > 0) ? m_ConfigHelper.m_sRootDirectory + '/' + sBack : sBack;
                    if (!findFile(sTemplateFile, sFound)) {
                        sTemplateFile = m_ConfigHelper.m_sProjectDirectory + m_sProjectName + '/' + sBack;
                        if (!findFile(sTemplateFile, sFound)) {
                            sTemplateFile = itFile->substr(0, itFile->rfind('/') + 1) + sBack;
                            if (!findFile(sTemplateFile, sFound)) {
                                outputError("Failed to find included file " + sBack);
                                return false;
                            }
                        }
                    }
                }
                //Add the file to the list
                if (find(vSearchFiles.begin(), vSearchFiles.end(), sTemplateFile) == vSearchFiles.end()) {
                    makeFileProjectRelative(sTemplateFile, sTemplateFile);
                    vSearchFiles.push_back(sTemplateFile);
                }
            }
            //Check for more
            uiFindPos = sFile.find(".c\"", uiFindPos + 1);
        }
    }

#if !FORCEALLDCE
    //Get a list of all files in current project directory (including subdirectories)
    bool bRecurse = (m_sProjectDir.compare(this->m_ConfigHelper.m_sRootDirectory) != 0);
    vSearchFiles.resize(0);
    findFiles(m_sProjectDir + "*.h", vSearchFiles, bRecurse);
    findFiles(m_sProjectDir + "*.c", vSearchFiles, bRecurse);
    findFiles(m_sProjectDir + "*.cpp", vSearchFiles, bRecurse);
    //Ensure we can add extra items to the list without needing reallocs
    if (vSearchFiles.capacity() < vSearchFiles.size() + 250) {
        vSearchFiles.reserve(vSearchFiles.size() + 250);
    }

    //Check all configurations are enabled early to avoid later lookups of unused functions
    for (map<string, DCEParams>::iterator itDCE = mFoundDCEUsage.begin(); itDCE != mFoundDCEUsage.end(); ) {
        outputProgramDCEsResolveDefine(itDCE->second.sDefine);
        if (itDCE->second.sDefine.compare("1") == 0) {
            vNonDCEUsage.insert(itDCE->first);
            //remove from the list
            mFoundDCEUsage.erase(itDCE++);
        } else {
            ++itDCE;
        }
    }
#endif

    //Now we need to find the declaration of each function
    map<string, DCEParams> mFoundDCEFunctions;
    map<string, DCEParams> mFoundDCEVariables;
    if (mFoundDCEUsage.size() > 0) {
        //Search through each included file
        for (StaticList::iterator itFile = vSearchFiles.begin(); itFile < vSearchFiles.end(); itFile++) {
            string sFile;
            if (!loadFromFile(*itFile, sFile)) {
                return false;
            }
            for (map<string, DCEParams>::iterator itDCE = mFoundDCEUsage.begin(); itDCE != mFoundDCEUsage.end(); ) {
                string sReturn;
                bool bIsFunc;
                if (outputProjectDCEsFindDeclarations(sFile, itDCE->first, *itFile, sReturn, bIsFunc)) {
                    //Get the declaration file
                    string sFileName;
                    makePathsRelative(*itFile, m_ConfigHelper.m_sRootDirectory, sFileName);
                    if (sFileName.at(0) == '.') {
                        sFileName = sFileName.substr(2);
                    }
                    if (bIsFunc) {
                        mFoundDCEFunctions[sReturn] = {itDCE->second.sDefine, sFileName};
                    } else {
                        mFoundDCEVariables[sReturn] = {itDCE->second.sDefine, sFileName};
                    }

                    //Remove it from the list
                    mFoundDCEUsage.erase(itDCE++);
                } else {
                    //Only increment the iterator when nothing has been found
                    // when we did find something we erased a value from the list so the iterator is still valid
                    ++itDCE;
                }
            }
        }
    }

    //Add any files requiring pre-processing to unfound list
    for (StaticList::iterator itPP = vPreProcFiles.begin(); itPP < vPreProcFiles.end(); itPP++) {
        mFoundDCEUsage[*itPP] = {"#", *itPP};
    }

    //Check if we failed to find any functions
    if (mFoundDCEUsage.size() > 0) {
        vector<string> vIncludeDirs2 = vIncludeDirs;
        string sTempFolder = sTempDirectory + m_sProjectName;
        if (!makeDirectory(sTempDirectory) || !makeDirectory(sTempFolder)) {
            outputError("Failed to create temporary working directory (" + sTempFolder + ")");
            return false;
        }
        //Get all the files that include functions
        map<string, vector<DCEParams>> mFunctionFiles;
        for (map<string, DCEParams>::iterator itDCE = mFoundDCEUsage.begin(); itDCE != mFoundDCEUsage.end(); itDCE++) {
            //Remove project dir from start of file
            uint uiPos = itDCE->second.sFile.find((m_sProjectDir.find("./") == 0) ? m_sProjectDir.substr(2) : m_sProjectDir);
            uiPos = (uiPos == string::npos) ? 0 : uiPos + m_sProjectDir.length();
            string sFile = sTempFolder + '/' + itDCE->second.sFile.substr(uiPos);
            if (sFile.find('/', sTempFolder.length() + 1) != string::npos) {
                string sFolder = sFile.substr(0, sFile.rfind('/'));
                if (!makeDirectory(sFolder)) {
                    outputError("Failed to create temporary working sub-directory (" + sFolder + ")");
                    return false;
                }
            }
            //Copy file to local working directory
            if (!copyFile(itDCE->second.sFile, sFile)) {
                outputError("Failed to copy dce file (" + itDCE->second.sFile + ") to temporary directory (" + sFile + ")");
                return false;
            }
            mFunctionFiles[sFile].push_back({itDCE->second.sDefine, itDCE->first});
        }
        map<string, vector<string>> mDirectoryObjects;
        const string asDCETags2[] = {"if (", "if(", "& ", "&", "| ", "|"};
        for (map<string, vector<DCEParams>>::iterator itDCE = mFunctionFiles.begin(); itDCE != mFunctionFiles.end(); itDCE++) {
            //Get subdirectory
            string sSubFolder = "";
            if (itDCE->first.find('/', sTempFolder.length() + 1) != string::npos) {
                sSubFolder = m_sProjectDir + itDCE->first.substr(sTempFolder.length() + 1, itDCE->first.rfind('/') - sTempFolder.length() - 1);
                if (find(vIncludeDirs2.begin(), vIncludeDirs2.end(), sSubFolder) == vIncludeDirs2.end()) {
                    //Need to add subdirectory to include list
                    vIncludeDirs2.push_back(sSubFolder);
{'fraction_non_alphanumeric': 0.11107931511422713, 'fraction_numerical': 0.006438689369008442, 'mean_word_length': 2.2338062924120914, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '948620', 'n_tokens_mistral': 6163, 'n_tokens_neox': 5889, 'n_words': 1545}