<commit_before>/*
 * Copyright (C) 2017 Black Duck Software, Inc.
 * http://www.blackducksoftware.com/
 *
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package com.blackducksoftware.integration.hub.detect.hub

import java.nio.charset.StandardCharsets

import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component

import com.blackducksoftware.integration.hub.builder.HubScanConfigBuilder
import com.blackducksoftware.integration.hub.dataservice.cli.CLIDataService
import com.blackducksoftware.integration.hub.detect.DetectConfiguration
import com.blackducksoftware.integration.hub.detect.model.DetectProject
import com.blackducksoftware.integration.hub.detect.util.DetectFileManager
import com.blackducksoftware.integration.hub.global.HubServerConfig
import com.blackducksoftware.integration.hub.model.request.ProjectRequest
import com.blackducksoftware.integration.hub.model.view.ProjectVersionView
import com.blackducksoftware.integration.hub.request.builder.ProjectRequestBuilder
import com.blackducksoftware.integration.hub.scan.HubScanConfig
import com.blackducksoftware.integration.util.ResourceUtil

@Component
class HubSignatureScanner {
    private final Logger logger = LoggerFactory.getLogger(HubSignatureScanner.class)

    @Autowired
    DetectConfiguration detectConfiguration

    @Autowired
    HubManager hubManager

    @Autowired
    DetectFileManager detectFileManager

    @Autowired
    OfflineScanner offlineScanner

    private Set<String> registeredPaths = []
    private Set<String> registeredPathsToExclude = []

    public void registerPathToScan(File file, String... fileNamesToExclude) {
        String matchingExcludedPath = detectConfiguration.hubSignatureScannerPathsToExclude.find {
            file.canonicalPath.startsWith(it)
        }

        if (matchingExcludedPath) {
            logger.info("Not registering excluded path ${file.canonicalPath} to scan")
        } else if (file.exists() && (file.isFile() || file.isDirectory())) {
            logger.info("Registering path ${file.canonicalPath} to scan")
            registeredPaths.add(file.canonicalPath)
            if (fileNamesToExclude) {
                for (String fileNameToExclude : fileNamesToExclude) {
                    File fileToExclude = detectFileManager.findFile(file, fileNameToExclude)
                    if (fileToExclude) {
                        String pattern = fileToExclude.getCanonicalPath().replace(file.canonicalPath, '')
                        if (pattern.contains('\\\\')) {
                            pattern = pattern.replace('\\\\', '/')
                        }
                        if (pattern.contains('\\')) {
                            pattern = pattern.replace('\\', '/')
                        }
                        pattern = pattern + '/'
                        registeredPathsToExclude.add(pattern)
                    }
                }
            }
        } else {
            logger.warn("Tried to register a scan for ${file.canonicalPath} but it doesn't appear to exist or it isn't a file or directory.")
        }
    }

    public ProjectVersionView scanPaths(HubServerConfig hubServerConfig, CLIDataService cliDataService, DetectProject detectProject) {
        ProjectVersionView projectVersionView = null
        if (detectProject.projectName && detectProject.projectVersionName && detectConfiguration.hubSignatureScannerPaths) {
            detectConfiguration.hubSignatureScannerPaths.each {
                projectVersionView = scanPath(cliDataService, hubServerConfig, new File(it).canonicalPath, detectProject)
            }
        } else {
            registeredPaths.each {
                logger.info("Attempting to scan ${it} for ${detectProject.projectName}/${detectProject.projectVersionName}")
                ProjectVersionView scanProject = scanPath(cliDataService, hubServerConfig, it, detectProject)
                if (!projectVersionView) {
                    projectVersionView = scanProject
                }
            }
        }
        return projectVersionView
    }

    public void scanPathsOffline(DetectProject detectProject) {
        if (detectProject.projectName && detectProject.projectVersionName && detectConfiguration.hubSignatureScannerPaths) {
            detectConfiguration.hubSignatureScannerPaths.each {
                scanPathOffline(new File(it).canonicalPath, detectProject)
            }
        } else {
            registeredPaths.each {
                logger.info("Attempting to scan ${it} for ${detectProject.projectName}/${detectProject.projectVersionName}")
                scanPathOffline(it, detectProject)
            }
        }
    }

    private ProjectVersionView scanPath(CLIDataService cliDataService, HubServerConfig hubServerConfig, String canonicalPath, DetectProject detectProject) {
        ProjectVersionView projectVersionView = null
        try {
            ProjectRequestBuilder builder = new ProjectRequestBuilder()
            builder.setProjectName(detectProject.projectName)
            builder.setVersionName(detectProject.projectVersionName)
            builder.setProjectLevelAdjustments(detectConfiguration.projectLevelMatchAdjustments)
            builder.setPhase(detectConfiguration.projectVersionPhase)
            builder.setDistribution(detectConfiguration.projectVersionDistribution)
            ProjectRequest projectRequest = builder.build()

            HubScanConfigBuilder hubScanConfigBuilder = createScanConfigBuilder(detectProject, canonicalPath)
            HubScanConfig hubScanConfig = hubScanConfigBuilder.build()

            String hubDetectVersion = ResourceUtil.getResourceAsString('version.txt', StandardCharsets.UTF_8)
            projectVersionView = cliDataService.installAndRunControlledScan(hubServerConfig, hubScanConfig, projectRequest, false, 'Hub-Detect', hubDetectVersion, hubDetectVersion)
            logger.info("${canonicalPath} was successfully scanned by the BlackDuck CLI.")
        } catch (Exception e) {
            logger.error("${detectProject.projectName}/${detectProject.projectVersionName} - ${canonicalPath} was not scanned by the BlackDuck CLI: ${e.message}")
        }
        return projectVersionView
    }

    private void scanPathOffline(String canonicalPath, DetectProject detectProject) {
        try {
            HubScanConfigBuilder hubScanConfigBuilder = createScanConfigBuilder(detectProject, canonicalPath)
            hubScanConfigBuilder.setDryRun(true)

            HubScanConfig hubScanConfig = hubScanConfigBuilder.build()

            offlineScanner.offlineScan(hubScanConfig)
        } catch (Exception e) {
            logger.error("${detectProject.projectName}/${detectProject.projectVersionName} - ${canonicalPath} was not scanned by the BlackDuck CLI: ${e.message}")
        }
    }

    private HubScanConfigBuilder createScanConfigBuilder(DetectProject detectProject, String canonicalPath) {
        File scannerDirectory = detectFileManager.createDirectory('signature_scanner')
        File toolsDirectory = detectFileManager.createDirectory(scannerDirectory, 'tools')

        HubScanConfigBuilder hubScanConfigBuilder = new HubScanConfigBuilder()
        hubScanConfigBuilder.scanMemory = detectConfiguration.hubSignatureScannerMemory
        hubScanConfigBuilder.toolsDir = toolsDirectory
        hubScanConfigBuilder.workingDirectory = scannerDirectory
        hubScanConfigBuilder.addScanTargetPath(canonicalPath)
        hubScanConfigBuilder.cleanupLogsOnSuccess = detectConfiguration.cleanupBomToolFiles
        hubScanConfigBuilder.dryRun = detectConfiguration.hubSignatureScannerDryRun

        final String codeLocationName = detectProject.getCodeLocationName(detectConfiguration.sourcePath, canonicalPath, detectFileManager.extractFinalPieceFromPath(detectConfiguration.sourcePath), detectConfiguration.getProjectCodeLocationPrefix(), 'Hub Detect Scan')
        hubScanConfigBuilder.codeLocationAlias = codeLocationName

        if (detectConfiguration.hubSignatureScannerExclusionPatterns) {
            hubScanConfigBuilder.setExcludePatterns(detectConfiguration.hubSignatureScannerExclusionPatterns)
        } else if (registeredPathsToExclude) {
            hubScanConfigBuilder.setExcludePatterns(registeredPathsToExclude as String[])
        }

        hubScanConfigBuilder
    }
}
<commit_msg>using the path to override cli location<commit_after>/*
 * Copyright (C) 2017 Black Duck Software, Inc.
 * http://www.blackducksoftware.com/
 *
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements. See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership. The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package com.blackducksoftware.integration.hub.detect.hub

import java.nio.charset.StandardCharsets

import org.slf4j.Logger
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component

import com.blackducksoftware.integration.hub.builder.HubScanConfigBuilder
import com.blackducksoftware.integration.hub.dataservice.cli.CLIDataService
import com.blackducksoftware.integration.hub.detect.DetectConfiguration
import com.blackducksoftware.integration.hub.detect.model.DetectProject
import com.blackducksoftware.integration.hub.detect.util.DetectFileManager
import com.blackducksoftware.integration.hub.global.HubServerConfig
import com.blackducksoftware.integration.hub.model.request.ProjectRequest
import com.blackducksoftware.integration.hub.model.view.ProjectVersionView
import com.blackducksoftware.integration.hub.request.builder.ProjectRequestBuilder
import com.blackducksoftware.integration.hub.scan.HubScanConfig
import com.blackducksoftware.integration.util.ResourceUtil

@Component
class HubSignatureScanner {
    private final Logger logger = LoggerFactory.getLogger(HubSignatureScanner.class)

    @Autowired
    DetectConfiguration detectConfiguration

    @Autowired
    HubManager hubManager

    @Autowired
    DetectFileManager detectFileManager

    @Autowired
    OfflineScanner offlineScanner

    private Set<String> registeredPaths = []
    private Set<String> registeredPathsToExclude = []

    public void registerPathToScan(File file, String... fileNamesToExclude) {
        String matchingExcludedPath = detectConfiguration.hubSignatureScannerPathsToExclude.find {
            file.canonicalPath.startsWith(it)
        }

        if (matchingExcludedPath) {
            logger.info("Not registering excluded path ${file.canonicalPath} to scan")
        } else if (file.exists() && (file.isFile() || file.isDirectory())) {
            logger.info("Registering path ${file.canonicalPath} to scan")
            registeredPaths.add(file.canonicalPath)
            if (fileNamesToExclude) {
                for (String fileNameToExclude : fileNamesToExclude) {
                    File fileToExclude = detectFileManager.findFile(file, fileNameToExclude)
                    if (fileToExclude) {
                        String pattern = fileToExclude.getCanonicalPath().replace(file.canonicalPath, '')
                        if (pattern.contains('\\\\')) {
                            pattern = pattern.replace('\\\\', '/')
                        }
                        if (pattern.contains('\\')) {
                            pattern = pattern.replace('\\', '/')
                        }
                        pattern = pattern + '/'
                        registeredPathsToExclude.add(pattern)
                    }
                }
            }
        } else {
            logger.warn("Tried to register a scan for ${file.canonicalPath} but it doesn't appear to exist or it isn't a file or directory.")
        }
    }

    public ProjectVersionView scanPaths(HubServerConfig hubServerConfig, CLIDataService cliDataService, DetectProject detectProject) {
        ProjectVersionView projectVersionView = null
        if (detectProject.projectName && detectProject.projectVersionName && detectConfiguration.hubSignatureScannerPaths) {
            detectConfiguration.hubSignatureScannerPaths.each {
                projectVersionView = scanPath(cliDataService, hubServerConfig, new File(it).canonicalPath, detectProject)
            }
        } else {
            registeredPaths.each {
                logger.info("Attempting to scan ${it} for ${detectProject.projectName}/${detectProject.projectVersionName}")
                ProjectVersionView scanProject = scanPath(cliDataService, hubServerConfig, it, detectProject)
                if (!projectVersionView) {
                    projectVersionView = scanProject
                }
            }
        }
        return projectVersionView
    }

    public void scanPathsOffline(DetectProject detectProject) {
        if (detectProject.projectName && detectProject.projectVersionName && detectConfiguration.hubSignatureScannerPaths) {
            detectConfiguration.hubSignatureScannerPaths.each {
                scanPathOffline(new File(it).canonicalPath, detectProject)
            }
        } else {
            registeredPaths.each {
                logger.info("Attempting to scan ${it} for ${detectProject.projectName}/${detectProject.projectVersionName}")
                scanPathOffline(it, detectProject)
            }
        }
    }

    private ProjectVersionView scanPath(CLIDataService cliDataService, HubServerConfig hubServerConfig, String canonicalPath, DetectProject detectProject) {
        ProjectVersionView projectVersionView = null
        try {
            ProjectRequestBuilder builder = new ProjectRequestBuilder()
            builder.setProjectName(detectProject.projectName)
            builder.setVersionName(detectProject.projectVersionName)
            builder.setProjectLevelAdjustments(detectConfiguration.projectLevelMatchAdjustments)
            builder.setPhase(detectConfiguration.projectVersionPhase)
            builder.setDistribution(detectConfiguration.projectVersionDistribution)
            ProjectRequest projectRequest = builder.build()

            HubScanConfigBuilder hubScanConfigBuilder = createScanConfigBuilder(detectProject, canonicalPath)
            HubScanConfig hubScanConfig = hubScanConfigBuilder.build()

            String hubDetectVersion = ResourceUtil.getResourceAsString('version.txt', StandardCharsets.UTF_8)
            projectVersionView = cliDataService.installAndRunControlledScan(hubServerConfig, hubScanConfig, projectRequest, false, 'Hub-Detect', hubDetectVersion, hubDetectVersion)
            logger.info("${canonicalPath} was successfully scanned by the BlackDuck CLI.")
        } catch (Exception e) {
            logger.error("${detectProject.projectName}/${detectProject.projectVersionName} - ${canonicalPath} was not scanned by the BlackDuck CLI: ${e.message}")
        }
        return projectVersionView
    }

    private void scanPathOffline(String canonicalPath, DetectProject detectProject) {
        try {
            HubScanConfigBuilder hubScanConfigBuilder = createScanConfigBuilder(detectProject, canonicalPath)
            hubScanConfigBuilder.setDryRun(true)

            if (detectConfiguration.hubSignatureScannerOfflineLocalPath) {
                hubScanConfigBuilder.toolsDir = detectConfiguration.hubSignatureScannerOfflineLocalPath
            }

            HubScanConfig hubScanConfig = hubScanConfigBuilder.build()

            offlineScanner.offlineScan(hubScanConfig)
        } catch (Exception e) {
            logger.error("${detectProject.projectName}/${detectProject.projectVersionName} - ${canonicalPath} was not scanned by the BlackDuck CLI: ${e.message}")
        }
    }

    private HubScanConfigBuilder createScanConfigBuilder(DetectProject detectProject, String canonicalPath) {
        File scannerDirectory = detectFileManager.createDirectory('signature_scanner')
        File toolsDirectory = detectFileManager.createDirectory(scannerDirectory, 'tools')

        HubScanConfigBuilder hubScanConfigBuilder = new HubScanConfigBuilder()
        hubScanConfigBuilder.scanMemory = detectConfiguration.hubSignatureScannerMemory
        hubScanConfigBuilder.toolsDir = toolsDirectory
        hubScanConfigBuilder.workingDirectory = scannerDirectory
        hubScanConfigBuilder.addScanTargetPath(canonicalPath)
        hubScanConfigBuilder.cleanupLogsOnSuccess = detectConfiguration.cleanupBomToolFiles
        hubScanConfigBuilder.dryRun = detectConfiguration.hubSignatureScannerDryRun

        final String codeLocationName = detectProject.getCodeLocationName(detectConfiguration.sourcePath, canonicalPath, detectFileManager.extractFinalPieceFromPath(detectConfiguration.sourcePath), detectConfiguration.getProjectCodeLocationPrefix(), 'Hub Detect Scan')
        hubScanConfigBuilder.codeLocationAlias = codeLocationName

        if (detectConfiguration.hubSignatureScannerExclusionPatterns) {
            hubScanConfigBuilder.setExcludePatterns(detectConfiguration.hubSignatureScannerExclusionPatterns)
        } else if (registeredPathsToExclude) {
            hubScanConfigBuilder.setExcludePatterns(registeredPathsToExclude as String[])
        }

        hubScanConfigBuilder
    }
}

{'fraction_non_alphanumeric': 0.07251047373509507, 'fraction_numerical': 0.0011816521645719197, 'mean_word_length': 3.2900921658986175, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '987742', 'n_tokens_mistral': 4539, 'n_tokens_neox': 4334, 'n_words': 948}