<commit_before>/*
 *  Copyright (c) 2018, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
 *
 *  WSO2 Inc. licenses this file to you under the Apache License,
 *  Version 2.0 (the "License"); you may not use this file except
 *  in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.wso2.security.tools.scanmanager.webapp.service;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.multipart.MultipartFile;
import org.wso2.security.tools.scanmanager.common.external.model.Scan;
import org.wso2.security.tools.scanmanager.common.external.model.ScanExternal;
import org.wso2.security.tools.scanmanager.common.external.model.ScanManagerScansResponse;
import org.wso2.security.tools.scanmanager.common.external.model.Scanner;
import org.wso2.security.tools.scanmanager.common.model.LogType;
import org.wso2.security.tools.scanmanager.common.model.ScanPriority;
import org.wso2.security.tools.scanmanager.common.model.ScanStatus;
import org.wso2.security.tools.scanmanager.common.model.ScanType;
import org.wso2.security.tools.scanmanager.webapp.config.ScanManagerWebappConfiguration;
import org.wso2.security.tools.scanmanager.webapp.exception.ScanManagerWebappException;
import org.wso2.security.tools.scanmanager.webapp.util.FTPUtil;
import org.wso2.security.tools.scanmanager.webapp.util.HTTPUtil;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;

import static org.wso2.security.tools.scanmanager.webapp.util.Constants.PAGE_PARAM_NAME;
import static org.wso2.security.tools.scanmanager.webapp.util.Constants.PRE_JOB_ID_PREFIX;

/**
 * Scan service implementation class.
 */
@Service
public class ScanServiceImpl implements ScanService {

    private LogService logService;

    private static final String SCAN_REQUEST_FILE_MAP_ATTRIBUTE_NAME = "fileMap";
    private static final String SCAN_REQUEST_PROPERTY_MAP_ATTRIBUTE_NAME = "propertyMap";
    private static final String SCAN_REQUEST_SCAN_NAME_ATTRIBUTE_NAME = "scanName";
    private static final String SCAN_REQUEST_SCAN_DESCRIPTION_ATTRIBUTE_NAME = "scanDescription";
    private static final String SCAN_REQUEST_SCAN_TYPE_ATTRIBUTE_NAME = "scanType";
    private static final String SCAN_REQUEST_PRODUCT_NAME_ATTRIBUTE_NAME = "productName";
    private static final String SCAN_REQUEST_SCANNER_ID_ATTRIBUTE_NAME = "scannerId";
    private static final String SCAN_ARTIFACT_DIRECTORY = "scans";

    private static final Integer ARTIFACT_DOWNLOAD_CONNECTION_TIMEOUT = 10000;
    private static final Integer ARTIFACT_DOWNLOAD_READ_TIMEOUT = 10000;

    // temporary hold the scans till the pre scan tasks are completed.
    private Map<String, Scan> waitingScans = new ConcurrentHashMap<>();

    @Autowired
    public ScanServiceImpl(LogService logService) {
        this.logService = logService;
    }

    @Override
    public Scan submitScan(Map<String, MultipartFile> fileMap, Map<String, String> parameterMap,
                           String scanDirectory) {
        Map<String, String> filesToBeDownloadedFromURL = new HashMap<>();
        Map<String, String> storedFileMap = new HashMap<>();
        File scanDirectoryLocation = null;

        // Add scan to the waiting scans till the pre scan tasks are completed.
        Scan scanWaiting = addScanToWaiting(parameterMap);
        logService.insert(scanWaiting, LogType.INFO, "Scan is waiting to be submitted to scan manager API.");
        try {
            scanDirectoryLocation = new File(SCAN_ARTIFACT_DIRECTORY + File.separator + scanDirectory);
            if (!scanDirectoryLocation.exists() && !scanDirectoryLocation.mkdirs()) {
                throw new ScanManagerWebappException("Error occurred while creating the scan directory");
            }

            // store the uploaded files in a temp scan directory.
            for (Map.Entry<String, MultipartFile> entry : fileMap.entrySet()) {
                String unifiedFileName = entry.getKey() + "." +
                        FilenameUtils.getExtension(entry.getValue().getOriginalFilename());
                Path unifiedFilePath = Paths.get(scanDirectoryLocation.toPath().toString(), unifiedFileName);
                writeToFile(entry.getValue(), unifiedFilePath);
                storedFileMap.put(entry.getKey(), unifiedFilePath.toString());
            }

            Iterator<Map.Entry<String, String>> iter = parameterMap.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry<String, String> param = iter.next();
                if (param.getValue().isEmpty()) {<commit_msg>Apply suggestions from code review<commit_after>/*
 *  Copyright (c) 2018, WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
 *
 *  WSO2 Inc. licenses this file to you under the Apache License,
 *  Version 2.0 (the "License"); you may not use this file except
 *  in compliance with the License.
 *  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.wso2.security.tools.scanmanager.webapp.service;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.HttpClientErrorException;
import org.springframework.web.multipart.MultipartFile;
import org.wso2.security.tools.scanmanager.common.external.model.Scan;
import org.wso2.security.tools.scanmanager.common.external.model.ScanExternal;
import org.wso2.security.tools.scanmanager.common.external.model.ScanManagerScansResponse;
import org.wso2.security.tools.scanmanager.common.external.model.Scanner;
import org.wso2.security.tools.scanmanager.common.model.LogType;
import org.wso2.security.tools.scanmanager.common.model.ScanPriority;
import org.wso2.security.tools.scanmanager.common.model.ScanStatus;
import org.wso2.security.tools.scanmanager.common.model.ScanType;
import org.wso2.security.tools.scanmanager.webapp.config.ScanManagerWebappConfiguration;
import org.wso2.security.tools.scanmanager.webapp.exception.ScanManagerWebappException;
import org.wso2.security.tools.scanmanager.webapp.util.FTPUtil;
import org.wso2.security.tools.scanmanager.webapp.util.HTTPUtil;

import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;

import static org.wso2.security.tools.scanmanager.webapp.util.Constants.PAGE_PARAM_NAME;
import static org.wso2.security.tools.scanmanager.webapp.util.Constants.PRE_JOB_ID_PREFIX;

/**
 * Scan service implementation class.
 */
@Service
public class ScanServiceImpl implements ScanService {

    private LogService logService;

    private static final String SCAN_REQUEST_FILE_MAP_ATTRIBUTE_NAME = "fileMap";
    private static final String SCAN_REQUEST_PROPERTY_MAP_ATTRIBUTE_NAME = "propertyMap";
    private static final String SCAN_REQUEST_SCAN_NAME_ATTRIBUTE_NAME = "scanName";
    private static final String SCAN_REQUEST_SCAN_DESCRIPTION_ATTRIBUTE_NAME = "scanDescription";
    private static final String SCAN_REQUEST_SCAN_TYPE_ATTRIBUTE_NAME = "scanType";
    private static final String SCAN_REQUEST_PRODUCT_NAME_ATTRIBUTE_NAME = "productName";
    private static final String SCAN_REQUEST_SCANNER_ID_ATTRIBUTE_NAME = "scannerId";
    private static final String SCAN_ARTIFACT_DIRECTORY = "scans";

    private static final Integer ARTIFACT_DOWNLOAD_CONNECTION_TIMEOUT = 10000;
    private static final Integer ARTIFACT_DOWNLOAD_READ_TIMEOUT = 10000;

    // Temporary hold the scans till the pre-scan tasks are completed.
    private Map<String, Scan> waitingScans = new ConcurrentHashMap<>();

    @Autowired
    public ScanServiceImpl(LogService logService) {
        this.logService = logService;
    }

    @Override
    public Scan submitScan(Map<String, MultipartFile> fileMap, Map<String, String> parameterMap,
                           String scanDirectory) {
        Map<String, String> filesToBeDownloadedFromURL = new HashMap<>();
        Map<String, String> storedFileMap = new HashMap<>();
        File scanDirectoryLocation = null;

        // Add scan to the waiting scans till the pre scan tasks are completed.
        Scan scanWaiting = addScanToWaiting(parameterMap);
        logService.insert(scanWaiting, LogType.INFO, "Scan is waiting to be submitted to scan manager API.");
        try {
            scanDirectoryLocation = new File(SCAN_ARTIFACT_DIRECTORY + File.separator + scanDirectory);
            if (!scanDirectoryLocation.exists() && !scanDirectoryLocation.mkdirs()) {
                throw new ScanManagerWebappException("Error occurred while creating the scan directory");
            }

            // Store the uploaded files in a temp scan directory.
            for (Map.Entry<String, MultipartFile> entry : fileMap.entrySet()) {
                String unifiedFileName = entry.getKey() + "." +
                        FilenameUtils.getExtension(entry.getValue().getOriginalFilename());
                Path unifiedFilePath = Paths.get(scanDirectoryLocation.toPath().toString(), unifiedFileName);
                writeToFile(entry.getValue(), unifiedFilePath);
                storedFileMap.put(entry.getKey(), unifiedFilePath.toString());
            }

            Iterator<Map.Entry<String, String>> iter = parameterMap.entrySet().iterator();
            while (iter.hasNext()) {
                Map.Entry<String, String> param = iter.next();
                if (param.getValue().isEmpty()) {
{'fraction_non_alphanumeric': 0.0918217221981826, 'fraction_numerical': 0.00640415404586759, 'mean_word_length': 4.1246119733924616, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2283366', 'n_tokens_mistral': 3247, 'n_tokens_neox': 3178, 'n_words': 767}