<commit_before>package edu.ucar.unidata.rosetta.controller;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MaxUploadSizeExceededException;
import org.springframework.web.servlet.HandlerExceptionResolver;
import org.springframework.web.servlet.ModelAndView;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import edu.ucar.unidata.rosetta.converters.TagUniversalFileFormat;
import edu.ucar.unidata.rosetta.domain.UploadedFile;
import edu.ucar.unidata.rosetta.util.RosettaProperties;
import edu.ucar.unidata.rosetta.util.ZipFileUtil;
import ucar.ma2.InvalidRangeException;

/**
 * Main controller for Rosetta application.
 */
@Controller
public class AutoConvertController implements HandlerExceptionResolver {

    protected static Logger logger = Logger.getLogger(AutoConvertController.class);
    @Autowired
    ServletContext servletContext;

    String APPLICATION_ZIP = "application/zip";

    private String getDownloadDir() {
        String downloadDir = "";
        downloadDir = RosettaProperties.getDownloadDir();
        return downloadDir;
    }

    private String getUploadDir() {
        String uploadDir = "";
        uploadDir = RosettaProperties.getUploadDir();
        return uploadDir;
    }


    /**
     * Accepts a POST request for an uploaded file, stores that file to disk, auto converts the
     * file, and, returns a zip file of the converted dataset ASCII file.
     *
     * @param file    The UploadedFile form backing object containing the file.
     * @param request The HttpServletRequest with which to glean the client IP address.
     * @return A String of the local file name for the ASCII file (or null for an error).
     */
    @RequestMapping(value = "/batchProcess", method = RequestMethod.POST, produces = "application/zip")
    @ResponseBody
    public Resource batchProcess(UploadedFile file, HttpServletRequest request, HttpServletResponse response) throws IOException {
        List<String> convertedFiles = new ArrayList<>();
        String uniqueId = createUniqueId(request);

        String filePath = FilenameUtils.concat(getUploadDir(), uniqueId);
        String extractToDir = FilenameUtils.concat(filePath, "extracted");
        File localFileDir = new File(filePath);
        if (!localFileDir.exists()) {
            localFileDir.mkdirs();
        }

        File uploadedFile = new File(FilenameUtils.concat(filePath, file.getFileName()));
        try (FileOutputStream outputStream = new FileOutputStream(uploadedFile)) {
            outputStream.write(file.getFile().getFileItem().get());
            outputStream.flush();
            outputStream.close();
        } catch (Exception exc) {
            logger.error("error while saving uploaded file to disk.");
        }

        File toDirectory = new File(extractToDir);
        if (!localFileDir.exists()) {
            localFileDir.mkdirs();
        }
        ArrayList<String> inventory = new ArrayList<>();
        if (uploadedFile.getName().endsWith(".zip")) {
            inventory = ZipFileUtil.unzipAndInventory(uploadedFile, toDirectory);
        }<commit_msg>ignore os generated files for batchProcess<commit_after>package edu.ucar.unidata.rosetta.controller;

import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.multipart.MaxUploadSizeExceededException;
import org.springframework.web.servlet.HandlerExceptionResolver;
import org.springframework.web.servlet.ModelAndView;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.stream.Stream;

import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import edu.ucar.unidata.rosetta.converters.TagUniversalFileFormat;
import edu.ucar.unidata.rosetta.domain.UploadedFile;
import edu.ucar.unidata.rosetta.util.RosettaProperties;
import edu.ucar.unidata.rosetta.util.ZipFileUtil;
import ucar.ma2.InvalidRangeException;

import static java.util.stream.Collectors.toList;

/**
 * Main controller for Rosetta application.
 */
@Controller
public class AutoConvertController implements HandlerExceptionResolver {

    protected static Logger logger = Logger.getLogger(AutoConvertController.class);
    @Autowired
    ServletContext servletContext;

    String APPLICATION_ZIP = "application/zip";

    private String getDownloadDir() {
        String downloadDir = "";
        downloadDir = RosettaProperties.getDownloadDir();
        return downloadDir;
    }

    private String getUploadDir() {
        String uploadDir = "";
        uploadDir = RosettaProperties.getUploadDir();
        return uploadDir;
    }

    private ArrayList<String> cleanupInventory(ArrayList<String> inventory) {
        // first pass based on typical .gitignore for OS generated files
        Stream<String> cleanInventory = inventory.stream()
                //.map(inventoryFile -> inventoryFile.toUpperCase())
                .filter(inventoryFile -> !StringUtils.containsIgnoreCase(inventoryFile, ".Spotlight-V100"))
                .filter(inventoryFile -> !StringUtils.containsIgnoreCase(inventoryFile, ".Trashes"))
                .filter(inventoryFile -> !StringUtils.containsIgnoreCase(inventoryFile, "ehthumbs.db"))
                .filter(inventoryFile -> !StringUtils.containsIgnoreCase(inventoryFile, "Thumbs.db"))
                .filter(inventoryFile -> !StringUtils.containsIgnoreCase(inventoryFile, "/__MACOSX/"))
                .filter(inventoryFile -> !StringUtils.containsIgnoreCase(inventoryFile, ".DS_STORE"));

        return new ArrayList<String>(cleanInventory.collect(toList()));
    }

    /**
     * Accepts a POST request for an uploaded file, stores that file to disk, auto converts the
     * file, and, returns a zip file of the converted dataset ASCII file.
     *
     * @param file    The UploadedFile form backing object containing the file.
     * @param request The HttpServletRequest with which to glean the client IP address.
     * @return A String of the local file name for the ASCII file (or null for an error).
     */
    @RequestMapping(value = "/batchProcess", method = RequestMethod.POST, produces = "application/zip")
    @ResponseBody
    public Resource batchProcess(UploadedFile file, HttpServletRequest request, HttpServletResponse response) throws IOException {
        List<String> convertedFiles = new ArrayList<>();
        String uniqueId = createUniqueId(request);



        String filePath = FilenameUtils.concat(getUploadDir(), uniqueId);
        String extractToDir = FilenameUtils.concat(filePath, "extracted");
        File localFileDir = new File(filePath);
        if (!localFileDir.exists()) {
            localFileDir.mkdirs();
        }

        File uploadedFile = new File(FilenameUtils.concat(filePath, file.getFileName()));
        try (FileOutputStream outputStream = new FileOutputStream(uploadedFile)) {
            outputStream.write(file.getFile().getFileItem().get());
            outputStream.flush();
            outputStream.close();
        } catch (Exception exc) {
            logger.error("error while saving uploaded file to disk.");
        }

        File toDirectory = new File(extractToDir);
        if (!localFileDir.exists()) {
            localFileDir.mkdirs();
        }
        ArrayList<String> inventory = new ArrayList<>();
        if (uploadedFile.getName().endsWith(".zip")) {
            inventory = ZipFileUtil.unzipAndInventory(uploadedFile, toDirectory);
        }

        // clean up inventory - some auto-generated OS files can really play havoc
        // if they are not accounted for. For example: "__MACOSX/"

        inventory = cleanupInventory(inventory);
{'fraction_non_alphanumeric': 0.09774755631109222, 'fraction_numerical': 0.0009562260943476413, 'mean_word_length': 3.780599288979177, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3439121', 'n_tokens_mistral': 2525, 'n_tokens_neox': 2470, 'n_words': 582}