<commit_before>/*
 *
 * Copyright SHMsoft, Inc. 
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/
package org.freeeed.search.web.controller;

import org.apache.log4j.Logger;
import org.freeeed.search.files.CaseFileService;
import org.freeeed.search.web.WebConstants;
import org.freeeed.search.web.model.Case;
import org.freeeed.search.web.model.solr.SolrDocument;
import org.freeeed.search.web.model.solr.SolrResult;
import org.freeeed.search.web.session.SolrSessionObject;
import org.freeeed.search.web.solr.SolrSearchService;
import org.springframework.web.servlet.ModelAndView;

import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpSession;
import java.io.*;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;

/**
 * Class CaseFileDownloadController.
 *
 * @author ilazarov.
 */
public class CaseFileDownloadController extends SecureController {
    private static final Logger log = Logger.getLogger(CaseFileDownloadController.class);

    private CaseFileService caseFileService;
    private SolrSearchService searchService;

    @Override
    public ModelAndView execute() {
        HttpSession session = this.request.getSession(true);
        SolrSessionObject solrSession = (SolrSessionObject)
                session.getAttribute(WebConstants.WEB_SESSION_SOLR_OBJECT);

        if (solrSession == null || solrSession.getSelectedCase() == null) {
            return new ModelAndView(WebConstants.CASE_FILE_DOWNLOAD);
        }

        Case selectedCase = solrSession.getSelectedCase();

        String action = (String) valueStack.get("action");

        log.debug("Action called: " + action);

        File toDownload = null;
        boolean htmlMode = false;

        String docPath = (String) valueStack.get("docPath");
        String uniqueId = (String) valueStack.get("uniqueId");

        try {
            if ("exportNative".equals(action)) {
                toDownload = caseFileService.getNativeFile(selectedCase.getName(), docPath, uniqueId);

            } else if ("exportImage".equals(action)) {
                toDownload = caseFileService.getImageFile(selectedCase.getName(), docPath, uniqueId);
            } else if ("exportHtml".equals(action)) {
                toDownload = caseFileService.getHtmlFile(selectedCase.getName(), docPath, uniqueId);
                htmlMode = true;
            } else if ("exportHtmlImage".equals(action)) {
                toDownload = caseFileService.getHtmlImageFile(selectedCase.getName(), docPath);
                htmlMode = true;
            } else if ("exportNativeAll".equals(action)) {
                String query = solrSession.buildSearchQuery();
                int rows = solrSession.getTotalDocuments();

                List<SolrDocument> docs = getDocumentPaths(query, 0, rows);

                toDownload = caseFileService.getNativeFiles(selectedCase.getName(), docs);

            } else if ("exportNativeAllFromSource".equals(action)) {
                String query = solrSession.buildSearchQuery();
                int rows = solrSession.getTotalDocuments();

                List<SolrDocument> docs = getDocumentPaths(query, 0, rows);

                String source = (String) valueStack.get("source");
                try {
                    source = URLDecoder.decode(source, "UTF-8");
                } catch (UnsupportedEncodingException e) {
                    log.error(e);
                }

                toDownload = caseFileService.getNativeFilesFromSource(source, docs);
            } else if ("exportImageAll".equals(action)) {
                String query = solrSession.buildSearchQuery();
                int rows = solrSession.getTotalDocuments();

                List<SolrDocument> docs = getDocumentPaths(query, 0, rows);
                toDownload = caseFileService.getImageFiles(selectedCase.getName(), docs);
            } else if ("exportLoadFile".equals(action)) {
                String query = solrSession.buildSearchQuery();
                int rows = solrSession.getTotalDocuments();
                byte[] result = getRawDocumentsWithAllTags(query, 0, rows);
                writeCSVReponse(result);
                return new ModelAndView(WebConstants.CASE_FILE_DOWNLOAD);
            }
        } catch (Exception e) {
            log.error("Problem sending cotent", e);
            valueStack.put("error", true);
        }

        if (toDownload != null) {
            try {
                int length;
                ServletOutputStream outStream = response.getOutputStream();
                String mimetype = "application/octet-stream";
                if (htmlMode) {
                    mimetype = "text/html";
                }

                response.setContentType(mimetype);
                response.setContentLength((int) toDownload.length());
                String fileName = toDownload.getName();

                if (!htmlMode) {
                    // sets HTTP header
                    response.setHeader("Content-Disposition", "attachment; filename=\""
                            + fileName + "\"");
                }

                byte[] byteBuffer = new byte[1024];
                DataInputStream in = new DataInputStream(new FileInputStream(toDownload));

                // reads the file's bytes and writes them to the response stream
                while ((length = in.read(byteBuffer)) != -1) {
                    outStream.write(byteBuffer, 0, length);
                }

                in.close();
                outStream.close();
            } catch (Exception e) {
                log.error("Problem sending cotent", e);
                valueStack.put("error", true);
            }
        } else {
            valueStack.put("error", true);
        }

        return new ModelAndView(WebConstants.CASE_FILE_DOWNLOAD);
    }

    private void writeCSVReponse(byte[] resultBytes) throws IOException {
        response.setContentType("text/csv");
        response.setHeader("Content-Disposition", "attachment; filename=result.csv");
        response.setContentLength(resultBytes.length);
        ServletOutputStream out = response.getOutputStream();
        out.write(resultBytes);
        out.close();
    }

    private byte[] getRawDocumentsWithAllTags(String query, int from, int rows) {
        if (!query.contains("tags-search-field")) {
            query += " AND tags-search-field:*";
        }
        return searchService.returnTaggedCSV(query, from, rows);
    }

    private List<SolrDocument> getDocumentPaths(String query, int from, int rows) {
        SolrResult solrResult = searchService.search(query, from, rows, null, false, "id,document_original_path,unique_id");
        List<SolrDocument> result = new ArrayList<SolrDocument>(solrResult.getTotalSize());
        result.addAll(solrResult.getDocuments().values());
        return result;
    }

    public void setCaseFileService(CaseFileService caseFileService) {
        this.caseFileService = caseFileService;
    }

    public void setSearchService(SolrSearchService searchService) {
        this.searchService = searchService;
    }
}<commit_msg>session search for load file export<commit_after>/*
 *
 * Copyright SHMsoft, Inc. 
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/
package org.freeeed.search.web.controller;

import org.apache.log4j.Logger;
import org.freeeed.search.files.CaseFileService;
import org.freeeed.search.web.WebConstants;
import org.freeeed.search.web.model.Case;
import org.freeeed.search.web.model.solr.SolrDocument;
import org.freeeed.search.web.model.solr.SolrResult;
import org.freeeed.search.web.session.SolrSessionObject;
import org.freeeed.search.web.solr.SolrSearchService;
import org.springframework.web.servlet.ModelAndView;

import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpSession;
import java.io.*;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.List;

/**
 * Class CaseFileDownloadController.
 *
 * @author ilazarov.
 */
public class CaseFileDownloadController extends SecureController {
    private static final Logger log = Logger.getLogger(CaseFileDownloadController.class);

    private CaseFileService caseFileService;
    private SolrSearchService searchService;

    @Override
    public ModelAndView execute() {
        HttpSession session = this.request.getSession(true);
        SolrSessionObject solrSession = (SolrSessionObject)
                session.getAttribute(WebConstants.WEB_SESSION_SOLR_OBJECT);

        if (solrSession == null || solrSession.getSelectedCase() == null) {
            return new ModelAndView(WebConstants.CASE_FILE_DOWNLOAD);
        }

        Case selectedCase = solrSession.getSelectedCase();

        String action = (String) valueStack.get("action");

        log.debug("Action called: " + action);

        File toDownload = null;
        boolean htmlMode = false;

        String docPath = (String) valueStack.get("docPath");
        String uniqueId = (String) valueStack.get("uniqueId");

        try {
            if ("exportNative".equals(action)) {
                toDownload = caseFileService.getNativeFile(selectedCase.getName(), docPath, uniqueId);

            } else if ("exportImage".equals(action)) {
                toDownload = caseFileService.getImageFile(selectedCase.getName(), docPath, uniqueId);
            } else if ("exportHtml".equals(action)) {
                toDownload = caseFileService.getHtmlFile(selectedCase.getName(), docPath, uniqueId);
                htmlMode = true;
            } else if ("exportHtmlImage".equals(action)) {
                toDownload = caseFileService.getHtmlImageFile(selectedCase.getName(), docPath);
                htmlMode = true;
            } else if ("exportNativeAll".equals(action)) {
                String query = solrSession.buildSearchQuery();
                int rows = solrSession.getTotalDocuments();

                List<SolrDocument> docs = getDocumentPaths(query, 0, rows);

                toDownload = caseFileService.getNativeFiles(selectedCase.getName(), docs);

            } else if ("exportNativeAllFromSource".equals(action)) {
                String query = solrSession.buildSearchQuery();
                int rows = solrSession.getTotalDocuments();

                List<SolrDocument> docs = getDocumentPaths(query, 0, rows);

                String source = (String) valueStack.get("source");
                try {
                    source = URLDecoder.decode(source, "UTF-8");
                } catch (UnsupportedEncodingException e) {
                    log.error(e);
                }

                toDownload = caseFileService.getNativeFilesFromSource(source, docs);
            } else if ("exportImageAll".equals(action)) {
                String query = solrSession.buildSearchQuery();
                int rows = solrSession.getTotalDocuments();

                List<SolrDocument> docs = getDocumentPaths(query, 0, rows);
                toDownload = caseFileService.getImageFiles(selectedCase.getName(), docs);
            } else if ("exportLoadFile".equals(action)) {
                String query = solrSession.buildSearchQuery();
                int rows = solrSession.getTotalDocuments();
                byte[] result = getRawDocumentsWithAllTags(query, 0, rows);
                writeCSVReponse(result);
                return new ModelAndView(WebConstants.CASE_FILE_DOWNLOAD);
            }
        } catch (Exception e) {
            log.error("Problem sending cotent", e);
            valueStack.put("error", true);
        }

        if (toDownload != null) {
            try {
                int length;
                ServletOutputStream outStream = response.getOutputStream();
                String mimetype = "application/octet-stream";
                if (htmlMode) {
                    mimetype = "text/html";
                }

                response.setContentType(mimetype);
                response.setContentLength((int) toDownload.length());
                String fileName = toDownload.getName();

                if (!htmlMode) {
                    // sets HTTP header
                    response.setHeader("Content-Disposition", "attachment; filename=\""
                            + fileName + "\"");
                }

                byte[] byteBuffer = new byte[1024];
                DataInputStream in = new DataInputStream(new FileInputStream(toDownload));

                // reads the file's bytes and writes them to the response stream
                while ((length = in.read(byteBuffer)) != -1) {
                    outStream.write(byteBuffer, 0, length);
                }

                in.close();
                outStream.close();
            } catch (Exception e) {
                log.error("Problem sending cotent", e);
                valueStack.put("error", true);
            }
        } else {
            valueStack.put("error", true);
        }

        return new ModelAndView(WebConstants.CASE_FILE_DOWNLOAD);
    }

    private void writeCSVReponse(byte[] resultBytes) throws IOException {
        response.setContentType("text/csv");
        response.setHeader("Content-Disposition", "attachment; filename=result.csv");
        response.setContentLength(resultBytes.length);
        ServletOutputStream out = response.getOutputStream();
        out.write(resultBytes);
        out.close();
    }

    private byte[] getRawDocumentsWithAllTags(String query, int from, int rows) {
        return searchService.returnTaggedCSV(query, from, rows);
    }

    private List<SolrDocument> getDocumentPaths(String query, int from, int rows) {
        SolrResult solrResult = searchService.search(query, from, rows, null, false, "id,document_original_path,unique_id");
        List<SolrDocument> result = new ArrayList<SolrDocument>(solrResult.getTotalSize());
        result.addAll(solrResult.getDocuments().values());
        return result;
    }

    public void setCaseFileService(CaseFileService caseFileService) {
        this.caseFileService = caseFileService;
    }

    public void setSearchService(SolrSearchService searchService) {
        this.searchService = searchService;
    }
}
{'fraction_non_alphanumeric': 0.10057300928670224, 'fraction_numerical': 0.0021076203648817756, 'mean_word_length': 2.4275395033860043, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1376655', 'n_tokens_mistral': 4005, 'n_tokens_neox': 3757, 'n_words': 884}