<commit_before>package com.metaweb.gridworks.commands.project;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Map.Entry;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.fileupload.util.Streams;
import org.apache.tools.bzip2.CBZip2InputStream;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.importers.Importer;
import com.metaweb.gridworks.importers.TsvCsvImporter;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.IOUtils;
import com.metaweb.gridworks.util.ParsingUtilities;

public class CreateProjectCommand extends Command {

    final static Logger logger = LoggerFactory.getLogger("create-project_command");

    static final private Map<String, Importer> importers = new HashMap<String, Importer>();

    private static final String[][] importerNames = {
        {"ExcelImporter", "com.metaweb.gridworks.importers.ExcelImporter"},
        {"XmlImporter", "com.metaweb.gridworks.importers.XmlImporter"},
        {"RdfTripleImporter", "com.metaweb.gridworks.importers.RdfTripleImporter"},
        {"MarcImporter", "com.metaweb.gridworks.importers.MarcImporter"},
        {"TsvCsvImporter", "com.metaweb.gridworks.importers.TsvCsvImporter"},
    };

    static {
        registerImporters(importerNames);
    }

    static public boolean registerImporters(String[][] importers) {
        boolean status = true;
        for (String[] importer : importerNames) {
            String importerName = importer[0];
            String className = importer[1];
            logger.debug("Loading command " + importerName + " class: " + className);
            Importer cmd;
            try {
                // TODO: May need to use the servlet container's class loader here
                cmd = (Importer) Class.forName(className).newInstance();
            } catch (InstantiationException e) {
                logger.error("Failed to load importer class " + className, e);
                status = false;
                continue;
            } catch (IllegalAccessException e) {
                logger.error("Failed to load importer class " + className, e);
                status = false;
                continue;
            } catch (ClassNotFoundException e) {
                logger.error("Failed to load importer class " + className, e);
                status = false;
                continue;
            }
            status |= registerImporter(importerName, cmd);
        }
        return status;
    }

    /**
     * Register a single importer.
     *
     * @param name
     *            importer verb for importer
     * @param commandObject
     *            object implementing the importer
     * @return true if importer was loaded and registered successfully
     */
    static public boolean registerImporter(String name,
            Importer importerObject) {
        if (importers.containsKey(name)) {
            return false;
        }
        importers.put(name, importerObject);
        return true;
    }

    // Currently only for test purposes
    static protected boolean unregisterImporter(String verb) {
        return importers.remove(verb) != null;
    }

    @Override
    public void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {

        ProjectManager.singleton.setBusy(true);
        try {
            /*
             * The uploaded file is in the POST body as a "file part". If
             * we call request.getParameter() then the POST body will get
             * read and we won't have a chance to parse the body ourselves.
             * This is why we have to parse the URL for parameters ourselves.
             * Don't call request.getParameter() before calling internalImport().
             */
            Properties options = ParsingUtilities.parseUrlParameters(request);

            Project project = new Project();

            internalImport(request, project, options);

            /*
             * The import process above populates options with parameters
             * in the POST body. That's why we're constructing the project
             * metadata object after calling internalImport().
             */
            ProjectMetadata pm = new ProjectMetadata();
            pm.setName(options.getProperty("project-name"));
            pm.setPassword(options.getProperty("project-password"));
            pm.setEncoding(options.getProperty("encoding"));
            pm.setEncodingConfidence(options.getProperty("encoding_confidence"));
            ProjectManager.singleton.registerProject(project, pm);

            project.update();

            redirect(response, "/project.html?project=" + project.id);
        } catch (Exception e) {
            redirect(response, "/error.html?redirect=index.html&msg=" +
                ParsingUtilities.encode("Failed to import file: " + e.getLocalizedMessage())
            );
            e.printStackTrace();
        } finally {
            ProjectManager.singleton.setBusy(false);
        }
    }

    protected void internalImport(
        HttpServletRequest    request,
        Project               project,
        Properties            options
    ) throws Exception {

        ServletFileUpload upload = new ServletFileUpload();
        String url = null;

        FileItemIterator iter = upload.getItemIterator(request);
        while (iter.hasNext()) {
            FileItemStream item = iter.next();
            String name = item.getFieldName().toLowerCase();<commit_msg>CreateProjectCommand no longer contains references to project.html, a file previously removed from the project.<commit_after>package com.metaweb.gridworks.commands.project;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Map.Entry;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.fileupload.util.Streams;
import org.apache.tools.bzip2.CBZip2InputStream;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import com.metaweb.gridworks.ProjectManager;
import com.metaweb.gridworks.ProjectMetadata;
import com.metaweb.gridworks.commands.Command;
import com.metaweb.gridworks.importers.Importer;
import com.metaweb.gridworks.importers.TsvCsvImporter;
import com.metaweb.gridworks.model.Project;
import com.metaweb.gridworks.util.IOUtils;
import com.metaweb.gridworks.util.ParsingUtilities;

public class CreateProjectCommand extends Command {

    final static Logger logger = LoggerFactory.getLogger("create-project_command");

    static final private Map<String, Importer> importers = new HashMap<String, Importer>();

    private static final String[][] importerNames = {
        {"ExcelImporter", "com.metaweb.gridworks.importers.ExcelImporter"},
        {"XmlImporter", "com.metaweb.gridworks.importers.XmlImporter"},
        {"RdfTripleImporter", "com.metaweb.gridworks.importers.RdfTripleImporter"},
        {"MarcImporter", "com.metaweb.gridworks.importers.MarcImporter"},
        {"TsvCsvImporter", "com.metaweb.gridworks.importers.TsvCsvImporter"},
    };

    static {
        registerImporters(importerNames);
    }

    static public boolean registerImporters(String[][] importers) {
        boolean status = true;
        for (String[] importer : importerNames) {
            String importerName = importer[0];
            String className = importer[1];
            logger.debug("Loading command " + importerName + " class: " + className);
            Importer cmd;
            try {
                // TODO: May need to use the servlet container's class loader here
                cmd = (Importer) Class.forName(className).newInstance();
            } catch (InstantiationException e) {
                logger.error("Failed to load importer class " + className, e);
                status = false;
                continue;
            } catch (IllegalAccessException e) {
                logger.error("Failed to load importer class " + className, e);
                status = false;
                continue;
            } catch (ClassNotFoundException e) {
                logger.error("Failed to load importer class " + className, e);
                status = false;
                continue;
            }
            status |= registerImporter(importerName, cmd);
        }
        return status;
    }

    /**
     * Register a single importer.
     *
     * @param name
     *            importer verb for importer
     * @param commandObject
     *            object implementing the importer
     * @return true if importer was loaded and registered successfully
     */
    static public boolean registerImporter(String name,
            Importer importerObject) {
        if (importers.containsKey(name)) {
            return false;
        }
        importers.put(name, importerObject);
        return true;
    }

    // Currently only for test purposes
    static protected boolean unregisterImporter(String verb) {
        return importers.remove(verb) != null;
    }

    @Override
    public void doPost(HttpServletRequest request, HttpServletResponse response)
            throws ServletException, IOException {

        ProjectManager.singleton.setBusy(true);
        try {
            /*
             * The uploaded file is in the POST body as a "file part". If
             * we call request.getParameter() then the POST body will get
             * read and we won't have a chance to parse the body ourselves.
             * This is why we have to parse the URL for parameters ourselves.
             * Don't call request.getParameter() before calling internalImport().
             */
            Properties options = ParsingUtilities.parseUrlParameters(request);

            Project project = new Project();

            internalImport(request, project, options);

            /*
             * The import process above populates options with parameters
             * in the POST body. That's why we're constructing the project
             * metadata object after calling internalImport().
             */
            ProjectMetadata pm = new ProjectMetadata();
            pm.setName(options.getProperty("project-name"));
            pm.setPassword(options.getProperty("project-password"));
            pm.setEncoding(options.getProperty("encoding"));
            pm.setEncodingConfidence(options.getProperty("encoding_confidence"));
            ProjectManager.singleton.registerProject(project, pm);

            project.update();

            redirect(response, "/project?project=" + project.id);
        } catch (Exception e) {
            redirect(response, "/error.html?redirect=index.html&msg=" +
                ParsingUtilities.encode("Failed to import file: " + e.getLocalizedMessage())
            );
            e.printStackTrace();
        } finally {
            ProjectManager.singleton.setBusy(false);
        }
    }

    protected void internalImport(
        HttpServletRequest    request,
        Project               project,
        Properties            options
    ) throws Exception {

        ServletFileUpload upload = new ServletFileUpload();
        String url = null;

        FileItemIterator iter = upload.getItemIterator(request);
        while (iter.hasNext()) {
            FileItemStream item = iter.next();
            String name = item.getFieldName().toLowerCase();
{'fraction_non_alphanumeric': 0.09348129639156316, 'fraction_numerical': 0.0008818990225619166, 'mean_word_length': 2.678832116788321, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7251688', 'n_tokens_mistral': 3565, 'n_tokens_neox': 3480, 'n_words': 852}