<commit_before>/*
 *  Copyright (C) 2010-2012 <NAME> (Akvo Foundation)
 *
 *  This file is part of Akvo FLOW.
 *
 *  Akvo FLOW is free software: you can redistribute it and modify it under the terms of
 *  the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
 *  either version 3 of the License or any later version.
 *
 *  Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Affero General Public License included below for more details.
 *
 *  The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
 */

package org.waterforpeople.mapping.dataexport;

import java.awt.GraphicsEnvironment;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;

import javax.swing.SwingUtilities;

import com.gallatinsystems.common.util.FileUtil;
import com.gallatinsystems.common.util.S3Util;
import com.gallatinsystems.common.util.ZipUtil;
import com.gallatinsystems.framework.dataexport.applet.DataImporter;
import com.gallatinsystems.framework.dataexport.applet.ProgressDialog;

/**
 * Utility to recursively search the file system for all zip and jpg files to upload. Some ignore
 * paths are hard coded based on the data upload by WSP in Liberia. This utility will also combine
 * all zip files in a single directory into a single ZIP taking care not to have any duplicate
 * survey instances. After uploading a zip file, this utility will call the server to kick off
 * processing
 * 
 * @author <NAME>
 */
public class SurveyBulkUploader implements DataImporter {

    private static final String NOTIFICATION_PATH = "/processor?action=submit&fileName=";
    private static final String UPLOAD_IMAGE_MODE = "uploadImageOnly";
    private static final String ZIP_ONLY_MODE = "processZipOnly";
    public static final String MODE_KEY = "mode";
    private static final String PROGRESS_FILE_NAME = "progress.txt";
    public static final String MERGE_ONLY_MODE = "mergeOnly";

    private static final String IMAGE_TEMP_DIR = "resized";
    private static final String OSX_RESOURCE_DIR = "__MACOSX";

    private static final String IMAGE_POLICY_KEY = "imagePolicy";
    private static final String IMAGE_SIG_KEY = "imageSig";
    private static final String DATA_POLICY_KEY = "dataPolicy";
    private static final String DATA_SIG_KEY = "dataSig";
    private static final String AWS_ID_KEY = "awsId";
    private static final String UPLOAD_BASE_KEY = "uploadBase";
    private static final String DEFAULT_LOCALE = "en";
    private static Map<String, String> UPLOADING;
    private static Map<String, String> COMPLETE;
    private List<File> filesToUpload;

    static {
        UPLOADING = new HashMap<String, String>();
        UPLOADING.put("en", "Uploading");
<commit_msg>[#690] WIP - Remove unused constants<commit_after>/*
 *  Copyright (C) 2010-2014 Stichting Akvo (Akvo Foundation)
 *
 *  This file is part of Akvo FLOW.
 *
 *  Akvo FLOW is free software: you can redistribute it and modify it under the terms of
 *  the GNU Affero General Public License (AGPL) as published by the Free Software Foundation,
 *  either version 3 of the License or any later version.
 *
 *  Akvo FLOW is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
 *  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 *  See the GNU Affero General Public License included below for more details.
 *
 *  The full license text can also be seen at <http://www.gnu.org/licenses/agpl.html>.
 */

package org.waterforpeople.mapping.dataexport;

import java.awt.GraphicsEnvironment;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;

import javax.swing.SwingUtilities;

import com.gallatinsystems.common.util.FileUtil;
import com.gallatinsystems.common.util.S3Util;
import com.gallatinsystems.common.util.ZipUtil;
import com.gallatinsystems.framework.dataexport.applet.DataImporter;
import com.gallatinsystems.framework.dataexport.applet.ProgressDialog;

/**
 * Utility to recursively search the file system for all zip and jpg files to upload. Some ignore
 * paths are hard coded based on the data upload by WSP in Liberia. This utility will also combine
 * all zip files in a single directory into a single ZIP taking care not to have any duplicate
 * survey instances. After uploading a zip file, this utility will call the server to kick off
 * processing
 * 
 * @author <NAME>
 */
public class SurveyBulkUploader implements DataImporter {

    private static final String NOTIFICATION_PATH = "/processor?action=submit&fileName=";
    private static final String UPLOAD_IMAGE_MODE = "uploadImageOnly";
    private static final String ZIP_ONLY_MODE = "processZipOnly";
    public static final String MODE_KEY = "mode";
    private static final String PROGRESS_FILE_NAME = "progress.txt";
    public static final String MERGE_ONLY_MODE = "mergeOnly";

    private static final String IMAGE_TEMP_DIR = "resized";
    private static final String OSX_RESOURCE_DIR = "__MACOSX";

    private static final String DEFAULT_LOCALE = "en";
    private static Map<String, String> UPLOADING;
    private static Map<String, String> COMPLETE;
    private List<File> filesToUpload;

    static {
        UPLOADING = new HashMap<String, String>();
        UPLOADING.put("en", "Uploading");

{'fraction_non_alphanumeric': 0.07587204629480791, 'fraction_numerical': 0.0036971547982639448, 'mean_word_length': 4.467486818980668, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '17106', 'n_tokens_mistral': 1783, 'n_tokens_neox': 1749, 'n_words': 649}