<commit_before>package com.ibm.streamsx.topology.internal.context;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Future;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import com.ibm.json.java.JSONObject;
import com.ibm.streamsx.topology.Topology;
import com.ibm.streamsx.topology.context.ContextProperties;
import com.ibm.streamsx.topology.internal.process.CompletedFuture;

public class ZippedToolkitStreamsContext extends ToolkitStreamsContext {
	
	@Override
	public Future<File> submit(Topology app, Map<String, Object> config) throws Exception {

        if (config == null)
            config = new HashMap<>();

        // If the toolkit path is not given, then create one in the
        // currrent directory.
        if (!config.containsKey(ContextProperties.TOOLKIT_DIR)) {
            config.put(ContextProperties.TOOLKIT_DIR, Files
                    .createTempDirectory(Paths.get(""), "tk").toAbsolutePath().toString());
        }
        
        File toolkitRoot = super.submit(app, config).get();
        Path zipOutPath = Paths.get(toolkitRoot.getAbsolutePath() + ".zip");
		pack(toolkitRoot.toPath(), zipOutPath);
        return new CompletedFuture<File>(zipOutPath.toFile());
	}
	
	@Override
	public Future<File> submit(JSONObject submission) throws Exception {
		JSONObject deployInfo = (JSONObject) submission.get(SUBMISSION_DEPLOY);
    	if (deployInfo == null)
    		submission.put("deploy", deployInfo = new JSONObject());
    	
        if (!deployInfo.containsKey(ContextProperties.TOOLKIT_DIR)) {
        	deployInfo.put(ContextProperties.TOOLKIT_DIR, Files
                    .createTempDirectory(Paths.get(""), "tk").toAbsolutePath().toString());
        }

        File toolkitRoot = super.submit(submission).get();
        Path zipOutPath = Paths.get(toolkitRoot.getAbsolutePath() + ".zip");
		pack(toolkitRoot.toPath(), zipOutPath);
        return new CompletedFuture<File>(zipOutPath.toFile());
	}
	
	public static void pack(final Path folder, final Path zipFilePath) throws IOException {
		String folderName = folder.getFileName().toString();
	    try (
	            FileOutputStream fos = new FileOutputStream(zipFilePath.toFile());
	            ZipOutputStream zos = new ZipOutputStream(fos)
	    ) {
	        Files.walkFileTree(folder, new SimpleFileVisitor<Path>() {
	            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
	                zos.putNextEntry(new ZipEntry(folderName + "/" + folder.relativize(file).toString()));
	                Files.copy(file, zos);
	                zos.closeEntry();
	                return FileVisitResult.CONTINUE;
	            }

	            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
	                zos.putNextEntry(new ZipEntry(folderName + "/" + folder.relativize(dir).toString() + "/"));
	                zos.closeEntry();
	                return FileVisitResult.CONTINUE;
	            }
	        });
	    }
	}

}<commit_msg>ZIPPED_TOOLKIT context now deletes toolkit unless specified.<commit_after>package com.ibm.streamsx.topology.internal.context;

import static com.ibm.streamsx.topology.context.ContextProperties.KEEP_ARTIFACTS;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Future;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import com.ibm.json.java.JSONObject;
import com.ibm.streamsx.topology.Topology;
import com.ibm.streamsx.topology.context.ContextProperties;
import com.ibm.streamsx.topology.internal.process.CompletedFuture;

public class ZippedToolkitStreamsContext extends ToolkitStreamsContext {
	
	@Override
	public Future<File> submit(Topology app, Map<String, Object> config) throws Exception {        
        File toolkitRoot = super.submit(app, config).get();
        Path zipOutPath = pack(toolkitRoot.toPath());
        if (!(config.containsKey(KEEP_ARTIFACTS) && (boolean)config.get(KEEP_ARTIFACTS)))
        	delete(toolkitRoot.toPath());
        return new CompletedFuture<File>(zipOutPath.toFile());
	}
	
	@Override
	public Future<File> submit(JSONObject submission) throws Exception {
        File toolkitRoot = super.submit(submission).get();
        Path zipOutPath = pack(toolkitRoot.toPath());
        
        JSONObject deployInfo = (JSONObject)  submission.get(SUBMISSION_DEPLOY);
        if (!(deployInfo.containsKey(KEEP_ARTIFACTS) && (boolean)deployInfo.get(KEEP_ARTIFACTS)))
        	delete(toolkitRoot.toPath());
        return new CompletedFuture<File>(zipOutPath.toFile());
	}
	
	// Deletes a file or directory, even if not empty. Returns false if
	// it could not be deleted.
	public static boolean delete(final Path folder) {
		try {
			Files.walkFileTree(folder, new SimpleFileVisitor<Path>(){
				@Override
				public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
					Files.delete(file);
			        return FileVisitResult.CONTINUE;
			    }

				@Override
			    public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
			    	Files.delete(dir);
			        return FileVisitResult.CONTINUE;
			    }
			});
		} catch (IOException e) {
			return false;
		}
		return true;
	}
	
	public static Path pack(final Path folder) throws IOException {
		Path zipFilePath = Paths.get(folder.toAbsolutePath().toString() + ".zip");
		String folderName = folder.getFileName().toString();
	    try (
	            FileOutputStream fos = new FileOutputStream(zipFilePath.toFile());
	            ZipOutputStream zos = new ZipOutputStream(fos)
	    ) {
	        Files.walkFileTree(folder, new SimpleFileVisitor<Path>() {
	            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
	                zos.putNextEntry(new ZipEntry(folderName + "/" + folder.relativize(file).toString()));
	                Files.copy(file, zos);
	                zos.closeEntry();
	                return FileVisitResult.CONTINUE;
	            }

	            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
	                zos.putNextEntry(new ZipEntry(folderName + "/" + folder.relativize(dir).toString() + "/"));
	                zos.closeEntry();
	                return FileVisitResult.CONTINUE;
	            }
	        });
	    }
	    return zipFilePath;
	}

}
{'fraction_non_alphanumeric': 0.10790446403184532, 'fraction_numerical': 0.0, 'mean_word_length': 3.2558983666061705, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6349930', 'n_tokens_mistral': 2133, 'n_tokens_neox': 1967, 'n_words': 401}