<commit_before>package org.loadosophia.jmeter;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.multipart.*;
import org.apache.commons.io.IOUtils;
import org.apache.jorphan.logging.LoggingManager;
import org.apache.log.Logger;

import java.io.*;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.zip.GZIPOutputStream;

public class LoadosophiaAPIClient {

    private static final Logger log = LoggingManager.getLoggerForClass();
    public static final String COLOR_NONE = "none";
    public static final String[] colors = {COLOR_NONE, "red", "green", "blue", "gray", "orange", "violet", "cyan", "black"};
    public static final String STATUS_DONE = "4";
    private final HttpClient httpClient = new HttpClient();
    private final StatusNotifierCallback notifier;
    private final String project;
    private final String address;
    private final String token;
    private final String colorFlag;
    private final String title;
    private final static int TIMEOUT = 5;

    public LoadosophiaAPIClient(StatusNotifierCallback informer, String aAddress, String aToken, String projectName, String aColorFlag, String aTitle) {
        project = projectName;
        address = aAddress;
        token = aToken;
        notifier = informer;
        colorFlag = aColorFlag;
        title = aTitle;

        httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(TIMEOUT * 1000);
        httpClient.getHttpConnectionManager().getParams().setSoTimeout(TIMEOUT * 1000);
    }

    public LoadosophiaUploadResults sendFiles(File targetFile, LinkedList<String> perfMonFiles) throws IOException {
        LoadosophiaUploadResults results = new LoadosophiaUploadResults();
        if (targetFile.length() == 0) {
            throw new IOException("Cannot send empty file to BM.Sense");
        }

        log.info("Preparing files to send");
        LinkedList<Part> partsList = new LinkedList<>();
        partsList.add(new StringPart("projectKey", project));
        partsList.add(new FilePart("jtl_file", new FilePartSource(gzipFile(targetFile))));

        Iterator<String> it = perfMonFiles.iterator();
        int index = 0;
        while (it.hasNext()) {
            File perfmonFile = new File(it.next());
            if (!perfmonFile.exists()) {
                log.warn("File not exists, skipped: " + perfmonFile.getAbsolutePath());
                continue;
            }
            partsList.add(new FilePart("perfmon_" + index, new FilePartSource(gzipFile(perfmonFile))));
            index++;
        }

        notifier.notifyAbout("Starting upload to BM.Sense");
        String[] fields = multipartPost(partsList, getUploaderURI(), HttpStatus.SC_OK);
        int queueID = Integer.parseInt(fields[0]);
        results.setQueueID(queueID);

        if (!title.trim().isEmpty() || !colorFlag.equals(COLOR_NONE)) {
            int testID = getTestByUpload(queueID);
            results.setTestID(testID);

            if (!title.trim().isEmpty()) {
                setTestTitle(testID, title.trim());
            }

            if (!colorFlag.equals(COLOR_NONE)) {
                setTestColor(testID, colorFlag);
            }

            results.setRedirectLink(address + "gui/" + testID + "/");
        } else {
            results.setRedirectLink(address + "api/file/status/" + queueID + "/?redirect=true");
        }
        return results;
    }

    public String startOnline() throws IOException {
        String uri = address + "api/active/receiver/start/";
        LinkedList<Part> partsList = new LinkedList<>();
        partsList.add(new StringPart("token", token));
        partsList.add(new StringPart("projectKey", project));
        partsList.add(new StringPart("title", title));
        String[] res = multipartPost(partsList, uri, HttpStatus.SC_CREATED);
        JSONObject obj = JSONObject.fromObject(res[0]);
        return address + "gui/active/" + obj.optString("OnlineID", "N/A") + "/";
    }

    public void sendOnlineData(JSONArray data) throws IOException {
        String uri = address + "api/active/receiver/data/";
        LinkedList<Part> partsList = new LinkedList<>();
        String dataStr = data.toString();
        log.debug("Sending active test data: " + dataStr);
        partsList.add(new StringPart("data", dataStr));
        multipartPost(partsList, uri, HttpStatus.SC_ACCEPTED);
    }

    public void endOnline(String redirectLink) throws IOException {
        String uri = address + "api/active/receiver/stop/";
        LinkedList<Part> partsList = new LinkedList<>();
        partsList.add(new StringPart("redirect", redirectLink));
        multipartPost(partsList, uri, HttpStatus.SC_RESET_CONTENT);
    }

    private File gzipFile(File src) throws IOException {
        // Never try to make it stream-like on the fly, because content-length still required
        // Create the GZIP output stream
        String outFilename = src.getAbsolutePath() + ".gz";
        notifier.notifyAbout("Gzipping " + src.getAbsolutePath());
        GZIPOutputStream out = new GZIPOutputStream(new FileOutputStream(outFilename));

        // Open the input file
        FileInputStream in = new FileInputStream(src);

        // Transfer bytes from the input file to the GZIP output stream
        byte[] buf = new byte[10240];
        int len;
        while ((len = in.read(buf)) > 0) {
            out.write(buf, 0, len);
        }
        in.close();

        // Complete the GZIP file
        out.finish();
        out.close();

        src.delete();

        return new File(outFilename);
    }

    private int getTestByUpload(int queueID) throws IOException {
        while (true) {
            try {
                Thread.sleep(5000); // TODO: parameterize it
            } catch (InterruptedException ex) {
                throw new RuntimeException("Failed to get test ID");
            }

            String[] status = getUploadStatus(queueID);
            if (status.length > 2 && !status[2].isEmpty()) {
                throw new RuntimeException("BM.Sense processing error: " + status[2]);
            }

            if (status[1].equals(STATUS_DONE)) {
                return Integer.parseInt(status[0]);
            }
        }
    }

    private void setTestTitle(int testID, String trim) throws IOException {
        String uri = address + "api/test/edit/title/" + testID + "/?title=" + URLEncoder.encode(trim, "UTF-8");
        multipartPost(new LinkedList<Part>(), uri, HttpStatus.SC_NO_CONTENT);
    }

    private void setTestColor(int testID, String colorFlag) throws IOException {
        String uri = address + "api/test/edit/color/" + testID + "/?color=" + colorFlag;
        multipartPost(new LinkedList<Part>(), uri, HttpStatus.SC_NO_CONTENT);
    }

    private String getUploaderURI() {
        return address + "api/file/upload/?format=csv";
    }

    protected String[] getUploadStatus(int queueID) throws IOException {
        String uri = address + "api/file/status/" + queueID + "/?format=csv";
        return multipartPost(new LinkedList<Part>(), uri, HttpStatus.SC_OK);
    }

    protected String[] multipartPost(LinkedList<Part> parts, String URL, int expectedSC) throws IOException {
        log.debug("Request POST: " + URL);
        parts.add(new StringPart("token", token));

        PostMethod postRequest = new PostMethod(URL);
        MultipartRequestEntity multipartRequest = new MultipartRequestEntity(parts.toArray(new Part[parts.size()]), postRequest.getParams());
        postRequest.setRequestEntity(multipartRequest);
        int result = httpClient.executeMethod(postRequest);
        InputStream respBody = postRequest.getResponseBodyAsStream();
        if (result != expectedSC) {
            if (respBody != null) {
                saveErrorResponse(respBody);
            }
            throw new HttpException("Request returned not " + expectedSC + " status code: " + result);
        }

        byte[] bytes;
        if (respBody != null) {
            bytes = IOUtils.toByteArray(respBody);
        } else {
            bytes = new byte[0];
        }
        String response = new String(bytes);
        return response.trim().split(";");
    }

    private void saveErrorResponse(InputStream respBody) throws IOException {
        String fname = File.createTempFile("error_", ".html").getAbsolutePath();
        notifier.notifyAbout("Saving server error response to: " + fname);
        FileOutputStream fos = new FileOutputStream(fname);
        FileChannel resultFile = fos.getChannel();
        resultFile.write(ByteBuffer.wrap(IOUtils.toByteArray(respBody)));
        resultFile.close();
    }
}
<commit_msg>Don't try to upload empty perfmon file<commit_after>package org.loadosophia.jmeter;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.multipart.*;
import org.apache.commons.io.IOUtils;
import org.apache.jorphan.logging.LoggingManager;
import org.apache.log.Logger;

import java.io.*;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.zip.GZIPOutputStream;

public class LoadosophiaAPIClient {

    private static final Logger log = LoggingManager.getLoggerForClass();
    public static final String COLOR_NONE = "none";
    public static final String[] colors = {COLOR_NONE, "red", "green", "blue", "gray", "orange", "violet", "cyan", "black"};
    public static final String STATUS_DONE = "4";
    private final HttpClient httpClient = new HttpClient();
    private final StatusNotifierCallback notifier;
    private final String project;
    private final String address;
    private final String token;
    private final String colorFlag;
    private final String title;
    private final static int TIMEOUT = 5;

    public LoadosophiaAPIClient(StatusNotifierCallback informer, String aAddress, String aToken, String projectName, String aColorFlag, String aTitle) {
        project = projectName;
        address = aAddress;
        token = aToken;
        notifier = informer;
        colorFlag = aColorFlag;
        title = aTitle;

        httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(TIMEOUT * 1000);
        httpClient.getHttpConnectionManager().getParams().setSoTimeout(TIMEOUT * 1000);
    }

    public LoadosophiaUploadResults sendFiles(File targetFile, LinkedList<String> perfMonFiles) throws IOException {
        LoadosophiaUploadResults results = new LoadosophiaUploadResults();
        if (targetFile.length() == 0) {
            throw new IOException("Cannot send empty file to BM.Sense");
        }

        log.info("Preparing files to send");
        LinkedList<Part> partsList = new LinkedList<>();
        partsList.add(new StringPart("projectKey", project));
        partsList.add(new FilePart("jtl_file", new FilePartSource(gzipFile(targetFile))));

        Iterator<String> it = perfMonFiles.iterator();
        int index = 0;
        while (it.hasNext()) {
            File perfmonFile = new File(it.next());
            if (!perfmonFile.exists()) {
                log.warn("File not exists, skipped: " + perfmonFile.getAbsolutePath());
                continue;
            }

            if (perfmonFile.length() == 0) {
                log.warn("Empty file skipped: " + perfmonFile.getAbsolutePath());
                continue;
            }

            partsList.add(new FilePart("perfmon_" + index, new FilePartSource(gzipFile(perfmonFile))));
            index++;
        }

        notifier.notifyAbout("Starting upload to BM.Sense");
        String[] fields = multipartPost(partsList, getUploaderURI(), HttpStatus.SC_OK);
        int queueID = Integer.parseInt(fields[0]);
        results.setQueueID(queueID);

        if (!title.trim().isEmpty() || !colorFlag.equals(COLOR_NONE)) {
            int testID = getTestByUpload(queueID);
            results.setTestID(testID);

            if (!title.trim().isEmpty()) {
                setTestTitle(testID, title.trim());
            }

            if (!colorFlag.equals(COLOR_NONE)) {
                setTestColor(testID, colorFlag);
            }

            results.setRedirectLink(address + "gui/" + testID + "/");
        } else {
            results.setRedirectLink(address + "api/file/status/" + queueID + "/?redirect=true");
        }
        return results;
    }

    public String startOnline() throws IOException {
        String uri = address + "api/active/receiver/start/";
        LinkedList<Part> partsList = new LinkedList<>();
        partsList.add(new StringPart("token", token));
        partsList.add(new StringPart("projectKey", project));
        partsList.add(new StringPart("title", title));
        String[] res = multipartPost(partsList, uri, HttpStatus.SC_CREATED);
        JSONObject obj = JSONObject.fromObject(res[0]);
        return address + "gui/active/" + obj.optString("OnlineID", "N/A") + "/";
    }

    public void sendOnlineData(JSONArray data) throws IOException {
        String uri = address + "api/active/receiver/data/";
        LinkedList<Part> partsList = new LinkedList<>();
        String dataStr = data.toString();
        log.debug("Sending active test data: " + dataStr);
        partsList.add(new StringPart("data", dataStr));
        multipartPost(partsList, uri, HttpStatus.SC_ACCEPTED);
    }

    public void endOnline(String redirectLink) throws IOException {
        String uri = address + "api/active/receiver/stop/";
        LinkedList<Part> partsList = new LinkedList<>();
        partsList.add(new StringPart("redirect", redirectLink));
        multipartPost(partsList, uri, HttpStatus.SC_RESET_CONTENT);
    }

    private File gzipFile(File src) throws IOException {
        // Never try to make it stream-like on the fly, because content-length still required
        // Create the GZIP output stream
        String outFilename = src.getAbsolutePath() + ".gz";
        notifier.notifyAbout("Gzipping " + src.getAbsolutePath());
        GZIPOutputStream out = new GZIPOutputStream(new FileOutputStream(outFilename));

        // Open the input file
        FileInputStream in = new FileInputStream(src);

        // Transfer bytes from the input file to the GZIP output stream
        byte[] buf = new byte[10240];
        int len;
        while ((len = in.read(buf)) > 0) {
            out.write(buf, 0, len);
        }
        in.close();

        // Complete the GZIP file
        out.finish();
        out.close();

        src.delete();

        return new File(outFilename);
    }

    private int getTestByUpload(int queueID) throws IOException {
        while (true) {
            try {
                Thread.sleep(5000); // TODO: parameterize it
            } catch (InterruptedException ex) {
                throw new RuntimeException("Failed to get test ID");
            }

            String[] status = getUploadStatus(queueID);
            if (status.length > 2 && !status[2].isEmpty()) {
                throw new RuntimeException("BM.Sense processing error: " + status[2]);
            }

            if (status[1].equals(STATUS_DONE)) {
                return Integer.parseInt(status[0]);
            }
        }
    }

    private void setTestTitle(int testID, String trim) throws IOException {
        String uri = address + "api/test/edit/title/" + testID + "/?title=" + URLEncoder.encode(trim, "UTF-8");
        multipartPost(new LinkedList<Part>(), uri, HttpStatus.SC_NO_CONTENT);
    }

    private void setTestColor(int testID, String colorFlag) throws IOException {
        String uri = address + "api/test/edit/color/" + testID + "/?color=" + colorFlag;
        multipartPost(new LinkedList<Part>(), uri, HttpStatus.SC_NO_CONTENT);
    }

    private String getUploaderURI() {
        return address + "api/file/upload/?format=csv";
    }

    protected String[] getUploadStatus(int queueID) throws IOException {
        String uri = address + "api/file/status/" + queueID + "/?format=csv";
        return multipartPost(new LinkedList<Part>(), uri, HttpStatus.SC_OK);
    }

    protected String[] multipartPost(LinkedList<Part> parts, String URL, int expectedSC) throws IOException {
        log.debug("Request POST: " + URL);
        parts.add(new StringPart("token", token));

        PostMethod postRequest = new PostMethod(URL);
        MultipartRequestEntity multipartRequest = new MultipartRequestEntity(parts.toArray(new Part[parts.size()]), postRequest.getParams());
        postRequest.setRequestEntity(multipartRequest);
        int result = httpClient.executeMethod(postRequest);
        InputStream respBody = postRequest.getResponseBodyAsStream();
        if (result != expectedSC) {
            if (respBody != null) {
                saveErrorResponse(respBody);
            }
            throw new HttpException("Request returned not " + expectedSC + " status code: " + result);
        }

        byte[] bytes;
        if (respBody != null) {
            bytes = IOUtils.toByteArray(respBody);
        } else {
            bytes = new byte[0];
        }
        String response = new String(bytes);
        return response.trim().split(";");
    }

    private void saveErrorResponse(InputStream respBody) throws IOException {
        String fname = File.createTempFile("error_", ".html").getAbsolutePath();
        notifier.notifyAbout("Saving server error response to: " + fname);
        FileOutputStream fos = new FileOutputStream(fname);
        FileChannel resultFile = fos.getChannel();
        resultFile.write(ByteBuffer.wrap(IOUtils.toByteArray(respBody)));
        resultFile.close();
    }
}

{'fraction_non_alphanumeric': 0.114806278125343, 'fraction_numerical': 0.0035671166721545386, 'mean_word_length': 2.776787564766839, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2564356', 'n_tokens_mistral': 5017, 'n_tokens_neox': 4903, 'n_words': 1198}