<commit_before>import java.io.*;
import java.net.MalformedURLException;
import javax.servlet.http.*;
import javax.servlet.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.gson.GsonBuilder;
import hex.genmodel.easy.exception.PredictException;
import hex.genmodel.easy.prediction.AbstractPrediction;
import hex.genmodel.easy.prediction.BinomialModelPrediction;
import hex.genmodel.easy.prediction.MultinomialModelPrediction;
import hex.genmodel.easy.*;
import hex.genmodel.*;

import com.google.gson.Gson;

public class PredictPythonServlet extends HttpServlet {
  private final Logger logger = Logging.getLogger(this.getClass());

  private static GenModel rawModel = ServletUtil.rawModel;
  private static EasyPredictModelWrapper model = ServletUtil.model;

  private static Process p = null;
  private static ProcessBuilder pb = null;
  private static OutputStream stdin;
  private static BufferedReader reader, err_reader;

  private static final Gson gson = new GsonBuilder().serializeSpecialFloatingPointValues().create();

  private static File servletPath = null;
  String[] colNames;

  public void init(ServletConfig servletConfig) throws ServletException {
    super.init(servletConfig);
    try {
      servletPath = new File(servletConfig.getServletContext().getResource("/").getPath());
      logger.debug("servletPath {}", servletPath);
      ServletUtil.loadModels(servletPath);
      model = ServletUtil.model;
      logger.debug("model {}", model);
    }
    catch (MalformedURLException e) {
      logger.error("init failed", e);
    }
  }

  private void startPython() throws Exception {
    logger.debug("startPython");
    String program = servletPath.getAbsolutePath() + "/WEB-INF/python.py";
    logger.debug("program {}", program);
    // start the python process
    try {
      // score.py
      pb = new ProcessBuilder("python", program);
      File pythonProcessDir = new File(servletPath, "/WEB-INF");
      pb.directory(pythonProcessDir);
      p = pb.start();
      stdin = p.getOutputStream();
      InputStream stdout = p.getInputStream();
      InputStream stderr = p.getErrorStream();
      reader = new BufferedReader(new InputStreamReader(stdout));
      err_reader = new BufferedReader(new InputStreamReader(stderr));
      logger.info("Python started");
    } catch (Exception ex) {
      logger.error("Python failed", ex);
      throw new Exception("Python failed");
    }
  }

  public void destroy() {
    if (p != null) {
      p.destroy();
      logger.info("Python destroyed");
    }
    super.destroy();
  }

  static private String jsonModel() {
    Gson gson = new Gson();
    String modelJson = gson.toJson(model);
    return modelJson;
  }

  private static final byte[] NewlineByteArray = "\n".getBytes();

  private synchronized String sendPython(String queryString) {
    String result = null;

    try {
      // restart if python failed
      if (p == null) {
        startPython();

      }
      try {
        logger.debug("send {}", queryString);
        stdin.write(queryString.getBytes());
        stdin.write(NewlineByteArray);
        stdin.flush();
        result = reader.readLine();
        logger.debug("result {}", result);
      }
      catch (IOException e) {
        String msg = "ERROR IOException in sendPython restarting python";
        result = msg;
        logger.error(msg, e);
        showStderr();
        // it failed so we restart it and retry
//        if (p != null) p.destroy();
//        startPython();
//        stdin.write(queryString.getBytes());
//        stdin.write(NewlineByteArray);
//        stdin.flush();
//        result = reader.readLine();
      }
//        showStderr();
    }
    catch (Exception ex) {
      logger.error("sendPython failed", ex);
      showStderr();
    }
    return result;
  }

  public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    long start = System.nanoTime();
    try {
      if (model == null)
        throw new Exception("No predictor model");

      String queryString = request.getQueryString();
      logger.debug("queryString {}", queryString);

      String result = sendPython(queryString.replaceAll("%20", " "));
      logger.debug("result {}", result);

      // should now be in sparse format from python
      RowData row = strMapToRowData(result);
      logger.debug("row: {}", row);

      AbstractPrediction pr = ServletUtil.predict(row);
      logger.debug("pr: {}", pr);

      // assemble json result
      String prJson = gson.toJson(pr);

      response.getWriter().write(prJson);
      response.setStatus(HttpServletResponse.SC_OK);

    }
    catch (Exception e) {
      // Prediction failed.
      logger.error("doGet failed", e);
      response.sendError(HttpServletResponse.SC_NOT_ACCEPTABLE, e.getMessage());
    }
    long done = System.nanoTime();
    ServletUtil.getPythonTimes.add(start, done);
    logger.debug("Python Get time {}", ServletUtil.getPythonTimes);
  }

  private void showStderr() {
    String line;
    try {
      while ((line=err_reader.readLine())!=null) {
        logger.info(line);
      }
    } catch (Exception ex2) {
      logger.error("showStderr failed", ex2);
    }
  }
<commit_msg>environemnt setup for python<commit_after>import java.io.*;
import java.net.MalformedURLException;
import javax.servlet.http.*;
import javax.servlet.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.gson.GsonBuilder;
import hex.genmodel.easy.exception.PredictException;
import hex.genmodel.easy.prediction.AbstractPrediction;
import hex.genmodel.easy.prediction.BinomialModelPrediction;
import hex.genmodel.easy.prediction.MultinomialModelPrediction;
import hex.genmodel.easy.*;
import hex.genmodel.*;

import com.google.gson.Gson;

public class PredictPythonServlet extends HttpServlet {
  private final Logger logger = Logging.getLogger(this.getClass());

  private static GenModel rawModel = ServletUtil.rawModel;
  private static EasyPredictModelWrapper model = ServletUtil.model;

  private static Process p = null;
  private static ProcessBuilder pb = null;
  private static OutputStream stdin;
  private static BufferedReader reader, err_reader;

  private static final Gson gson = new GsonBuilder().serializeSpecialFloatingPointValues().create();

  private static File servletPath = null;
  String[] colNames;

  public void init(ServletConfig servletConfig) throws ServletException {
    super.init(servletConfig);
    try {
      servletPath = new File(servletConfig.getServletContext().getResource("/").getPath());
      logger.debug("servletPath {}", servletPath);
      ServletUtil.loadModels(servletPath);
      model = ServletUtil.model;
      logger.debug("model {}", model);
    }
    catch (MalformedURLException e) {
      logger.error("init failed", e);
    }
  }

  private void setupVirtualPythonEnv(String envName, String envFile) throws Exception {
    logger.debug("setup virtual conda environment");
    pb = new ProcessBuilder("source", "activate", envName);
    pb.start();
    int returnCode = pb.waitFor();
    if (returnCode == 0) {
      logger.info("python virtual environment {} already exists", envName);
    }
    else {
      if (envFile == null) {
        logger.info("creating python virtual environment {}", envName);
        pb = new ProcessBuilder("conda", "create", "-n", envName);
      }
      else {
        logger.info("creating python virtual environment {} with environment file {}", envName, envFile);
        pb = new ProcessBuilder("conda", "env", "create", "-f", envFile, "-n", envName);
      }
      pb.redirectErrorStream(true); // send stderr to stdout
      InputStream stdout = p.getInputStream();
      reader = new BufferedReader(new InputStreamReader(stdout));
      pb.start();
      int returnCode = pb.waitFor();
      if (returnCode == 0) {
        logger.info("python environment {} created", envName);
      }
      else {
        logger.error("Failed to create environemnt {}", envName);
        showStdout(reader);
        throw new Exception("Can't created virtual python environment " + envName + " env file " + envFile);
      }
    }

  }

  private void startPython() throws Exception {
    logger.debug("startPython");
    String program = servletPath.getAbsolutePath() + "/WEB-INF/python.py";
    logger.debug("program {}", program);
    // start the python process
    try {
      // score.py
      pb = new ProcessBuilder("python", program);
      File pythonProcessDir = new File(servletPath, "/WEB-INF");
      pb.directory(pythonProcessDir);
      p = pb.start();
      stdin = p.getOutputStream();
      InputStream stdout = p.getInputStream();
      InputStream stderr = p.getErrorStream();
      reader = new BufferedReader(new InputStreamReader(stdout));
      err_reader = new BufferedReader(new InputStreamReader(stderr));
      logger.info("Python started");
    } catch (Exception ex) {
      logger.error("Python failed", ex);
      throw new Exception("Python failed");
    }
  }

  public void destroy() {
    if (p != null) {
      p.destroy();
      logger.info("Python destroyed");
    }
    super.destroy();
  }

  static private String jsonModel() {
    Gson gson = new Gson();
    String modelJson = gson.toJson(model);
    return modelJson;
  }

  private static final byte[] NewlineByteArray = "\n".getBytes();

  private synchronized String sendPython(String queryString) {
    String result = null;

    try {
      // restart if python failed
      if (p == null) {
        logger.info("python process not started, trying again");
        startPython();

      }
      try {
        logger.debug("send {}", queryString);
        stdin.write(queryString.getBytes());
        stdin.write(NewlineByteArray);
        stdin.flush();
        result = reader.readLine();
        logger.debug("result {}", result);
      }
      catch (IOException e) {
        String msg = "ERROR IOException in sendPython";
        result = msg;
        logger.error(msg, e);
        showStderr();
        // it failed so we restart it and retry
//        if (p != null) p.destroy();
//        startPython();
//        stdin.write(queryString.getBytes());
//        stdin.write(NewlineByteArray);
//        stdin.flush();
//        result = reader.readLine();
      }
//        showStderr(err_reader);
    }
    catch (Exception ex) {
      logger.error("sendPython failed", ex);
      showStderr(err_reader);
    }
    return result;
  }

  public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    long start = System.nanoTime();
    try {
      if (model == null)
        throw new Exception("No predictor model");

      String queryString = request.getQueryString();
      logger.debug("queryString {}", queryString);

      String result = sendPython(queryString.replaceAll("%20", " "));
      logger.debug("result {}", result);

      // should now be in sparse format from python
      RowData row = strMapToRowData(result);
      logger.debug("row: {}", row);

      AbstractPrediction pr = ServletUtil.predict(row);
      logger.debug("pr: {}", pr);

      // assemble json result
      String prJson = gson.toJson(pr);

      response.getWriter().write(prJson);
      response.setStatus(HttpServletResponse.SC_OK);

    }
    catch (Exception e) {
      // Prediction failed.
      logger.error("doGet failed", e);
      response.sendError(HttpServletResponse.SC_NOT_ACCEPTABLE, e.getMessage());
    }
    long done = System.nanoTime();
    ServletUtil.getPythonTimes.add(start, done);
    logger.debug("Python Get time {}", ServletUtil.getPythonTimes);
  }

  private void showStderr(BufferedReader err_reader) {
    String line;
    try {
      while ((line=err_reader.readLine())!=null) {
        logger.info(line);
      }
    } catch (Exception ex2) {
      logger.error("showStderr failed", ex2);
    }
  }

  private void showStdout(BufferedReader reader) {
    String line;
    try {
      while ((line=reader.readLine())!=null) {
        logger.info(line);
      }
    } catch (Exception ex2) {
      logger.error("showStdout failed", ex2);
    }
  }

{'fraction_non_alphanumeric': 0.12431369335409326, 'fraction_numerical': 0.0013111529951651234, 'mean_word_length': 3.0450778919456414, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1221597', 'n_tokens_mistral': 3507, 'n_tokens_neox': 3427, 'n_words': 837}