<commit_before>package se.kth.hopsworks.rest;

import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import javax.enterprise.context.RequestScoped;
import javax.ws.rs.GET;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.AccessControlException;
import se.kth.hopsworks.filters.AllowedRoles;
import se.kth.hopsworks.hdfs.fileoperations.DistributedFileSystemOps;
import se.kth.hopsworks.hdfs.fileoperations.DistributedFsService;

@RequestScoped
@TransactionAttribute(TransactionAttributeType.NEVER)
public class DownloadService {

  private static final Logger LOG
          = Logger.getLogger(DownloadService.class.getName());

  @EJB
  private DistributedFsService dfs;

  private String path;
  private String username;

  public DownloadService() {
  }

  public void setPath(String path) {
    this.path = path;
  }

  public void setUsername(String username) {
    this.username = username;
  }

  @GET
  @Produces(MediaType.APPLICATION_OCTET_STREAM)
  @AllowedRoles(roles = {AllowedRoles.DATA_SCIENTIST, AllowedRoles.DATA_OWNER})
  public Response downloadFromHDFS() throws AppException, AccessControlException {
    FSDataInputStream stream;
    DistributedFileSystemOps dfso = null;
    try {
      if (username != null) {
        dfso = dfs.getDfsOps(username);
        stream = dfso.open(new Path(this.path));
      } else {
        dfso = dfs.getDfsOps();
        stream = dfso.open(new Path(this.path));
      }
      Response.ResponseBuilder response = Response.ok((Object) stream);
      response.header("Content-disposition", "attachment;");

      return response.build();
    } catch (AccessControlException ex) {
      throw new AccessControlException(
              "Permission denied: You can not download the file ");
    } catch (IOException ex) {
      LOG.log(Level.SEVERE, null, ex);
      throw new AppException(Response.Status.BAD_REQUEST.getStatusCode(),
              "File does not exist: " + this.path);
    } 
//    finally {
//      if (dfso != null) {
//        dfso.close();
//      }
//    }
  }
}<commit_msg>Change download function to close the stream and the file system object when the download is finished.<commit_after>package se.kth.hopsworks.rest;

import java.io.IOException;
import java.io.OutputStream;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.TransactionAttribute;
import javax.ejb.TransactionAttributeType;
import javax.enterprise.context.RequestScoped;
import javax.ws.rs.GET;
import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.StreamingOutput;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.AccessControlException;
import se.kth.hopsworks.filters.AllowedRoles;
import se.kth.hopsworks.hdfs.fileoperations.DistributedFileSystemOps;
import se.kth.hopsworks.hdfs.fileoperations.DistributedFsService;

@RequestScoped
@TransactionAttribute(TransactionAttributeType.NEVER)
public class DownloadService {

  private static final Logger LOG
          = Logger.getLogger(DownloadService.class.getName());

  @EJB
  private DistributedFsService dfs;

  private String path;
  private String username;

  public DownloadService() {
  }

  public void setPath(String path) {
    this.path = path;
  }

  public void setUsername(String username) {
    this.username = username;
  }

  @GET
  @Produces(MediaType.APPLICATION_OCTET_STREAM)
  @AllowedRoles(roles = {AllowedRoles.DATA_SCIENTIST, AllowedRoles.DATA_OWNER})
  public Response downloadFromHDFS() throws AppException, AccessControlException {
    FSDataInputStream stream;
    DistributedFileSystemOps dfso = null;
    try {
      if (username != null) {
        dfso = dfs.getDfsOps(username);
        stream = dfso.open(new Path(this.path));
      } else {
        dfso = dfs.getDfsOps();
        stream = dfso.open(new Path(this.path));
      }    
      Response.ResponseBuilder response = Response.ok(buildOutputStream(stream,
              dfso));
      response.header("Content-disposition", "attachment;");
      
      return response.build();
    } catch (AccessControlException ex) {
      throw new AccessControlException(
              "Permission denied: You can not download the file ");
    } catch (IOException ex) {
      LOG.log(Level.SEVERE, null, ex);
      throw new AppException(Response.Status.BAD_REQUEST.getStatusCode(),
              "File does not exist: " + this.path);
    } 
  }
  
  private StreamingOutput buildOutputStream(final FSDataInputStream stream,
          final DistributedFileSystemOps dfso){
     StreamingOutput output = new StreamingOutput() {
        @Override
        public void write(OutputStream out) throws IOException,
                WebApplicationException {
          int length;
          byte[] buffer = new byte[1024];
          while((length = stream.read(buffer)) != -1) {
            out.write(buffer, 0, length);
          }
          out.flush();
          stream.close();
          dfso.close();
        }
      };
      
      return output;
  }
}
{'fraction_non_alphanumeric': 0.1028927132918345, 'fraction_numerical': 0.0010984987184181618, 'mean_word_length': 3.4234817813765184, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6957573', 'n_tokens_mistral': 1623, 'n_tokens_neox': 1591, 'n_words': 340}