<commit_before>/*
 * Copyright (C) 2012 <NAME>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package fr.iscpif.gridscale.storage

import java.io.File
import java.net.MalformedURLException
import java.net.URI
import org.apache.axis.client.Call
import org.glite.voms.contact.VOMSProxyInit
import org.glite.voms.contact.VOMSRequestOptions
import org.glite.voms.contact.VOMSServerInfo
import org.globus.gsi.GSIConstants.DelegationType
import org.globus.ftp.GridFTPClient
import org.globus.gsi.GSIConstants.CertificateType
import org.globus.axis.gsi.GSIConstants
import org.ietf.jgss.GSSCredential
import collection.JavaConversions._
import org.globus.util.Util
import org.globus.gsi.X509Credential
import org.globus.gsi.gssapi._
import org.ogf.srm22._
import org.globus.axis.transport._
import org.apache.axis.configuration.SimpleProvider
import org.apache.axis.SimpleTargetedChain
import org.apache.axis.client.Stub
import TStatusCode._
import collection.JavaConversions._
import java.util.concurrent.TimeoutException
import org.globus.gsi.gssapi.auth.HostAuthorization
import org.globus.io.streams._

object SRMStorage {
  
  Call.setTransportForProtocol("httpg", classOf[GSIHTTPTransport])
  
  def init = Unit
  
  def gridFtpPort(p: Int) = 
    if(p == -1) 2811 else p
 
  type RequestStatus = { 
    def getReturnStatus(): TReturnStatus
  }
   
  type RequestStatusWithToken = RequestStatus {
    def getRequestToken(): String
  }
    
}

import SRMStorage._

trait SRMStorage extends Storage with RecursiveRmDir {
  
  SRMStorage.init
  
  type A = GlobusGSSCredentialImpl
  
  def host: String
  def port: Int
  def basePath: String
  
  def timeout = 120
  def sleepTime = 10000
  def lsSizeMax = 1000
  def SERVICE_PROTOCOL = "httpg"
  def SERVICE_PATH = "/srm/managerv2" 
  def transferProtocols = Array("gsiftp")
  
  def version(implicit credential: GlobusGSSCredentialImpl) = stub.srmPing(new SrmPingRequest).getVersionInfo
  
  def list(absolutePath: String)(implicit credential: GlobusGSSCredentialImpl): Seq[(String, FileType)] = {
    def recList(offset: Int, res: List[Seq[(String, FileType)]] = List.empty): Seq[(String, FileType)] = {
      val ls = list(absolutePath, offset, lsSizeMax)
      if(ls.size < lsSizeMax) (ls :: res).reverse.flatten
      else recList(offset + lsSizeMax, ls :: res)
    }
    recList(0)
  }
  
  def list(absolutePath: String, offset: Int, size: Int)(implicit credential: GlobusGSSCredentialImpl) = {
    val uri = toSrmURI(absolutePath)
    val request = new SrmLsRequest
    request.setArrayOfSURLs(new ArrayOfAnyURI(Array(uri)))
    request.setAllLevelRecursive(false)
    request.setFullDetailedList(false)
    request.setOffset(offset)
    request.setCount(size)
    
    type SRMLsRS = RequestStatus {
      def getDetails(): ArrayOfTMetaDataPathDetail
    }
    
    def requestStatus(implicit credential: GlobusGSSCredentialImpl) = stub.srmLs(request)
    
    val childs = complete[SRMLsRS](requestStatus) { 
      token => 
      val status = new SrmStatusOfLsRequestRequest
      status.setRequestToken(token)
      stub.srmStatusOfLsRequest(status)
    }.getDetails.getPathDetailArray
    
    (for {
      child <- childs
      if(child.getArrayOfSubPaths != null)
      pd <- child.getArrayOfSubPaths.getPathDetailArray 
    } yield {
          val t = pd.getType match {
            case TFileType.DIRECTORY => DirectoryType
            case TFileType.FILE => FileType
            case TFileType.LINK => LinkType
          }
          new File(pd.getPath).getName -> t
      
    }).toSeq
  }
  
  
  def makeDir(path: String)(implicit credential: GlobusGSSCredentialImpl) = {
    val uri = this.toSrmURI(path)
    val request = new SrmMkdirRequest
    request.setSURL(uri)
    val requestStatus = stub.srmMkdir(request)
    if(requestStatus.getReturnStatus.getStatusCode != SRM_SUCCESS) throwError(requestStatus)
  }
  <commit_msg>Make sure nothing is null.<commit_after>/*
 * Copyright (C) 2012 <NAME>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

package fr.iscpif.gridscale.storage

import java.io.File
import java.net.MalformedURLException
import java.net.URI
import org.apache.axis.client.Call
import org.glite.voms.contact.VOMSProxyInit
import org.glite.voms.contact.VOMSRequestOptions
import org.glite.voms.contact.VOMSServerInfo
import org.globus.gsi.GSIConstants.DelegationType
import org.globus.ftp.GridFTPClient
import org.globus.gsi.GSIConstants.CertificateType
import org.globus.axis.gsi.GSIConstants
import org.ietf.jgss.GSSCredential
import collection.JavaConversions._
import org.globus.util.Util
import org.globus.gsi.X509Credential
import org.globus.gsi.gssapi._
import org.ogf.srm22._
import org.globus.axis.transport._
import org.apache.axis.configuration.SimpleProvider
import org.apache.axis.SimpleTargetedChain
import org.apache.axis.client.Stub
import TStatusCode._
import collection.JavaConversions._
import java.util.concurrent.TimeoutException
import org.globus.gsi.gssapi.auth.HostAuthorization
import org.globus.io.streams._

object SRMStorage {
  
  Call.setTransportForProtocol("httpg", classOf[GSIHTTPTransport])
  
  def init = Unit
  
  def gridFtpPort(p: Int) = 
    if(p == -1) 2811 else p
 
  type RequestStatus = { 
    def getReturnStatus(): TReturnStatus
  }
   
  type RequestStatusWithToken = RequestStatus {
    def getRequestToken(): String
  }
    
}

import SRMStorage._

trait SRMStorage extends Storage with RecursiveRmDir {
  
  SRMStorage.init
  
  type A = GlobusGSSCredentialImpl
  
  def host: String
  def port: Int
  def basePath: String
  
  def timeout = 120
  def sleepTime = 10000
  def lsSizeMax = 1000
  def SERVICE_PROTOCOL = "httpg"
  def SERVICE_PATH = "/srm/managerv2" 
  def transferProtocols = Array("gsiftp")
  
  def version(implicit credential: GlobusGSSCredentialImpl) = stub.srmPing(new SrmPingRequest).getVersionInfo
  
  def list(absolutePath: String)(implicit credential: GlobusGSSCredentialImpl): Seq[(String, FileType)] = {
    def recList(offset: Int, res: List[Seq[(String, FileType)]] = List.empty): Seq[(String, FileType)] = {
      val ls = list(absolutePath, offset, lsSizeMax)
      if(ls.size < lsSizeMax) (ls :: res).reverse.flatten
      else recList(offset + lsSizeMax, ls :: res)
    }
    recList(0)
  }
  
  def list(absolutePath: String, offset: Int, size: Int)(implicit credential: GlobusGSSCredentialImpl) = {
    val uri = toSrmURI(absolutePath)
    val request = new SrmLsRequest
    request.setArrayOfSURLs(new ArrayOfAnyURI(Array(uri)))
    request.setAllLevelRecursive(false)
    request.setFullDetailedList(false)
    request.setOffset(offset)
    request.setCount(size)
    
    type SRMLsRS = RequestStatus {
      def getDetails(): ArrayOfTMetaDataPathDetail
    }
    
    def requestStatus(implicit credential: GlobusGSSCredentialImpl) = stub.srmLs(request)
    
    val childs = complete[SRMLsRS](requestStatus) { 
      token => 
      val status = new SrmStatusOfLsRequestRequest
      status.setRequestToken(token)
      stub.srmStatusOfLsRequest(status)
    }.getDetails.getPathDetailArray
    
    (for {
        child <- childs
        if(child.getArrayOfSubPaths != null && child.getArrayOfSubPaths.getPathDetailArray != null)
          pd <- child.getArrayOfSubPaths.getPathDetailArray 
      } yield {
        val t = pd.getType match {
          case TFileType.DIRECTORY => DirectoryType
          case TFileType.FILE => FileType
          case TFileType.LINK => LinkType
        }
        new File(pd.getPath).getName -> t
      }).toSeq
  }
  
  
  def makeDir(path: String)(implicit credential: GlobusGSSCredentialImpl) = {
    val uri = this.toSrmURI(path)
    val request = new SrmMkdirRequest
    request.setSURL(uri)
    val requestStatus = stub.srmMkdir(request)
    if(requestStatus.getReturnStatus.getStatusCode != SRM_SUCCESS) throwError(requestStatus)
  }
  
{'fraction_non_alphanumeric': 0.08734442119176121, 'fraction_numerical': 0.006388368763079634, 'mean_word_length': 3.7119875454073687, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1089501', 'n_tokens_mistral': 2936, 'n_tokens_neox': 2865, 'n_words': 704}