<commit_before>/*
 * Copyright 2010 Pentaho Corporation.  All rights reserved.
 *                   
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 * http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 *
 * @author <NAME>
 */
package org.pentaho.s3.vfs;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;

import org.apache.commons.io.IOUtils;
import org.apache.commons.vfs.FileName;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSystemException;
import org.apache.commons.vfs.FileType;
import org.apache.commons.vfs.provider.AbstractFileObject;
import org.jets3t.service.S3Service;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;

public class S3FileObject extends AbstractFileObject implements FileObject {

  private S3Service service = null;
  private S3Bucket bucket = null;

  protected S3FileObject(final FileName name, final S3FileSystem fileSystem) throws FileSystemException {
    super(name, fileSystem);
    service = fileSystem.getS3Service();
  }

  protected String getS3BucketName() throws Exception {
    String bucketName = getName().getPath();
    if (bucketName.indexOf("/", 1) > 1) {
      // this file is a file, to get the bucket, remove the name from the path
      bucketName = bucketName.substring(1, bucketName.indexOf(getName().getBaseName()) - 1);
    } else {
      // this file is a bucket
      bucketName = bucketName.replaceAll("/", "");
    }
    return bucketName;
  }

  protected S3Bucket getS3Bucket() throws Exception {
    if (bucket == null) {
      String bucketName = getS3BucketName();
      // subtract out the name
      bucket = service.getBucket(bucketName);
    }
    return bucket;
  }

  protected S3Object getS3Object(boolean deleteIfAlreadyExists) throws Exception {
    try {
      if (getName().getPath().indexOf("/", 1) == -1) {
        return null;
      }
      String name = getName().getPath().substring(getName().getPath().indexOf("/", 1) + 1);
      if (!name.equals("")) {
        try {
          S3Object object = service.getObject(getS3Bucket(), name);
          if (deleteIfAlreadyExists) {
            bucket = getS3Bucket();
            bucket = service.createBucket(getS3BucketName());
            service.deleteObject(getS3Bucket(), name);
            object = new S3Object(name);
          }
          return object;
        } catch (Exception e) {
          S3Object object = new S3Object(name);
          if (deleteIfAlreadyExists) {
            bucket = getS3Bucket();
            bucket = service.createBucket(getS3BucketName());
            service.deleteObject(getS3Bucket(), name);
          }
          return object;
        }
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
    return null;
  }

  protected long doGetContentSize() throws Exception {
    return getS3Object(false).getContentLength();
  }

  protected OutputStream doGetOutputStream(final boolean append) throws Exception {
    final ByteArrayOutputStream output = new ByteArrayOutputStream();
    final PipedInputStream pis = new PipedInputStream();

    final Thread t = new Thread(new Runnable() {
      public void run() {
        try {
          IOUtils.copy(pis, output);
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
    });
    t.start();

    final PipedOutputStream pos = new PipedOutputStream() {

      public void write(byte[] b) throws IOException {
        System.out.println("3 writing begin");
        super.write(b);
        System.out.println("3writing end");
      }

      public void write(byte[] b, int off, int len) throws IOException {
        System.out.println("2writing begin");
        super.write(b, off, len);
        System.out.println("2writing end");
      }

      public void write(int b) throws IOException {
        System.out.println("1writing begin");
        super.write(b);
        System.out.println("1writing end");
      }

      public void close() throws IOException {
        System.out.println("------ closing begin --------");
        super.close();
        try {
          // wait for reader to finish
          t.join();
          S3Object s3Object = getS3Object(true);
          byte[] bytes = output.toByteArray();
          s3Object.setContentLength(bytes.length);
          s3Object.setDataInputStream(new ByteArrayInputStream(bytes));
          service.putObject(getS3Bucket(), s3Object);
        } catch (Exception e) {
          e.printStackTrace();
        }
        System.out.println("------ closing end --------");
      }
    };
    pis.connect(pos);

    return pos;
  }

  public void close() throws FileSystemException {
    try {
      System.out.println("closing " + getName().getURI());
      getS3Object(false).closeDataInputStream();
      super.close();
    } catch (Exception e) {
    }
  }

  protected InputStream doGetInputStream() throws Exception {
    return getS3Object(false).getDataInputStream();
  }

  protected FileType doGetType() throws Exception {
    S3Bucket bucket = null;
    S3Object object = null;
    try {
      bucket = getS3Bucket();
    } catch (Exception ex) {
    }
    try {
      object = getS3Object(false);
    } catch (Exception ex) {
    }

    if (getName().getPath().equals("") || getName().getPath().equals("/")) {
      return FileType.FOLDER;
    }

    if (bucket == null && object == null) {
      return FileType.IMAGINARY;
    } else if (bucket != null && object == null) {
      return FileType.FOLDER;
    } else if (object.getBucketName() != null && object.getLastModifiedDate() != null) {
      return FileType.FILE;
    }
    return FileType.IMAGINARY;
  }

  public void doCreateFolder() throws Exception {
    bucket = service.createBucket(getS3BucketName());
  }

  public boolean canRenameTo(FileObject newfile) {
    try {
      // we cannot rename buckets
      if (getType().equals(FileType.FOLDER)) {
        return false;
      }
    } catch (Exception e) {
    }
    return super.canRenameTo(newfile);
  }

  public void doDelete() throws Exception {
    if (getName().getPath().equals("") || getName().getPath().equals("/")) {
      return;
    }
    if (getType() == FileType.FILE) {
      service.deleteObject(getS3Bucket(), getName().getBaseName());
    } else if (getType() == FileType.FOLDER) {
      service.deleteBucket(getS3Bucket());
    }
  }

  protected void doRename(FileObject newfile) throws Exception {
    if (getType().equals(FileType.FOLDER)) {
      throw new UnsupportedOperationException("Bucket renaming is not permitted");
    }
    S3Object s3Object = getS3Object(false);
    s3Object.setKey(newfile.getName().getBaseName());
    service.renameObject(getS3BucketName(), getName().getBaseName(), s3Object);
  }

  protected long doGetLastModifiedTime() throws Exception {
    if (getType() == FileType.FOLDER) {
      return -1;
    }
    return getS3Object(false).getLastModifiedDate().getTime();
  }

  protected void doSetLastModifiedTime(long modtime) throws Exception {
  }

  protected String[] doListChildren() throws Exception {
    S3Bucket bucket = getS3Bucket();
    if (bucket == null && (getName().getPath().equals("") || getName().getPath().equals("/"))) {
      S3Bucket[] buckets = service.listAllBuckets();
      String[] children = new String[buckets.length];
      for (int i = 0; i < buckets.length; i++) {
        children[i] = buckets[i].getName();
      }
      return children;
    } else {
      S3Object[] s3Children = service.listObjects(getS3Bucket());
      String[] children = new String[s3Children.length];
      for (int i = 0; i < s3Children.length; i++) {
        children[i] = s3Children[i].getKey();
      }
      return children;
    }
  }

}
<commit_msg>[CLEANUP] PDI-4445 Adding errors / codes for common problems.<commit_after>/*
 * Copyright 2010 Pentaho Corporation.  All rights reserved.
 *                   
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License. 
 * You may obtain a copy of the License at 
 * 
 * http://www.apache.org/licenses/LICENSE-2.0 
 *   
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, 
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 * See the License for the specific language governing permissions and 
 * limitations under the License. 
 *
 * @author <NAME>
 */
package org.pentaho.s3.vfs;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;

import org.apache.commons.io.IOUtils;
import org.apache.commons.vfs.FileName;
import org.apache.commons.vfs.FileObject;
import org.apache.commons.vfs.FileSystemException;
import org.apache.commons.vfs.FileType;
import org.apache.commons.vfs.provider.AbstractFileObject;
import org.jets3t.service.S3Service;
import org.jets3t.service.model.S3Bucket;
import org.jets3t.service.model.S3Object;

public class S3FileObject extends AbstractFileObject implements FileObject {

  private S3Service service = null;
  private S3Bucket bucket = null;

  protected S3FileObject(final FileName name, final S3FileSystem fileSystem) throws FileSystemException {
    super(name, fileSystem);
    service = fileSystem.getS3Service();
  }

  protected String getS3BucketName() throws Exception {
    String bucketName = getName().getPath();
    if (bucketName.indexOf("/", 1) > 1) {
      // this file is a file, to get the bucket, remove the name from the path
      bucketName = bucketName.substring(1, bucketName.indexOf(getName().getBaseName()) - 1);
    } else {
      // this file is a bucket
      bucketName = bucketName.replaceAll("/", "");
    }
    return bucketName;
  }

  protected S3Bucket getS3Bucket() throws Exception {
    if (bucket == null) {
      String bucketName = getS3BucketName();
      // subtract out the name
      bucket = service.getBucket(bucketName);
    }
    return bucket;
  }

  protected S3Object getS3Object(boolean deleteIfAlreadyExists) throws Exception {
    try {
      if (getName().getPath().indexOf("/", 1) == -1) {
        return null;
      }
      String name = getName().getPath().substring(getName().getPath().indexOf("/", 1) + 1);
      if (!name.equals("")) {
        try {
          S3Object object = service.getObject(getS3Bucket(), name);
          if (deleteIfAlreadyExists) {
            bucket = getS3Bucket();
            bucket = service.createBucket(getS3BucketName());
            service.deleteObject(getS3Bucket(), name);
            object = new S3Object(name);
          }
          return object;
        } catch (Exception e) {
          S3Object object = new S3Object(name);
          if (deleteIfAlreadyExists) {
            bucket = getS3Bucket();
            bucket = service.createBucket(getS3BucketName());
            service.deleteObject(getS3Bucket(), name);
          }
          return object;
        }
      }
    } catch (Exception ex) {
      ex.printStackTrace();
    }
    return null;
  }

  protected long doGetContentSize() throws Exception {
    return getS3Object(false).getContentLength();
  }

  protected OutputStream doGetOutputStream(final boolean append) throws Exception {
    final ByteArrayOutputStream output = new ByteArrayOutputStream();
    final PipedInputStream pis = new PipedInputStream();

    final Thread t = new Thread(new Runnable() {
      public void run() {
        try {
          IOUtils.copy(pis, output);
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
    });
    t.start();

    final PipedOutputStream pos = new PipedOutputStream() {
      public void close() throws IOException {
        super.close();
        try {
          // wait for reader to finish
          t.join();
          S3Object s3Object = getS3Object(true);
          byte[] bytes = output.toByteArray();
          s3Object.setContentLength(bytes.length);
          s3Object.setDataInputStream(new ByteArrayInputStream(bytes));
          service.putObject(getS3Bucket(), s3Object);
        } catch (Exception e) {
          e.printStackTrace();
        }
      }
    };
    pis.connect(pos);

    return pos;
  }

  public void close() throws FileSystemException {
    try {
      getS3Object(false).closeDataInputStream();
      super.close();
    } catch (Exception e) {
    }
  }

  protected InputStream doGetInputStream() throws Exception {
    return getS3Object(false).getDataInputStream();
  }

  protected FileType doGetType() throws Exception {
    S3Bucket bucket = null;
    S3Object object = null;
    try {
      bucket = getS3Bucket();
    } catch (Exception ex) {
    }
    try {
      object = getS3Object(false);
    } catch (Exception ex) {
    }

    if (getName().getPath().equals("") || getName().getPath().equals("/")) {
      return FileType.FOLDER;
    }

    if (bucket == null && object == null) {
      return FileType.IMAGINARY;
    } else if (bucket != null && object == null) {
      return FileType.FOLDER;
    } else if (object.getBucketName() != null && object.getLastModifiedDate() != null) {
      return FileType.FILE;
    }
    return FileType.IMAGINARY;
  }

  public void doCreateFolder() throws Exception {
    if (getS3Object(false) == null) {
      bucket = service.createBucket(getS3BucketName());
    } else {
      throw new FileSystemException("vfs.provider/create-folder-not-supported.error");
    }
  }

  public boolean canRenameTo(FileObject newfile) {
    try {
      // we cannot rename buckets
      if (getType().equals(FileType.FOLDER)) {
        return false;
      }
    } catch (Exception e) {
    }
    return super.canRenameTo(newfile);
  }

  public void doDelete() throws Exception {
    if (getName().getPath().equals("") || getName().getPath().equals("/")) {
      return;
    }
    if (getType() == FileType.FILE) {
      service.deleteObject(getS3Bucket(), getName().getBaseName());
    } else if (getType() == FileType.FOLDER) {
      service.deleteBucket(getS3Bucket());
    }
  }

  protected void doRename(FileObject newfile) throws Exception {
    if (getType().equals(FileType.FOLDER)) {
      throw new FileSystemException("vfs.provider/rename-not-supported.error");
    }
    S3Object s3Object = getS3Object(false);
    s3Object.setKey(newfile.getName().getBaseName());
    service.renameObject(getS3BucketName(), getName().getBaseName(), s3Object);
  }

  protected long doGetLastModifiedTime() throws Exception {
    if (getType() == FileType.FOLDER) {
      return -1;
    }
    return getS3Object(false).getLastModifiedDate().getTime();
  }

  protected void doSetLastModifiedTime(long modtime) throws Exception {
  }

  protected String[] doListChildren() throws Exception {
    S3Bucket bucket = getS3Bucket();
    if (bucket == null && (getName().getPath().equals("") || getName().getPath().equals("/"))) {
      S3Bucket[] buckets = service.listAllBuckets();
      String[] children = new String[buckets.length];
      for (int i = 0; i < buckets.length; i++) {
        children[i] = buckets[i].getName();
      }
      return children;
    } else {
      S3Object[] s3Children = service.listObjects(getS3Bucket());
      String[] children = new String[s3Children.length];
      for (int i = 0; i < s3Children.length; i++) {
        children[i] = s3Children[i].getKey();
      }
      return children;
    }
  }

}

{'fraction_non_alphanumeric': 0.11768230571034813, 'fraction_numerical': 0.010485948828569716, 'mean_word_length': 2.5322751322751325, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3741178', 'n_tokens_mistral': 4792, 'n_tokens_neox': 4654, 'n_words': 1166}