<commit_before>/*
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE and NOTICE files at the root of the source
 * tree and available online at
 *
 *     http://duracloud.org/license/
 */
package org.duracloud.audit.reader.impl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.text.MessageFormat;
import java.util.Iterator;

import org.apache.commons.io.IOUtils;
import org.duracloud.audit.AuditLogUtil;
import org.duracloud.audit.reader.AuditLogReader;
import org.duracloud.audit.reader.AuditLogReaderException;
import org.duracloud.audit.reader.AuditLogReaderNotEnabledException;
import org.duracloud.error.ContentStoreException;
import org.duracloud.s3storage.S3StorageProvider;
import org.duracloud.storage.domain.AuditConfig;
import org.duracloud.storage.error.StorageException;
import org.duracloud.storage.provider.StorageProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;

/**
 * 
 * @author <NAME> 
 *         Date: Sept. 17, 2014
 * 
 */
public class AuditLogReaderImpl implements AuditLogReader {
    private static Logger log =
        LoggerFactory.getLogger(AuditLogReaderImpl.class);

    private AuditConfig auditConfig;
    
    private StorageProvider storageProvider;

    public AuditLogReaderImpl() {
    }

    @Override
    public void initialize(AuditConfig auditConfig){
        this.auditConfig = auditConfig;
    }
    
    @Override
    public InputStream getAuditLog(final String account, final String storeId, final String spaceId)
        throws AuditLogReaderException {
        
        checkEnabled();
        
        this.storageProvider = getStorageProvider();
        final String auditBucket = auditConfig.getAuditLogSpaceId();

        String prefix = MessageFormat.format("{0}/{1}/{2}/",account, storeId, spaceId);
        final PipedInputStream is = new PipedInputStream(10 * 1024);
        final PipedOutputStream os;
        try {
            os = new PipedOutputStream(is);
        } catch (IOException e) {
            throw new AuditLogReaderException(e);
        }
        
        try {
             final Iterator<String> it =
                this.storageProvider.getSpaceContents(auditBucket, prefix);
            if (!it.hasNext()) {
                os.write((AuditLogUtil.getHeader() + "\n").getBytes());
                os.close();
            }

            new Thread(new Runnable() {
                @Override
                public void run() {
                    try {
                        int count = 0;

                        
                        while (it.hasNext()) {
                            String contentId = it.next();
                            writeToOutputStream(auditBucket,
                                                storageProvider,
                                                os,
                                                count,
                                                contentId);

                            count++;
                        }

                        os.close();

                    } catch (ContentStoreException | IOException ex) {
                        log.error(MessageFormat.format("failed to complete audit log read routine for space: storeId={0}, spaceId={1}",
                                                       storeId,
                                                       spaceId),
                                  ex);
                    }
                }
            }).start();
        } catch (StorageException | IOException e) {
            throw new AuditLogReaderException(e);
        } 

        return is;
    }

    private void checkEnabled() throws AuditLogReaderNotEnabledException{
        if(auditConfig.getAuditLogSpaceId() == null ||
           auditConfig.getAuditQueueName() == null){
            throw new AuditLogReaderNotEnabledException();
        }
    }

    protected StorageProvider getStorageProvider() {
        AWSCredentials creds = new DefaultAWSCredentialsProviderChain().getCredentials();
        return new S3StorageProvider(creds.getAWSAccessKeyId(), creds.getAWSSecretKey());
    }

    protected void writeToOutputStream(String auditSpaceId,
                                       StorageProvider storageProvider,
                                       final PipedOutputStream os,
                                       int count,
                                       String contentId)
        throws ContentStoreException,
            IOException {
        InputStream is = storageProvider.getContent(auditSpaceId, contentId);
        BufferedReader reader =
            new BufferedReader(new InputStreamReader(is));<commit_msg>Resolves invalid credential issue with previous commit for #19.<commit_after>/*
 * The contents of this file are subject to the license and copyright
 * detailed in the LICENSE and NOTICE files at the root of the source
 * tree and available online at
 *
 *     http://duracloud.org/license/
 */
package org.duracloud.audit.reader.impl;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.text.MessageFormat;
import java.util.Iterator;

import org.apache.commons.io.IOUtils;
import org.duracloud.audit.AuditLogUtil;
import org.duracloud.audit.reader.AuditLogReader;
import org.duracloud.audit.reader.AuditLogReaderException;
import org.duracloud.audit.reader.AuditLogReaderNotEnabledException;
import org.duracloud.error.ContentStoreException;
import org.duracloud.s3storage.S3StorageProvider;
import org.duracloud.storage.domain.AuditConfig;
import org.duracloud.storage.error.StorageException;
import org.duracloud.storage.provider.StorageProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.internal.AWSS3V4Signer;

/**
 * 
 * @author <NAME> 
 *         Date: Sept. 17, 2014
 * 
 */
public class AuditLogReaderImpl implements AuditLogReader {
    private static Logger log =
        LoggerFactory.getLogger(AuditLogReaderImpl.class);

    private AuditConfig auditConfig;
    
    private StorageProvider storageProvider;

    public AuditLogReaderImpl() {
    }

    @Override
    public void initialize(AuditConfig auditConfig){
        this.auditConfig = auditConfig;
    }
    
    @Override
    public InputStream getAuditLog(final String account, final String storeId, final String spaceId)
        throws AuditLogReaderException {
        
        checkEnabled();
        
        this.storageProvider = getStorageProvider();
        final String auditBucket = auditConfig.getAuditLogSpaceId();

        String prefix = MessageFormat.format("{0}/{1}/{2}/",account, storeId, spaceId);
        final PipedInputStream is = new PipedInputStream(10 * 1024);
        final PipedOutputStream os;
        try {
            os = new PipedOutputStream(is);
        } catch (IOException e) {
            throw new AuditLogReaderException(e);
        }
        
        try {
             final Iterator<String> it =
                this.storageProvider.getSpaceContents(auditBucket, prefix);
            if (!it.hasNext()) {
                os.write((AuditLogUtil.getHeader() + "\n").getBytes());
                os.close();
            }

            new Thread(new Runnable() {
                @Override
                public void run() {
                    try {
                        int count = 0;

                        
                        while (it.hasNext()) {
                            String contentId = it.next();
                            writeToOutputStream(auditBucket,
                                                storageProvider,
                                                os,
                                                count,
                                                contentId);

                            count++;
                        }

                        os.close();

                    } catch (ContentStoreException | IOException ex) {
                        log.error(MessageFormat.format("failed to complete audit log read routine for space: storeId={0}, spaceId={1}",
                                                       storeId,
                                                       spaceId),
                                  ex);
                    }
                }
            }).start();
        } catch (StorageException | IOException e) {
            throw new AuditLogReaderException(e);
        } 

        return is;
    }

    private void checkEnabled() throws AuditLogReaderNotEnabledException{
        if(auditConfig.getAuditLogSpaceId() == null ||
           auditConfig.getAuditQueueName() == null){
            throw new AuditLogReaderNotEnabledException();
        }
    }

    protected StorageProvider getStorageProvider() {
        AWSCredentials creds = new DefaultAWSCredentialsProviderChain().getCredentials();
        AmazonS3Client s3client = new AmazonS3Client();
        return new S3StorageProvider(s3client, creds.getAWSAccessKeyId(), null);
    }

    protected void writeToOutputStream(String auditSpaceId,
                                       StorageProvider storageProvider,
                                       final PipedOutputStream os,
                                       int count,
                                       String contentId)
        throws ContentStoreException,
            IOException {
        InputStream is = storageProvider.getContent(auditSpaceId, contentId);
        BufferedReader reader =
            new BufferedReader(new InputStreamReader(is));
{'fraction_non_alphanumeric': 0.07762466541092496, 'fraction_numerical': 0.005650837711906415, 'mean_word_length': 1.8037798777098388, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4232961', 'n_tokens_mistral': 2511, 'n_tokens_neox': 2417, 'n_words': 544}