<commit_before>package com.eqt.ssc.tools;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.List;
import java.util.zip.GZIPInputStream;

import org.apache.commons.io.IOUtils;

import com.amazonaws.auth.ClasspathPropertiesFileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;

/***
 * utility that will take an s3 bucket and mirror it locally.
 * 
 * @author gman
 * 
 */
public class S3Downloader {

	public static void main(String[] args) {

		if (args.length < 2) {
			System.out.println("USAGE: localPath bucketname <bucketPrefix>");
			System.exit(1);
		}

		String localPath = args[0];
		String bucketName = args[1];
		String bucketPrefix = "";
<commit_msg>tweaking the downloader util class.<commit_after>package com.eqt.ssc.tools;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;

import org.apache.commons.io.IOUtils;

import com.amazonaws.auth.ClasspathPropertiesFileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;

/***
 * utility that will take an s3 bucket and mirror it locally.
 * 
 * @author gman
 * 
 */
public class S3Downloader {

	public static void main(String[] args) {

		if (args.length < 2) {
			System.out.println("USAGE: localPath bucketname <bucketPrefix>");
			System.exit(1);
		}

		String localPath = args[0];
		String bucketName = args[1];
		String bucketPrefix = "";

{'fraction_non_alphanumeric': 0.11480075901328274, 'fraction_numerical': 0.016129032258064516, 'mean_word_length': 4.9914772727272725, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5626568', 'n_tokens_mistral': 705, 'n_tokens_neox': 690, 'n_words': 140}