<commit_before>#
# This file will download all of the objects in a bucket that are
# associated with a specific BibleId/FileId
#
import boto3
import io
import os

media = "text/ENGESV"
mediaLen = len(media)
filename1 = "about.html"
filename2 = "info.json"
target = "/Users/garygriswold/ShortSands/DBL/FCBH_info"

session = boto3.Session(profile_name='FCBH_BibleApp')
client = session.client('s3')

input = io.open("metadata/FCBH/dbp_dev.txt", mode="r", encoding="utf-8")
for line in input:
	if line[0:mediaLen] == media:
		line = line.strip()
		row = line.split("/")
		last = row[-1]
		if last == filename1 or last == filename2:
			directory = target + "/" + row[1]
			if not os.path.exists(directory):
				os.makedirs(directory)
			path = directory + "/" + last
			try:
				client.download_file('dbp-prod', line, path)
				print "Sucessfully downloaded", path
			except:
				print "Failed ", line

input.close()<commit_msg>Versions Add DBPDownloadInfo.<commit_after>#
# This file will download all of the objects in a bucket that are
# associated with a specific BibleId/FileId
#
import boto3
import io
import os 

searchFile = "info.json"
target = "/Users/garygriswold/ShortSands/DBL/FCBH_info/"

session = boto3.Session(profile_name='FCBH_BibleApp')
client = session.client('s3')

input = io.open("metadata/FCBH/dbp_prod.txt", mode="r", encoding="utf-8")
for line in input:
	line = line.strip()
	if line.endswith(searchFile):
		filename = target + line.replace("/", "|")
		try:
			client.download_file('dbp-prod', line, filename)
			print "Done ", line
		except:
			print "Error Failed ", line

input.close()
{'fraction_non_alphanumeric': 0.13842333954065797, 'fraction_numerical': 0.00931098696461825, 'mean_word_length': 4.302631578947368, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6635717', 'n_tokens_mistral': 603, 'n_tokens_neox': 555, 'n_words': 147}