<commit_before>#!/usr/bin/python

from urllib2 import urlopen, URLError, HTTPError
import sys
import os
import zlib
import shutil
import tempfile
from optparse import OptionParser

foundSqlite = False
foundSqlite3 = False

# figure out which sqlite module to use
# in Python 2.4 an old version is present
# which does not allow proper read out of
# long int and therefore cannot merge catalogs
try:
	import sqlite3 as sqlite
	foundSqlite3 = True
except:
	pass
if not foundSqlite3:
	try:
		import sqlite
		foundSqlite = True
	except ImportError, e:
		pass


def doHttpRequest(url):
	response = urlopen(url)
	return response.read()


def getRootCatalogName(repositoryUrl):
	result = ""
	try:
		result = doHttpRequest(repositoryUrl + "/.cvmfspublished")
	except:
		printError("cannot load .cvmfspublished")

	lines = result.split('\n')
	if len(lines) < 1:
		printError(".cvmfspublished is malformed")

	return lines[0][1:]


def createDirectory(directory):
	d = os.path.dirname(directory)
	if not os.path.exists(d):
		os.makedirs(d)


def getCatalogFilePath(catalogName, catalogDirectory):
	return catalogDirectory + "/" + catalogName[0:2] + "/" + catalogName[2:] + "C"


def downloadObject(repositoryUrl, objectName, resultDirectory, beVerbose):
	subdir   = objectName[0:2]
	filename = objectName[2:]
	url      = repositoryUrl + "/data/" + subdir + "/" + filename
	destDir  = resultDirectory + "/" + subdir + "/"
	dest     = destDir + filename

	# create target directory if not existing and open output file
	createDirectory(destDir)
	outputFile = open(dest, 'wb')

	# download the catalog
	try:
		f = urlopen(url)
		meta = f.info()
		fileSize = int(meta.getheaders("Content-Length")[0])

		if beVerbose:
			print "retrieving " + objectName + " - " + str(fileSize) + " bytes"

		local_file = open(dest, "wb")
		try:
			local_file.write(f.read())
		finally:
			local_file.close()
	except HTTPError, e:
		printError("HTTP: " + e.code + url)
	except URLError, e:
		printError("URL:" + e.reason + url)


def downloadCatalog(repositoryUrl, catalogName, catalogDirectory, beVerbose):
	downloadObject(repositoryUrl, catalogName + "C", catalogDirectory, beVerbose)


def decompressCatalog(filename, destination):
	str_object1 = open(filename, 'rb').read()
	str_object2 = zlib.decompress(str_object1)
	f = open(destination, 'wb')
	f.write(str_object2)
	f.close()


def findNestedCatalogs(catalogName, catalogDirectory, repositoryUrl, beVerbose, getDirtab):
	catalogFile = getCatalogFilePath(catalogName, catalogDirectory)
	tempFile    = tempfile.NamedTemporaryFile('wb')
	decompressCatalog(catalogFile, tempFile.name)

	dbHandle = sqlite.connect(tempFile.name)
	cursor = dbHandle.cursor()
	catalogs = []

	# nested catalog references
	cursor.execute("SELECT sha1 FROM nested_catalogs")
	result = cursor.fetchall()
	for catalog in result:
		catalogs.append(catalog[0])

	# dirtab entry
	if getDirtab:
		cursor.execute("SELECT hash FROM catalog WHERE name = '.cvmfsdirtab'")<commit_msg>use pysqlite2 as sqlite3 in Python < 2.5<commit_after>#!/usr/bin/python

from urllib2 import urlopen, URLError, HTTPError
import sys
import os
import zlib
import shutil
import tempfile
from optparse import OptionParser

# figure out which sqlite module to use
# in Python 2.4 there is dbapi2 in pysqlite2 which will become sqlite3
# in the standard library later on but was not at this time (software archeology)
try:
	import sqlite3
except:
	from pysqlite2 import dbapi2 as sqlite3


def doHttpRequest(url):
	response = urlopen(url)
	return response.read()


def getRootCatalogName(repositoryUrl):
	result = ""
	try:
		result = doHttpRequest(repositoryUrl + "/.cvmfspublished")
	except:
		printError("cannot load .cvmfspublished")

	lines = result.split('\n')
	if len(lines) < 1:
		printError(".cvmfspublished is malformed")

	return lines[0][1:]


def createDirectory(directory):
	d = os.path.dirname(directory)
	if not os.path.exists(d):
		os.makedirs(d)


def getCatalogFilePath(catalogName, catalogDirectory):
	return catalogDirectory + "/" + catalogName[0:2] + "/" + catalogName[2:] + "C"


def downloadObject(repositoryUrl, objectName, resultDirectory, beVerbose):
	subdir   = objectName[0:2]
	filename = objectName[2:]
	url      = repositoryUrl + "/data/" + subdir + "/" + filename
	destDir  = resultDirectory + "/" + subdir + "/"
	dest     = destDir + filename

	# create target directory if not existing and open output file
	createDirectory(destDir)
	outputFile = open(dest, 'wb')

	# download the catalog
	try:
		f = urlopen(url)
		meta = f.info()
		fileSize = int(meta.getheaders("Content-Length")[0])

		if beVerbose:
			print "retrieving " + objectName + " - " + str(fileSize) + " bytes"

		local_file = open(dest, "wb")
		try:
			local_file.write(f.read())
		finally:
			local_file.close()
	except HTTPError, e:
		printError("HTTP: " + e.code + url)
	except URLError, e:
		printError("URL:" + e.reason + url)


def downloadCatalog(repositoryUrl, catalogName, catalogDirectory, beVerbose):
	downloadObject(repositoryUrl, catalogName + "C", catalogDirectory, beVerbose)


def decompressCatalog(filename, destination):
	str_object1 = open(filename, 'rb').read()
	str_object2 = zlib.decompress(str_object1)
	f = open(destination, 'wb')
	f.write(str_object2)
	f.close()


def findNestedCatalogs(catalogName, catalogDirectory, repositoryUrl, beVerbose, getDirtab):
	catalogFile = getCatalogFilePath(catalogName, catalogDirectory)
	tempFile    = tempfile.NamedTemporaryFile('wb')
	decompressCatalog(catalogFile, tempFile.name)

	dbHandle = sqlite3.connect(tempFile.name)
	cursor = dbHandle.cursor()
	catalogs = []

	# nested catalog references
	cursor.execute("SELECT sha1 FROM nested_catalogs")
	result = cursor.fetchall()
	for catalog in result:
		catalogs.append(catalog[0])

	# dirtab entry
	if getDirtab:
		cursor.execute("SELECT hash FROM catalog WHERE name = '.cvmfsdirtab'")
{'fraction_non_alphanumeric': 0.11372549019607843, 'fraction_numerical': 0.009207161125319692, 'mean_word_length': 4.877755511022044, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2645210', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1937, 'n_words': 504}