<commit_before># standard modules
import os
import getpass
import re
import stat
import shlex
import shutil
import subprocess
import tempfile
import warnings
try:
	import ConfigParser as cp
except:
	import configparser as cp

# third-party modules
import pandas as pd
try:
	import paramiko
	HAS_PARAMIKO = True
except:
	HAS_PARAMIKO = False
	pass

# custom modules
import DatabaseProvider


class LocationGroup(object):
	""" Bundles a set of locations to abstract access to raw data no matter where the data is stored.

	Behaves like a set of buckets.
	"""
	def __init__(self):
		self._hosts = dict()
		self._buckets = None

	def from_file(self, filename=None):
		""" Reads the user-defined configuration of storage location.

		:param filename: Filename to read the config from. If not present, `.iago.conf` in the home directory will be used instead.
		"""
		config = cp.ConfigParser()
		if filename is None:
			filenames = [os.path.expanduser('~/.iago.conf')]
		else:
			filenames = [filename]
		config.read(filenames)

		self._hosts = dict()
		for hostalias in config.sections():
			skip = False
			try:
				skip = config.getboolean(hostalias, 'skip')
			except cp.NoOptionError:
				pass
			if not skip:
				self._hosts[hostalias] = config.get(hostalias, 'url')

	def get_bucket_list(self, force_update=False, with_status=False):
		for hostalias, host in self._hosts.iteritems():
			if not isinstance(host, LocationProvider):
				host = LocationProvider.create(host)
				self._hosts[hostalias] = host
			else:
				if not force_update:
					continue
			try:
				df = host.get_bucket_list()
			except ValueError as e:
				warnings.warn('Unable to load host %s: %s' % (hostalias, str(e)))
				continue

			df['hostalias'] = hostalias
			if self._buckets is None:
				self._buckets = df
			else:
				self._buckets = self._buckets[self._buckets.hostalias != hostalias]
				self._buckets = self._buckets.append(df)
		if self._buckets is None:
			return None
		self._buckets = self._buckets.reset_index(drop=True)

		if with_status:
			return self._get_bucket_status(self._buckets)
		else:
			return self._buckets

	def _get_bucket_status(self, bucket_list=None):
		""" Checks for bucket database status.

		:param bucket_list: Pandas dataframe with buckets to query.
		:return: Pandas dataframe.
		"""
		if bucket_list is None:
			bucket_list = self.get_bucket_list()

		bucket_list['hasMeta'] = False
		bucket_list['hasDB'] = False
		for idx, bucket in bucket_list.iterrows():
			lp = self._hosts[bucket.hostalias]
			if lp.has_file(bucket.rawname, '.iago.conf'):
				bucket_list.loc[idx, 'hasMeta'] = True
			if lp.has_file(bucket.rawname, 'iagodb.json'):
				bucket_list.loc[idx, 'hasDB'] = True

		return bucket_list

	def _get_bucket_from_name_or_id(self, bucket):
		if bucket in self._buckets.id.values:
			rows = self._buckets[self._buckets.id == bucket]
		elif bucket in self._buckets.name.values:
			rows = self._buckets[self._buckets.name == bucket]

		if len(rows) > 1:
			raise ValueError('Bucket name or id %s is not unique' % bucket)
		if len(rows) == 0:
			raise ValueError('No bucket with this id or name: %s' % bucket)

		return rows.iloc[0]

	def fetch_database(self, bucket):
		row = self._get_bucket_from_name_or_id(bucket)
		try:
			fh = self._hosts[row.hostalias].open_file(row.rawname, 'iagodb.json')
		except:
			fh = None
		try:
			fn, ephemeral = self._hosts[row.hostalias].local_file(row.rawname, 'iagodb.h5')
			if fh is not None:
				fh.close()
				fh = None
		except:
			fn = None

		if fh is None and fn is None:
			raise RuntimeError('Unable to open remote database.')

		db = DatabaseProvider.DB()
		db.read(handle=fh, name=fn)
		if ephemeral:
			os.remove(fn)
		return db

	def build_database(self, bucket):<commit_msg>Database discovery: detect file type. Closes #16<commit_after># standard modules
import os
import getpass
import re
import stat
import shlex
import shutil
import subprocess
import tempfile
import warnings
try:
	import ConfigParser as cp
except:
	import configparser as cp

# third-party modules
import pandas as pd
try:
	import paramiko
	HAS_PARAMIKO = True
except:
	HAS_PARAMIKO = False
	pass

# custom modules
import DatabaseProvider


class LocationGroup(object):
	""" Bundles a set of locations to abstract access to raw data no matter where the data is stored.

	Behaves like a set of buckets.
	"""
	def __init__(self):
		self._hosts = dict()
		self._buckets = None

	def from_file(self, filename=None):
		""" Reads the user-defined configuration of storage location.

		:param filename: Filename to read the config from. If not present, `.iago.conf` in the home directory will be used instead.
		"""
		config = cp.ConfigParser()
		if filename is None:
			filenames = [os.path.expanduser('~/.iago.conf')]
		else:
			filenames = [filename]
		config.read(filenames)

		self._hosts = dict()
		for hostalias in config.sections():
			skip = False
			try:
				skip = config.getboolean(hostalias, 'skip')
			except cp.NoOptionError:
				pass
			if not skip:
				self._hosts[hostalias] = config.get(hostalias, 'url')

	def get_bucket_list(self, force_update=False, with_status=False):
		for hostalias, host in self._hosts.iteritems():
			if not isinstance(host, LocationProvider):
				host = LocationProvider.create(host)
				self._hosts[hostalias] = host
			else:
				if not force_update:
					continue
			try:
				df = host.get_bucket_list()
			except ValueError as e:
				warnings.warn('Unable to load host %s: %s' % (hostalias, str(e)))
				continue

			df['hostalias'] = hostalias
			if self._buckets is None:
				self._buckets = df
			else:
				self._buckets = self._buckets[self._buckets.hostalias != hostalias]
				self._buckets = self._buckets.append(df)
		if self._buckets is None:
			return None
		self._buckets = self._buckets.reset_index(drop=True)

		if with_status:
			return self._get_bucket_status(self._buckets)
		else:
			return self._buckets

	def _get_bucket_status(self, bucket_list=None):
		""" Checks for bucket database status.

		:param bucket_list: Pandas dataframe with buckets to query.
		:return: Pandas dataframe.
		"""
		if bucket_list is None:
			bucket_list = self.get_bucket_list()

		bucket_list['hasMeta'] = False
		bucket_list['hasDB'] = False
		for idx, bucket in bucket_list.iterrows():
			lp = self._hosts[bucket.hostalias]
			if lp.has_file(bucket.rawname, '.iago.conf'):
				bucket_list.loc[idx, 'hasMeta'] = True
			if lp.has_file(bucket.rawname, 'iagodb.json'):
				bucket_list.loc[idx, 'hasDB'] = True

		return bucket_list

	def _get_bucket_from_name_or_id(self, bucket):
		if bucket in self._buckets.id.values:
			rows = self._buckets[self._buckets.id == bucket]
		elif bucket in self._buckets.name.values:
			rows = self._buckets[self._buckets.name == bucket]

		if len(rows) > 1:
			raise ValueError('Bucket name or id %s is not unique' % bucket)
		if len(rows) == 0:
			raise ValueError('No bucket with this id or name: %s' % bucket)

		return rows.iloc[0]

	def fetch_database(self, bucket):
		row = self._get_bucket_from_name_or_id(bucket)
		format = None
		try:
			fh = self._hosts[row.hostalias].open_file(row.rawname, 'iagodb.json')
			format = 'json'
		except:
			fh = None
		try:
			fn, ephemeral = self._hosts[row.hostalias].local_file(row.rawname, 'iagodb.h5')
			format = 'hdf5'
			if fh is not None:
				fh.close()
				fh = None
		except:
			fn = None

		if fh is None and fn is None:
			raise RuntimeError('Unable to open remote database.')

		db = DatabaseProvider.DB()
		db.read(handle=fh, name=fn, format=format)
		if ephemeral:
			os.remove(fn)
		return db

	def build_database(self, bucket):
{'fraction_non_alphanumeric': 0.10594111461619347, 'fraction_numerical': 0.0014458464773922186, 'mean_word_length': 3.7915617128463475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5974854', 'n_tokens_mistral': 2960, 'n_tokens_neox': 2539, 'n_words': 734}