<commit_before>#!/usr/bin/env python

import sys
from hbasetable import HBaseTable


class UidTable(HBaseTable):
	def table(self):
		return "uid"
	
	def create(self):
		return """
		{NAME => 'name', VERSIONS => 1,COMPRESSION => 'NONE'},
		{NAME => 'id',   VERSIONS => 1,COMPRESSION => 'NONE'}
		"""
	
	def validate(self, schema):
		return True


class DimensionHourTable(HBaseTable):
	def table(self):
		return "dimension-hour"

	def create(self):
		return """
		{NAME => 'md',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		{NAME => 'ml',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		{NAME => 'mc',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		if not("'ml'" in schema):
			self.alter("{NAME => 'ml',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}")
		
		if not("'mc'" in schema):
			self.alter("{NAME => 'mc',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}")

		return True


class DimensionDayTable(HBaseTable):
	def table(self):
		return "dimension-day"

	def create(self):
		return """
		{NAME => 'tc', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'tl', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'h',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'hc', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class VisitorTable(HBaseTable):
	def table(self):
		return "visitor"

	def create(self):
		return """
		{NAME => 'i', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 's', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class SessionTable(HBaseTable):
	def table(self):
		return "session"

	def create(self):
		return """
		{NAME => 'hp', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'i',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'p',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class DcsidLookupTable(HBaseTable):
	def table(self):
		return "dcsid-lookup"

	def create(self):
		return """
		{NAME => 'id',   VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'name', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class DimensionLookupTable(HBaseTable):
	def table(self):
		return "dimension-lookup"

	def create(self):
		return """
		{NAME => 'id',   VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'name', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class NextIdTable(HBaseTable):
	def table(self):
		return "next-id"

	def create(self):
		return """
		{NAME => 'id', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class MetadataTable(HBaseTable):
	def table(self):
		return "metadata"

	def create(self):
		return """
		{NAME=>'meta'}
		"""

	def validate(self, schema):
		return True


HBaseTable.manage(UidTable())
HBaseTable.manage(DimensionHourTable())
HBaseTable.manage(DimensionDayTable())
HBaseTable.manage(VisitorTable())
HBaseTable.manage(SessionTable())
HBaseTable.manage(DcsidLookupTable())
HBaseTable.manage(DimensionLookupTable())
HBaseTable.manage(NextIdTable())
HBaseTable.manage(MetadataTable())



if (len(sys.argv) != 3):
	print "usage:   ./create_tables.py <dc> <pod>"
	print "example: ./create_tables.py pdx V"
	sys.exit(-1)
	

HBaseTable.dc = sys.argv[1]
HBaseTable.pod = sys.argv[2]

print "dc: " + HBaseTable.dc
print "pod: " + HBaseTable.pod

sys.exit(0 if HBaseTable.validate() else -1)


<commit_msg>fixing hbase dimension-hour schema<commit_after>#!/usr/bin/env python

import sys
from hbasetable import HBaseTable


class UidTable(HBaseTable):
	def table(self):
		return "uid"
	
	def create(self):
		return """
		{NAME => 'name', VERSIONS => 1,COMPRESSION => 'NONE'},
		{NAME => 'id',   VERSIONS => 1,COMPRESSION => 'NONE'}
		"""
	
	def validate(self, schema):
		return True


class DimensionHourTable(HBaseTable):
	def table(self):
		return "dimension-hour"

	def create(self):
		return """
		{NAME => 'md',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'ml',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'mc',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		if not("'ml'" in schema):
			self.alter("{NAME => 'ml',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}")
		
		if not("'mc'" in schema):
			self.alter("{NAME => 'mc',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}")

		return True


class DimensionDayTable(HBaseTable):
	def table(self):
		return "dimension-day"

	def create(self):
		return """
		{NAME => 'tc', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'tl', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'h',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'hc', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class VisitorTable(HBaseTable):
	def table(self):
		return "visitor"

	def create(self):
		return """
		{NAME => 'i', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 's', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class SessionTable(HBaseTable):
	def table(self):
		return "session"

	def create(self):
		return """
		{NAME => 'hp', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'i',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'p',  VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class DcsidLookupTable(HBaseTable):
	def table(self):
		return "dcsid-lookup"

	def create(self):
		return """
		{NAME => 'id',   VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'name', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class DimensionLookupTable(HBaseTable):
	def table(self):
		return "dimension-lookup"

	def create(self):
		return """
		{NAME => 'id',   VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'},
		{NAME => 'name', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class NextIdTable(HBaseTable):
	def table(self):
		return "next-id"

	def create(self):
		return """
		{NAME => 'id', VERSIONS => 1, COMPRESSION => 'NONE', BLOOMFILTER => 'ROW'}
		"""

	def validate(self, schema):
		return True


class MetadataTable(HBaseTable):
	def table(self):
		return "metadata"

	def create(self):
		return """
		{NAME=>'meta'}
		"""

	def validate(self, schema):
		return True


HBaseTable.manage(UidTable())
HBaseTable.manage(DimensionHourTable())
HBaseTable.manage(DimensionDayTable())
HBaseTable.manage(VisitorTable())
HBaseTable.manage(SessionTable())
HBaseTable.manage(DcsidLookupTable())
HBaseTable.manage(DimensionLookupTable())
HBaseTable.manage(NextIdTable())
HBaseTable.manage(MetadataTable())



if (len(sys.argv) != 3):
	print "usage:   ./create_tables.py <dc> <pod>"
	print "example: ./create_tables.py pdx V"
	sys.exit(-1)
	

HBaseTable.dc = sys.argv[1]
HBaseTable.pod = sys.argv[2]

print "dc: " + HBaseTable.dc
print "pod: " + HBaseTable.pod

sys.exit(0 if HBaseTable.validate() else -1)



{'fraction_non_alphanumeric': 0.18909137908750162, 'fraction_numerical': 0.006979449398991857, 'mean_word_length': 4.430175438596491, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 173, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7315590', 'n_tokens_mistral': 3038, 'n_tokens_neox': 2746, 'n_words': 689}