# Path: arbalest/core.py
# class PipelineStep(object):
#     def run(self):
#         pass
# 
#     def validate(self):
#         pass
# 
# Path: arbalest/redshift/manifest.py
# class Manifest(object):
#     def __init__(self, metadata, source, schema, bucket):
#         self.metadata = metadata
#         self.source = source
#         self.schema = schema
#         self.bucket = bucket
#         self.file_name = '{0}_manifest.json'.format(schema.table)
#         self.journal_file_name = '{0}_journal.json'.format(schema.table)
# 
#     @property
#     def all_keys(self):
#         return [k.name for k in self.bucket.list(self.source) if
#                 not k.name.endswith('/')]
# 
#     @property
#     def manifest_key(self):
#         return normalize_path('{0}/{1}'.format(self.metadata, self.file_name))
# 
#     @property
#     def journal_key(self):
#         return normalize_path(
#             '{0}/{1}'.format(self.metadata, self.journal_file_name))
# 
#     @property
#     def manifest_url(self):
#         return 's3://{0}{1}'.format(self.bucket.name, self.manifest_key)
# 
#     def journal(self):
#         journal = self.bucket.get(self.journal_key)
#         if journal.exists():
#             return json.loads(journal.get_contents_as_string())
#         else:
#             return []
# 
#     def get(self):
#         updated_journal = self.all_keys
#         journal = self.journal()
#         keys = list(set(updated_journal) - set(journal))
# 
#         return {
#             'manifest': {
#                 'entries': [
#                     {'url': 's3://{0}/{1}'.format(self.bucket.name, key),
#                      'mandatory': True} for key in keys
#                 ]
#             },
#             'updated_journal': updated_journal
#         }
# 
#     def save(self):
#         manifest = self.get()
#         self.bucket.save(self.manifest_key, json.dumps(manifest['manifest']))
#         return manifest['updated_journal']
# 
#     def commit(self, saved_keys):
#         self.bucket.save(self.journal_key, json.dumps(saved_keys))
# 
#     def exists(self):
#         return self.bucket.get(self.manifest_key).exists()
# 
#     def journal_exists(self):
#         return self.bucket.get(self.journal_key).exists()
# 
# Path: arbalest/redshift/runner.py
# class S3JsonStepRunner(object):
#     def __init__(self, metadata, schema, bucket, table):
#         self.metadata = metadata
#         self.bucket = bucket
#         self.table = table
#         self.schema = schema
# 
#     @property
#     def schema_key(self):
#         return normalize_path(
#             '{0}/{1}'.format(self.metadata, self.schema.file_name))
# 
#     @property
#     def schema_url(self):
#         return 's3://{0}{1}'.format(self.bucket.name, self.schema_key)
# 
#     def stage(self):
#         self.bucket.save(self.schema_key, json.dumps(self.schema.paths()))
#         self.table.database.open()
# 
#     def commit(self):
#         self.table.database.commit()
#         self.bucket.delete(self.schema_key)
# 
#     def rollback(self):
#         self.table.database.rollback()
#         self.bucket.delete(self.schema_key)
# 
# Path: arbalest/s3.py
# def normalize_path(path):
#     return posixpath.normpath(path)


from psycopg2.extensions import AsIs
from arbalest.core import PipelineStep
from arbalest.redshift.manifest import Manifest
from arbalest.redshift.runner import S3JsonStepRunner
from arbalest.s3 import normalize_path

    @property
    def validate_sql(self):
        return self.copy_sql + " NOLOAD"

    def run(self):
        self.runner.stage()
        self.__execute(self.copy_sql)
        self.__promote()
        self.runner.commit()

    def validate(self):
        self.runner.stage()
        self.__execute(self.validate_sql)
        self.__promote()
        self.runner.rollback()

    def __execute(self, sql):
        self.table.stage_update()
        self.sql.execute((
            sql, self.schema.update_table, self.source_url,
            self.aws_access_key_id,
            self.aws_secret_access_key, self.schema_url, self.max_error_count))

    def __promote(self):
        if self.table.exists():
            self.table.drop()
        self.table.promote_update()



# Path: arbalest/core.py
# class PipelineStep(object):
#     def run(self):
#         pass
# 
#     def validate(self):
#         pass
# 
# Path: arbalest/redshift/manifest.py
# class Manifest(object):
#     def __init__(self, metadata, source, schema, bucket):
#         self.metadata = metadata
#         self.source = source
#         self.schema = schema
#         self.bucket = bucket
#         self.file_name = '{0}_manifest.json'.format(schema.table)
#         self.journal_file_name = '{0}_journal.json'.format(schema.table)
# 
#     @property
#     def all_keys(self):
#         return [k.name for k in self.bucket.list(self.source) if
#                 not k.name.endswith('/')]
# 
#     @property
#     def manifest_key(self):
#         return normalize_path('{0}/{1}'.format(self.metadata, self.file_name))
# 
#     @property
#     def journal_key(self):
#         return normalize_path(
#             '{0}/{1}'.format(self.metadata, self.journal_file_name))
# 
#     @property
#     def manifest_url(self):
#         return 's3://{0}{1}'.format(self.bucket.name, self.manifest_key)
# 
#     def journal(self):
#         journal = self.bucket.get(self.journal_key)
#         if journal.exists():
#             return json.loads(journal.get_contents_as_string())
#         else:
#             return []
# 
#     def get(self):
#         updated_journal = self.all_keys
#         journal = self.journal()
#         keys = list(set(updated_journal) - set(journal))
# 
#         return {
#             'manifest': {
#                 'entries': [
#                     {'url': 's3://{0}/{1}'.format(self.bucket.name, key),
#                      'mandatory': True} for key in keys
#                 ]
#             },
#             'updated_journal': updated_journal
#         }
# 
#     def save(self):
#         manifest = self.get()
#         self.bucket.save(self.manifest_key, json.dumps(manifest['manifest']))
#         return manifest['updated_journal']
# 
#     def commit(self, saved_keys):
#         self.bucket.save(self.journal_key, json.dumps(saved_keys))
# 
#     def exists(self):
#         return self.bucket.get(self.manifest_key).exists()
# 
#     def journal_exists(self):
#         return self.bucket.get(self.journal_key).exists()
# 
# Path: arbalest/redshift/runner.py
# class S3JsonStepRunner(object):
#     def __init__(self, metadata, schema, bucket, table):
#         self.metadata = metadata
#         self.bucket = bucket
#         self.table = table
#         self.schema = schema
# 
#     @property
#     def schema_key(self):
#         return normalize_path(
#             '{0}/{1}'.format(self.metadata, self.schema.file_name))
# 
#     @property
#     def schema_url(self):
#         return 's3://{0}{1}'.format(self.bucket.name, self.schema_key)
# 
#     def stage(self):
#         self.bucket.save(self.schema_key, json.dumps(self.schema.paths()))
#         self.table.database.open()
# 
#     def commit(self):
#         self.table.database.commit()
#         self.bucket.delete(self.schema_key)
# 
#     def rollback(self):
#         self.table.database.rollback()
#         self.bucket.delete(self.schema_key)
# 
# Path: arbalest/s3.py
# def normalize_path(path):
#     return posixpath.normpath(path)


# Path: arbalest/redshift/step.py
from psycopg2.extensions import AsIs
from arbalest.core import PipelineStep
from arbalest.redshift.manifest import Manifest
from arbalest.redshift.runner import S3JsonStepRunner
from arbalest.s3 import normalize_path


    @property
    def validate_sql(self):
        return self.copy_sql + " NOLOAD"

    def run(self):
        self.runner.stage()
        self.__execute(self.copy_sql)
        self.__promote()
        self.runner.commit()

    def validate(self):
        self.runner.stage()
        self.__execute(self.validate_sql)
        self.__promote()
        self.runner.rollback()

    def __execute(self, sql):
        self.table.stage_update()
        self.sql.execute((
            sql, self.schema.update_table, self.source_url,
            self.aws_access_key_id,
            self.aws_secret_access_key, self.schema_url, self.max_error_count))

    def __promote(self):
        if self.table.exists():
            self.table.drop()
        self.table.promote_update()



class ManifestCopyFromS3JsonStep(PipelineStep):