# Path: lib/csv_stream.py
# class CSVNormalizerStream(io.IOBase):
#     '''
#     Filter for applying a function to each line of a CSV file, compatible with iostreams
#     '''
#     def __init__(self, infile, func):
#         '''
#         :param infile: A stream that reads a CSV file. e.g: a file
#         :param func: A function to apply to each CSV row. It takes an array of fields and returns an array of fields.
#         '''
#         self._csvreader = csv.reader(infile)
#         self._buffer = ''
#         self._func = func
# 
#     def read(self, nbytes):
#         try:
#             while len(self._buffer) < nbytes:
#                 self.getline()
#             out, self._buffer = self._buffer[:nbytes], self._buffer[nbytes:]
#             return out
#         except StopIteration:
#             out = self._buffer
#             self._buffer = ''
#             return out
# 
#     def getline(self):
#         line = next(self._csvreader)
#         clean_line = self._func(line)
#         self._buffer += (','.join(clean_line) + ' \n')
# 
# Path: lib/copy.py
# def copy_from_csv(session, table_name, columns, csv_stream):
#     '''
#     Creates a table, loading the data from a .csv file.
# 
#     :param session: A SQL Alchemy session
#     :param table_name: Output table name
#     :param columns: Dictionary of columns, keys are named, values are types.
#     :param csv_stream: A stream that reads a CSV file. e.g: a file or a
#                        :class:`CSVNormalizerStream <lib.csv_stream.CSVNormalizerStream>`
#     '''
#     with session.connection().connection.cursor() as cursor:
#         cursor.execute('CREATE TABLE {output} ({cols})'.format(
#             output=table_name,
#             cols=', '.join(['{name} {type}'.format(name=k, type=v) for k, v in columns.items()])
#         ))
# 
#         cursor.copy_expert(
#             'COPY {table} ({cols}) FROM stdin WITH (FORMAT CSV, HEADER)'.format(
#                 cols=', '.join(columns.keys()),
#                 table=table_name),
#             csv_stream)
# 
# Path: tasks/base_tasks.py
# class RepoFileUnzipTask(RepoFileUncompressTask):
#     '''
#     Download a zip file to location {output}.zip and unzip it to the folder
#     {output}.
#     '''
#     compressed_extension = 'zip'
# 
#     def uncompress(self):
#         uncompress_file(self.output().path)
# 
# class TempTableTask(Task):
#     '''
#     A Task that generates a table that will not be referred to in metadata.
# 
#     This is useful for intermediate processing steps that can benefit from the
#     session guarantees of the ETL, as well as automatic table naming.
# 
#     :param force: Optional Boolean, ``False`` by default.  If ``True``, will
#                   overwrite output table even if it exists already.
#     '''
# 
#     force = BoolParameter(default=False, significant=False)
# 
#     def on_failure(self, ex):
#         session_rollback(self, ex)
#         super(TempTableTask, self).on_failure(ex)
# 
#     def on_success(self):
#         session_commit(self)
# 
#     def target_tablename(self):
#         '''
#         Can be overriden to expose a different table name
#         :return:
#         '''
#         return unqualified_task_id(self.task_id)[:MAX_PG_IDENTIFIER_LENGTH]
# 
#     def run(self):
#         '''
#         Must be overriden by subclass.  Should create and populate a table
#         named from ``self.output().table``
# 
#         If this completes without exceptions, the :func:`~.util.current_session
#         will be committed; if there is an exception, it will be rolled back.
#         '''
#         raise Exception('Must override `run`')
# 
#     def output(self):
#         '''
#         By default, returns a :class:`~.targets.TableTarget` whose associated
#         table lives in a special-purpose schema in Postgres derived using
#         :func:`~.util.classpath`.
#         '''
#         return PostgresTarget(classpath(self), self.target_tablename())
# 
# Path: tasks/meta.py
# def current_session():
#     '''
#     Returns the session relevant to the currently operating :class:`Task`, if
#     any.  Outside the context of a :class:`Task`, this can still be used for
#     manual session management.
#     '''
#     return _current_session.get()
# 
# Path: tasks/util.py
# def copyfile(src, dst):
#     # From http://luigi.readthedocs.io/en/stable/_modules/luigi/local_target.html#LocalTarget.makedirs
#     normpath = os.path.normpath(dst)
#     parentfolder = os.path.dirname(normpath)
#     if parentfolder:
#         try:
#             os.makedirs(parentfolder, exist_ok=True)
#         except OSError as e:
#             LOGGER.error('Error making dirs for copyfile: %s', e)
#             pass
# 
#     shutil.copyfile(src, dst)
# 
# Path: tasks/uk/census/metadata.py
# def sanitize_identifier(colid):
#     return DISALLOWED_CHARACTERS_RE.sub('_', '_'.join(colid.split(';')[0].split(':')[-2:]))


from collections import OrderedDict
from luigi import Parameter
from lib.csv_stream import CSVNormalizerStream
from lib.copy import copy_from_csv
from tasks.base_tasks import RepoFileUnzipTask, TempTableTask
from tasks.meta import current_session
from tasks.util import copyfile
from .metadata import sanitize_identifier
import csv
import os
# http://www.scotlandscensus.gov.uk/ods-web/data-warehouse.html#bulkdatatab






class DownloadScotlandLocal(RepoFileUnzipTask):
    URL = 'http://www.scotlandscensus.gov.uk/ods-web/download/getDownloadFile.html?downloadFileIds=Output%20Area%20blk'

    def get_url(self):
        return self.URL



# Path: lib/csv_stream.py
# class CSVNormalizerStream(io.IOBase):
#     '''
#     Filter for applying a function to each line of a CSV file, compatible with iostreams
#     '''
#     def __init__(self, infile, func):
#         '''
#         :param infile: A stream that reads a CSV file. e.g: a file
#         :param func: A function to apply to each CSV row. It takes an array of fields and returns an array of fields.
#         '''
#         self._csvreader = csv.reader(infile)
#         self._buffer = ''
#         self._func = func
# 
#     def read(self, nbytes):
#         try:
#             while len(self._buffer) < nbytes:
#                 self.getline()
#             out, self._buffer = self._buffer[:nbytes], self._buffer[nbytes:]
#             return out
#         except StopIteration:
#             out = self._buffer
#             self._buffer = ''
#             return out
# 
#     def getline(self):
#         line = next(self._csvreader)
#         clean_line = self._func(line)
#         self._buffer += (','.join(clean_line) + ' \n')
# 
# Path: lib/copy.py
# def copy_from_csv(session, table_name, columns, csv_stream):
#     '''
#     Creates a table, loading the data from a .csv file.
# 
#     :param session: A SQL Alchemy session
#     :param table_name: Output table name
#     :param columns: Dictionary of columns, keys are named, values are types.
#     :param csv_stream: A stream that reads a CSV file. e.g: a file or a
#                        :class:`CSVNormalizerStream <lib.csv_stream.CSVNormalizerStream>`
#     '''
#     with session.connection().connection.cursor() as cursor:
#         cursor.execute('CREATE TABLE {output} ({cols})'.format(
#             output=table_name,
#             cols=', '.join(['{name} {type}'.format(name=k, type=v) for k, v in columns.items()])
#         ))
# 
#         cursor.copy_expert(
#             'COPY {table} ({cols}) FROM stdin WITH (FORMAT CSV, HEADER)'.format(
#                 cols=', '.join(columns.keys()),
#                 table=table_name),
#             csv_stream)
# 
# Path: tasks/base_tasks.py
# class RepoFileUnzipTask(RepoFileUncompressTask):
#     '''
#     Download a zip file to location {output}.zip and unzip it to the folder
#     {output}.
#     '''
#     compressed_extension = 'zip'
# 
#     def uncompress(self):
#         uncompress_file(self.output().path)
# 
# class TempTableTask(Task):
#     '''
#     A Task that generates a table that will not be referred to in metadata.
# 
#     This is useful for intermediate processing steps that can benefit from the
#     session guarantees of the ETL, as well as automatic table naming.
# 
#     :param force: Optional Boolean, ``False`` by default.  If ``True``, will
#                   overwrite output table even if it exists already.
#     '''
# 
#     force = BoolParameter(default=False, significant=False)
# 
#     def on_failure(self, ex):
#         session_rollback(self, ex)
#         super(TempTableTask, self).on_failure(ex)
# 
#     def on_success(self):
#         session_commit(self)
# 
#     def target_tablename(self):
#         '''
#         Can be overriden to expose a different table name
#         :return:
#         '''
#         return unqualified_task_id(self.task_id)[:MAX_PG_IDENTIFIER_LENGTH]
# 
#     def run(self):
#         '''
#         Must be overriden by subclass.  Should create and populate a table
#         named from ``self.output().table``
# 
#         If this completes without exceptions, the :func:`~.util.current_session
#         will be committed; if there is an exception, it will be rolled back.
#         '''
#         raise Exception('Must override `run`')
# 
#     def output(self):
#         '''
#         By default, returns a :class:`~.targets.TableTarget` whose associated
#         table lives in a special-purpose schema in Postgres derived using
#         :func:`~.util.classpath`.
#         '''
#         return PostgresTarget(classpath(self), self.target_tablename())
# 
# Path: tasks/meta.py
# def current_session():
#     '''
#     Returns the session relevant to the currently operating :class:`Task`, if
#     any.  Outside the context of a :class:`Task`, this can still be used for
#     manual session management.
#     '''
#     return _current_session.get()
# 
# Path: tasks/util.py
# def copyfile(src, dst):
#     # From http://luigi.readthedocs.io/en/stable/_modules/luigi/local_target.html#LocalTarget.makedirs
#     normpath = os.path.normpath(dst)
#     parentfolder = os.path.dirname(normpath)
#     if parentfolder:
#         try:
#             os.makedirs(parentfolder, exist_ok=True)
#         except OSError as e:
#             LOGGER.error('Error making dirs for copyfile: %s', e)
#             pass
# 
#     shutil.copyfile(src, dst)
# 
# Path: tasks/uk/census/metadata.py
# def sanitize_identifier(colid):
#     return DISALLOWED_CHARACTERS_RE.sub('_', '_'.join(colid.split(';')[0].split(':')[-2:]))


# Path: tasks/uk/census/scotland.py
from collections import OrderedDict
from luigi import Parameter
from lib.csv_stream import CSVNormalizerStream
from lib.copy import copy_from_csv
from tasks.base_tasks import RepoFileUnzipTask, TempTableTask
from tasks.meta import current_session
from tasks.util import copyfile
from .metadata import sanitize_identifier
import csv
import os

# http://www.scotlandscensus.gov.uk/ods-web/data-warehouse.html#bulkdatatab






class DownloadScotlandLocal(RepoFileUnzipTask):
    URL = 'http://www.scotlandscensus.gov.uk/ods-web/download/getDownloadFile.html?downloadFileIds=Output%20Area%20blk'

    def get_url(self):
        return self.URL



class ImportScotland(TempTableTask):