# Path: pemi/pipes/patterns.py
# class TargetPipe(pemi.Pipe):
#     '''
#     A target pipe takes data provided to it, encodes it into a structure that can be
#     understand by some external target, and then loads the data into that external target.
#     '''
# 
#     def __init__(self, *, schema, **params):
#         super().__init__(**params)
# 
#         self.schema = schema
# 
#         self.source(
#             pemi.PdDataSubject,
#             name='main',
#             schema=self.schema
#         )
# 
#         self.target(
#             pemi.PdDataSubject,
#             name='errors'
#         )
# 
#         self.target(
#             pemi.PdDataSubject,
#             name='response'
#         )
# 
#     def encode(self):
#         # e.g., CsvTargetEncoder.encode()
#         # return source_main_data_encoded_for_loader
#         raise NotImplementedError
# 
#     def load(self, encoded_data):
#         # e.g., S3Loader.load()
#         # return results_from_load_operation
#         raise NotImplementedError
# 
#     def flow(self):
#         self.load(self.encode())


import os
import re
import pandas as pd
import pemi
from pemi.pipes.patterns import TargetPipe



def default_column_normalizer(name):
    name = str(name)
    name = re.sub(r'\s+', '_', name).lower()
    name = re.sub(r'[^a-z0-9_]', '_', name)
    return name


class StrConverter(dict):
    def __contains__(self, key):
        return True

    def __getitem__(self, key):
        return str

class LocalCsvFileSourcePipe(pemi.Pipe):
    def __init__(self, *, paths, schema=None, csv_opts=None,
                 filename_field=None, filename_full_path=False, normalize_columns=True, **params):
        super().__init__(**params)

        self.paths = paths
        self.schema = schema
        self.filename_field = filename_field
        self.filename_full_path = filename_full_path
        self.csv_opts = self._build_csv_opts(csv_opts or {})


# Path: pemi/pipes/patterns.py
# class TargetPipe(pemi.Pipe):
#     '''
#     A target pipe takes data provided to it, encodes it into a structure that can be
#     understand by some external target, and then loads the data into that external target.
#     '''
# 
#     def __init__(self, *, schema, **params):
#         super().__init__(**params)
# 
#         self.schema = schema
# 
#         self.source(
#             pemi.PdDataSubject,
#             name='main',
#             schema=self.schema
#         )
# 
#         self.target(
#             pemi.PdDataSubject,
#             name='errors'
#         )
# 
#         self.target(
#             pemi.PdDataSubject,
#             name='response'
#         )
# 
#     def encode(self):
#         # e.g., CsvTargetEncoder.encode()
#         # return source_main_data_encoded_for_loader
#         raise NotImplementedError
# 
#     def load(self, encoded_data):
#         # e.g., S3Loader.load()
#         # return results_from_load_operation
#         raise NotImplementedError
# 
#     def flow(self):
#         self.load(self.encode())


# Path: pemi/pipes/csv.py
import os
import re
import pandas as pd
import pemi
from pemi.pipes.patterns import TargetPipe




def default_column_normalizer(name):
    name = str(name)
    name = re.sub(r'\s+', '_', name).lower()
    name = re.sub(r'[^a-z0-9_]', '_', name)
    return name


class StrConverter(dict):
    def __contains__(self, key):
        return True

    def __getitem__(self, key):
        return str

class LocalCsvFileSourcePipe(pemi.Pipe):
    def __init__(self, *, paths, schema=None, csv_opts=None,
                 filename_field=None, filename_full_path=False, normalize_columns=True, **params):
        super().__init__(**params)

        self.paths = paths
        self.schema = schema
        self.filename_field = filename_field
        self.filename_full_path = filename_full_path
        self.csv_opts = self._build_csv_opts(csv_opts or {})


        if callable(normalize_columns):