<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
# PyLint tests that will never be applied by Trump.
#
# ... is not callable, ignored because a property that returns a callable
#                       becomes callable.
# pylint: disable-msg=E1102

# missing parameter, ignored because a SQLAlchemy function is wrapped.
#                    it's a documented issue with that team.
# pylint: disable-msg=E1120

# Used * or ** magic, we're not getting rid of this, it's imperative to Trump.
# pylint: disable-msg=W0142

# Too many/few arguments, ignored, because its confusing and doesn't make
#                         sense to refactor templates.
#
# pylint: disable-msg=R0913
# pylint: disable-msg=R0903

"""
Trump's Object Relational Model is the glue to the framework, used to create
a Symbol's tags, alias, meta data, data feeds and their sources, munging,
error handling and validity instructions.
"""

# SQLAQ - running the uninstall script, then this script, in the same session
#        causes an error:
#
#        sqlalchemy.exc.InvalidRequestError: When initializing mapper
#        Mapper|Feed|_feeds, expression 'FeedMeta' failed to locate a name
#        ("name 'FeedMeta' is not defined"). If this is a class name, consider
#        adding this relationship() to the <class 'trump.orm.Feed'> class
#        after both dependent classes have been defined
#
#        Why?


import datetime as dt

import pandas as pd
from sqlalchemy import event, Table, Column, ForeignKey, ForeignKeyConstraint,\
    String, Integer, Float, Boolean, DateTime, MetaData, func
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship, aliased, backref
from sqlalchemy.orm.session import object_session
from sqlalchemy.exc import ProgrammingError, NoSuchTableError
from sqlalchemy.sql import and_, or_
from sqlalchemy import create_engine

from indexing import indexingtypes
from validity import validitychecks
from datadef import datadefs

from trump.tools import ReprMixin, ProxyDict, isinstanceofany, \
    BitFlag, BitFlagType, ReprObjType, DuckTypeMixin

from trump.extensions.symbol_aggs import FeedAggregator, sorted_feed_cols
from trump.templating import bFeed, pab, pnab
from trump.options import read_config, read_settings
from trump.converting import FXConverter

from handling import Handler

from reporting.objects import TrumpReport, FeedReport, SymbolReport, \
    ReportPoint

BitFlag.associate_with(BitFlagType)

try:
    ENGINE_STR = read_config(sect='readwrite', sett='engine')
except:
    print ("Problem reading trump.cfg.  Continuing using an in-memory "
           "SQLlite database. Trump was not designed to work in-memory, "
           "because, What's the point of non-persistent persistent objects?")
    ENGINE_STR = "sqlite://"

rbd = read_config(sect='options', sett='raise_by_default')
if rbd.upper() == 'TRUE':
    rbd = BitFlag(1)
else:
    rbd = None

# Bind the engine to the metadata of the Base class so that the
# declaratives can be accessed through a DBSession instance

Base = declarative_base()

ADO = "all, delete-orphan"
CC = {'onupdate': "CASCADE", 'ondelete': "CASCADE"}

class SymbolManager(object):

    """
    The SymbolManager maintains the SQLAlchemy database session, and 
    provides access to object creation, deletion, searching, and 
    overrides/failsafes.
    """

    def __init__(self, engine_or_eng_str=None, loud=False, echo=False):
        """
        Parameters
        ----------
        engine_or_eng_str : str or None, optional
            Pass a SQLAlchemy engine, or a string.  Without one,
            it will use the string provided in trump/options/trump.cfg
            If it fails to get a value there, an in-memory SQLlite
            session would be created.
        loud : bool, optional
            Print information such as engine string used, defaults to False
        echo : bool, optional
            If a new engine is created, it will pass this to it'safes
            constructor, enabling SQLAlchemy's echo mode.
            
        Returns
        -------
        SymbolManager
        """
        if engine_or_eng_str is None:
            engine = create_engine(ENGINE_STR, echo=echo)
        elif isinstance(engine_or_eng_str, (str, unicode)):
            engine = create_engine(engine_or_eng_str, echo=echo)
        else:
            engine = engine_or_eng_str
        
        Base.metadata.bind = engine
        DBSession = sessionmaker(bind=engine)
        
        self.loud = loud
        if loud:
            print "Using engine: {}".format(ENGINE_STR)

        self.ses = DBSession()
        
    def finish(self):
        """ Closes the session with the database.

        Call at the end of a trump session. It also 
        calls SessionManager.complete().
        """
        self.complete()
        self.ses.close()

    def create(self, name, description=None, units=None,
               agg_method="priority_fill", overwrite=False):
        """ Create, or get if exists, a Symbol.
        
        Parameters
        ----------
        name : str
            A symbol's name is a primary key, used across
            the Trump ORM.
        description : str, optional
            An arbitrary string, used to store user information
            related to the symbol.
        units : str, optional
            This is a string used to denote the units of the final
            data Series.
        agg_method : str, optional
            The aggregation method, used to calculate
            the final feed.  Defaults to priority_fill.
        overwrite : bool, optional
            Set to True, to force deletion an existing symbol.
            defaults to False.
            
        Returns
        -------
        Symbol
        """
        sym = self.try_to_get(name)

        if sym is not None:
            if overwrite:
                print "Deleting {}".format(sym.name)
                self.ses.delete(sym)
                self.ses.commit()
            else:
                msg = 'Symbol {} already exists.\n' + \
                      'Consider setting overwrite to True.'
                msg = msg.format(name)
                raise Exception(msg)

        sym = Symbol(name, description, units, agg_method)
        
        self.ses.add(sym)

        print "Creating {}".format(sym.name)
        sym.add_alias(name)

        sym.handle = SymbolHandle(sym=sym)
        self.ses.commit()

        return sym

    def delete(self, symbol):
        """
        Deletes a Symbol.
        
        Parameters
        ----------
        symbol : str or Symbol
        """
        if isinstance(symbol, str):
            sym = self.get(symbol)
        elif isinstance(symbol, Symbol):
            sym = symbol
        else:
            raise Exception("Invalid symbol {}".format((repr(symbol))))
        self.ses.delete(sym)
        self.ses.commit()

    def complete(self):
        """Commits any changes to the database.
        In general, most of Trump API's auto-commits
        or does so internally.  
        
        This is necessary when working directly with SQLAlchemy
        exposed attributes.
        """
        self.ses.commit()

    def exists(self, symbol):
        """Checks to if a symbol exists, by name.
        
        Parameters
        ----------
        symbol : str or Symbol

        Returns
        -------
        bool
        """

        if isinstance(symbol, str):
            sym = symbol
        elif isinstance(symbol, Symbol):
            sym = symbol.name

        syms = self.ses.query(Symbol).filter(Symbol.name == sym).all()
        if len(syms) == 0:
            return False
        else:
            return True

    def get(self, symbol):
        """ Gets a Symbol based on name, which is expected to exist. 
        
        Parameters
        ----------
        symbol : str or Symbol
        
        Returns
        -------
        Symbol
        
        Raises
        ------
        Exception
            If it does not exist. Use .try_to_get(), 
            if the symbol may or may not exist.
        """
        syms = self.try_to_get(symbol)
        if syms is None:
            raise Exception("Symbol {} does not exist".format(symbol))
        else:
            return syms

    def try_to_get(self, symbol):
        """ Gets a Symbol based on name, which may or may not exist.
        
        Parameters
        ----------
        symbol : str
        
        Returns
        -------
        Symbol or None.  
        
        Note
        ----
        Use .get(), if the symbol should exist, and an exception 
        is needed if it doesn't.

        """
        syms = self.ses.query(Symbol).filter(Symbol.name == symbol).all()
        if len(syms) == 0:
            return None
        else:
            return syms[0]

    def search_tag(self, tag, symbols=True, feeds=False):
        """ Get a list of Symbols by searching a tag or partial tag.

        Parameters
        ----------
        tag : str
            The tag to search.  Appending '%' will use SQL's "LIKE"
            functionality.
        symbols : bool, optional
            Search for Symbol's based on their tags.
        feeds : bool, optional
            Search for Symbol's based on their Feeds' tags.
        
        Returns
        -------
        List of Symbols or empty list
        
        """

        syms = []
        
        if isinstance(tag, (str, unicode)):
            tags = [tag]
        else:
            tags = tag

        if symbols:
            crits = []
            for tag in tags:
                if "%" in tag:
                    crit = SymbolTag.tag.like(tag)
                else:
                    crit = SymbolTag.tag == tag
                crits.append(crit)

            qry = self.ses.query(SymbolTag)
            qry = qry.filter(or_(*crits))
            syms = qry.all()
            
            syms = [tagged.symbol for tagged in syms]
        if feeds:
            crits = []
            for tag in tags:
                if "%" in tag:
                    crit = FeedTag.tag.like(tag)
                else:
                    crit = FeedTag.tag == tag
                crits.append(crit)
                    
            qry = self.ses.query(Symbol).select_from(FeedTag)
            qry = qry.join(FeedTag.feed).join(Feed.symbol)
            qry = qry.filter(or_(*crits))
            fds = qry.distinct()
            
            syms = syms + [sym for sym in fds]
            return list(set(syms))
        return syms

    def search_meta(self, **avargs):
        """Search list of Symbol objects by by querying specific 
        meta attributes and their respective values.
        
        Parameters
        ----------
        avargs
            The attributes and values passed as key word arguments.
            If more than one criteria is specified, AND logic is applied.
            Appending '%' to values will use SQL's "LIKE" functionality.
        
        Example
        -------
        >>> sm.search_meta(geography='Canada', sector='Gov%')
            
        Returns
        -------
        List of Symbols or empty list
        """
        
        qry = self.ses.query(Symbol).join(SymbolMeta.symbol)


        for attr, value in avargs.iteritems():
            SMA = aliased(SymbolMeta)
            if "%" in value:
                acrit = SMA.value.like(value)
            else:
                acrit = SMA.value == value
            
            crit = and_(acrit, SMA.attr == attr)

            qry = qry.filter(crit).join(SMA, SMA.symname == SymbolMeta.symname)

        qry = qry.order_by(Symbol.name)
        return qry.all()

    def bulk_cache_of_tag(self, tag):
        """ Caches all the symbols by a certain tag.

        For now, there is no different, than 
        caching each symbol individually.  In the future,
        this functionality could have speed improvements.

        Parameters
        ----------
        tag : str
            Use '%' to enable SQL's "LIKE" functionality.

        Returns
        -------
        TrumpReport
        """

        syms = self.search_tag(tag)
        
        name = 'Bulk Cache of Symbols tagged {}'.format(tag)
        tr = TrumpReport(name)
        for sym in syms:
            sr = sym.cache()
            tr.add_symbolreport(sr)
        
        return tr
        
    def build_view_from_tag(self, tag):
        """
        Build a view of group of Symbols based on their tag.
        
        Parameters
        ----------
        tag : str
            Use '%' to enable SQL's "LIKE" functionality.

        """
        
        syms = self.search_tag(tag)
        
        names = [sym.name for sym in syms]
        
        subs = ["SELECT indx, '{}' AS symbol, final FROM {}".format(s, s) for s in names]
        
        qry = " UNION ALL ".join(subs)
        
        qry = "CREATE VIEW {} AS {};".format(tag, qry)

        self.ses.execute("DROP VIEW IF EXISTS {};".format(tag))
        self.ses.commit()        
        self.ses.execute(qry)
        self.ses.commit()
    def _add_orfs(self, which, symbol, ind, val, dt_log=None, user=None, comment=None):
        """
        Appends a single indexed-value pair, to a symbol object, to be
        used during the final steps of the aggregation of the datatable.

        See add_override and add_fail_safe.
        
        Parameters
        ----------
        which : str
            Fail Safe or Override?
        symbol : Symbol or str
            The Symbol to apply the fail safe
        ind : obj
            The index value where the fail safe should be applied
        val : obj
            The data value which will be used in the fail safe
        dt_log : datetime
            A log entry, for saving when this fail safe was created.
        user : str
            A string representing which user made the fail safe
        comment : str
            A string to store any notes related to this fail safe.
        """
        if not isinstance(symbol, (str, unicode)):
            symbol = symbol.name

        if not dt_log:
            dt_log = dt.datetime.now()

        if which.lower() == 'override':
            qry = self.ses.query(func.max(Override.ornum).label('max_ornum'))
            use = Override
        elif which.lower() == 'failsafe':
            qry = self.ses.query(func.max(FailSafe.fsnum).label('max_fsnum'))
            use = FailSafe
            
        qry = qry.filter_by(symname = symbol)
        
        cur_num = qry.one()
        
        if cur_num[0] is None:
            next_num = 0
        else:
            next_num = cur_num[0] + 1            

        tmp = use(symname=symbol,
                   ind=ind,
                   val=val,
                   dt_log=dt_log,
                   user=user,
                   comment=comment,
                   ornum=next_num)
                                             
        self.ses.add(tmp)
        self.ses.commit()
        
    def add_override(self, symbol, ind, val, dt_log=None, user=None, comment=None):
        """
        Appends a single indexed-value pair, to a symbol object, to be
        used during the final steps of the aggregation of the datatable.

        With default settings Overrides, get applied with highest priority.
        
        Parameters
        ----------
        symbol : Symbol or str
            The Symbol to override
        ind : obj
            The index value where the override should be applied
        val : obj
            The data value which will be used in the override
        dt_log : datetime
            A log entry, for saving when this override was created.
        user : str
            A string representing which user made the override
        comment : str
            A string to store any notes related to this override.
        """
        self._add_orfs('override', symbol, ind, val, dt_log, user, comment)


    def add_fail_safe(self, symbol, ind, val,
                      dt_log=None, user=None, comment=None):
        """
        Appends a single indexed-value pair, to a symbol object, to be
        used during the final steps of the aggregation of the datatable.

        With default settings FailSafes, get applied with lowest priority.
        
        Parameters
        ----------
        symbol : Symbol or str
            The Symbol to apply the fail safe
        ind : obj
            The index value where the fail safe should be applied
        val : obj
            The data value which will be used in the fail safe
        dt_log : datetime
            A log entry, for saving when this fail safe was created.
        user : str
            A string representing which user made the fail safe
        comment : str
            A string to store any notes related to this fail safe.
        """
        self._add_orfs('failsafe', symbol, ind, val, dt_log, user, comment)

class ConversionManager(SymbolManager):
    """
    A ConversionManager handles the conversion of previously instantiated
    symbols, based on the object's units and the conversion manager
    setup.  The conversion is performed adhoc, in python
    only usage.  That is, nothing about the conversion persists
    in the Trump framework.  Only the final series is converted.
    """
    def __init__(self, engine_or_eng_str=None, system='FX', tag=None):
        """
        Parameters
        ----------
        engine_or_eng_str : str or None
            Pass a SQLAlchemy engine, or a string.  Without one,
            it will use the defaul provided in trump/options/trump.cfg
            If it fails to get a value there, an in-memory SQLlite
            session would be created.
        system : str, optional
            Uses the FX conversion system logic by default.
            Currently, no other systems are implemented.  Eg. metric-only,
            imperial-metric, etc.
            
            Other systems can be added after instantiation of the
            ConversionManager, but the one specified at instantiation
            will be used as default.
            
        tag : str, optional
            Tag for the set of feeds to use for conversion.  Only necessary,
            if the conversion system relies on it.  For FX, it's needed, to 
            specify the set of feeds to use.       

            Other tags can be added after instantiation of the
            ConversionManager, but the one specified at instantiation
            will be used as default.

        """
        super(ConversionManager, self).__init__(engine_or_eng_str)
        
        self.default_system = system
        self.default_tag = tag
        
        self.converters = {}
        self.add_converter(system, tag)

    def add_converter(self, system, tag):
        if system not in self.converters:
            self.converters[system] = {}
            
        if tag not in self.converters[system]:
            if system == 'FX':
                if tag is None:
                    raise Exception("Must specify a tag for FX Conversion")
                conversion_syms = self.search_tag(tag)
                conv = FXConverter()
                conv.use_trump_data(conversion_syms)
                self.converters[system][tag] = conv
                
    def get_converted(self, symbol, units='CAD', system=None, tag=None):
        """
        Uses a Symbol's Dataframe, to build a new Dataframe,
        with the data converted to the new units
        
        Parameters
        ----------
        symbol : str
            String representing a symbol
        units : str, optional
            Specify the units to convert the symbol to, default to CAD 
        system : str, optional
            If None, the default system specified at instantiation
            is used.  System defines which conversion approach to take.
        tag : str, optional
            Tags define which set of conversion data is used.  If None, the
            default tag specified at instantiation is used.  
        """
        sym = self.get(symbol)
        
        system = system or self.default_system
        tag = tag or self.default_tag
        
        conv = self.converters[system][tag]

        return conv.convert(sym.df, sym.units, units)
        

class Symbol(Base, ReprMixin):
    __tablename__ = '_symbols'

    name = Column('name', String, primary_key=True)
    description = Column('description', String)
    units = Column('units', String)
    agg_method = Column('agg_method', String)

    index = relationship('Index', uselist=False, backref='_symbols',
                         cascade=ADO)
    dtype = relationship('SymbolDataDef', uselist=False, backref='_symbols',
                         cascade=ADO)

    handle = relationship("SymbolHandle", uselist=False, backref='_symbols',
                         cascade=ADO)
                         
    tags = relationship("SymbolTag", cascade=ADO)
    aliases = relationship("SymbolAlias", cascade=ADO)
    validity = relationship("SymbolValidity", cascade=ADO)
    feeds = relationship("Feed", cascade=ADO)
    meta = relationship("SymbolMeta", lazy='dynamic', cascade=ADO)
    
    def __init__(self, name, description=None, units=None,
                 agg_method="PRIORITY_FILL",
                 indexname="UNNAMED", indeximp="DatetimeIndexImp"):
        """A Trump Symbol persistently objectifies indexed data

        Use the SymbolManager class to create or retrieve existing symbols.

        Parameters
        ----------
        name : str
            The name of the symbol to be added to the database, serves
            as a primary key across the trump installation.
        description : str, optional
            a description of the symbol, just for notes.
        units : str, optional
            a string representing the units for the data.
        agg_method : str, default PRIORITY_FILL
            the method used for aggregating feeds, see
            trump.extensions.symbol_aggs.py for the list of available options.
        indexname : str
            a proprietary name assigned to the index.
        indeximp : str
            a string representing an index implementer (one of the classes in indexing.py)

        """
        
        self.name = name
        self.description = description
        self.units = units

        self.index = Index(indexname, indeximp, sym=name)
        self.dtype = SymbolDataDef("SkipDataDef", sym=name)
        
        self.agg_method = agg_method
        self.datatable = None
        self.datatable_exists = False
    
    def set_indexing(self, index_template):
        """
        Update a symbol's indexing strategy
        
        Parameters
        ----------
        index_template : bIndex or bIndex-like
            An index template used to overwrite all 
            details about the symbol's current index.

        """
        objs = object_session(self)
        self.index.name = index_template.name
        self.index.indimp = index_template.imp_name
        self.index.case = index_template.case
        self.index.setkwargs(**index_template.kwargs)<commit_msg>Enable changing of index datatype without delete<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
# PyLint tests that will never be applied by Trump.
#
# ... is not callable, ignored because a property that returns a callable
#                       becomes callable.
# pylint: disable-msg=E1102

# missing parameter, ignored because a SQLAlchemy function is wrapped.
#                    it's a documented issue with that team.
# pylint: disable-msg=E1120

# Used * or ** magic, we're not getting rid of this, it's imperative to Trump.
# pylint: disable-msg=W0142

# Too many/few arguments, ignored, because its confusing and doesn't make
#                         sense to refactor templates.
#
# pylint: disable-msg=R0913
# pylint: disable-msg=R0903

"""
Trump's Object Relational Model is the glue to the framework, used to create
a Symbol's tags, alias, meta data, data feeds and their sources, munging,
error handling and validity instructions.
"""

# SQLAQ - running the uninstall script, then this script, in the same session
#        causes an error:
#
#        sqlalchemy.exc.InvalidRequestError: When initializing mapper
#        Mapper|Feed|_feeds, expression 'FeedMeta' failed to locate a name
#        ("name 'FeedMeta' is not defined"). If this is a class name, consider
#        adding this relationship() to the <class 'trump.orm.Feed'> class
#        after both dependent classes have been defined
#
#        Why?


import datetime as dt

import pandas as pd
from sqlalchemy import event, Table, Column, ForeignKey, ForeignKeyConstraint,\
    String, Integer, Float, Boolean, DateTime, MetaData, func
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship, aliased, backref
from sqlalchemy.orm.session import object_session
from sqlalchemy.exc import ProgrammingError, NoSuchTableError
from sqlalchemy.sql import and_, or_
from sqlalchemy import create_engine

from indexing import indexingtypes
from validity import validitychecks
from datadef import datadefs

from trump.tools import ReprMixin, ProxyDict, isinstanceofany, \
    BitFlag, BitFlagType, ReprObjType, DuckTypeMixin

from trump.extensions.symbol_aggs import FeedAggregator, sorted_feed_cols
from trump.templating import bFeed, pab, pnab
from trump.options import read_config, read_settings
from trump.converting import FXConverter

from handling import Handler

from reporting.objects import TrumpReport, FeedReport, SymbolReport, \
    ReportPoint

BitFlag.associate_with(BitFlagType)

try:
    ENGINE_STR = read_config(sect='readwrite', sett='engine')
except:
    print ("Problem reading trump.cfg.  Continuing using an in-memory "
           "SQLlite database. Trump was not designed to work in-memory, "
           "because, What's the point of non-persistent persistent objects?")
    ENGINE_STR = "sqlite://"

rbd = read_config(sect='options', sett='raise_by_default')
if rbd.upper() == 'TRUE':
    rbd = BitFlag(1)
else:
    rbd = None

# Bind the engine to the metadata of the Base class so that the
# declaratives can be accessed through a DBSession instance

Base = declarative_base()

ADO = "all, delete-orphan"
CC = {'onupdate': "CASCADE", 'ondelete': "CASCADE"}

class SymbolManager(object):

    """
    The SymbolManager maintains the SQLAlchemy database session, and 
    provides access to object creation, deletion, searching, and 
    overrides/failsafes.
    """

    def __init__(self, engine_or_eng_str=None, loud=False, echo=False):
        """
        Parameters
        ----------
        engine_or_eng_str : str or None, optional
            Pass a SQLAlchemy engine, or a string.  Without one,
            it will use the string provided in trump/options/trump.cfg
            If it fails to get a value there, an in-memory SQLlite
            session would be created.
        loud : bool, optional
            Print information such as engine string used, defaults to False
        echo : bool, optional
            If a new engine is created, it will pass this to it'safes
            constructor, enabling SQLAlchemy's echo mode.
            
        Returns
        -------
        SymbolManager
        """
        if engine_or_eng_str is None:
            engine = create_engine(ENGINE_STR, echo=echo)
        elif isinstance(engine_or_eng_str, (str, unicode)):
            engine = create_engine(engine_or_eng_str, echo=echo)
        else:
            engine = engine_or_eng_str
        
        Base.metadata.bind = engine
        DBSession = sessionmaker(bind=engine)
        
        self.loud = loud
        if loud:
            print "Using engine: {}".format(ENGINE_STR)

        self.ses = DBSession()
        
    def finish(self):
        """ Closes the session with the database.

        Call at the end of a trump session. It also 
        calls SessionManager.complete().
        """
        self.complete()
        self.ses.close()

    def create(self, name, description=None, units=None,
               agg_method="priority_fill", overwrite=False):
        """ Create, or get if exists, a Symbol.
        
        Parameters
        ----------
        name : str
            A symbol's name is a primary key, used across
            the Trump ORM.
        description : str, optional
            An arbitrary string, used to store user information
            related to the symbol.
        units : str, optional
            This is a string used to denote the units of the final
            data Series.
        agg_method : str, optional
            The aggregation method, used to calculate
            the final feed.  Defaults to priority_fill.
        overwrite : bool, optional
            Set to True, to force deletion an existing symbol.
            defaults to False.
            
        Returns
        -------
        Symbol
        """
        sym = self.try_to_get(name)

        if sym is not None:
            if overwrite:
                print "Deleting {}".format(sym.name)
                self.ses.delete(sym)
                self.ses.commit()
            else:
                msg = 'Symbol {} already exists.\n' + \
                      'Consider setting overwrite to True.'
                msg = msg.format(name)
                raise Exception(msg)

        sym = Symbol(name, description, units, agg_method)
        
        self.ses.add(sym)

        print "Creating {}".format(sym.name)
        sym.add_alias(name)

        sym.handle = SymbolHandle(sym=sym)
        self.ses.commit()

        return sym

    def delete(self, symbol):
        """
        Deletes a Symbol.
        
        Parameters
        ----------
        symbol : str or Symbol
        """
        if isinstance(symbol, str):
            sym = self.get(symbol)
        elif isinstance(symbol, Symbol):
            sym = symbol
        else:
            raise Exception("Invalid symbol {}".format((repr(symbol))))
        self.ses.delete(sym)
        self.ses.commit()

    def complete(self):
        """Commits any changes to the database.
        In general, most of Trump API's auto-commits
        or does so internally.  
        
        This is necessary when working directly with SQLAlchemy
        exposed attributes.
        """
        self.ses.commit()

    def exists(self, symbol):
        """Checks to if a symbol exists, by name.
        
        Parameters
        ----------
        symbol : str or Symbol

        Returns
        -------
        bool
        """

        if isinstance(symbol, str):
            sym = symbol
        elif isinstance(symbol, Symbol):
            sym = symbol.name

        syms = self.ses.query(Symbol).filter(Symbol.name == sym).all()
        if len(syms) == 0:
            return False
        else:
            return True

    def get(self, symbol):
        """ Gets a Symbol based on name, which is expected to exist. 
        
        Parameters
        ----------
        symbol : str or Symbol
        
        Returns
        -------
        Symbol
        
        Raises
        ------
        Exception
            If it does not exist. Use .try_to_get(), 
            if the symbol may or may not exist.
        """
        syms = self.try_to_get(symbol)
        if syms is None:
            raise Exception("Symbol {} does not exist".format(symbol))
        else:
            return syms

    def try_to_get(self, symbol):
        """ Gets a Symbol based on name, which may or may not exist.
        
        Parameters
        ----------
        symbol : str
        
        Returns
        -------
        Symbol or None.  
        
        Note
        ----
        Use .get(), if the symbol should exist, and an exception 
        is needed if it doesn't.

        """
        syms = self.ses.query(Symbol).filter(Symbol.name == symbol).all()
        if len(syms) == 0:
            return None
        else:
            return syms[0]

    def search_tag(self, tag, symbols=True, feeds=False):
        """ Get a list of Symbols by searching a tag or partial tag.

        Parameters
        ----------
        tag : str
            The tag to search.  Appending '%' will use SQL's "LIKE"
            functionality.
        symbols : bool, optional
            Search for Symbol's based on their tags.
        feeds : bool, optional
            Search for Symbol's based on their Feeds' tags.
        
        Returns
        -------
        List of Symbols or empty list
        
        """

        syms = []
        
        if isinstance(tag, (str, unicode)):
            tags = [tag]
        else:
            tags = tag

        if symbols:
            crits = []
            for tag in tags:
                if "%" in tag:
                    crit = SymbolTag.tag.like(tag)
                else:
                    crit = SymbolTag.tag == tag
                crits.append(crit)

            qry = self.ses.query(SymbolTag)
            qry = qry.filter(or_(*crits))
            syms = qry.all()
            
            syms = [tagged.symbol for tagged in syms]
        if feeds:
            crits = []
            for tag in tags:
                if "%" in tag:
                    crit = FeedTag.tag.like(tag)
                else:
                    crit = FeedTag.tag == tag
                crits.append(crit)
                    
            qry = self.ses.query(Symbol).select_from(FeedTag)
            qry = qry.join(FeedTag.feed).join(Feed.symbol)
            qry = qry.filter(or_(*crits))
            fds = qry.distinct()
            
            syms = syms + [sym for sym in fds]
            return list(set(syms))
        return syms

    def search_meta(self, **avargs):
        """Search list of Symbol objects by by querying specific 
        meta attributes and their respective values.
        
        Parameters
        ----------
        avargs
            The attributes and values passed as key word arguments.
            If more than one criteria is specified, AND logic is applied.
            Appending '%' to values will use SQL's "LIKE" functionality.
        
        Example
        -------
        >>> sm.search_meta(geography='Canada', sector='Gov%')
            
        Returns
        -------
        List of Symbols or empty list
        """
        
        qry = self.ses.query(Symbol).join(SymbolMeta.symbol)


        for attr, value in avargs.iteritems():
            SMA = aliased(SymbolMeta)
            if "%" in value:
                acrit = SMA.value.like(value)
            else:
                acrit = SMA.value == value
            
            crit = and_(acrit, SMA.attr == attr)

            qry = qry.filter(crit).join(SMA, SMA.symname == SymbolMeta.symname)

        qry = qry.order_by(Symbol.name)
        return qry.all()

    def bulk_cache_of_tag(self, tag):
        """ Caches all the symbols by a certain tag.

        For now, there is no different, than 
        caching each symbol individually.  In the future,
        this functionality could have speed improvements.

        Parameters
        ----------
        tag : str
            Use '%' to enable SQL's "LIKE" functionality.

        Returns
        -------
        TrumpReport
        """

        syms = self.search_tag(tag)
        
        name = 'Bulk Cache of Symbols tagged {}'.format(tag)
        tr = TrumpReport(name)
        for sym in syms:
            sr = sym.cache()
            tr.add_symbolreport(sr)
        
        return tr
        
    def build_view_from_tag(self, tag):
        """
        Build a view of group of Symbols based on their tag.
        
        Parameters
        ----------
        tag : str
            Use '%' to enable SQL's "LIKE" functionality.

        """
        
        syms = self.search_tag(tag)
        
        names = [sym.name for sym in syms]
        
        subs = ["SELECT indx, '{}' AS symbol, final FROM {}".format(s, s) for s in names]
        
        qry = " UNION ALL ".join(subs)
        
        qry = "CREATE VIEW {} AS {};".format(tag, qry)

        self.ses.execute("DROP VIEW IF EXISTS {};".format(tag))
        self.ses.commit()        
        self.ses.execute(qry)
        self.ses.commit()
    def _add_orfs(self, which, symbol, ind, val, dt_log=None, user=None, comment=None):
        """
        Appends a single indexed-value pair, to a symbol object, to be
        used during the final steps of the aggregation of the datatable.

        See add_override and add_fail_safe.
        
        Parameters
        ----------
        which : str
            Fail Safe or Override?
        symbol : Symbol or str
            The Symbol to apply the fail safe
        ind : obj
            The index value where the fail safe should be applied
        val : obj
            The data value which will be used in the fail safe
        dt_log : datetime
            A log entry, for saving when this fail safe was created.
        user : str
            A string representing which user made the fail safe
        comment : str
            A string to store any notes related to this fail safe.
        """
        if not isinstance(symbol, (str, unicode)):
            symbol = symbol.name

        if not dt_log:
            dt_log = dt.datetime.now()

        if which.lower() == 'override':
            qry = self.ses.query(func.max(Override.ornum).label('max_ornum'))
            use = Override
        elif which.lower() == 'failsafe':
            qry = self.ses.query(func.max(FailSafe.fsnum).label('max_fsnum'))
            use = FailSafe
            
        qry = qry.filter_by(symname = symbol)
        
        cur_num = qry.one()
        
        if cur_num[0] is None:
            next_num = 0
        else:
            next_num = cur_num[0] + 1            

        tmp = use(symname=symbol,
                   ind=ind,
                   val=val,
                   dt_log=dt_log,
                   user=user,
                   comment=comment,
                   ornum=next_num)
                                             
        self.ses.add(tmp)
        self.ses.commit()
        
    def add_override(self, symbol, ind, val, dt_log=None, user=None, comment=None):
        """
        Appends a single indexed-value pair, to a symbol object, to be
        used during the final steps of the aggregation of the datatable.

        With default settings Overrides, get applied with highest priority.
        
        Parameters
        ----------
        symbol : Symbol or str
            The Symbol to override
        ind : obj
            The index value where the override should be applied
        val : obj
            The data value which will be used in the override
        dt_log : datetime
            A log entry, for saving when this override was created.
        user : str
            A string representing which user made the override
        comment : str
            A string to store any notes related to this override.
        """
        self._add_orfs('override', symbol, ind, val, dt_log, user, comment)


    def add_fail_safe(self, symbol, ind, val,
                      dt_log=None, user=None, comment=None):
        """
        Appends a single indexed-value pair, to a symbol object, to be
        used during the final steps of the aggregation of the datatable.

        With default settings FailSafes, get applied with lowest priority.
        
        Parameters
        ----------
        symbol : Symbol or str
            The Symbol to apply the fail safe
        ind : obj
            The index value where the fail safe should be applied
        val : obj
            The data value which will be used in the fail safe
        dt_log : datetime
            A log entry, for saving when this fail safe was created.
        user : str
            A string representing which user made the fail safe
        comment : str
            A string to store any notes related to this fail safe.
        """
        self._add_orfs('failsafe', symbol, ind, val, dt_log, user, comment)

class ConversionManager(SymbolManager):
    """
    A ConversionManager handles the conversion of previously instantiated
    symbols, based on the object's units and the conversion manager
    setup.  The conversion is performed adhoc, in python
    only usage.  That is, nothing about the conversion persists
    in the Trump framework.  Only the final series is converted.
    """
    def __init__(self, engine_or_eng_str=None, system='FX', tag=None):
        """
        Parameters
        ----------
        engine_or_eng_str : str or None
            Pass a SQLAlchemy engine, or a string.  Without one,
            it will use the defaul provided in trump/options/trump.cfg
            If it fails to get a value there, an in-memory SQLlite
            session would be created.
        system : str, optional
            Uses the FX conversion system logic by default.
            Currently, no other systems are implemented.  Eg. metric-only,
            imperial-metric, etc.
            
            Other systems can be added after instantiation of the
            ConversionManager, but the one specified at instantiation
            will be used as default.
            
        tag : str, optional
            Tag for the set of feeds to use for conversion.  Only necessary,
            if the conversion system relies on it.  For FX, it's needed, to 
            specify the set of feeds to use.       

            Other tags can be added after instantiation of the
            ConversionManager, but the one specified at instantiation
            will be used as default.

        """
        super(ConversionManager, self).__init__(engine_or_eng_str)
        
        self.default_system = system
        self.default_tag = tag
        
        self.converters = {}
        self.add_converter(system, tag)

    def add_converter(self, system, tag):
        if system not in self.converters:
            self.converters[system] = {}
            
        if tag not in self.converters[system]:
            if system == 'FX':
                if tag is None:
                    raise Exception("Must specify a tag for FX Conversion")
                conversion_syms = self.search_tag(tag)
                conv = FXConverter()
                conv.use_trump_data(conversion_syms)
                self.converters[system][tag] = conv
                
    def get_converted(self, symbol, units='CAD', system=None, tag=None):
        """
        Uses a Symbol's Dataframe, to build a new Dataframe,
        with the data converted to the new units
        
        Parameters
        ----------
        symbol : str
            String representing a symbol
        units : str, optional
            Specify the units to convert the symbol to, default to CAD 
        system : str, optional
            If None, the default system specified at instantiation
            is used.  System defines which conversion approach to take.
        tag : str, optional
            Tags define which set of conversion data is used.  If None, the
            default tag specified at instantiation is used.  
        """
        sym = self.get(symbol)
        
        system = system or self.default_system
        tag = tag or self.default_tag
        
        conv = self.converters[system][tag]

        return conv.convert(sym.df, sym.units, units)
        

class Symbol(Base, ReprMixin):
    __tablename__ = '_symbols'

    name = Column('name', String, primary_key=True)
    description = Column('description', String)
    units = Column('units', String)
    agg_method = Column('agg_method', String)

    index = relationship('Index', uselist=False, backref='_symbols',
                         cascade=ADO)
    dtype = relationship('SymbolDataDef', uselist=False, backref='_symbols',
                         cascade=ADO)

    handle = relationship("SymbolHandle", uselist=False, backref='_symbols',
                         cascade=ADO)
                         
    tags = relationship("SymbolTag", cascade=ADO)
    aliases = relationship("SymbolAlias", cascade=ADO)
    validity = relationship("SymbolValidity", cascade=ADO)
    feeds = relationship("Feed", cascade=ADO)
    meta = relationship("SymbolMeta", lazy='dynamic', cascade=ADO)
    
    def __init__(self, name, description=None, units=None,
                 agg_method="PRIORITY_FILL",
                 indexname="UNNAMED", indeximp="DatetimeIndexImp"):
        """A Trump Symbol persistently objectifies indexed data

        Use the SymbolManager class to create or retrieve existing symbols.

        Parameters
        ----------
        name : str
            The name of the symbol to be added to the database, serves
            as a primary key across the trump installation.
        description : str, optional
            a description of the symbol, just for notes.
        units : str, optional
            a string representing the units for the data.
        agg_method : str, default PRIORITY_FILL
            the method used for aggregating feeds, see
            trump.extensions.symbol_aggs.py for the list of available options.
        indexname : str
            a proprietary name assigned to the index.
        indeximp : str
            a string representing an index implementer (one of the classes in indexing.py)

        """
        
        self.name = name
        self.description = description
        self.units = units

        self.index = Index(indexname, indeximp, sym=name)
        self.dtype = SymbolDataDef("SkipDataDef", sym=name)
        
        self.agg_method = agg_method
        self.datatable = None
        self.datatable_exists = False
    
    def set_indexing(self, index_template):
        """
        Update a symbol's indexing strategy
        
        Parameters
        ----------
        index_template : bIndex or bIndex-like
            An index template used to overwrite all 
            details about the symbol's current index.

        """
        objs = object_session(self)
        
        if self.index.indimp != index_template.imp_name:
            self._refresh_datatable_schema()
        
        self.index.name = index_template.name
        self.index.indimp = index_template.imp_name
        self.index.case = index_template.case
        self.index.setkwargs(**index_template.kwargs)
{'fraction_non_alphanumeric': 0.08607198748043818, 'fraction_numerical': 0.0012433811391943748, 'mean_word_length': 1.7070566388115134, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6384077', 'n_tokens_mistral': 12347, 'n_tokens_neox': 11835, 'n_words': 4245}