import ast
import hashlib
from json import dumps, loads
import os
from Queue import Queue
import random
import sys
import time
import threading
import urllib

import deserialize
from processor import Processor, print_log
from utils import *
from storage import Storage
from utils import logger

class BlockchainProcessor(Processor):

    def __init__(self, config, shared):
        Processor.__init__(self)

        self.mtimes = {} # monitoring
        self.shared = shared
        self.config = config
        self.up_to_date = False

        self.watch_lock = threading.Lock()
        self.watch_blocks = []
        self.watch_headers = []
        self.watched_addresses = {}

        self.history_cache = {}
        self.max_cache_size = 100000
        self.chunk_cache = {}
        self.cache_lock = threading.Lock()
        self.headers_data = ''
        self.headers_path = config.get('leveldb', 'path')

        self.mempool_values = {}
        self.mempool_addresses = {}
        self.mempool_hist = {}
        self.mempool_hashes = set([])
        self.mempool_lock = threading.Lock()

        self.address_queue = Queue()

        try:
            self.test_reorgs = config.getboolean('leveldb', 'test_reorgs')   # simulate random blockchain reorgs
        except:
            self.test_reorgs = False
        self.storage = Storage(config, shared, self.test_reorgs)

        self.dblock = threading.Lock()

        self.bitcoind_url = 'http://%s:%s@%s:%s/' % (
            config.get('bitcoind', 'bitcoind_user'),
            config.get('bitcoind', 'bitcoind_password'),
            config.get('bitcoind', 'bitcoind_host'),
            config.get('bitcoind', 'bitcoind_port'))

        self.sent_height = 0
        self.sent_header = None

        # catch_up headers
        self.init_headers(self.storage.height)

        self.blockchain_thread = threading.Thread(target = self.do_catch_up)
        self.blockchain_thread.start()

    def do_catch_up(self):

        self.header = self.block2header(self.bitcoind('getblock', [self.storage.last_hash]))
        self.header['utxo_root'] = self.storage.get_root_hash().encode('hex')
        self.catch_up(sync=False)
        print_log("Blockchain is up to date.")
        self.memorypool_update()
        print_log("Memory pool initialized.")

        while not self.shared.stopped():
            self.main_iteration()
            if self.shared.paused():
                print_log("bitcoind is responding")
                self.shared.unpause()
            time.sleep(10)

    def mtime(self, name):
        now = time.time()
        if name != '':
            delta = now - self.now
            t = self.mtimes.get(name, 0)
            self.mtimes[name] = t + delta
        self.now = now

    def print_mtime(self):
        s = ''
        for k, v in self.mtimes.items():
            s += k+':'+"%.2f"%v+' '
        print_log(s)

    def wait_on_bitcoind(self):
        self.shared.pause()
        time.sleep(10)
        if self.shared.stopped():
            # this will end the thread
            raise

    def bitcoind(self, method, params=[]):
        postdata = dumps({"method": method, 'params': params, 'id': 'jsonrpc'})
        while True:
            try:
                respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
            except:
                print_log("cannot reach bitcoind...")
                self.wait_on_bitcoind()
            else:
                r = loads(respdata)
                if r['error'] is not None:
                    if r['error'].get('code') == -28:
                        print_log("bitcoind still warming up...")
                        self.wait_on_bitcoind()
                        continue
                    raise BaseException(r['error'])
                break
        return r.get('result')

    def block2header(self, b):
        return {
            "block_height": b.get('height'),
            "version": b.get('version'),
            "prev_block_hash": b.get('previousblockhash'),
            "merkle_root": b.get('merkleroot'),
            "timestamp": b.get('time'),
            "bits": int(b.get('bits'), 16),
            "nonce": b.get('nonce'),
        }

    def get_header(self, height):
        block_hash = self.bitcoind('getblockhash', [height])
        b = self.bitcoind('getblock', [block_hash])
        return self.block2header(b)

    def init_headers(self, db_height):
        self.chunk_cache = {}
        self.headers_filename = os.path.join(self.headers_path, 'blockchain_headers')

        if os.path.exists(self.headers_filename):
            height = os.path.getsize(self.headers_filename)/80 - 1   # the current height
            if height > 0:
                prev_hash = self.hash_header(self.read_header(height))
            else:
                prev_hash = None
        else:
            open(self.headers_filename, 'wb').close()
            prev_hash = None
            height = -1

        if height < db_height:
            print_log("catching up missing headers:", height, db_height)

        try:
            while height < db_height:
                height += 1
                header = self.get_header(height)
                if height > 1:
                    if prev_hash != header.get('prev_block_hash'):
                        # The prev_hash block is orphaned, go back
                        print_log("reorganizing, a block in file is orphaned:", prev_hash)
                        # Go to the parent of the orphaned block
                        height -= 2
                        prev_hash = self.hash_header(self.read_header(height))
                        continue

                self.write_header(header, sync=False)
                prev_hash = self.hash_header(header)
                if (height % 1000) == 0:
                    print_log("headers file:", height)
        except KeyboardInterrupt:
            self.flush_headers()
            sys.exit()

        self.flush_headers()

    def hash_header(self, header):
        return rev_hex(Hash(header_to_string(header).decode('hex')).encode('hex'))

    def read_header(self, block_height):
        if os.path.exists(self.headers_filename):
            with open(self.headers_filename, 'rb') as f:
                f.seek(block_height * 80)
                h = f.read(80)
            if len(h) == 80:
                h = header_from_string(h)
                return h

    def read_chunk(self, index):
        with open(self.headers_filename, 'rb') as f:
            f.seek(index*2016*80)
            chunk = f.read(2016*80)
        return chunk.encode('hex')

    def write_header(self, header, sync=True):
        if not self.headers_data:
            self.headers_offset = header.get('block_height')

        self.headers_data += header_to_string(header).decode('hex')
        if sync or len(self.headers_data) > 40*100:
            self.flush_headers()

        with self.cache_lock:
            chunk_index = header.get('block_height')/2016
            if self.chunk_cache.get(chunk_index):
                self.chunk_cache.pop(chunk_index)

    def pop_header(self):
        # we need to do this only if we have not flushed
        if self.headers_data:
            self.headers_data = self.headers_data[:-40]

    def flush_headers(self):
        if not self.headers_data:
            return
        with open(self.headers_filename, 'rb+') as f:
            f.seek(self.headers_offset*80)
            f.write(self.headers_data)
        self.headers_data = ''

    def get_chunk(self, i):
        # store them on disk; store the current chunk in memory
        with self.cache_lock:
            chunk = self.chunk_cache.get(i)
            if not chunk:
                chunk = self.read_chunk(i)
                self.chunk_cache[i] = chunk

        return chunk

    def get_mempool_transaction(self, txid):
        try:
            raw_tx = self.bitcoind('getrawtransaction', [txid, 0])
        except:
            return None

        vds = deserialize.BCDataStream()
        vds.write(raw_tx.decode('hex'))
        try:
            return deserialize.parse_Transaction(vds, is_coinbase=False)
        except:
            print_log("ERROR: cannot parse", txid)
            return None

    def get_history(self, addr, cache_only=False):
        with self.cache_lock:
            hist = self.history_cache.get(addr)
        if hist is not None:
            return hist
        if cache_only:
            return -1

        with self.dblock:
            hist = self.storage.get_history(addr)

        # add memory pool
        with self.mempool_lock:


"""Translation helper functions."""
from __future__ import unicode_literals

import gettext as gettext_module
import os
import re
import sys
import warnings
from collections import OrderedDict
from threading import local

from django.apps import apps
from django.conf import settings
from django.conf.locale import LANG_INFO
from django.core.exceptions import AppRegistryNotReady
from django.core.signals import setting_changed
from django.dispatch import receiver
from django.utils import lru_cache, six
from django.utils._os import upath
from django.utils.deprecation import RemovedInDjango19Warning
from django.utils.encoding import force_text
from django.utils.safestring import SafeData, mark_safe
from django.utils.six import StringIO
from django.utils.translation import (
    LANGUAGE_SESSION_KEY, TranslatorCommentWarning, trim_whitespace,
)

# Translations are cached in a dictionary for every language.
# The active translations are stored by threadid to make them thread local.
_translations = {}
_active = local()

# The default translation is based on the settings file.
_default = None

# magic gettext number to separate context from message
CONTEXT_SEPARATOR = "\x04"

# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9
# and RFC 3066, section 2.1
accept_language_re = re.compile(r'''
        ([A-Za-z]{1,8}(?:-[A-Za-z0-9]{1,8})*|\*)      # "en", "en-au", "x-y-z", "es-419", "*"
        (?:\s*;\s*q=(0(?:\.\d{,3})?|1(?:.0{,3})?))?   # Optional "q=1.00", "q=0.8"
        (?:\s*,\s*|$)                                 # Multiple accepts per header.
        ''', re.VERBOSE)

language_code_re = re.compile(r'^[a-z]{1,8}(?:-[a-z0-9]{1,8})*$', re.IGNORECASE)

language_code_prefix_re = re.compile(r'^/([\w-]+)(/|$)')

# some browsers use deprecated locales. refs #18419
_DJANGO_DEPRECATED_LOCALES = {
    'zh-cn': 'zh-hans',
    'zh-tw': 'zh-hant',
}

@receiver(setting_changed)
def reset_cache(**kwargs):
    """
    Reset global state when LANGUAGES setting has been changed, as some
    languages should no longer be accepted.
    """
    if kwargs['setting'] in ('LANGUAGES', 'LANGUAGE_CODE'):
        check_for_language.cache_clear()
        get_languages.cache_clear()
        get_supported_language_variant.cache_clear()

def to_locale(language, to_lower=False):
    """
    Turns a language name (en-us) into a locale name (en_US). If 'to_lower' is
    True, the last component is lower-cased (en_us).
    """
    p = language.find('-')
    if p >= 0:
        if to_lower:
            return language[:p].lower() + '_' + language[p + 1:].lower()
        else:
            # Get correct locale for sr-latn
            if len(language[p + 1:]) > 2:
                return language[:p].lower() + '_' + language[p + 1].upper() + language[p + 2:].lower()
            return language[:p].lower() + '_' + language[p + 1:].upper()
    else:
        return language.lower()

def to_language(locale):
    """Turns a locale name (en_US) into a language name (en-us)."""
    p = locale.find('_')
    if p >= 0:
        return locale[:p].lower() + '-' + locale[p + 1:].lower()
    else:
        return locale.lower()

class DjangoTranslation(gettext_module.GNUTranslations):
    """
    This class sets up the GNUTranslations context with regard to output
    charset.

    This translation object will be constructed out of multiple GNUTranslations
    objects by merging their catalogs. It will construct an object for the
    requested language and add a fallback to the default language, if it's
    different from the requested language.
    """
    def __init__(self, language):
        """Create a GNUTranslations() using many locale directories"""
        gettext_module.GNUTranslations.__init__(self)
        self.set_output_charset('utf-8')  # For Python 2 gettext() (#25720)

        self.__language = language
        self.__to_language = to_language(language)
        self.__locale = to_locale(language)

        self._init_translation_catalog()
        self._add_installed_apps_translations()
        self._add_local_translations()
        self._add_fallback()

    def __repr__(self):
        return "<DjangoTranslation lang:%s>" % self.__language

    def _new_gnu_trans(self, localedir, use_null_fallback=True):
        """
        Returns a mergeable gettext.GNUTranslations instance.

        A convenience wrapper. By default gettext uses 'fallback=False'.
        Using param `use_null_fallback` to avoid confusion with any other
        references to 'fallback'.
        """
        translation = gettext_module.translation(
            domain='django',
            localedir=localedir,
            languages=[self.__locale],
            codeset='utf-8',
            fallback=use_null_fallback)
        if not hasattr(translation, '_catalog'):
            # provides merge support for NullTranslations()
            translation._catalog = {}
            translation._info = {}
            translation.plural = lambda n: int(n != 1)
        return translation

    def _init_translation_catalog(self):
        """Creates a base catalog using global django translations."""
        settingsfile = upath(sys.modules[settings.__module__].__file__)
        localedir = os.path.join(os.path.dirname(settingsfile), 'locale')
        use_null_fallback = True
        if self.__language == settings.LANGUAGE_CODE:
            # default lang should be present and parseable, if not
            # gettext will raise an IOError (refs #18192).
            use_null_fallback = False
        translation = self._new_gnu_trans(localedir, use_null_fallback)
        self.plural = translation.plural
        self._info = translation._info.copy()
        self._catalog = translation._catalog.copy()

    def _add_installed_apps_translations(self):
        """Merges translations from each installed app."""
        try:
            app_configs = reversed(list(apps.get_app_configs()))
        except AppRegistryNotReady:
            raise AppRegistryNotReady(
                "The translation infrastructure cannot be initialized before the "
                "apps registry is ready. Check that you don't make non-lazy "
                "gettext calls at import time.")
        for app_config in app_configs:
            localedir = os.path.join(app_config.path, 'locale')
            translation = self._new_gnu_trans(localedir)
            self.merge(translation)

    def _add_local_translations(self):
        """Merges translations defined in LOCALE_PATHS."""
        for localedir in reversed(settings.LOCALE_PATHS):
            translation = self._new_gnu_trans(localedir)
            self.merge(translation)

    def _add_fallback(self):
        """Sets the GNUTranslations() fallback with the default language."""
        # Don't set a fallback for the default language or any English variant
        # (as it's empty, so it'll ALWAYS fall back to the default language)
        if self.__language == settings.LANGUAGE_CODE or self.__language.startswith('en'):
            return
        default_translation = translation(settings.LANGUAGE_CODE)
        self.add_fallback(default_translation)

    def merge(self, other):
        """Merge another translation into this catalog."""
        self._catalog.update(other._catalog)

    def language(self):
        """Returns the translation language."""
        return self.__language

    def to_language(self):
        """Returns the translation language name."""
        return self.__to_language

def translation(language):
    """
    Returns a translation object.
    """
    global _translations
    if language not in _translations:
        _translations[language] = DjangoTranslation(language)
    return _translations[language]

def activate(language):
    """
    Fetches the translation object for a given language and installs it as the
    current translation object for the current thread.
    """
    if not language:
        return
    if language in _DJANGO_DEPRECATED_LOCALES:
        msg = ("The use of the language code '%s' is deprecated. "
               "Please use the '%s' translation instead.")
        warnings.warn(msg % (language, _DJANGO_DEPRECATED_LOCALES[language]),
                      RemovedInDjango19Warning, stacklevel=2)
    _active.value = translation(language)

def deactivate():
    """
    Deinstalls the currently active translation object so that further _ calls
    will resolve against the default translation object, again.
    """
    if hasattr(_active, "value"):
        del _active.value

def deactivate_all():
    """
    Makes the active translation object a NullTranslations() instance. This is
    useful when we want delayed translations to appear as the original string
    for some reason.
    """
    _active.value = gettext_module.NullTranslations()
    _active.value.to_language = lambda *args: None

def get_language():
    """Returns the currently selected language."""
    t = getattr(_active, "value", None)
    if t is not None:
        try:
            return t.to_language()
        except AttributeError:
            pass
    # If we don't have a real translation object, assume it's the default language.
    return settings.LANGUAGE_CODE

def get_language_bidi():
    """
    Returns selected language's BiDi layout.

    * False = left-to-right layout
    * True = right-to-left layout
    """
    lang = get_language()
    if lang is None:
        return False
    else:
        base_lang = get_language().split('-')[0]
        return base_lang in settings.LANGUAGES_BIDI

def catalog():
    """
    Returns the current active catalog for further processing.
    This can be used if you need to modify the catalog or want to access the
    whole message catalog instead of just translating one string.
    """
    global _default

    t = getattr(_active, "value", None)
    if t is not None:
        return t
    if _default is None:
        _default = translation(settings.LANGUAGE_CODE)
    return _default

def do_translate(message, translation_function):
    """
    Translates 'message' using the given 'translation_function' name -- which
    will be either gettext or ugettext. It uses the current thread to find the
    translation object to use. If no current translation is activated, the
    message will be run through the default translation object.
    """
    global _default

    # str() is allowing a bytestring message to remain bytestring on Python 2
    eol_message = message.replace(str('\r\n'), str('\n')).replace(str('\r'), str('\n'))

    if len(eol_message) == 0:
        # Returns an empty value of the corresponding type if an empty message
        # is given, instead of metadata, which is the default gettext behavior.
        result = type(message)("")
    else:
        _default = _default or translation(settings.LANGUAGE_CODE)
        translation_object = getattr(_active, "value", _default)

        result = getattr(translation_object, translation_function)(eol_message)

    if isinstance(message, SafeData):
        return mark_safe(result)

    return result

def gettext(message):
    """
    Returns a string of the translation of the message.

    Returns a string on Python 3 and an UTF-8-encoded bytestring on Python 2.
    """
    return do_translate(message, 'gettext')

if six.PY3:
    ugettext = gettext
else:
    def ugettext(message):
        return do_translate(message, 'ugettext')

def pgettext(context, message):
    msg_with_ctxt = "%s%s%s" % (context, CONTEXT_SEPARATOR, message)
    result = ugettext(msg_with_ctxt)
    if CONTEXT_SEPARATOR in result:
        # Translation not found
        # force unicode, because lazy version expects unicode
        result = force_text(message)
    return result

def gettext_noop(message):
    """
    Marks strings for translation but doesn't translate them now. This can be
    used to store strings in global variables that should stay in the base
    language (because they might be used externally) and will be translated
    later.
    """
    return message

def do_ntranslate(singular, plural, number, translation_function):
    global _default

    t = getattr(_active, "value", None)
    if t is not None:
        return getattr(t, translation_function)(singular, plural, number)
    if _default is None:
        _default = translation(settings.LANGUAGE_CODE)
    return getattr(_default, translation_function)(singular, plural, number)

def ngettext(singular, plural, number):
    """
    Returns a string of the translation of either the singular or plural,
    based on the number.

    Returns a string on Python 3 and an UTF-8-encoded bytestring on Python 2.
    """
    return do_ntranslate(singular, plural, number, 'ngettext')

if six.PY3:
    ungettext = ngettext
else:
    def ungettext(singular, plural, number):
        """
        Returns a unicode strings of the translation of either the singular or
        plural, based on the number.
        """
        return do_ntranslate(singular, plural, number, 'ungettext')

def npgettext(context, singular, plural, number):
    msgs_with_ctxt = ("%s%s%s" % (context, CONTEXT_SEPARATOR, singular),
                      "%s%s%s" % (context, CONTEXT_SEPARATOR, plural),
                      number)
    result = ungettext(*msgs_with_ctxt)
    if CONTEXT_SEPARATOR in result:
        # Translation not found
        result = ungettext(singular, plural, number)
    return result

def all_locale_paths():
    """
    Returns a list of paths to user-provides languages files.
    """
    globalpath = os.path.join(
        os.path.dirname(upath(sys.modules[settings.__module__].__file__)), 'locale')
    return [globalpath] + list(settings.LOCALE_PATHS)

@lru_cache.lru_cache(maxsize=1000)
def check_for_language(lang_code):
    """
    Checks whether there is a global language file for the given language
    code. This is used to decide whether a user-provided language is
    available.

    lru_cache should have a maxsize to prevent from memory exhaustion attacks,
    as the provided language codes are taken from the HTTP request. See also
    <https://www.djangoproject.com/weblog/2007/oct/26/security-fix/>.
    """
    # First, a quick check to make sure lang_code is well-formed (#21458)
    if lang_code is None or not language_code_re.search(lang_code):
        return False
    for path in all_locale_paths():
        if gettext_module.find('django', path, [to_locale(lang_code)]) is not None:
            return True
    return False

@lru_cache.lru_cache()
def get_languages():
    """
    Cache of settings.LANGUAGES in an OrderedDict for easy lookups by key.
    """
    return OrderedDict(settings.LANGUAGES)

@lru_cache.lru_cache(maxsize=1000)
def get_supported_language_variant(lang_code, strict=False):
    """
    Returns the language-code that's listed in supported languages, possibly
    selecting a more generic variant. Raises LookupError if nothing found.

    If `strict` is False (the default), the function will look for an alternative
    country-specific variant when the currently checked is not found.

    lru_cache should have a maxsize to prevent from memory exhaustion attacks,
    as the provided language codes are taken from the HTTP request. See also
    <https://www.djangoproject.com/weblog/2007/oct/26/security-fix/>.
    """
    if lang_code:
        # If 'fr-ca' is not supported, try special fallback or language-only 'fr'.
        possible_lang_codes = [lang_code]
        try:
            possible_lang_codes.extend(LANG_INFO[lang_code]['fallback'])
        except KeyError:
            pass
        generic_lang_code = lang_code.split('-')[0]
        possible_lang_codes.append(generic_lang_code)
        supported_lang_codes = get_languages()

        for code in possible_lang_codes:
            if code in supported_lang_codes and check_for_language(code):
                return code
        if not strict:
            # if fr-fr is not supported, try fr-ca.
            for supported_code in supported_lang_codes:
                if supported_code.startswith(generic_lang_code + '-'):
                    return supported_code
    raise LookupError(lang_code)

def get_language_from_path(path, strict=False):
    """
    Returns the language-code if there is a valid language-code
    found in the `path`.

    If `strict` is False (the default), the function will look for an alternative
    country-specific variant when the currently checked is not found.
    """
    regex_match = language_code_prefix_re.match(path)
    if not regex_match:
        return None
    lang_code = regex_match.group(1)
    try:
        return get_supported_language_variant(lang_code, strict=strict)
    except LookupError:
        return None

def get_language_from_request(request, check_path=False):
    """
    Analyzes the request to find what language the user wants the system to
    show. Only languages listed in settings.LANGUAGES are taken into account.
    If the user requests a sublanguage where we have a main language, we send
    out the main language.

    If check_path is True, the URL path prefix will be checked for a language
    code, otherwise this is skipped for backwards compatibility.
    """
    if check_path:
        lang_code = get_language_from_path(request.path_info)
        if lang_code is not None:
            return lang_code

    supported_lang_codes = get_languages()

    if hasattr(request, 'session'):
        lang_code = request.session.get(LANGUAGE_SESSION_KEY)
        if lang_code in supported_lang_codes and lang_code is not None and check_for_language(lang_code):
            return lang_code

    lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)

    try:
        return get_supported_language_variant(lang_code)
    except LookupError:
        pass

    accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')
    for accept_lang, unused in parse_accept_lang_header(accept):
        if accept_lang == '*':
            break

        if not language_code_re.search(accept_lang):
            continue

        try:
            return get_supported_language_variant(accept_lang)
        except LookupError:
            continue

    try:
        return get_supported_language_variant(settings.LANGUAGE_CODE)
    except LookupError:
        return settings.LANGUAGE_CODE

dot_re = re.compile(r'\S')

def blankout(src, char):
    """
    Changes every non-whitespace character to the given char.
    Used in the templatize function.
    """
    return dot_re.sub(char, src)

context_re = re.compile(r"""^\s+.*context\s+((?:"[^"]*?")|(?:'[^']*?'))\s*""")
inline_re = re.compile(r"""^\s*trans\s+((?:"[^"]*?")|(?:'[^']*?'))(\s+.*context\s+((?:"[^"]*?")|(?:'[^']*?')))?\s*""")
block_re = re.compile(r"""^\s*blocktrans(\s+.*context\s+((?:"[^"]*?")|(?:'[^']*?')))?(?:\s+|$)""")
endblock_re = re.compile(r"""^\s*endblocktrans$""")
plural_re = re.compile(r"""^\s*plural$""")
constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?'))\)""")
one_percent_re = re.compile(r"""(?<!%)%(?!%)""")

def templatize(src, origin=None):
    """
    Turns a Django template into something that is understood by xgettext. It
    does so by translating the Django translation tags into standard gettext
    function invocations.
    """
    from django.template.base import (Lexer, TOKEN_TEXT, TOKEN_VAR,
        TOKEN_BLOCK, TOKEN_COMMENT, TRANSLATOR_COMMENT_MARK)
    src = force_text(src, settings.FILE_CHARSET)
    out = StringIO('')
    message_context = None
    intrans = False
    inplural = False
    trimmed = False
    singular = []
    plural = []
    incomment = False
    comment = []
    lineno_comment_map = {}
    comment_lineno_cache = None

    def join_tokens(tokens, trim=False):
        message = ''.join(tokens)
        if trim:
            message = trim_whitespace(message)
        return message

    for t in Lexer(src, origin).tokenize():
        if incomment:
            if t.token_type == TOKEN_BLOCK and t.contents == 'endcomment':
                content = ''.join(comment)
                translators_comment_start = None
                for lineno, line in enumerate(content.splitlines(True)):
                    if line.lstrip().startswith(TRANSLATOR_COMMENT_MARK):
                        translators_comment_start = lineno
                for lineno, line in enumerate(content.splitlines(True)):
                    if translators_comment_start is not None and lineno >= translators_comment_start:
                        out.write(' # %s' % line)
                    else:
                        out.write(' #\n')
                incomment = False
                comment = []
            else:
                comment.append(t.contents)
        elif intrans:
            if t.token_type == TOKEN_BLOCK:
                endbmatch = endblock_re.match(t.contents)
                pluralmatch = plural_re.match(t.contents)
                if endbmatch:
                    if inplural:
                        if message_context:
                            out.write(' npgettext(%r, %r, %r,count) ' % (
                                message_context,
                                join_tokens(singular, trimmed),
                                join_tokens(plural, trimmed)))
                        else:
                            out.write(' ngettext(%r, %r, count) ' % (
                                join_tokens(singular, trimmed),
                                join_tokens(plural, trimmed)))
                        for part in singular:
                            out.write(blankout(part, 'S'))
                        for part in plural:
                            out.write(blankout(part, 'P'))
                    else:
                        if message_context:
                            out.write(' pgettext(%r, %r) ' % (
                                message_context,
                                join_tokens(singular, trimmed)))
                        else:
                            out.write(' gettext(%r) ' % join_tokens(singular,
                                                                    trimmed))
                        for part in singular:
                            out.write(blankout(part, 'S'))
                    message_context = None
                    intrans = False
                    inplural = False
                    singular = []
                    plural = []
                elif pluralmatch:
                    inplural = True
                else:
                    filemsg = ''
                    if origin:
                        filemsg = 'file %s, ' % origin
                    raise SyntaxError(
                        "Translation blocks must not include other block tags: "
                        "%s (%sline %d)" % (t.contents, filemsg, t.lineno)
                    )
            elif t.token_type == TOKEN_VAR:
                if inplural:
                    plural.append('%%(%s)s' % t.contents)
                else:
                    singular.append('%%(%s)s' % t.contents)
            elif t.token_type == TOKEN_TEXT:
                contents = one_percent_re.sub('%%', t.contents)
                if inplural:
                    plural.append(contents)
                else:
                    singular.append(contents)

        else:
            # Handle comment tokens (`{# ... #}`) plus other constructs on
            # the same line:
            if comment_lineno_cache is not None:
                cur_lineno = t.lineno + t.contents.count('\n')
                if comment_lineno_cache == cur_lineno:
                    if t.token_type != TOKEN_COMMENT:
                        for c in lineno_comment_map[comment_lineno_cache]:
                            filemsg = ''
                            if origin:
                                filemsg = 'file %s, ' % origin
                            warn_msg = ("The translator-targeted comment '%s' "
                                "(%sline %d) was ignored, because it wasn't the last item "
                                "on the line.") % (c, filemsg, comment_lineno_cache)
                            warnings.warn(warn_msg, TranslatorCommentWarning)
                        lineno_comment_map[comment_lineno_cache] = []
                else:
                    out.write('# %s' % ' | '.join(lineno_comment_map[comment_lineno_cache]))
                comment_lineno_cache = None

            if t.token_type == TOKEN_BLOCK:
                imatch = inline_re.match(t.contents)
                bmatch = block_re.match(t.contents)
                cmatches = constant_re.findall(t.contents)
                if imatch:
                    g = imatch.group(1)
                    if g[0] == '"':
                        g = g.strip('"')
                    elif g[0] == "'":
                        g = g.strip("'")
                    g = one_percent_re.sub('%%', g)
                    if imatch.group(2):
                        # A context is provided
                        context_match = context_re.match(imatch.group(2))
                        message_context = context_match.group(1)
                        if message_context[0] == '"':
                            message_context = message_context.strip('"')
                        elif message_context[0] == "'":
                            message_context = message_context.strip("'")
                        out.write(' pgettext(%r, %r) ' % (message_context, g))
                        message_context = None
                    else:
                        out.write(' gettext(%r) ' % g)
                elif bmatch:
                    for fmatch in constant_re.findall(t.contents):
                        out.write(' _(%s) ' % fmatch)
                    if bmatch.group(1):
                        # A context is provided
                        context_match = context_re.match(bmatch.group(1))


from datetime import datetime
from reporter import Reporter
from urlparse import urlparse
from BeautifulSoup import BeautifulSoup
import ConfigParser
import logging
import requests
import socket
import traceback
from inspect import getargspec, getmodule
import subprocess
import json

def exec_helper(cmd, args=None):
    '''Use this function to call helpers, not to perform checks!
    The example use-case would a Testcase that required to get an
    authentication tokenout of a mailbox to complete a login procedure.
    The email-fetching would be done as a helper
    '''
    if not args:
        params = cmd
    else:
        params = [cmd] + args # becomes [cmd, arg1, arg2]
    try:
        output = subprocess.Popen(params, stdout=subprocess.PIPE).communicate()[0] # use Popen instead of subprocess.get_output for Python2.6 compatibility
        try:
            res = json.loads(output)
        except ValueError:
            return {"result":"Fail", "message":"Invalid JSON data", "data":""}
        if 'result' in res and 'message' in res and 'data' in res:
            return res
        else:
            return {"result":"Fail", "message":"Incomplete JSON data. Your helper should return a dict with the keys result, message and data.", "data":""}
    except subprocess.CalledProcessError as e: # raised when? for Popen?
        return {"result":"Fail", "message":"The helper script returned with a non-zero returnvalue", "data": e.output}

class PassiveTest():
    secure_only = False
    insecure_only = False

    def analyze(self, response, results):
        return None

    def result(self, state, message, data):
        return {'state' : state,  'message' : message, 'data' : data }

class ActiveTest():
    new_session = False # enable  (e.g. from cli) to enforce new session generation
    secure_only = False
    insecure_only = False
    run_passives = True
    description = "The base class for an Active Test."
    sessions = {}

    def __init__(self):
        if hasattr(self, "setup"):
            self.setup()

    #def get_url(self, url, status = True):
    #    try:
    #        sess = self.sessions[self.url]
    #    except KeyError:
    #        sess = requests.session()
    #    #print "Issue request towards %s using %s" % (url, sess.cookies)
    #    r = sess.get(url, allow_redirects = False)
    #    print url, r.status_code, status
    #    if status:
    #        r.raise_for_status()
    #    return r

    def execute(self, url, predecessor=None):
        self.url = url
        if self.url not in self.sessions or self.new_session:
            self.sessions[url] = requests.session() # Create per-target session
        try:
            if "pred" in getargspec(self.do_test).args:
                resulttuple = self.do_test(url, predecessor)
            else:
                resulttuple = self.do_test(url)
        except Exception, e:
            tb = traceback.format_exc()
            resulttuple = (ActiveTest().result("Error", e, tb), None)

        return resulttuple

    def result(self, state, message, data):
        return { 'state' : state, 'message' : message, 'data' : data, 'passive' : {}}

class HtmlTest(PassiveTest):
    description = 'allow easy analysis of html source code'
    def analyze(self, response):
        if 'text/html' in response.headers['content-type']:
            soup = BeautifulSoup(response.content)
            return self.analyze_html(response, soup)
        else:
            result = self.result("Skip", "Content-type is not html "+ response.headers['content-type'], None)
            return result

    def analyze_html(self, response, soup):
        """ implement this method in subclass"""
        pass

class Scanner():
    logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s')
    logger = logging.getLogger("Garmr-Scanner")
    logger.setLevel(logging.DEBUG)

    def __init__(self):
        self.resolve_target = True
        self.force_passives = False
        self._disabled_tests_ = []
        self._passive_tests_ = []
        self._active_tests_ = []
        self._finished_active_tests_ = []
        self._targets_ = []
        self._protos_ = ["http", "https"]
        Scanner.logger.debug("Scanner initialized.")
        self.reporter = Reporter()
        self.modules = []

    def do_passive_scan(self, passiveclass, is_ssl, response):
        if passiveclass.secure_only and not is_ssl:
            Scanner.logger.debug("\t\t[%s] Skip Test invalid for http scheme" % passiveclass)
            passive_result = PassiveTest().result("Skip", "This check is only applicable to SSL requests.", None)
            start = datetime.now()
            passive_result['start'] = start
            passive_result['end'] = start
            passive_result["duration"] = 0
        else:
            start = datetime.now()
            passive = passiveclass()
            passive_result = passive.analyze(response)
            end = datetime.now()
            td = end - start
            passive_result['start'] = start
            passive_result['end'] = end
            passive_result['duration'] = float((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6)) / 10**6
            Scanner.logger.info("\t\t[%s] %s %s" % (passiveclass, passive_result['state'], passive_result['message']))
        return passive_result

    def do_active_scan(self, testclass, is_ssl, target):
        ''' instantiate the class and run it against the specified target, if applicable '''
        if (testclass.secure_only and not is_ssl):
            Scanner.logger.info("\t[Skip] [%s] (reason: secure_only)" % testclass)
            result = ActiveTest().result("Skip", "This check is only applicable to SSL requests", None)
            result['start'] = datetime.now()
            result['end'] = result['start']
            result['duration'] = 0
            return result
        elif (testclass.insecure_only and is_ssl):
            Scanner.logger.info("\t[Skip] [%s] (reason: insecure_only)" % testclass)
            result = ActiveTest().result("Skip", "This check is only applicable to SSL requests", None)
            result['start'] = datetime.now()
            result['end'] = result['start']
            result['duration'] = 0
            return result
        elif str(testclass).split('.')[-1] in self._disabled_tests_:
            Scanner.logger.info("\t[Skip] [%s] (reason: disabled)" % testclass)
            result = ActiveTest().result("Skip", "This check was marked as disabled.", None)
            result['start'] = datetime.now()
            result['end'] = result['start']
            result['duration'] = 0
            return result
        start = datetime.now()
        test = testclass() # from now on we have an instance of the class
        if "pred" in getargspec(test.do_test).args:
            # Check if class accepts this parameter. avoids rewriting.
            predecessor_key = "%s.%s" % (getmodule(self._finished_active_tests_[-1]).__name__, self._finished_active_tests_[-1].__name__)
            predecessor_results = self.results[predecessor_key]
            result, response = test.execute(target, predecessor=predecessor_results)
        else:
            result, response = test.execute(target)
        end = datetime.now()
        td = end - start
        result['response'] = response
        result['start'] = start
        result['end'] = end
        result['duration'] = float((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6)) / 10**6
        Scanner.logger.info("\t[%s] %s %s" % (testclass, result['state'], result['message']))
        self.reporter.write_active(testclass, result)
        if (result['state'] == "Error"):
            Scanner.logger.error(result['data'])
        if response != None and test.run_passives:
            result['passive'] = {}
            self.reporter.start_passives()
            for passive_testclass in self._passive_tests_:
		passive_testclass_key = "%s.%s" % (getmodule(passive_testclass).__name__, passive_testclass.__name__)
                result["passive"][passive_testclass_key] = self.do_passive_scan(passive_testclass, is_ssl, response)
                self.reporter.write_passive(passive_testclass, result["passive"][passive_testclass_key])
            self.reporter.end_passives()
        return result

    def scan_target(self, target):
        ''' iterate over registered tests and deligate for scan '''
        self.reporter.write_target(target)
        Scanner.logger.info("[%s] scanning:" % target)
        url = urlparse(target)
        is_ssl = url.scheme == "https"
        self.results = {}
        self.reporter.start_actives()
        self.active_tests_stack = self._active_tests_
        while len(self.active_tests_stack) > 0:
            testclass = self.active_tests_stack[0]
            self.active_tests_stack = self.active_tests_stack[1:]
	    testclass_key = "%s.%s" % (getmodule(testclass).__name__, testclass.__name__)
            self.results[testclass_key] = self.do_active_scan(testclass, is_ssl, target)
            if hasattr(testclass, 'events'): #TODO enforce every test to have event dict present?
                events_lower = dict([(k.lower(),v) for k,v in testclass.events.items()])
                if self.results[testclass_key]['state'].lower() in events_lower and events_lower[self.results[testclass_key]['state'].lower()] != None:
                   nexttest = events_lower[self.results[testclass_key]['state'].lower()]
                   Scanner.logger.info("\t[%s] Instantiated because %s declares it as its successor (the event was '%s')" %  (nexttest, testclass, self.results[testclass_key]['state']))
                   self.active_tests_stack.append(nexttest) # we have to hand over the response!!1, # important: we hand over an instance, not the class
            self._finished_active_tests_.append(testclass)
        self.reporter.end_actives()
        return self.results

    def run_scan(self):
        ''' iterate over target and deligate to list of tests '''
        results = {}
        self.reporter.start_report()
        self.reporter.start_targets()
        if len(self._targets_) == 0:
            Scanner.logger.error('No targets configured.')
            return
        for target in self._targets_:
            try:
                results[target] = self.scan_target(target)
            except:
                Scanner.logger.error(traceback.format_exc())
        self.reporter.end_targets()
	result = self.reporter.end_report(results)
	if (self.output == "-"):
	 print result
	else:
         file = open(self.output, "w")
         file.write(self.reporter.end_report(results))
         file.close()

    def register_target(self, url):
        ''' add target to the scanning engine '''
        u = urlparse(url)
        valid = u.netloc != "" and u.scheme in self._protos_
        reason = "%s%s" % ("[bad netloc]" if u.netloc == "" else "", "" if u.scheme in self._protos_ else "[bad scheme]")

        # todo - support ipv6 urls
        host = u.netloc.split(':')[0]
        if (self.resolve_target):
            try:
                socket.getaddrinfo(host, None)
            except socket.gaierror:
                valid = False
                reason = "%s[dns]" % reason
        else:
            valid = True
        if valid:
            self._targets_.append(url)
            Scanner.logger.debug("[target]: %s" % url)
            return
        Scanner.logger.error("%s is not a valid target (reason: %s)" % (url, reason))

    def configure_check(self, check_name, key, value):
        if check_name in map(lambda x: str(x), self._active_tests_):
            index = map(lambda x: str(x), self._active_tests_).index(check_name)
            check = self._active_tests_[index]
        if check_name in map(lambda x: str(x), self._passive_tests_):
            index = map(lambda x: str(x), self._active_tests_).index(check_name)
            check = self._active_tests_[index]
        else:
            raise Exception("The requested check is not available (%s)" % check_name)
        if hasattr(check, "config") == False:
            raise Exception("This check cannot be configured.")
        if check.config.has_key(key) == False:
            raise Exception("%s is not a valid configuration for %s", key, check_name)
        check.config[key] = value
        Scanner.logger.debug("\t%s.%s=%s" % (check_name, key, value))

    def disable_check(self, check_name):
        ''' add a previously added test to a blacklist of test that are to be skipped '''
        if check_name in map(lambda x: str(x).split('.')[-1], self._active_tests_) or check_name in map(lambda x: str(x).split('.')[-1], self._passive_tests_):
            self._disabled_tests_.append(check_name)
            Scanner.logger.debug("\t%s disabled.", check_name)
        else:
            print "The requested check is not available (%s)" % check_name
            print "The list of available checks is %s for actives and %s for passives" % (map(lambda x: str(x).split('.')[-1], self._active_tests_), map(lambda x: str(x).split('.')[-1], self._passive_tests_))
            Scanner.logger.debug("\t%s NOT disabled, because it could not be found.", check_name)

    def register_check(self, test):
        ''' add a test to the scanner '''
        module = test.__module__

        if module not in self.modules:
            self.modules.append(module)

        if hasattr(test, "execute"):
            self._active_tests_.append( test)
            Scanner.logger.debug("Added %s to active tests." % test)
            return len(self._active_tests_)
        if hasattr(test, "analyze"):
            self._passive_tests_.append( test)
            Scanner.logger.debug("Added %s to passive tests." % test)
            return len(self._passive_tests_)
        raise Exception('test is not a valid test type')

    def save_configuration(self, path):
        pass #XXX defunct
        # write out a configuration file.
        config = ConfigParser.RawConfigParser()
        config.add_section("Garmr")
        config.set("Garmr", "force-passives", self.force_passives)
        config.set("Garmr", "module", ", ".join(self.modules))
        config.set("Garmr", "reporter", self.reporter.__class__)
        config.set("Garmr", "output", self.output)
        config.set("Garmr", "dns", self.resolve_target)

        if len(self._targets_) > 0:
            config.add_section("Targets")
            for i,target in enumerate(self._targets_):
                config.set("Targets", "%s"%i, target)

        for index, check in enumerate(self._active_tests_):
            check = str(check)
            config.add_section(check)
            if check not in self._disabled_tests_:
                config.set(check, "enabled", True)
            else:
                config.set(check, "enabled", False)
            if hasattr(self._active_tests_[index], "config"):
                for key in self._active_tests_[index].config.keys():
                    config.set(check, key, self._active_tests_[index].config[key])

        for index, check in enumerate(self._passive_tests_):


from types import *
import pdb
#import dbg
#from GasTypes import *
#from token import *

class Recurse(Exception):
  def __init__(self,item):
    Exception.__init__(self)
    self.item = item

class Marker:
    ctype = "Marker"
    def __init__(self,name):
      self.name = name

    def __str__(self):
      return ""  # Return nothing so the markers are not output as text in the document

    def __repr__(self):
      return "<PyHtmlGen.document.Marker instance %s>" % self.name

Start = Marker("Start")
End   = Marker("End")

After   = 0
Before  = 1
Replace = 2

class Doc:
  "A generated document"
  ctype = "document.Doc"
  def __init__(self,name=None):
    self.doc = [Start,End]
    self.markers = {Start.name:0,End.name:1}
    self.markerByName = {Start.name:Start,End.name:End}
    self.markerIndex = 0
    self.subdocIndex = 0
    self.parent = None
    self.anonMarker = "mark "
    self.xforms     = []
    self.scratchPad = {}   # Any thing can store arbitrary data here for lookup during subsequent processing

  def SubDocFactory(self):
    "Derived classes should overload this so the the subdoc can be derived classes too"
    return Doc()

  def SubDoc(self):
    ret = self.SubDocFactory()
    ret.parent = self
    self.subdocIndex += 1
    ret.anonMarker = "s%d %s" % (self.subdocIndex, self.anonMarker)
    return ret

  def AddXform(self,typeOrPredicate, func):
    """Add a transformation function that is applied to all objects of type 'entity'.
       For example, this can be used to implement internationalization (i18n), by surrounding all
       text with a call to a server-side transformation function.

       @param typeOrPredicate A type or obj.__class__ to test equality or instance respectively, or a function that evaluates to True if the transformation should be applied
       @param func return a transformed object when complete, or raise Recurse(obj) to recurse
    """
    self.xforms.append((typeOrPredicate,func))

  def xform(self,s):
    """ Internal function that implements the installed transformations """
    while 1:
     try:
      for (tp,f) in self.xforms:
        if type(tp) is FunctionType:
          if tp(s): s = f(s); break
        elif isInstanceOf(s, tp): s = f(s); break
        elif tp is type(s): s = f(s); break
      break
     except Recurse, r:
      s = r.item
    return s

  def genDoc(self,lst):
    if type(lst) != type([]): lst = [lst]  # change into canonical form

    for item in lst:
      if type(item) == type(lst): # handle recursive lists
        self.genDoc(item)
      else:
        item = self.xform(item)  # User-installed transforms can override default generation
        if type(item) is InstanceType:
          if "gen" in dir(item):
            item.gen(self)
          elif "__str__" in dir(item):
            print "WARNING: __str__ function depreciated in favor of gen in object: ", item, "\n"
            assert(0)
            #self.Insert([chunkStr(str(item))],self.body,Before)
          else:
            raise AttributeError, "Obj: %s has neither a gen or __str__ function, so it cannot be converted into html." % item
        elif item is not None:  # A "simple" item just gets converted to a string and then output
          #self.Insert([chunkStr(str(item))],self.body,Before)
          self.Insert([str(item)],self.body,Before)

# Private
  def Marker2Index(self,marker):
    if type(marker) in StringTypes:  # Accept strings or markers
      name = marker
      marker = self.markerByName[name]
    else:
      name = marker.name

    if not self.markers.has_key(name):
      if self.parent is None:
        # raise KeyError, "Marker does not exist in the document"
        self.markers[name] = 0
      else:
        return self.parent.Marker2Index(marker)

    oldidx = self.markers[name]
    idx = oldidx
    while idx < len(self.doc):
      if self.doc[idx] == marker:
        self.markers[name] = idx
        return (self,idx)
      idx += 1
    assert(not "Marker should have been found, because it is in our dictionary")
    """ # Can only happen if you can delete chunks of the doc
    idx = oldidx
    while idx > 0:
      idx -= 1
      if self.doc[idx] == Marker:
        return idx
    """

# Public

  def Insert(self,tokenOrList,marker,whence=Before):
    if type(tokenOrList) == type([]):
      toks = tokenOrList
    else:
      toks = [tokenOrList]

    (d,idx) = self.Marker2Index(marker)
    if whence == Replace:
      del d.markers[marker]
      del d.doc[idx]
    elif whence == After:
      idx += 1

    d.doc[idx:idx] = toks

#   Code to check for an obj in the doc, if you find that something is not being converted.
#    for tok in toks:
#      if type(tok) is InstanceType:
#        assert(not tok.ctype == "chunkTag")

  # After creating a marker, you must insert it using the above fn
  def NewMarker(self,Name=None):
    if not Name:
      Name = self.anonMarker + str(self.markerIndex)
    if self.markers.has_key(Name):
      raise KeyError, "Attempted to create a new marker with the same name as an existing marker"
    self.markers[Name]=0
    marker = Marker(Name)
    self.markerByName[Name] = marker
    return marker

  def __str__(self):
    ret = "".join([str(tok) for tok in self.doc])
    return ret

#    ret = ""
#    for tok in self.doc:
#      ret += str(tok)
#    return ret

class DocSections(Doc):
  """This looks like a document but it actually just keeps a dictionary of markers and associated data.  Unlike a dict, there is no sequential positioning for all the markers.  This is useful when generating sub-documents and then merging them all into a single doc."""
  ctype = "document.Doc"
  def __init__(self,name=None):
    Doc.__init__(self,name=name)
    self.sections = {}
    self.markers = {}
    self.markerByName = {}
    self.markerIndex = 0
    self.subdocIndex = 0
    self.parent = None
    self.anonMarker = "mark "
    self.scratchPad = {}   # Any thing can store arbitrary data here for lookup during subsequent processing

  def update(self,doc):
    for (k,v) in self.sections.items():
      doc.Insert(v[0] + v[1],k.name,Before)

  def getSection(self,marker):
    t = self.sections[marker]
    return (t[0] + t[1])

  def SubDocFactory(self):
    "Derived classes should overload this so the the subdoc can be derived classes too"
    return DocSections()

  def __getattr__(self,name):
    "The assumption here is that a member variable defining a section is being accessed"
    if self.markerByName.has_key(name): return self.markerByName[name]
    else: return self.NewMarker(name)

# Public

  def Insert(self,tokenOrList,marker,whence=Before):
    if type(tokenOrList) == type([]):
      toks = tokenOrList
    else:
      toks = [tokenOrList]

    d = self


# -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

import functools as ft
import logging
import superdesk
from flask import current_app as app
from eve.utils import config, ParsedRequest
from copy import deepcopy
from apps.packages import PackageService
from superdesk.celery_task_utils import get_lock_id
from superdesk.utc import utcnow
from .archive import SOURCE as ARCHIVE
from superdesk.metadata.item import ITEM_STATE, CONTENT_STATE, ITEM_TYPE, CONTENT_TYPE, ASSOCIATIONS, MEDIA_TYPES
from superdesk.lock import lock, unlock, remove_locks
from superdesk.notification import push_notification
from superdesk import get_resource_service
from bson.objectid import ObjectId
from datetime import timedelta

logger = logging.getLogger(__name__)

def log_exeption(fn):
    @ft.wraps(fn)
    def inner(*a, **kw):
        try:
            return fn(*a, **kw)
        except Exception as e:
            logger.exception(e)
    return inner

class RemoveExpiredContent(superdesk.Command):
    log_msg = ''

    def run(self):
        now = utcnow()
        self.log_msg = 'Expiry Time: {}.'.format(now)
        logger.info('{} Starting to remove expired content at.'.format(self.log_msg))
        lock_name = get_lock_id('archive', 'remove_expired')

        if not lock(lock_name, expire=610):
            logger.info('{} Remove expired content task is already running.'.format(self.log_msg))
            return

        logger.info('{} Removing expired content for expiry.'.format(self.log_msg))
        # both functions should be called, even the first one throw exception,
        # so they are wrapped with log_exeption
        self._remove_expired_publish_queue_items()
        self._remove_expired_items(now)
        unlock(lock_name)

        push_notification('content:expired')
        logger.info('{} Completed remove expired content.'.format(self.log_msg))

        remove_locks()

    @log_exeption
    def _remove_expired_publish_queue_items(self):
        expire_interval = app.config.get('PUBLISH_QUEUE_EXPIRY_MINUTES', 0)
        if expire_interval:
            expire_time = utcnow() - timedelta(minutes=expire_interval)
            logger.info('{} Removing publish queue items created before {}'.format(self.log_msg, str(expire_time)))

            get_resource_service('publish_queue').delete({'_id': {'$lte': ObjectId.from_datetime(expire_time)}})

    @log_exeption
    def _remove_expired_items(self, expiry_datetime):
        """Remove the expired items.

        :param datetime expiry_datetime: expiry datetime
        :param str log_msg: log message to be prefixed
        """
        logger.info('{} Starting to remove published expired items.'.format(self.log_msg))
        archive_service = get_resource_service(ARCHIVE)
        published_service = get_resource_service('published')
        items_to_remove = set()
        items_to_be_archived = dict()
        items_having_issues = dict()

        for expired_items in archive_service.get_expired_items(expiry_datetime):
            if len(expired_items) == 0:
                logger.info('{} No items found to expire.'.format(self.log_msg))
                return

            # delete spiked items
            self.delete_spiked_items(expired_items)

            # get killed items
            killed_items = {item.get(config.ID_FIELD): item
                            for item in expired_items if item.get(ITEM_STATE) in {CONTENT_STATE.KILLED}}

            # check if killed items imported to legal
            items_having_issues.update(self.check_if_items_imported_to_legal_archive(killed_items))

            # filter out the killed items not imported to legal.
            killed_items = {item_id: item for item_id, item in killed_items.items()
                            if item_id not in items_having_issues}

            # Get the not killed and spiked items
            not_killed_items = {item.get(config.ID_FIELD): item for item in expired_items
                                if item.get(ITEM_STATE) not in {CONTENT_STATE.KILLED, CONTENT_STATE.SPIKED}}

            log_msg_format = "{{'_id': {_id}, 'unique_name': {unique_name}, 'version': {_current_version}, " \
                             "'expired_on': {expiry}}}."

            # Processing items to expire
            for item_id, item in not_killed_items.items():
                item.setdefault(config.VERSION, 1)
                item.setdefault('expiry', expiry_datetime)
                item.setdefault('unique_name', '')
                expiry_msg = log_msg_format.format(**item)
                logger.info('{} Processing expired item. {}'.format(self.log_msg, expiry_msg))

                processed_items = dict()
                if item_id not in items_to_be_archived and item_id not in items_having_issues and \
                        self._can_remove_item(item, processed_items):
                    # item can be archived and removed from the database
                    logger.info('{} Removing item. {}'.format(self.log_msg, expiry_msg))
                    logger.info('{} Items to be removed. {}'.format(self.log_msg, processed_items))
                    issues = self.check_if_items_imported_to_legal_archive(processed_items)
                    if issues:
                        items_having_issues.update(processed_items)
                    else:
                        items_to_be_archived.update(processed_items)

            # all items to expire
            items_to_expire = deepcopy(items_to_be_archived)

            # check once again in items imported to legal
            items_having_issues.update(self.check_if_items_imported_to_legal_archive(items_to_expire))
            if items_having_issues:
                # remove items not imported to legal
                items_to_expire = {item_id: item for item_id, item in items_to_expire.items()
                                   if item_id not in items_having_issues}

                # remove items not imported to legal from archived items
                items_to_be_archived = {item_id: item for item_id, item in items_to_be_archived.items()
                                        if item_id not in items_having_issues}

                # items_to_be_archived might contain killed items
                for item_id, item in items_to_be_archived.items():
                    if item.get(ITEM_STATE) == CONTENT_STATE.KILLED:
                        killed_items[item_id] = item

                # remove killed items from the items_to_be_archived
                items_to_be_archived = {item_id: item for item_id, item in items_to_be_archived.items()
                                        if item.get(ITEM_STATE) != CONTENT_STATE.KILLED}

            # add killed items to items to expire
            items_to_expire.update(killed_items)

            # get the filter conditions
            logger.info('{} filter conditions.'.format(self.log_msg))
            req = ParsedRequest()
            filter_conditions = list(get_resource_service('content_filters').get(req=req,
                                                                                lookup={'is_archived_filter': True}))

            # move to archived collection
            logger.info('{} Archiving items.'.format(self.log_msg))
            for item_id, item in items_to_be_archived.items():
                self._move_to_archived(item, filter_conditions)

            for item_id, item in killed_items.items():
                # delete from the published collection and queue
                msg = log_msg_format.format(**item)
                try:
                    published_service.delete_by_article_id(item_id)
                    logger.info('{} Deleting killed item from published. {}'.format(self.log_msg, msg))
                    items_to_remove.add(item_id)
                except Exception:
                    logger.exception('{} Failed to delete killed item from published. {}'.format(self.log_msg, msg))

            if items_to_remove:
                logger.info('{} Deleting articles.: {}'.format(self.log_msg, items_to_remove))
                archive_service.delete_by_article_ids(list(items_to_remove))

            for item_id, item in items_having_issues.items():
                msg = log_msg_format.format(**item)
                try:
                    archive_service.system_update(item.get(config.ID_FIELD), {'expiry_status': 'invalid'}, item)
                    logger.info('{} Setting item expiry status. {}'.format(self.log_msg, msg))
                except Exception:
                    logger.exception('{} Failed to set expiry status for item. {}'.format(self.log_msg, msg))

            logger.info('{} Deleting killed from archive.'.format(self.log_msg))

    def _can_remove_item(self, item, processed_item=None):
        """Recursively checks if the item can be removed.

        :param dict item: item to be remove
        :param set processed_item: processed items
        :return: True if item can be removed, False otherwise.
        """

        if processed_item is None:
            processed_item = dict()

        item_refs = []
        package_service = PackageService()
        archive_service = get_resource_service(ARCHIVE)

        if item.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
            # Get the item references for is package
            item_refs = package_service.get_residrefs(item)

        if item.get(ITEM_TYPE) in [CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED]:
            broadcast_items = get_resource_service('archive_broadcast').get_broadcast_items_from_master_story(item)
            # If master story expires then check if broadcast item is included in a package.
            # If included in a package then check the package expiry.
            item_refs.extend([broadcast_item.get(config.ID_FIELD) for broadcast_item in broadcast_items])

            if item.get('rewrite_of'):
                item_refs.append(item.get('rewrite_of'))

            if item.get('rewritten_by'):
                item_refs.append(item.get('rewritten_by'))

        # get the list of associated item ids
        if item.get(ITEM_TYPE) in MEDIA_TYPES:
            item_refs.extend(self._get_associated_items(item))
        else:
            item_refs.extend(self._get_associated_media_id(item))

        # get item reference where this referred
        item_refs.extend(package_service.get_linked_in_package_ids(item))

        # check item refs in the ids to remove set
        is_expired = item.get('expiry') and item.get('expiry') < utcnow()

        if is_expired:
            # now check recursively for all references
            if item.get(config.ID_FIELD) in processed_item:
                return is_expired

            processed_item[item.get(config.ID_FIELD)] = item
            if item_refs:
                archive_items = archive_service.get_from_mongo(req=None, lookup={'_id': {'$in': item_refs}})
                for archive_item in archive_items:
                    is_expired = self._can_remove_item(archive_item, processed_item)
                    if not is_expired:
                        break

        return is_expired

    def _get_associated_media_id(self, item):
        """Get the associated media id
        :param dict item:
        :return list: list of associated item ids
        """
        ids = []
        for key, associated_item in (item.get(ASSOCIATIONS) or {}).items():
            if associated_item:
                ids.append(associated_item.get(config.ID_FIELD))
        return ids

    def _get_associated_items(self, item):
        """Get the list of item ids where the media item is associated.
        :param dict item: media item
        :return list: list of associated item ids
        """
        if item.get(ITEM_TYPE) not in MEDIA_TYPES:
            return []

        associated_items = list(get_resource_service('media_references').get(req=None,
                                                                             lookup={
                                                                                'associated_id':
                                                                                item.get(config.ID_FIELD)
                                                                             }))

        ids = set()
        for assoc in associated_items:
            ids.add(assoc['item_id'])

        # extra query just to ensure that item is not deleted
        # get the associated items from the archive collection
        archive_docs = list(get_resource_service(ARCHIVE).get_from_mongo(req=None,
                                                                         lookup={'_id': {'$in': list(ids)}}))
        return [doc.get(config.ID_FIELD) for doc in archive_docs]

    def _move_to_archived(self, item, filter_conditions):
        """Moves all the published version of an article to archived.

        Deletes all published version of an article in the published collection

        :param str item_id: item_id of the document
        :param list filter_conditions: list of filter conditions
        """


import pytest
pytest.importorskip('numpy')

import numpy as np
import dask
import dask.array as da
from dask.optimization import fuse
from dask.utils import SerializableLock
from dask.array.core import getter, getter_nofancy
from dask.array.optimization import (getitem, optimize, optimize_slices,
                                     fuse_slice)
from dask.array.utils import assert_eq

def test_fuse_getitem():
    pairs = [((getter, (getter, 'x', slice(1000, 2000)), slice(15, 20)),
              (getter, 'x', slice(1015, 1020))),

             ((getitem, (getter, 'x', (slice(1000, 2000), slice(100, 200))),
                        (slice(15, 20), slice(50, 60))),
              (getter, 'x', (slice(1015, 1020), slice(150, 160)))),

             ((getitem, (getter_nofancy, 'x', (slice(1000, 2000), slice(100, 200))),
                        (slice(15, 20), slice(50, 60))),
              (getter_nofancy, 'x', (slice(1015, 1020), slice(150, 160)))),

             ((getter, (getter, 'x', slice(1000, 2000)), 10),
              (getter, 'x', 1010)),

             ((getitem, (getter, 'x', (slice(1000, 2000), 10)),
                        (slice(15, 20),)),
              (getter, 'x', (slice(1015, 1020), 10))),

             ((getitem, (getter_nofancy, 'x', (slice(1000, 2000), 10)),
                        (slice(15, 20),)),
              (getter_nofancy, 'x', (slice(1015, 1020), 10))),

             ((getter, (getter, 'x', (10, slice(1000, 2000))),
               (slice(15, 20), )),
              (getter, 'x', (10, slice(1015, 1020)))),

             ((getter, (getter, 'x', (slice(1000, 2000), slice(100, 200))),
               (slice(None, None), slice(50, 60))),
              (getter, 'x', (slice(1000, 2000), slice(150, 160)))),

             ((getter, (getter, 'x', (None, slice(None, None))),
               (slice(None, None), 5)),
              (getter, 'x', (None, 5))),

             ((getter, (getter, 'x', (slice(1000, 2000), slice(10, 20))),
               (slice(5, 10),)),
              (getter, 'x', (slice(1005, 1010), slice(10, 20)))),

             ((getitem, (getitem, 'x', (slice(1000, 2000),)),
               (slice(5, 10), slice(10, 20))),
              (getitem, 'x', (slice(1005, 1010), slice(10, 20)))),

             ((getter, (getter, 'x', slice(1000, 2000), False, False), slice(15, 20)),
              (getter, 'x', slice(1015, 1020))),

             ((getter, (getter, 'x', slice(1000, 2000)), slice(15, 20), False, False),
              (getter, 'x', slice(1015, 1020))),

             ((getter, (getter_nofancy, 'x', slice(1000, 2000), False, False),
               slice(15, 20), False, False),
              (getter_nofancy, 'x', slice(1015, 1020), False, False)),
             ]

    for inp, expected in pairs:
        result = optimize_slices({'y': inp})
        assert result == {'y': expected}

def test_fuse_getitem_lock():
    lock1 = SerializableLock()
    lock2 = SerializableLock()

    pairs = [((getter, (getter, 'x', slice(1000, 2000), True, lock1), slice(15, 20)),
              (getter, 'x', slice(1015, 1020), True, lock1)),

             ((getitem, (getter, 'x', (slice(1000, 2000), slice(100, 200)), True, lock1),
                        (slice(15, 20), slice(50, 60))),
              (getter, 'x', (slice(1015, 1020), slice(150, 160)), True, lock1)),

             ((getitem, (getter_nofancy, 'x', (slice(1000, 2000), slice(100, 200)), True, lock1),
                        (slice(15, 20), slice(50, 60))),
              (getter_nofancy, 'x', (slice(1015, 1020), slice(150, 160)), True, lock1)),

             ((getter, (getter, 'x', slice(1000, 2000), True, lock1), slice(15, 20), True, lock2),
              (getter, (getter, 'x', slice(1000, 2000), True, lock1), slice(15, 20), True, lock2))]

    for inp, expected in pairs:
        result = optimize_slices({'y': inp})
        assert result == {'y': expected}

def test_optimize_with_getitem_fusion():
    dsk = {'a': 'some-array',
           'b': (getter, 'a', (slice(10, 20), slice(100, 200))),
           'c': (getter, 'b', (5, slice(50, 60)))}

    result = optimize(dsk, ['c'])
    expected_task = (getter, 'some-array', (15, slice(150, 160)))
    assert any(v == expected_task for v in result.values())
    assert len(result) < len(dsk)

def test_optimize_slicing():
    dsk = {'a': (range, 10),
           'b': (getter, 'a', (slice(None, None, None),)),
           'c': (getter, 'b', (slice(None, None, None),)),
           'd': (getter, 'c', (slice(0, 5, None),)),
           'e': (getter, 'd', (slice(None, None, None),))}

    expected = {'e': (getter, (range, 10), (slice(0, 5, None),))}
    result = optimize_slices(fuse(dsk, [], rename_keys=False)[0])
    assert result == expected

    # protect output keys
    expected = {'c': (getter, (range, 10), (slice(0, None, None),)),
                'd': (getter, 'c', (slice(0, 5, None),)),
                'e': (getter, 'd', (slice(None, None, None),))}
    result = optimize_slices(fuse(dsk, ['c', 'd', 'e'], rename_keys=False)[0])

    assert result == expected

def test_fuse_slice():
    assert fuse_slice(slice(10, 15), slice(0, 5, 2)) == slice(10, 15, 2)

    assert (fuse_slice((slice(100, 200),), (None, slice(10, 20))) ==
            (None, slice(110, 120)))
    assert (fuse_slice((slice(100, 200),), (slice(10, 20), None)) ==
            (slice(110, 120), None))
    assert (fuse_slice((1,), (None,)) ==
            (1, None))
    assert (fuse_slice((1, slice(10, 20)), (None, None, 3, None)) ==
            (1, None, None, 13, None))

    with pytest.raises(NotImplementedError):
        fuse_slice(slice(10, 15, 2), -1)
    # Regression test for #3076
    with pytest.raises(NotImplementedError):
        fuse_slice(None, np.array([0, 0]))

def test_fuse_slice_with_lists():
    assert fuse_slice(slice(10, 20, 2), [1, 2, 3]) == [12, 14, 16]
    assert fuse_slice([10, 20, 30, 40, 50], [3, 1, 2]) == [40, 20, 30]
    assert fuse_slice([10, 20, 30, 40, 50], 3) == 40
    assert fuse_slice([10, 20, 30, 40, 50], -1) == 50
    assert fuse_slice([10, 20, 30, 40, 50], slice(1, None, 2)) == [20, 40]
    assert fuse_slice((slice(None), slice(0, 10), [1, 2, 3]),
                      (slice(None), slice(1, 5), slice(None))) == (slice(0, None), slice(1, 5), [1, 2, 3])
    assert fuse_slice((slice(None), slice(None), [1, 2, 3]),
                      (slice(None), slice(1, 5), 1)) == (slice(0, None), slice(1, 5), 2)

def test_nonfusible_fancy_indexing():
    nil = slice(None)
    cases = [# x[:, list, :][int, :, :]
             ((nil, [1, 2, 3], nil), (0, nil, nil)),
             # x[int, :, :][:, list, :]
             ((0, nil, nil), (nil, [1, 2, 3], nil)),
             # x[:, list, :, :][:, :, :, int]
             ((nil, [1, 2], nil, nil), (nil, nil, nil, 0))]

    for a, b in cases:
        with pytest.raises(NotImplementedError):
            fuse_slice(a, b)

def test_hard_fuse_slice_cases():
    dsk = {'x': (getter, (getter, 'x', (None, slice(None, None))),
                 (slice(None, None), 5))}
    assert optimize_slices(dsk) == {'x': (getter, 'x', (None, 5))}

def test_dont_fuse_numpy_arrays():
    x = np.ones(10)
    for chunks in [(5,), (10,)]:
        y = da.from_array(x, chunks=(10,))

        dsk = y.__dask_optimize__(y.dask, y.__dask_keys__())
        assert sum(isinstance(v, np.ndarray) for v in dsk.values()) == 1

def test_minimize_data_transfer():
    x = np.ones(100)
    y = da.from_array(x, chunks=25)
    z = y + 1
    dsk = z.__dask_optimize__(z.dask, z.__dask_keys__())

    keys = list(dsk)
    results = dask.get(dsk, keys)
    big_key = [k for k, r in zip(keys, results) if r is x][0]
    dependencies, dependents = dask.core.get_deps(dsk)
    deps = dependents[big_key]

    assert len(deps) == 4
    for dep in deps:
        assert dsk[dep][0] in (getitem, getter)
        assert dsk[dep][1] == big_key

def test_fuse_slices_with_alias():
    dsk = {'x': np.arange(16).reshape((4, 4)),


# -*- coding: utf-8 -*-

from openerp.osv import fields, osv

class sessionpos(osv.Model):

    def _fun_difference(self,cr,uid,ids,fields,args,context=None):
        res={}
        for session in self.browse(cr,uid,ids,context=context):
            total=0
            totali=0
            totali=session.cash_register_balance_end
            totalf=session.cash_register_balance_end_real
            for order in session.order_ids:
                flag=False
                for producto in order.lines:
                    if producto.product_id.expense_pdt:
                        print producto.product_id.name
                        flag=True
                if flag==True:
                    totali-=(order.amount_total*2)

            total= (totali - totalf)
            res[session.id]=total

            if total<0:
                total=-total
            else:
                total=-total

            if session.state!='closed':
                self.write(cr,uid,session.id,{'difference2':total},context=context)
                self.write(cr,uid,session.id,{'money_close':totali},context=context)
                self.write(cr,uid,session.id,{'money_reported':totalf},context=context)
        return res

    def _calc_vb(self,cr,uid,ids,fields,args,context=None):
        res={}
        flag=False
        for session in self.browse(cr,uid,ids,context=context):
            total=0
            for order in session.order_ids:
                flag=False
                for producto in order.lines:
                    if producto.product_id.expense_pdt or producto.product_id.income_pdt:
                        flag=True
                if flag==False:
                    total+=order.amount_total
            res[session.id]=total
        return res

    def _calc_statements_total(self,cr,uid,ids,fields,args,context=None):
        res={}
        for session in self.browse(cr,uid,ids,context=context):
            total=0
            for st in session.statement_ids:
                total += st.total_entry_encoding
            res[session.id]=total
        return res

    def _calc_isv(self,cr,uid,ids,fields,args,context=None):
        res={}
        for session in self.browse(cr,uid,ids,context=context):
            total=0
            for order in session.order_ids:
                total+=order.amount_tax
            res[session.id]=total
        return res

    def _calc_subtotal(self,cr,uid,ids,fields,args,context=None):
        res={}
        for session in self.browse(cr,uid,ids,context=context):
            total=session.venta_bruta-session.isv
            res[session.id]=total
        return res

    def _calc_no_facturas(self,cr,uid,ids,fields,args,context=None):
        res={}
        array=[]
        count=0
        for session in self.browse(cr,uid,ids,context=context):
            for order in session.order_ids:
                count+=1
                array.append(order.pos_reference)
            if array:
                res[session.id]=str(count) + " facturas "+str(array[len(array)-1])+" A "+str(array[0])

        return res

    def _calc_discount(self,cr,uid,ids,fields,args,context=None):
        res={}
        for session in self.browse(cr,uid,ids,context=context):
            des_total=0
            for order in session.order_ids:
                discount=0
                for desc in order.lines:
                    discount+=desc.price_unit*(desc.discount/100)
                des_total+=discount
            res[session.id]=des_total
        return res

    def _calc_money_incoming(self,cr,uid,ids,fields,args,context=None):
        res={}
        for session in self.browse(cr,uid,ids,context=context):
            total=0
            counttotal=0

            for order in session.order_ids:
                total2=0
                count=0
                for desc in order.lines:
                    if desc.product_id.income_pdt:
                        count+=1
                        total2+=desc.price_subtotal_incl
                total+=total2
                counttotal+=count
            res[session.id]=str(counttotal) + " Entrada(s) "+" Total Entradas "+ str(total)
        return res

    def _calc_money_outgoing(self,cr,uid,ids,fields,args,context=None):
        res={}
        for session in self.browse(cr,uid,ids,context=context):
            total=0
            counttotal=0
            for order in session.order_ids:
                total2=0
                count=0
                for desc in order.lines:
                    if desc.product_id.expense_pdt:
                        count+=1
                        total2+=desc.price_subtotal_incl
                total+=total2
                counttotal+=count
            res[session.id]=str(counttotal) + " Salida(s) "+"  Total Salidas "+ str(total)
        return res

    def _calc_tickets(self, cr, uid, ids, name, args, context=None):
        res = {}
        for session in self.browse(cr,uid,ids,context=context):
            res[session.id] = {
                'tickets_num': 0,
                'ticket_first_id': None,
                'ticket_last_id': None,
            }
            if session.order_ids:
                res[session.id]['tickets_num'] = len(session.order_ids)
                res[session.id]['ticket_first_id'] = session.order_ids[-1]
                res[session.id]['ticket_last_id'] = session.order_ids[0]
        return res

    def summary_by_product(self, cr, uid, ids, context=None):
        assert len(ids) == 1, 'This option should only be used for a single id at a time.'
        products = {} # product_id -> data
        for session in self.browse(cr,uid,ids,context=context):
            for order in session.order_ids:
                for line in order.lines:
                    id = line.product_id.id
                    if id not in products:
                        products[id] = {'product':line.product_id.name,
                                        'qty': 0,
                                        'total': 0}
                    products[id]['qty'] += line.qty
                    products[id]['total'] += line.price_subtotal_incl
        return products.values()

    def summary_by_tax(self, cr, uid, ids, context=None):
        assert len(ids) == 1, 'This option should only be used for a single id at a time.'
        account_tax_obj = self.pool.get('account.tax')
        cur_obj = self.pool.get('res.currency')
        res = {} # tax_id -> data
        for session in self.browse(cr,uid,ids,context=context):
            for order in session.order_ids:
                for line in order.lines:
                    taxes_ids = [ tax for tax in line.product_id.taxes_id if tax.company_id.id == line.order_id.company_id.id ]

                    price = line.price_unit * (1 - (line.discount or 0.0) / 100.0)
                    taxes = account_tax_obj.compute_all(cr, uid, taxes_ids, price, line.qty, product=line.product_id, partner=line.order_id.partner_id or False)
                    cur = line.order_id.pricelist_id.currency_id

                    for tax in taxes['taxes']:
                        id = tax['tax_code_id']
                        if id not in res:
                            t = account_tax_obj.browse(cr, uid, id, context=context)
                            tax_rule = ''
                            if t.type == 'percent':
                                tax_rule = str(100*t.amount) + '%'
                            else:
                                tax_rule = str(t.amount)
                            res[id] = {'name': tax['name'],
                                       'base': 0,
                                       'tax': tax_rule,
                                       'total': 0,
                                   }
                        res[id]['base'] += price*line.qty
                        res[id]['total'] += tax['amount']
                        #cur_obj.round(cr, uid, cur, taxes['amount'])

        return res.values()
    def _calc_tax(self, cr, uid, ids, name, args, context=None):
        account_tax_obj = self.pool.get('account.tax')
        cur_obj = self.pool.get('res.currency')
        res = {}
        for session in self.browse(cr,uid,ids,context=context):
            res[session.id] = {'tax_base_total':0}
            for order in session.order_ids:
                for line in order.lines:
                    taxes_ids = [ tax for tax in line.product_id.taxes_id if tax.company_id.id == line.order_id.company_id.id ]

                    price = line.price_unit * (1 - (line.discount or 0.0) / 100.0)
                    taxes = account_tax_obj.compute_all(cr, uid, taxes_ids, price, line.qty, product=line.product_id, partner=line.order_id.partner_id or False)
                    cur = line.order_id.pricelist_id.currency_id

                    res[session.id]['tax_base_total'] += taxes['total']
        return res

    def _calc_sales(self, cr, uid, ids, name, args, context=None):
        res = {}
        for session in self.browse(cr,uid,ids,context=context):