from __future__ import division

import hashlib
import random
import warnings

import p2pool
from p2pool.util import math, pack

def hash256(data):
    return pack.IntType(256).unpack(hashlib.sha256(hashlib.sha256(data).digest()).digest())

def hash160(data):
    if data == '04ffd03de44a6e11b9917f3a29f9443283d9871c9d743ef30d5eddcd37094b64d1b3d8090496b53256786bf5c82932ec23c3b74d9f05a6f95a8b5529352656664b'.decode('hex'):
        return 0x384f570ccc88ac2e7e00b026d1690a3fca63dd0 # hack for people who don't have openssl - this is the only value that p2pool ever hashes
    return pack.IntType(160).unpack(hashlib.new('ripemd160', hashlib.sha256(data).digest()).digest())

class ChecksummedType(pack.Type):
    def __init__(self, inner, checksum_func=lambda data: hashlib.sha256(hashlib.sha256(data).digest()).digest()[:4]):
        self.inner = inner
        self.checksum_func = checksum_func

    def read(self, file):
        obj, file = self.inner.read(file)
        data = self.inner.pack(obj)

        calculated_checksum = self.checksum_func(data)
        checksum, file = pack.read(file, len(calculated_checksum))
        if checksum != calculated_checksum:
            raise ValueError('invalid checksum')

        return obj, file

    def write(self, file, item):
        data = self.inner.pack(item)
        return (file, data), self.checksum_func(data)

class FloatingInteger(object):
    __slots__ = ['bits', '_target']

    @classmethod
    def from_target_upper_bound(cls, target):
        n = math.natural_to_string(target)
        if n and ord(n[0]) >= 128:
            n = '\x00' + n
        bits2 = (chr(len(n)) + (n + 3*chr(0))[:3])[::-1]
        bits = pack.IntType(32).unpack(bits2)
        return cls(bits)

    def __init__(self, bits, target=None):
        self.bits = bits
        self._target = None
        if target is not None and self.target != target:
            raise ValueError('target does not match')

    @property
    def target(self):
        res = self._target
        if res is None:
            res = self._target = math.shift_left(self.bits & 0x00ffffff, 8 * ((self.bits >> 24) - 3))
        return res

    def __hash__(self):
        return hash(self.bits)

    def __eq__(self, other):
        return self.bits == other.bits

    def __ne__(self, other):
        return not (self == other)

    def __cmp__(self, other):
        assert False

    def __repr__(self):
        return 'FloatingInteger(bits=%s, target=%s)' % (hex(self.bits), hex(self.target))

class FloatingIntegerType(pack.Type):
    _inner = pack.IntType(32)

    def read(self, file):
        bits, file = self._inner.read(file)
        return FloatingInteger(bits), file

    def write(self, file, item):
        return self._inner.write(file, item.bits)

address_type = pack.ComposedType([
    ('services', pack.IntType(64)),
    ('address', pack.IPV6AddressType()),
    ('port', pack.IntType(16, 'big')),
])

def is_segwit_tx(tx):
    return tx.get('marker', -1) == 0 and tx.get('flag', -1) >= 1

tx_in_type = pack.ComposedType([
    ('previous_output', pack.PossiblyNoneType(dict(hash=0, index=2**32 - 1), pack.ComposedType([
        ('hash', pack.IntType(256)),
        ('index', pack.IntType(32)),
    ]))),
    ('script', pack.VarStrType()),
    ('sequence', pack.PossiblyNoneType(2**32 - 1, pack.IntType(32))),
])

tx_out_type = pack.ComposedType([
    ('value', pack.IntType(64)),
    ('script', pack.VarStrType()),
])

tx_id_type = pack.ComposedType([
    ('version', pack.IntType(32)),
    ('tx_ins', pack.ListType(tx_in_type)),
    ('tx_outs', pack.ListType(tx_out_type)),
    ('lock_time', pack.IntType(32))
])

class TransactionType(pack.Type):
    _int_type = pack.IntType(32)
    _varint_type = pack.VarIntType()
    _witness_type = pack.ListType(pack.VarStrType())
    _wtx_type = pack.ComposedType([
        ('flag', pack.IntType(8)),
        ('tx_ins', pack.ListType(tx_in_type)),
        ('tx_outs', pack.ListType(tx_out_type))
    ])
    _ntx_type = pack.ComposedType([
        ('tx_outs', pack.ListType(tx_out_type)),
        ('lock_time', _int_type)
    ])
    _write_type = pack.ComposedType([
        ('version', _int_type),
        ('marker', pack.IntType(8)),
        ('flag', pack.IntType(8)),
        ('tx_ins', pack.ListType(tx_in_type)),
        ('tx_outs', pack.ListType(tx_out_type))
    ])

    def read(self, file):
        version, file = self._int_type.read(file)
        marker, file = self._varint_type.read(file)
        if marker == 0:
            next, file = self._wtx_type.read(file)
            witness = [None]*len(next['tx_ins'])
            for i in xrange(len(next['tx_ins'])):
                witness[i], file = self._witness_type.read(file)
            locktime, file = self._int_type.read(file)
            return dict(version=version, marker=marker, flag=next['flag'], tx_ins=next['tx_ins'], tx_outs=next['tx_outs'], witness=witness, lock_time=locktime), file
        else:
            tx_ins = [None]*marker
            for i in xrange(marker):
                tx_ins[i], file = tx_in_type.read(file)
            next, file = self._ntx_type.read(file)
            return dict(version=version, tx_ins=tx_ins, tx_outs=next['tx_outs'], lock_time=next['lock_time']), file

    def write(self, file, item):
        if is_segwit_tx(item):
            assert len(item['tx_ins']) == len(item['witness'])
            res = self._write_type.pack(item)
            for w in item['witness']:
                res += self._witness_type.pack(w)
            res += self._int_type.pack(item['lock_time'])
            return file, res
        return tx_id_type.write(file, item)

tx_type = TransactionType()

merkle_link_type = pack.ComposedType([
    ('branch', pack.ListType(pack.IntType(256))),
    ('index', pack.IntType(32)),
])

merkle_tx_type = pack.ComposedType([
    ('tx', tx_id_type), # used only in aux_pow_type
    ('block_hash', pack.IntType(256)),
    ('merkle_link', merkle_link_type),
])

block_header_type = pack.ComposedType([
    ('version', pack.IntType(32)),
    ('previous_block', pack.PossiblyNoneType(0, pack.IntType(256))),
    ('merkle_root', pack.IntType(256)),
    ('timestamp', pack.IntType(32)),
    ('bits', FloatingIntegerType()),
    ('nonce', pack.IntType(32)),
])

block_type = pack.ComposedType([
    ('header', block_header_type),
    ('txs', pack.ListType(tx_type)),
])

stripped_block_type = pack.ComposedType([
    ('header', block_header_type),
    ('txs', pack.ListType(tx_id_type)),
])

# merged mining

aux_pow_type = pack.ComposedType([
    ('merkle_tx', merkle_tx_type),
    ('merkle_link', merkle_link_type),
    ('parent_block_header', block_header_type),
])

aux_pow_coinbase_type = pack.ComposedType([
    ('merkle_root', pack.IntType(256, 'big')),
    ('size', pack.IntType(32)),
    ('nonce', pack.IntType(32)),
])

def make_auxpow_tree(chain_ids):
    for size in (2**i for i in xrange(31)):
        if size < len(chain_ids):
            continue
        res = {}
        for chain_id in chain_ids:
            pos = (1103515245 * chain_id + 1103515245 * 12345 + 12345) % size
            if pos in res:
                break
            res[pos] = chain_id
        else:
            return res, size
    raise AssertionError()

# merkle trees

merkle_record_type = pack.ComposedType([
    ('left', pack.IntType(256)),
    ('right', pack.IntType(256)),
])

def merkle_hash(hashes):
    if not hashes:
        return 0
    hash_list = list(hashes)
    while len(hash_list) > 1:
        hash_list = [hash256(merkle_record_type.pack(dict(left=left, right=right)))
            for left, right in zip(hash_list[::2], hash_list[1::2] + [hash_list[::2][-1]])]
    return hash_list[0]

def calculate_merkle_link(hashes, index):
    # XXX optimize this

    hash_list = [(lambda _h=h: _h, i == index, []) for i, h in enumerate(hashes)]

    while len(hash_list) > 1:
        hash_list = [
            (
                lambda _left=left, _right=right: hash256(merkle_record_type.pack(dict(left=_left(), right=_right()))),
                left_f or right_f,
                (left_l if left_f else right_l) + [dict(side=1, hash=right) if left_f else dict(side=0, hash=left)],
            )
            for (left, left_f, left_l), (right, right_f, right_l) in
                zip(hash_list[::2], hash_list[1::2] + [hash_list[::2][-1]])
        ]

    res = [x['hash']() for x in hash_list[0][2]]

    assert hash_list[0][1]
    if p2pool.DEBUG:
        new_hashes = [random.randrange(2**256) if x is None else x
            for x in hashes]
        assert check_merkle_link(new_hashes[index], dict(branch=res, index=index)) == merkle_hash(new_hashes)
    assert index == sum(k*2**i for i, k in enumerate([1-x['side'] for x in hash_list[0][2]]))

    return dict(branch=res, index=index)

def check_merkle_link(tip_hash, link):
    if link['index'] >= 2**len(link['branch']):
        raise ValueError('index too large')
    return reduce(lambda c, (i, h): hash256(merkle_record_type.pack(
        dict(left=h, right=c) if (link['index'] >> i) & 1 else
        dict(left=c, right=h)
    )), enumerate(link['branch']), tip_hash)

# targets

def target_to_average_attempts(target):
    assert 0 <= target and isinstance(target, (int, long)), target
    if target >= 2**256: warnings.warn('target >= 2**256!')
    return 2**256//(target + 1)

def average_attempts_to_target(average_attempts):
    assert average_attempts > 0
    return min(int(2**256/average_attempts - 1 + 0.5), 2**256-1)

def target_to_difficulty(target):
    assert 0 <= target and isinstance(target, (int, long)), target
    if target >= 2**256: warnings.warn('target >= 2**256!')
    return (0xffff0000 * 2**(256-64) + 1)/(target + 1)

def difficulty_to_target(difficulty):
    assert difficulty >= 0
    if difficulty == 0: return 2**256-1
    return min(int((0xffff0000 * 2**(256-64) + 1)/difficulty - 1 + 0.5), 2**256-1)

# human addresses

base58_alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'

def base58_encode(bindata):


"""Helper classes for tests."""

# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
__license__ = "MIT"

import pickle
import copy
import functools
import unittest
from unittest import TestCase
from lib.bs4 import BeautifulSoup
from lib.bs4.element import (
    CharsetMetaAttributeValue,
    Comment,
    ContentMetaAttributeValue,
    Doctype,
    SoupStrainer,
)

from bs4.builder import HTMLParserTreeBuilder
default_builder = HTMLParserTreeBuilder

class SoupTest(unittest.TestCase):

    @property
    def default_builder(self):
        return default_builder()

    def soup(self, markup, **kwargs):
        """Build a Beautiful Soup object from markup."""
        builder = kwargs.pop('builder', self.default_builder)
        return BeautifulSoup(markup, builder=builder, **kwargs)

    def document_for(self, markup):
        """Turn an HTML fragment into a document.

        The details depend on the builder.
        """
        return self.default_builder.test_fragment_to_document(markup)

    def assertSoupEquals(self, to_parse, compare_parsed_to=None):
        builder = self.default_builder
        obj = BeautifulSoup(to_parse, builder=builder)
        if compare_parsed_to is None:
            compare_parsed_to = to_parse

        self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))

    def assertConnectedness(self, element):
        """Ensure that next_element and previous_element are properly
        set for all descendants of the given element.
        """
        earlier = None
        for e in element.descendants:
            if earlier:
                self.assertEqual(e, earlier.next_element)
                self.assertEqual(earlier, e.previous_element)
            earlier = e

class HTMLTreeBuilderSmokeTest(object):

    """A basic test of a treebuilder's competence.

    Any HTML treebuilder, present or future, should be able to pass
    these tests. With invalid markup, there's room for interpretation,
    and different parsers can handle it differently. But with the
    markup in these tests, there's not much room for interpretation.
    """

    def test_pickle_and_unpickle_identity(self):
        # Pickling a tree, then unpickling it, yields a tree identical
        # to the original.
        tree = self.soup("foo")
        dumped = pickle.dumps(tree, 2)
        loaded = pickle.loads(dumped)
        self.assertEqual(loaded.__class__, BeautifulSoup)
        self.assertEqual(loaded.decode(), tree.decode())

    def assertDoctypeHandled(self, doctype_fragment):
        """Assert that a given doctype string is handled correctly."""
        doctype_str, soup = self._document_with_doctype(doctype_fragment)

        # Make sure a Doctype object was created.
        doctype = soup.contents[0]
        self.assertEqual(doctype.__class__, Doctype)
        self.assertEqual(doctype, doctype_fragment)
        self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)

        # Make sure that the doctype was correctly associated with the
        # parse tree and that the rest of the document parsed.
        self.assertEqual(soup.p.contents[0], 'foo')

    def _document_with_doctype(self, doctype_fragment):
        """Generate and parse a document with the given doctype."""
        doctype = '<!DOCTYPE %s>' % doctype_fragment
        markup = doctype + '\n
foo
'
        soup = self.soup(markup)
        return doctype, soup

    def test_normal_doctypes(self):
        """Make sure normal, everyday HTML doctypes are handled correctly."""
        self.assertDoctypeHandled("html")
        self.assertDoctypeHandled(
            'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')

    def test_empty_doctype(self):
        soup = self.soup("<!DOCTYPE>")
        doctype = soup.contents[0]
        self.assertEqual("", doctype.strip())

    def test_public_doctype_with_url(self):
        doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
        self.assertDoctypeHandled(doctype)

    def test_system_doctype(self):
        self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')

    def test_namespaced_system_doctype(self):
        # We can handle a namespaced doctype with a system ID.
        self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')

    def test_namespaced_public_doctype(self):
        # Test a namespaced doctype with a public id.
        self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')

    def test_real_xhtml_document(self):
        """A real XHTML document should come out more or less the same as it went in."""
        markup = b"""<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">

<title>Hello.</title>
Goodbye.
"""
        soup = self.soup(markup)
        self.assertEqual(
            soup.encode("utf-8").replace(b"\n", b""),
            markup.replace(b"\n", b""))

    def test_processing_instruction(self):
        # We test both Unicode and bytestring to verify that
        # process_markup correctly sets processing_instruction_class
        # even when the markup is already Unicode and there is no
        # need to process anything.
        markup = u"""<?PITarget PIContent?>"""
        soup = self.soup(markup)
        self.assertEqual(markup, soup.decode())

        markup = b"""<?PITarget PIContent?>"""
        soup = self.soup(markup)
        self.assertEqual(markup, soup.encode("utf8"))

    def test_deepcopy(self):
        """Make sure you can copy the tree builder.

        This is important because the builder is part of a
        BeautifulSoup object, and we want to be able to copy that.
        """
        copy.deepcopy(self.default_builder)

    def test_p_tag_is_never_empty_element(self):
        """A
 tag is never designated as an empty-element tag.

        Even if the markup shows it as an empty-element tag, it
        shouldn't be presented that way.
        """
        soup = self.soup("
")
        self.assertFalse(soup.p.is_empty_element)
        self.assertEqual(str(soup.p), "

")

    def test_unclosed_tags_get_closed(self):
        """A tag that's not closed by the end of the document should be closed.

        This applies to all tags except empty-element tags.
        """
        self.assertSoupEquals("
", "

")
        self.assertSoupEquals("", "")

        self.assertSoupEquals("
", "
")

    def test_br_is_always_empty_element_tag(self):
        """A
 tag is designated as an empty-element tag.

        Some parsers treat

 as one
 tag, some parsers as
        two tags, but it should always be an empty-element tag.
        """
        soup = self.soup("

")
        self.assertTrue(soup.br.is_empty_element)
        self.assertEqual(str(soup.br), "
")

    def test_nested_formatting_elements(self):
        self.assertSoupEquals("")

    def test_double_head(self):
        html = '''<!DOCTYPE html>

<title>Ordinary HEAD element test</title>

Hello, world!

'''
        soup = self.soup(html)
        self.assertEqual("text/javascript", soup.find('script')['type'])

    def test_comment(self):
        # Comments are represented as Comment objects.
        markup = "
foobaz
"
        self.assertSoupEquals(markup)

        soup = self.soup(markup)
        comment = soup.find(text="foobar")
        self.assertEqual(comment.__class__, Comment)

        # The comment is properly integrated into the tree.
        foo = soup.find(text="foo")
        self.assertEqual(comment, foo.next_element)
        baz = soup.find(text="baz")
        self.assertEqual(comment, baz.previous_element)

    def test_preserved_whitespace_in_pre_and_textarea(self):
        """Whitespace must be preserved in
 and <textarea> tags,
        even if that would mean not prettifying the markup.
        """
        pre_markup = "

"
        textarea_markup = "<textarea> woo\nwoo  </textarea>"
        self.assertSoupEquals(pre_markup)
        self.assertSoupEquals(textarea_markup)

        soup = self.soup(pre_markup)
        self.assertEqual(soup.pre.prettify(), pre_markup)

        soup = self.soup(textarea_markup)
        self.assertEqual(soup.textarea.prettify(), textarea_markup)

        soup = self.soup("<textarea></textarea>")
        self.assertEqual(soup.textarea.prettify(), "<textarea></textarea>")

    def test_nested_inline_elements(self):
        """Inline elements can be nested indefinitely."""
        b_tag = "Inside a B tag"
        self.assertSoupEquals(b_tag)

        nested_b_tag = "
A nested tag
"
        self.assertSoupEquals(nested_b_tag)

        double_nested_b_tag = "
A doubly nested tag
"
        self.assertSoupEquals(nested_b_tag)

    def test_nested_block_level_elements(self):
        """Block elements can be nested."""
        soup = self.soup('

Foo

')
        blockquote = soup.blockquote
        self.assertEqual(blockquote.p.b.string, 'Foo')
        self.assertEqual(blockquote.b.string, 'Foo')

    def test_correctly_nested_tables(self):
        """One table can go inside another one."""
        markup = ('
'
                  '
'
                  "Here's another table:"
                  '
'
                  '
foo
'
                  '
	')

        self.assertSoupEquals(
            markup,
            '

Here\'s another table:'
            '

foo

'
            '

')

        self.assertSoupEquals(
            "

Foo

"
            "

Bar

"
            "

Baz

")

    def test_deeply_nested_multivalued_attribute(self):
        # html5lib can set the attributes of the same tag many times
        # as it rearranges the tree. This has caused problems with
        # multivalued attributes.
        markup = '

'
        soup = self.soup(markup)
        self.assertEqual(["css"], soup.div.div['class'])

    def test_multivalued_attribute_on_html(self):
        # html5lib uses a different API to set the attributes ot the
        #  tag. This has caused problems with multivalued
        # attributes.
        markup = ''
        soup = self.soup(markup)
        self.assertEqual(["a", "b"], soup.html['class'])

    def test_angle_brackets_in_attribute_values_are_escaped(self):
        self.assertSoupEquals('<a b="">', '')

    def test_entities_in_attributes_converted_to_unicode(self):
        expect = u'

'
        self.assertSoupEquals('

', expect)
        self.assertSoupEquals('

', expect)
        self.assertSoupEquals('

', expect)
        self.assertSoupEquals('

', expect)

    def test_entities_in_text_converted_to_unicode(self):
        expect = u'
pi\N{LATIN SMALL LETTER N WITH TILDE}ata
'
        self.assertSoupEquals("
piñata
", expect)
        self.assertSoupEquals("
piñata
", expect)
        self.assertSoupEquals("
piñata
", expect)
        self.assertSoupEquals("
piñata
", expect)

    def test_quot_entity_converted_to_quotation_mark(self):
        self.assertSoupEquals("
I said "good day!"
",
                              '
I said "good day!"
')

    def test_out_of_range_entity(self):
        expect = u"\N{REPLACEMENT CHARACTER}"
        self.assertSoupEquals("&#10000000000000;", expect)
        self.assertSoupEquals("&#x10000000000000;", expect)
        self.assertSoupEquals("&#1000000000;", expect)

    def test_multipart_strings(self):
        "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
        soup = self.soup("
\nfoo

")
        self.assertEqual("p", soup.h2.string.next_element.name)
        self.assertEqual("p", soup.p.name)
        self.assertConnectedness(soup)

    def test_head_tag_between_head_and_body(self):
        "Prevent recurrence of a bug in the html5lib treebuilder."
        content = """
  <link></link>
  foo

"""
        soup = self.soup(content)
        self.assertNotEqual(None, soup.html.body)
        self.assertConnectedness(soup)

    def test_multiple_copies_of_a_tag(self):
        "Prevent recurrence of a bug in the html5lib treebuilder."
        content = """<!DOCTYPE html>

"""
        soup = self.soup(content)
        self.assertConnectedness(soup.article)

    def test_basic_namespaces(self):
        """Parsers don't need to *understand* namespaces, but at the
        very least they should not choke on namespaces or lose
        data."""

        markup = b'<mathml:msqrt>4</mathml:msqrt>'
        soup = self.soup(markup)
        self.assertEqual(markup, soup.encode())
        html = soup.html
        self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
        self.assertEqual(
            'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
        self.assertEqual(
            'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])

    def test_multivalued_attribute_value_becomes_list(self):
        markup = b''
        soup = self.soup(markup)
        self.assertEqual(['foo', 'bar'], soup.a['class'])

    #
    # Generally speaking, tests below this point are more tests of
    # Beautiful Soup than tests of the tree builders. But parsers are
    # weird, so we run these tests separately for every tree builder
    # to detect any differences between them.
    #

    def test_can_parse_unicode_document(self):
        # A seemingly innocuous document... but it's in Unicode! And
        # it contains characters that can't be represented in the
        # encoding found in the  declaration! The horror!
        markup = u'<meta encoding="euc-jp">Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
        soup = self.soup(markup)
        self.assertEqual(u'Sacr\xe9 bleu!', soup.body.string)

    def test_soupstrainer(self):
        """Parsers should be able to work with SoupStrainers."""
        strainer = SoupStrainer("b")
        soup = self.soup("A bold <meta/> statement",
                         parse_only=strainer)
        self.assertEqual(soup.decode(), "bold")

    def test_single_quote_attribute_values_become_double_quotes(self):
        self.assertSoupEquals("<foo attr='bar'></foo>",
                              '<foo attr="bar"></foo>')

    def test_attribute_values_with_nested_quotes_are_left_alone(self):
        text = """<foo attr='bar "brawls" happen'>a</foo>"""
        self.assertSoupEquals(text)

    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
        text = """<foo attr='bar "brawls" happen'>a</foo>"""
        soup = self.soup(text)
        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
        self.assertSoupEquals(
            soup.foo.decode(),
            """<foo attr="Brawls happen at "Bob\'s Bar"">a</foo>""")

    def test_ampersand_in_attribute_value_gets_escaped(self):
        self.assertSoupEquals('<this is="really messed up & stuff"></this>',
                              '<this is="really messed up & stuff"></this>')

        self.assertSoupEquals(
            'foo',
            'foo')

    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
        self.assertSoupEquals('')

    def test_entities_in_strings_converted_during_parsing(self):
        # Both XML and HTML entities are converted to Unicode characters
        # during parsing.
        text = "
<<sacré bleu!>>
"
        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
"
        self.assertSoupEquals(text, expected)

    def test_smart_quotes_converted_on_the_way_in(self):
        # Microsoft smart quotes are converted to Unicode characters during
        # parsing.
        quote = b"
\x91Foo\x92
"
        soup = self.soup(quote)
        self.assertEqual(
            soup.p.string,
            u"\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")

    def test_non_breaking_spaces_converted_on_the_way_in(self):
        soup = self.soup("  ")
        self.assertEqual(soup.a.string, u"\N{NO-BREAK SPACE}" * 2)

    def test_entities_converted_on_the_way_out(self):
        text = "
<<sacré bleu!>>
"
        expected = u"
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
".encode("utf-8")
        soup = self.soup(text)
        self.assertEqual(soup.p.encode("utf-8"), expected)

    def test_real_iso_latin_document(self):
        # Smoke test of interrelated functionality, using an
        # easy-to-understand document.

        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
        unicode_html = u'<meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/>
Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!
'

        # That's because we're going to encode it into ISO-Latin-1, and use
        # that to test.
        iso_latin_html = unicode_html.encode("iso-8859-1")

        # Parse the ISO-Latin-1 HTML.
        soup = self.soup(iso_latin_html)
        # Encode it to UTF-8.
        result = soup.encode("utf-8")

        # What do we expect the result to look like? Well, it would
        # look like unicode_html, except that the META tag would say
        # UTF-8 instead of ISO-Latin-1.
        expected = unicode_html.replace("ISO-Latin-1", "utf-8")

        # And, of course, it would be in UTF-8, not Unicode.
        expected = expected.encode("utf-8")

        # Ta-da!
        self.assertEqual(result, expected)

    def test_real_shift_jis_document(self):
        # Smoke test to make sure the parser can handle a document in
        # Shift-JIS encoding, without choking.
        shift_jis_html = (
            b'
'
            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
            b'
')
        unicode_html = shift_jis_html.decode("shift-jis")
        soup = self.soup(unicode_html)

        # Make sure the parse tree is correctly encoded to various
        # encodings.
        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))

    def test_real_hebrew_document(self):
        # A real-world test to make sure we can convert ISO-8859-9 (a
        # Hebrew encoding) to UTF-8.
        hebrew_document = b'<title>Hebrew (ISO 8859-8) in Visual Directionality</title>
Hebrew (ISO 8859-8) in Visual Directionality
\xed\xe5\xec\xf9'
        soup = self.soup(
            hebrew_document, from_encoding="iso8859-8")
        # Some tree builders call it iso8859-8, others call it iso-8859-9.
        # That's not a difference we really care about.


#!/usr/bin/env python3

#--------[ Imports ]-----------------------------------------------------------
from os.path import isfile
from os.path import getsize
from os import remove
from io import StringIO
from shutil import copy
import struct
from io import StringIO

import base64
import hashlib
from Crypto import Random
from Crypto.Cipher import AES

#--------[ Methods ]-----------------------------------------------------------

#### check_file_structure #######################################
#                                                               #
# input:  file path to todo file                                #
# output: 0 if file passes, else the step it failed on          #
#                                                               #
# What to check for:                                            #
#   1) File exists                                              #
#   2) File begins with a project                               #
#   3) All lines are a project or a task                        #
#                                                               #
#################################################################
def check_file_structure(filename, verbose=False):
    if verbose: print("Checking file structure of '{}'...".format(filename))
# 1) Check if file exists
    if verbose: print("\nDoes todo file exist...?")
    if not isfile(filename):
        if verbose:
            ans=input("Would you like to create a new todo file? ")
            if ans == "yes" or ans == "y":
                create_file = open(filename, mode='w')
                create_file.close()
                print("Todo file created at " + filename)
            else:
                return 1
        else:
            return 1
    else:
        if verbose: print("   The file exists")
    #If the file is empty, it passes the test
    if verbose: print("\nIs the file empty...?")
    if getsize(filename) == 0:
        if verbose: print("   The file is empty")
        return 0
    else:
        if verbose: print("   The file is not empty!")

# 2) Check if the file begins with a project
    if verbose: print("\nDoes the file begin with a project...?")
    with open(filename, mode = 'r+') as f:
        first_line = f.readline()
        if first_line[0:2] != "PR":
            if verbose:
                print("   The todo file must begin with a project.")
                ans=input("   Would you like create a 'Unsorted' project? ")
                if ans == "yes" or ans =="y":
                    f.seek(0)
                    data = f.read()
                    f.seek(0)
                    f.write("PR  Unsorted\n" + data)
                    print("   'Unsorted' project created")
                else:
                    return 2
            else:
                return 2
        else:
            if verbose: print("   The file begins with a project")

# 3) Check if all lines are valid
    if verbose: print("\nAre all lines valid...?")
    with open(filename, mode='r') as f:
        line_number = 0
        for line in f:
            line_number += 1
            kc = line[0:2]
            if kc!="PR" and kc!="[ " and kc!="[X":
                if verbose:
                    print("   Invalid line: " + str(line_number))
                    print("   Please fix this line")
                    return 3
                else:
                    return 3
        if verbose: print("   All lines are valid")

    if verbose: print("\nFile structure check complete. PASS")
    return 0;

#### encrypt_file ###############################################
#                                                               #
# input:  file to be encrypted                                  #
#         file to wirte to                                      #
#         passphrase used to generate the encrytion key         #
#         //iv used in AES CBC cipher                           #
# output: 0 if file passes, else the step it failed on          #
#                                                               #
# Encrypt a static file with given key and iv. Should be used   #
# as an initial encryption of the file. This method does not    #
# an IV as it will generate one and store it at the start of    #
# the output file.                                              #
#                                                               #
#################################################################
def encrypt_file(filename_in, filename_out, passphrase):
    """
    print("[+] Running encrypt_file with arguments:")
    print("     filename_in : " + filename_in)
    print("     filename_out: " + filename_out)
    print("     passphare   : " + passphrase)
    print("")
    """

    #Check if file in and file out are the same file
    if filename_in == filename_out:
        SAME_FILENAME = True
        filename_out = filename_out+".tmp"
    else:
        SAME_FILENAME = False

    # Initial Setup
    iv = Random.new().read(AES.block_size)

    if not filename_out:
        filename_out = filename_in + '.enc'
    cipher = AES.new ( hashlib.md5(passphrase.encode('utf-8')).digest(),
                       AES.MODE_CBC,
                       iv )

    # Produce MD5 hash of file
    file_hash=hashlib.md5()
    with open(filename_in, 'rb') as infile:
        while True:
            chunk = infile.read(AES.block_size)
            file_hash.update(chunk)
            if len(chunk) == 0:
                break

    # Open files to encrypt and write file
    with open(filename_in, 'rb') as infile:
        with open(filename_out, 'wb') as outfile:
            outfile.write(file_hash.digest())
#            outfile.write(struct.pack('<Q', filesize))
            outfile.write(iv)

            while True:
                chunk = infile.read(AES.block_size)
                if len(chunk) == 0:
                    break
                outfile.write(cipher.encrypt(_pad(chunk)))

    if SAME_FILENAME:
        copy(filename_out, filename_in)
        remove(filename_out)

    return 0;

#### decrypt_file ###############################################
#                                                               #
# input:  file to be decrypted                                  #
#         file to wirte to                                      #
#         passphrase used to generate the encryption key        #
#         //iv used in AES CBC cipher                           #
# output: 0 if file passes, else the step it failed on          #
#                                                               #
# Decrypt a static file with given key and iv. Should be used   #
# as a final decryption of the file.                            #
#                                                               #
#################################################################
def decrypt_file(filename_in, filename_out, passphrase):
    """
    print("[+] Running decrypt_file with arguments:")
    print("     filename_in : " + filename_in)
    print("     filename_out: " + filename_out)
    print("     passphare   : " + passphrase)
    print("")
    """

    #Check if file in and file out are the same file
    if filename_in == filename_out:
        SAME_FILENAME = True
        filename_out = filename_out+".tmp"
    else:
        SAME_FILENAME = False

    # Initial Setup
    if not filename_out:
        filename_out = filename_in + '.dec'

    with open(filename_in, 'rb') as infile:
        file_hash = infile.read(AES.block_size)
#        original_size=struct.unpack('<Q',infile.read(struct.calcsize('Q')))[0]
        iv = infile.read(AES.block_size)
        cipher = AES.new ( hashlib.md5(passphrase.encode('utf-8')).digest(),
                           AES.MODE_CBC,
                           iv )

        # Decrypt File


"""
Celery task for CSV student answer export.
"""
import time

from celery.task import task
from celery.utils.log import get_task_logger
from lms.djangoapps.instructor_task.models import ReportStore
from opaque_keys.edx.keys import CourseKey
from student.models import user_by_anonymous_id
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError

from .mcq import MCQBlock, RatingBlock
from problem_builder.answer import AnswerBlock
from .questionnaire import QuestionnaireAbstractBlock
from .sub_api import sub_api

logger = get_task_logger(__name__)

@task()
def export_data(course_id, source_block_id_str, block_types, user_ids, match_string):
    """
    Exports student answers to all MCQ questions to a CSV file.
    """
    start_timestamp = time.time()

    logger.debug("Beginning data export")
    try:
        course_key = CourseKey.from_string(course_id)
        src_block = modulestore().get_items(course_key, qualifiers={'name': source_block_id_str}, depth=0)[0]
    except IndexError:
        raise ValueError("Could not find the specified Block ID.")
    course_key_str = unicode(course_key)

    type_map = {cls.__name__: cls for cls in [MCQBlock, RatingBlock, AnswerBlock]}

    if not block_types:
        block_types = tuple(type_map.values())
    else:
        block_types = tuple(type_map[class_name] for class_name in block_types)

    # Build an ordered list of blocks to include in the export
    blocks_to_include = []

    def scan_for_blocks(block):
        """ Recursively scan the course tree for blocks of interest """
        if isinstance(block, block_types):
            blocks_to_include.append(block)
        elif block.has_children:
            for child_id in block.children:
                try:
                    scan_for_blocks(block.runtime.get_block(child_id))
                except ItemNotFoundError:
                    # Blocks may refer to missing children. Don't break in this case.
                    pass

    scan_for_blocks(src_block)

    # Define the header row of our CSV:
    rows = []
    rows.append(["Section", "Subsection", "Unit", "Type", "Question", "Answer", "Username"])

    # Collect results for each block in blocks_to_include
    for block in blocks_to_include:
        if not user_ids:
            results = _extract_data(course_key_str, block, None, match_string)
            rows += results
        else:
            for user_id in user_ids:
                results = _extract_data(course_key_str, block, user_id, match_string)
                rows += results

    # Generate the CSV:
    filename = u"pb-data-export-{}.csv".format(time.strftime("%Y-%m-%d-%H%M%S", time.gmtime(start_timestamp)))
    report_store = ReportStore.from_config(config_name='GRADES_DOWNLOAD')
    report_store.store_rows(course_key, filename, rows)

    generation_time_s = time.time() - start_timestamp
    logger.debug("Done data export - took {} seconds".format(generation_time_s))

    return {
        "error": None,
        "report_filename": filename,
        "start_timestamp": start_timestamp,
        "generation_time_s": generation_time_s,
        "display_data": [] if len(rows) == 1 else rows[1:1001]  # Limit to preview of 1000 items
    }

def _extract_data(course_key_str, block, user_id, match_string):
    """
    Extract results for `block`.
    """
    rows = []

    # Extract info for "Section", "Subsection", and "Unit" columns
    section_name, subsection_name, unit_name = _get_context(block)

    # Extract info for "Type" column
    block_type = _get_type(block)

    # Extract info for "Question" column
    block_question = _get_question(block)

    # Extract info for "Answer" and "Username" columns
    # - Get all of the most recent student submissions for this block:
    submissions = _get_submissions(course_key_str, block, user_id)

    # - For each submission, look up student's username and answer:
    answer_cache = {}
    for submission in submissions:
        username = _get_username(submission, user_id)
        answer = _get_answer(block, submission, answer_cache)

        # Short-circuit if answer does not match search criteria
        if not match_string.lower() in answer.lower():
            continue

        rows.append([section_name, subsection_name, unit_name, block_type, block_question, answer, username])

    return rows

def _get_context(block):
    """
    Return section, subsection, and unit names for `block`.
    """
    block_names_by_type = {}
    block_iter = block
    while block_iter:
        block_iter_type = block_iter.scope_ids.block_type
        block_names_by_type[block_iter_type] = block_iter.display_name_with_default
        block_iter = block_iter.get_parent() if block_iter.parent else None
    section_name = block_names_by_type.get('chapter', '')
    subsection_name = block_names_by_type.get('sequential', '')
    unit_name = block_names_by_type.get('vertical', '')
    return section_name, subsection_name, unit_name

def _get_type(block):
    """
    Return type of `block`.
    """
    return block.scope_ids.block_type

def _get_question(block):
    """
    Return question for `block`; default to question ID if `question` is not set.
    """
    return block.question or block.name

def _get_submissions(course_key_str, block, user_id):
    """
    Return submissions for `block`.
    """
    # Load the actual student submissions for `block`.
    # Note this requires one giant query that retrieves all student submissions for `block` at once.
    block_id = unicode(block.scope_ids.usage_id.replace(branch=None, version_guid=None))
    block_type = _get_type(block)
    if block_type == 'pb-answer':
        block_id = block.name  # item_id of Long Answer submission matches question ID and not block ID
    if not user_id:
        return sub_api.get_all_submissions(course_key_str, block_id, block_type)
    else:
        student_dict = {
            'student_id': user_id,
            'item_id': block_id,
            'course_id': course_key_str,
            'item_type': block_type,
        }
        return sub_api.get_submissions(student_dict, limit=1)

def _get_username(submission, user_id):
    """
    Return username of student who provided `submission`.

    If the anonymous id of the submission can't be resolved into a username, the anonymous
    id is returned.
    """
    # If the student ID key doesn't exist, we're dealing with a single student and know the ID already.
    student_id = submission.get('student_id', user_id)
    user = user_by_anonymous_id(student_id)
    if user is None:
        return student_id
    return user.username

def _get_answer(block, submission, answer_cache):
    """
    Return answer associated with `submission` to `block`.

    `answer_cache` is a dict that is reset for each block.
    """
    answer = submission['answer']
    if isinstance(block, QuestionnaireAbstractBlock):
        # Convert from answer ID to answer label
        if answer not in answer_cache:


# -*- coding: utf-8 -*-
''' accpy.visualize.plot
author:     felix.kramer(at)physik.hu-berlin.de

'best'         : 0, (only implemented for axes legends)
'upper right'  : 1,
'upper left'   : 2,
'lower left'   : 3,
'lower right'  : 4,
'right'        : 5,
'center left'  : 6,
'center right' : 7,
'lower center' : 8,
'upper center' : 9,
'center'       : 10,
'''
from __future__ import division
from numpy import (nanmax, nanmin, concatenate, empty, linspace, array, arange,
                   abs as npabs, sqrt, sin, cos, arccos as acos, nanmean)
from itertools import product
from matplotlib.figure import Figure
from matplotlib.pyplot import cm
from matplotlib.gridspec import GridSpec
from .stringformat import SI, SId
from .lattice import drawlattice
from ..simulate import const
from ..simulate.rmatrices import UCS2R
from ..simulate.tracking import trackpart

def plot(ax, x, y, ls, xlabel, xunit, ylabel, yunit, label, col=False,
         setlim=True, rescaleX=True, rescaleY=True, xprefix=None, mx=None,
         yprefix=None, my=None):
    if xprefix is None:
        xprefix, mx = SId(nanmean(abs(x)))
    if yprefix is None:
        yprefix, my = SId(nanmean(abs(y)))
    if rescaleX:
        x = x/mx  # carefull! numpy.ndarrays are mutable!!!
        if xunit != '':
            xunit = ' / ('+xprefix+xunit+')'
    elif xunit != '':
            xunit = ' / ('+xunit+')'
    if rescaleY:
        y = y/my
        if yunit != '':
            yunit = ' / ('+yprefix+yunit+')'
    elif yunit != '':
            yunit = ' / ('+yunit+')'
    if col is False:
        ax.plot(x, y, ls, label=label)
    else:
        ax.plot(x, y, ls, color=col, label=label)
    if xlabel != '':
        ax.set_xlabel(xlabel+xunit)
    if ylabel != '':
        ax.set_ylabel(ylabel+yunit)
    if setlim:
        epsy = (max(y)-min(y))*0.15
        ax.set_xlim([min(x), max(x)])
        ax.set_ylim([min(y)-epsy, max(y)+epsy])
    return x, y, yprefix, my

def Mplot(ax, x, ys, lss, xlabel, xunit, ylabel, yunit, labels, rescaleX=True, rescaleY=True):
    colors = getcolors(len(ys))
    xprefix, mx = SId(nanmean(x))
    yprefix, my = SId(nanmean(ys))
    if rescaleX:
        if xunit != '':
            xunit = ' / ('+xprefix+xunit+')'
    elif xunit != '':
            xunit = ' / ('+xunit+')'
    if rescaleY:
        if yunit != '':
            yunit = ' / ('+yprefix+yunit+')'
    elif yunit != '':
            yunit = ' / ('+yunit+')'
    if labels == '':
        labels = ['' for i in range(len(ys))]
    if type(x) != type([]):
        xs = [x for i in range(len(ys))]
    else:
        xs = x
    for x, y, ls, lab, col in zip(xs, ys, lss, labels, colors):
        if rescaleY:
            y = y/my
        if rescaleX:
            x = x/mx  # carefull! numpy.ndarrays are mutable!!!
        ax.plot(x, y, ls, color=col, label=lab)
    if xlabel != '':
        ax.set_xlabel(xlabel+xunit)
    if ylabel != '':
        ax.set_ylabel(ylabel+yunit)
    ax.set_xlim([min(x), max(x)])
    return mx, my

def legplot(ax, linestyles, labels, loc=0):
    colors = getcolors(len(labels))
    for ls, lab, col in zip(linestyles, labels, colors):
        ax.plot([], [], ls, label=lab, color=col)
    ax.axis('off')
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)
    ax.legend(fancybox=True, loc=loc)
    return

def getcolors(number):
    return list(cm.rainbow(linspace(0, 1, number)))

def plotoptic(UC, diagnostics, s, xtwiss, ytwiss, xdisp):
    figs = [Figure() for i in range(4)]
    ax = [figs[i].add_subplot(1, 1, 1) for i in range(4)]
    drawlattice(ax[0], UC, diagnostics, [xtwiss[0, 0, :], xtwiss[0, 1, :]], 0)
    ax[0].plot(s, xtwiss[0, 0, :], '-r', label=r'$\beta_x$')
    ax[0].plot(s, -xtwiss[0, 1, :], '-c', label=r'$\alpha_x$')
    ax[0].plot([], [], '-m', label=r'$\gamma_x$')
    ax[0].set_ylabel(r'betatron function $\beta_x$ / (m)')
    ax2 = ax[0].twinx()
    ax2.plot(s, xtwiss[1, 1, :], '-m')
    ax2.set_ylabel(r'gamma function $\gamma_x$ / (m)', color='m')
    ax2.tick_params(axis='y', colors='m')
    drawlattice(ax[1], UC, diagnostics, [ytwiss[0, 0, :], ytwiss[0, 1, :]], 0)
    ax[1].plot(s, ytwiss[0, 0, :], '-b', label=r'$\beta_y$')
    ax[1].plot(s, -ytwiss[0, 1, :], '-c', label=r'$\alpha_y$')
    ax[1].plot([], [], '-m', label=r'$\gamma_y$')
    ax[1].set_ylabel(r'betatron function $\beta_y$ / (m)')
    ax2 = ax[1].twinx()
    ax2.plot(s, ytwiss[1, 1, :], '-m', label=r'$\gamma_y$')
    ax2.set_ylabel(r'gamma function $\gamma_y$ / (m)', color='m')
    ax2.tick_params(axis='y', colors='m')
    drawlattice(ax[2], UC, diagnostics, [xdisp[0, :]], 0)
    ax[2].plot(s, xdisp[0, :], '-g', label=r'$D_x$')
    ax[2].plot([], [], '-m', label=r'$D_x^\prime$')
    ax[2].set_ylabel(r'dispersion function $D_x$ / (m)')
    ax2 = ax[2].twinx()
    ax2.plot(s, xdisp[1, :], '-m', label=r'$D_x^\prime$')
    ax2.set_ylabel(r'derived dispersion function $D_x^\prime$ / (m)', color='m')
    ax2.tick_params(axis='y', colors='m')
    drawlattice(ax[3], UC, diagnostics, [xtwiss[0, 0, :], ytwiss[0, 0, :]], 0)
    ax[3].plot(s, xtwiss[0, 0, :], '-r', label=r'$\beta_x$')
    ax[3].plot(s, ytwiss[0, 0, :], '-b', label=r'$\beta_y$')
    ax[3].plot([], [], '-g', label=r'$D_x$')
    ax[3].set_ylabel(r'betatron function $\beta_{x,y}$ / (m)')
    ax2 = ax[3].twinx()
    ax2.plot(s, xdisp[0, :], '-g', label=r'$D_x$')
    ax2.set_ylabel(r'dispersion function $D_x$ / (m)', color='g')
    ax2.tick_params(axis='y', colors='g')
    [ax[i].set_xlabel(r'orbit position s / (m)') for i in range(4)]
    [ax[i].set_xlim([0, nanmax(s)]) for i in range(4)]
    legs = [ax[i].legend(fancybox=True, loc=0) for i in range(4)]
    [legs[i].get_frame().set_alpha(0.5) for i in range(4)]
    return figs

def plotbeamsigma(UC, diagnostics, s, sigx, sigy):
    fig = Figure()
    ax = fig.add_subplot(1, 1, 1)
    rel = abs(nanmean(sigy)/nanmean(sigx))
    if rel > 100 or rel < 1e-2:
        drawlattice(ax, UC, diagnostics, [sigx], 0)
        ax.plot(s, sigx, '-r', label=r'$\sigma_x$')
        ax.plot([], [], '-b', label=r'$\sigma_y$')
        ax.set_ylabel(r'Beam extent $\sigma_x$ / (m)')
        ax.set_xlabel(r'orbit position s / (m)')
        ax2 = ax.twinx()
        ax2.plot(s, sigy, '-b')
        ax2.tick_params(axis='y', colors='b')
        ax2.set_ylabel(r'Beam extent $\sigma_y$ / (m)', color='b')
    else:
        drawlattice(ax, UC, diagnostics, [sigx, sigy], 0)
        ax.plot(s, sigx, '-r', label=r'$\sigma_x$')
        ax.plot(s, sigy, '-b', label=r'$\sigma_y$')
        ax.set_xlabel(r'orbit position s / (m)')
        ax.set_ylabel(r'Beam extent $\sigma_u$ / (m)')
    ax.set_xlim([min(s), max(s)])
    leg = ax.legend(fancybox=True, loc=2)
    leg.get_frame().set_alpha(0.5)
    return fig

def plotopticpars_closed(xtwiss, xdisp, ytwiss, gamma, Qx, Xx, Jx, emiteqx,
                         tau_x, Qy, Xy, Jy, E, emiteqy, tau_y, alpha_mc,
                         eta_mc, gamma_tr, Q_s, Js, sigma_E, sigma_tau,
                         sigma_s, tau_s, U_rad, P_ges, E_c, lambda_c):
    fig = Figure()
    ax = fig.add_subplot(1, 1, 1)
    ax.axis('off')
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)
    radpars = ''.join(
        [r'------------------------------------''\n',
         r'Radial parameters''\n',
         r'------------------------------------''\n',
         r'$\beta_{x,max} = %g m$''\n' % nanmax(xtwiss[0, 0, :]),
         r'$\beta_{x,min} = %g m$''\n' % nanmin(xtwiss[0, 0, :]),
         r'$\alpha_{x,max} = %g$''\n' % nanmax(-xtwiss[0, 1, :]),
         r'$\alpha_{x,min} = %g$''\n' % nanmin(-xtwiss[0, 1, :]),
         r'$\gamma_{x,max} = %g$''\n' % nanmax(xtwiss[1, 1, :]),
         r'$\gamma_{x,min} = %g$''\n' % nanmin(xtwiss[1, 1, :]),
         r'$D_{x,max} = %g m$''\n' % nanmax(xdisp[0, :]),
         r'$D_{x,min} = %g m$''\n' % nanmin(xdisp[0, :]),
         r'$D_{x,max}^\prime = %g$''\n' % nanmax(xdisp[1, :]),
         r'$D_{x,min}^\prime = %g$''\n' % nanmin(xdisp[1, :]),
         r'$Q_x = %g$''\n' % Qx,
         r'$\xi_{x,nat} = %g$''\n' % Xx,
         r'$J_x = %g$''\n' % Jx,
         r'$\epsilon_x = %g \pi rad m$''\n' % emiteqx,
         r'$\tau_x = %e s$' % tau_x])
    axipars = ''.join(
        [r'------------------------------------''\n',
         r'Axial parameters''\n',
         r'------------------------------------''\n',
         r'$\beta_{y,max} = %g m$''\n' % nanmax(ytwiss[0, 0, :]),
         r'$\beta_{y,min} = %g m$''\n' % nanmin(ytwiss[0, 0, :]),
         r'$\alpha_{y,max} = %g$''\n' % nanmax(-ytwiss[0, 1, :]),
         r'$\alpha_{y,min} = %g$''\n' % nanmin(-ytwiss[0, 1, :]),
         r'$\gamma_{x,max} = %g$''\n' % nanmax(ytwiss[1, 1, :]),
         r'$\gamma_{x,min} = %g$''\n' % nanmin(ytwiss[1, 1, :]),
         r'$Q_y = %g$''\n' % Qy,
         r'$\xi_{y,nat} = %g$''\n' % Xy,
         r'$J_y = %g$''\n' % Jy,
         r'$\epsilon_y = %g \pi rad m$''\n' % emiteqy,
         r'$\tau_y = %e s$' % tau_y])
    lonpars = ''.join(
        [r'------------------------------------''\n',
         r'Longitudinal parameters''\n',
         r'------------------------------------''\n',
         r'$E = %g eV$''\n' % E,
         r'$\gamma_{lorentz} = %g$''\n' % gamma,
         r'$\alpha_{p} = %g $''\n' % alpha_mc,
         r'$\eta_{slip} = %g $''\n' % eta_mc,
         r'$\gamma_{tr} = %g $''\n' % gamma_tr,
         r'$Q_s = %g$''\n' % Q_s,
         r'$J_s = %g$''\n' % Js,
         r'$\sigma_{E} = %e \%% $''\n' % (sigma_E*100),
         r'$\sigma_{\tau} = %g s$''\n' % sigma_tau,
         r'$\sigma_{s} = %g m$''\n' % sigma_s,
         r'$\tau_{s} = %e s$''\n' % tau_s,
         r'$E_{loss} = %g eV$''\n' % U_rad,
         r'$P_{rad} = %g W$''\n' % P_ges,
         r'$E_{crit} = %g eV$''\n' % E_c,
         r'$\lambda_{crit} = %g m$' % lambda_c])
    ax.text(0.025, 0.975, radpars, horizontalalignment='left',
            verticalalignment='top', transform=ax.transAxes)
    ax.text(0.35, 0.975, axipars, horizontalalignment='left',
            verticalalignment='top', transform=ax.transAxes)
    ax.text(0.675, 0.975, lonpars, horizontalalignment='left',
            verticalalignment='top', transform=ax.transAxes)
    return fig

def plotopticpars_open(xtwiss, xdisp, ytwiss, gamma, E):
    fig = Figure()
    ax = fig.add_subplot(1, 1, 1)
    ax.axis('off')
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)
    radpars = ''.join(
        [r'------------------------------------''\n',
         r'Radial parameters''\n',
         r'------------------------------------''\n',
         r'$\beta_{x,max} = %g m$''\n' % nanmax(xtwiss[0, 0, :]),
         r'$\beta_{x,min} = %g m$''\n' % nanmin(xtwiss[0, 0, :]),
         r'$\alpha_{x,max} = %g$''\n' % nanmax(-xtwiss[0, 1, :]),
         r'$\alpha_{x,min} = %g$''\n' % nanmin(-xtwiss[0, 1, :]),
         r'$\gamma_{x,max} = %g$''\n' % nanmax(xtwiss[1, 1, :]),


# ActivitySim
# See full license in LICENSE.txt.

import os
import logging
import warnings

import numpy as np
import pandas as pd

from activitysim.core import skim_dictionary
from activitysim.core import inject
from activitysim.core import util
from activitysim.core import config
from activitysim.core import pathbuilder
from activitysim.core import mem
from activitysim.core import tracing

from activitysim.core.skim_dictionary import NOT_IN_SKIM_ZONE_ID
from activitysim.core.skim_dict_factory import NumpyArraySkimFactory
from activitysim.core.skim_dict_factory import MemMapSkimFactory

skim_factories = {
    'NumpyArraySkimFactory': NumpyArraySkimFactory,
    'MemMapSkimFactory': MemMapSkimFactory,
}

logger = logging.getLogger(__name__)

LOS_SETTINGS_FILE_NAME = 'network_los.yaml'

ONE_ZONE = 1
TWO_ZONE = 2
THREE_ZONE = 3

DEFAULT_SETTINGS = {
    'rebuild_tvpb_cache': True,
    'zone_system': ONE_ZONE,
    'skim_dict_factory': 'NumpyArraySkimFactory'
}

TRACE_TRIMMED_MAZ_TO_TAP_TABLES = True

class Network_LOS(object):
    """
    ::

      singleton object to manage skims and skim-related tables

      los_settings_file_name: str         # e.g. 'network_los.yaml'
      skim_dtype_name:str                 # e.g. 'float32'

      dict_factory_name: str              # e.g. 'NumpyArraySkimFactory'
      zone_system: str                    # str (ONE_ZONE, TWO_ZONE, or THREE_ZONE)
      skim_time_periods = None            # list of str e.g. ['AM', 'MD', 'PM''

      skims_info: dict                    # dict of SkimInfo keyed by skim_tag
      skim_buffers: dict                  # if multiprocessing, dict of multiprocessing.Array buffers keyed by skim_tag
      skim_dicts: dice                    # dict of SkimDict keyed by skim_tag

      # TWO_ZONE and THREE_ZONE
      maz_taz_df: pandas.DataFrame        # DataFrame with two columns, MAZ and TAZ, mapping MAZ to containing TAZ
      maz_to_maz_df: pandas.DataFrame     # maz_to_maz attributes for MazSkimDict sparse skims
                                          # indexed by synthetic omaz/dmaz index for faster get_mazpairs lookup)
      maz_ceiling: int                    # max maz_id + 1 (to compute synthetic omaz/dmaz index by get_mazpairs)
      max_blend_distance: dict            # dict of int maz_to_maz max_blend_distance values keyed by skim_tag

      # THREE_ZONE only
      tap_df: pandas.DataFrame
      tap_lines_df: pandas.DataFrame      # if specified in settings, list of transit lines served, indexed by TAP
                                          # use to prune maz_to_tap_dfs to drop more distant TAPS with redundant service
                                          # since a TAP can serve multiple lines, tap_lines_df TAP index is not unique
      maz_to_tap_dfs: dict                # dict of maz_to_tap DataFrames indexed by access mode (e.g. 'walk', 'drive')
                                          # maz_to_tap dfs have OMAZ and DMAZ columns plus additional attribute columns
      tap_tap_uid: TapTapUidCalculator
    """

    def __init__(self, los_settings_file_name=LOS_SETTINGS_FILE_NAME):

        # Note: we require all skims to be of same dtype so they can share buffer - is that ok?
        # fixme is it ok to require skims be all the same type? if so, is this the right choice?
        self.skim_dtype_name = 'float32'
        self.zone_system = None
        self.skim_time_periods = None
        self.skims_info = {}
        self.skim_dicts = {}

        # TWO_ZONE and THREE_ZONE
        self.maz_taz_df = None
        self.maz_to_maz_df = None
        self.maz_ceiling = None
        self.max_blend_distance = {}

        # THREE_ZONE only
        self.tap_lines_df = None
        self.maz_to_tap_dfs = {}
        self.tvpb = None

        self.los_settings_file_name = los_settings_file_name
        self.load_settings()

        # dependency injection of skim factory (of type specified in skim_dict_factory setting)
        skim_dict_factory_name = self.setting('skim_dict_factory')
        assert skim_dict_factory_name in skim_factories, \
            f"Unrecognized skim_dict_factory setting '{skim_dict_factory_name}"
        self.skim_dict_factory = skim_factories[skim_dict_factory_name](network_los=self)
        logger.info(f"Network_LOS using skim_dict_factory: {type(self.skim_dict_factory).__name__}")

        # load SkimInfo for all skims for this zone_system (TAZ for ONE_ZONE and TWO_ZONE, TAZ and MAZ for THREE_ZONE)
        self.load_skim_info()

    @property
    def rebuild_tvpb_cache(self):
        # setting as property here so others don't need to know default
        assert self.zone_system == THREE_ZONE, f"Should not even be asking about rebuild_tvpb_cache if not THREE_ZONE"
        return self.setting('rebuild_tvpb_cache')

    def setting(self, keys, default='<REQUIRED>'):

        # if they dont specify a default, check the default defaults
        default = DEFAULT_SETTINGS.get(keys, '<REQUIRED>') if default == '<REQUIRED>' else default

        # get setting value for single key or dot-delimited key path (e.g. 'maz_to_maz.tables')
        key_list = keys.split('.')
        s = self.los_settings
        for key in key_list[:-1]:
            s = s.get(key)
            assert isinstance(s, dict), f"expected key '{key}' not found in '{keys}' in {self.los_settings_file_name}"
        key = key_list[-1]  # last key
        if default == '<REQUIRED>':
            assert key in s, f"Expected setting {keys} not found in in {LOS_SETTINGS_FILE_NAME}"
        return s.get(key, default)

    def load_settings(self):
        """
        Read setting file and initialize object variables (see class docstring for list of object variables)
        """

        try:
            self.los_settings = config.read_settings_file(self.los_settings_file_name, mandatory=True)
        except config.SettingsFileNotFound as e:

            print(f"los_settings_file_name {self.los_settings_file_name} not found - trying global settings")
            print(f"skims_file: {config.setting('skims_file')}")
            print(f"skim_time_periods: {config.setting('skim_time_periods')}")
            print(f"source_file_paths: {config.setting('source_file_paths')}")
            print(f"inject.get_injectable('configs_dir') {inject.get_injectable('configs_dir')}")

            # look for legacy 'skims_file' setting in global settings file
            if config.setting('skims_file'):

                warnings.warn("Support for 'skims_file' setting in global settings file will be removed."
                              "Use 'taz_skims' in network_los.yaml config file instead.", FutureWarning)

                # in which case, we also expect to find skim_time_periods in settings file
                skim_time_periods = config.setting('skim_time_periods')
                assert skim_time_periods is not None, "'skim_time_periods' setting not found."
                warnings.warn("Support for 'skim_time_periods' setting in global settings file will be removed."
                              "Put 'skim_time_periods' in network_los.yaml config file instead.", FutureWarning)

                self.los_settings = {
                    'taz_skims': config.setting('skims_file'),
                    'zone_system': ONE_ZONE,
                    'skim_time_periods': skim_time_periods
                }

            else:
                raise e

        # validate skim_time_periods
        self.skim_time_periods = self.setting('skim_time_periods')
        if 'hours' in self.skim_time_periods:
            self.skim_time_periods['periods'] = self.skim_time_periods.pop('hours')
            warnings.warn('support for `skim_time_periods` key `hours` will be removed in '
                          'future verions. Use `periods` instead',
                          FutureWarning)
        assert 'periods' in self.skim_time_periods, "'periods' key not found in network_los.skim_time_periods"
        assert 'labels' in self.skim_time_periods, "'labels' key not found in network_los.skim_time_periods"

        self.zone_system = self.setting('zone_system')
        assert self.zone_system in [ONE_ZONE, TWO_ZONE, THREE_ZONE], \
            f"Network_LOS: unrecognized zone_system: {self.zone_system}"

        if self.zone_system in [TWO_ZONE, THREE_ZONE]:
            # maz_to_maz_settings
            self.max_blend_distance = self.setting('maz_to_maz.max_blend_distance', default={})
            if isinstance(self.max_blend_distance, int):
                self.max_blend_distance = {'DEFAULT': self.max_blend_distance}
            self.blend_distance_skim_name = self.setting('maz_to_maz.blend_distance_skim_name', default=None)

        # validate skim_time_periods
        self.skim_time_periods = self.setting('skim_time_periods')
        assert {'periods', 'labels'}.issubset(set(self.skim_time_periods.keys()))

    def load_skim_info(self):
        """
        read skim info from omx files into SkimInfo, and store in self.skims_info dict keyed by skim_tag

        ONE_ZONE and TWO_ZONE systems have only TAZ skims
        THREE_ZONE systems have both TAZ and TAP skims
        """
        assert self.skim_dict_factory is not None
        # load taz skim_info
        self.skims_info['taz'] = self.skim_dict_factory.load_skim_info('taz')

        if self.zone_system == THREE_ZONE:
            # load tap skim_info
            self.skims_info['tap'] = self.skim_dict_factory.load_skim_info('tap')

        if self.zone_system == THREE_ZONE:
            # load this here rather than in load_data as it is required during multiprocessing to size TVPBCache
            self.tap_df = pd.read_csv(config.data_file_path(self.setting('tap'), mandatory=True)).sort_values('TAP')
            self.tvpb = pathbuilder.TransitVirtualPathBuilder(self)  # dependent on self.tap_df

    def load_data(self):
        """
        Load tables and skims from files specified in network_los settigns
        """

        # load maz tables
        if self.zone_system in [TWO_ZONE, THREE_ZONE]:

            # maz
            file_name = self.setting('maz')
            self.maz_taz_df = pd.read_csv(config.data_file_path(file_name, mandatory=True))
            self.maz_taz_df = self.maz_taz_df[['MAZ', 'TAZ']].sort_values(by='MAZ')  # only fields we need

            self.maz_ceiling = self.maz_taz_df.MAZ.max() + 1

            # maz_to_maz_df
            maz_to_maz_tables = self.setting('maz_to_maz.tables')
            maz_to_maz_tables = [maz_to_maz_tables] if isinstance(maz_to_maz_tables, str) else maz_to_maz_tables
            for file_name in maz_to_maz_tables:

                df = pd.read_csv(config.data_file_path(file_name, mandatory=True))


# -*- coding: utf-8 -*-

"""
Various useful shortcuts.
"""

import os
import sys
import subprocess
import random
import re
import hashlib
import threading

__all__ = (
    "force_list",
    "force_tuple",
    "allof",
    "anyof",
    "noneof",
    "oneof",
    "is_empty",
    "iter_chunks",
    "random_string",
    "hex_string",
    "touch",
    "read_from_file",
    "save",
    "save_to_file",
    "find_symlinks_to",
    "run",
    "compute_file_checksums",
    "parse_checksum_line",
    "read_checksum_file",
    "split_path",
    "relative_path",
    "makedirs",
)

def force_list(value):
    """Convert a value to list.

    @rtype: list
    """
    if type(value) is list:
        return value
    if type(value) in (tuple, set):
        return list(value)
    return [value]

def force_tuple(value):
    """Convert a value to tuple.

    @rtype: tuple
    """
    if type(value) is tuple:
        return value
    if type(value) in (list, set):
        return tuple(value)
    return (value, )

def allof(*args, **kwargs):
    """Test if all values are True.

    @rtype: bool
    """
    for i in list(args) + kwargs.values():
        if not i:
            return False
    return True

def anyof(*args, **kwargs):
    """Test if at least one of the values is True.

    @rtype: bool
    """
    for i in list(args) + kwargs.values():
        if i:
            return True
    return False

def noneof(*args, **kwargs):
    """Test if all values are False.

    @rtype: bool
    """
    for i in list(args) + kwargs.values():
        if i:
            return False
    return True

def oneof(*args, **kwargs):
    """Test if just one of the values is True.

    @rtype: bool
    """
    found = False
    for i in list(args) + kwargs.values():
        if i:
            if found:
                return False
            found = True
    return found

def is_empty(value):
    """Test if value is None, empty string, list, tuple or dict.

    @rtype: bool
    """
    if value is None:
        return True
    if type(value) in (list, tuple, dict):
        return len(value) == 0
    if not value:
        return True
    return False

def iter_chunks(input_list, chunk_size):
    """Iterate through input_list and yield chunk_size-d chunks."""

    # TODO: any idea how to detect files better? This works for StringIO at least...
    is_file = type(input_list) is file
    is_file |= hasattr(input_list, "read") and hasattr(input_list, "close") and hasattr(input_list, "seek")
    if is_file:
        while 1:
            chunk = input_list.read(chunk_size)
            if not chunk:
                break
            yield chunk
        return

    # TODO: any idea how to detect generators better?
    if hasattr(input_list, "__iter__") and hasattr(input_list, "next"):
        chunk = []
        for i in input_list:
            chunk.append(i)
            if len(chunk) == chunk_size:
                yield chunk
                chunk = []
        if chunk:
            yield chunk
        return

    can_slice = hasattr(input_list, "__getslice__")
    for i in xrange(0, len(input_list), chunk_size):
        if can_slice:
            # regular list
            yield input_list[i:i + chunk_size]
        else:
            # xrange, etc.
            chunk = []
            for j in xrange(i, i + chunk_size):
                if j < len(input_list):
                    chunk.append(input_list[j])
            yield chunk

def random_string(length=32, alphabet=None):
    """Return random string of given lenght which consists of characters from the alphabet.

    @rtype: str
    """
    alphabet = alphabet or "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
    return "".join([ random.choice(alphabet) for i in xrange(length) ])

def hex_string(string):
    """Convert a string to a string of hex digits.

    @rtype: str
    """
    return "".join(( "%02x" % ord(i) for i in string ))

def touch(filename, mode=0644):
    """Touch a file."""
    save_to_file(filename, "", append=True, mode=mode)

def read_from_file(filename, lines=None, re_filter=None):
    """Read a text file."""
    result = []
    fo = open(filename, "rt")

    if re_filter is not None:
        re_filter_compiled = re.compile(re_filter)

    for i, line in enumerate(fo):
        if lines is not None and i+1 not in lines:
            continue


# This file is part of fedmsg.
# Copyright (C) 2012 - 2014 Red Hat, Inc.
#
# fedmsg is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# fedmsg is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with fedmsg; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors:  Ralph Bean <rbean@redhat.com>
#           Luke Macken <lmacken@redhat.com>
#
""" :mod:`fedmsg.meta` handles the conversion of fedmsg messages
(dict-like json objects) into internationalized human-readable
strings:  strings like ``"nirik voted on a tag in tagger"`` and
``"lmacken commented on a bodhi update."``

The intent is to use the module 1) in the ``fedmsg-irc`` bot and 2) in the
gnome-shell desktop notification widget.  The sky is the limit, though.

The primary entry point is :func:`fedmsg.meta.msg2repr` which takes a dict and
returns the string representation.  Portions of that string are in turn
produced by :func:`fedmsg.meta.msg2title`, :func:`fedmsg.meta.msg2subtitle`,
and :func:`fedmsg.meta.msg2link`.

Message processing is handled by a list of MessageProcessors (instances of
:class:`fedmsg.meta.base.BaseProcessor`) which are discovered on a setuptools
**entry-point**.  Messages for which no MessageProcessor exists are
handled gracefully.

The original deployment of fedmsg in `Fedora Infrastructure` uses metadata
providers/message processors from a plugin called
`fedmsg_meta_fedora_infrastructure
<https://github.com/fedora-infra/fedmsg_meta_fedora_infrastructure>`_.
If you'd like to add your own processors for your own deployment, you'll need
to extend :class:`fedmsg.meta.base.BaseProcessor` and override the appropriate
methods.  If you package up your processor and expose it on the ``fedmsg.meta``
entry-point, your new class will need to be added to the
:data:`fedmsg.meta.processors` list at runtime.

End users can have multiple plugin sets installed simultaneously.

"""

import six

# gettext is used for internationalization.  I have tested that it can produce
# the correct files, but I haven't submitted it to anyone for translation.
import gettext
t = gettext.translation('fedmsg', 'locale', fallback=True)

if six.PY3:
    _ = t.gettext
else:
    _ = t.ugettext

from fedmsg.meta.default import DefaultProcessor
from fedmsg.meta.base import BaseConglomerator

import logging
log = logging.getLogger("fedmsg")

class ProcessorsNotInitialized(Exception):
    def __iter__(self):
        raise self
    __len__ = __iter__

    def __nonzero__(self):
        return False
    __bool__ = __nonzero__

processors = ProcessorsNotInitialized("You must first call "
                                      "fedmsg.meta.make_processors(**config)")

def make_processors(**config):
    """ Initialize all of the text processors.

    You'll need to call this once before using any of the other functions in
    this module.

        >>> import fedmsg.config
        >>> import fedmsg.meta
        >>> config = fedmsg.config.load_config([], None)
        >>> fedmsg.meta.make_processors(**config)
        >>> text = fedmsg.meta.msg2repr(some_message_dict, **config)

    """
    global processors

    # If they're already initialized, then fine.
    if processors:
        return

    import pkg_resources
    processors = []
    for processor in pkg_resources.iter_entry_points('fedmsg.meta'):
        try:
            processors.append(processor.load()(_, **config))
        except Exception as e:
            log.warn("Failed to load %r processor." % processor.name)
            log.exception(e)

    # This should always be last
    processors.append(DefaultProcessor(_, **config))

    # By default we have three builtin processors:  Default, Logger, and
    # Announce.  If these are the only three, then we didn't find any
    # externally provided ones.  calls to msg2subtitle and msg2link likely will
    # not work the way the user is expecting.
    if len(processors) == 3:
        log.warn("No fedmsg.meta plugins found.  fedmsg.meta.msg2* crippled")

def msg2processor(msg, **config):
    """ For a given message return the text processor that can handle it.

    This will raise a :class:`fedmsg.meta.ProcessorsNotInitialized` exception
    if :func:`fedmsg.meta.make_processors` hasn't been called yet.
    """
    for processor in processors:
        if processor.handle_msg(msg, **config) is not None:
            return processor
    else:
        return processors[-1]  # DefaultProcessor

def legacy_condition(cls):
    def _wrapper(f):
        def __wrapper(msg, legacy=False, **config):
            try:
                return f(msg, **config)
            except KeyError:
                if legacy:
                    return cls()
                else:
                    raise

        __wrapper.__doc__ = f.__doc__
        __wrapper.__name__ = f.__name__
        return __wrapper
    return _wrapper

def with_processor():
    def _wrapper(f):
        def __wrapper(msg, processor=None, **config):
            if not processor:
                processor = msg2processor(msg, **config)

            return f(msg, processor=processor, **config)

        __wrapper.__doc__ = f.__doc__
        __wrapper.__name__ = f.__name__
        return __wrapper
    return _wrapper

def conglomerate(messages, subject=None, lexers=False, **config):
    """ Return a list of messages with some of them grouped into conglomerate
    messages.  Conglomerate messages represent several other messages.

    For example, you might pass this function a list of 40 messages.
    38 of those are git.commit messages, 1 is a bodhi.update message, and 1 is
    a badge.award message.  This function could return a list of three
    messages, one representing the 38 git commit messages, one representing the
    bodhi.update message, and one representing the badge.award message.

    The ``subject`` argument is optional and will return "subjective"
    representations if possible (see msg2subjective(...)).

    Functionality is provided by fedmsg.meta plugins on a "best effort" basis.
    """

    # First, give every registered processor a chance to do its work
    for processor in processors:
        messages = processor.conglomerate(messages, subject=subject, **config)

    # Then, just fake it for every other ungrouped message.
    for i, message in enumerate(messages):
        # If these were successfully grouped, then skip
        if 'msg_ids' in message:
            continue

        # For ungrouped ones, replace them with a fake conglomerate
        messages[i] = BaseConglomerator.produce_template(
            [message], subject=subject, lexers=lexers, **config)
        # And fill out the fields that fully-implemented conglomerators would
        # normally fill out.
        messages[i].update({
            'link': msg2link(message, **config),
            'subtitle': msg2subtitle(message, **config),
            'subjective': msg2subjective(message, subject=subject, **config),
            'secondary_icon': msg2secondary_icon(message, **config),
        })

    return messages

@legacy_condition(six.text_type)
@with_processor()
def msg2repr(msg, processor, **config):
    """ Return a human-readable or "natural language" representation of a
    dict-like fedmsg message.  Think of this as the 'top-most level' function
    in this module.

    """
    fmt = u"{title} -- {subtitle} {link}"
    title = msg2title(msg, **config)
    subtitle = processor.subtitle(msg, **config)
    link = processor.link(msg, **config) or ''
    return fmt.format(**locals())

@legacy_condition(six.text_type)
@with_processor()
def msg2title(msg, processor, **config):
    """ Return a 'title' or primary text associated with a message. """
    return processor.title(msg, **config)

@legacy_condition(six.text_type)
@with_processor()
def msg2subtitle(msg, processor, **config):
    """ Return a 'subtitle' or secondary text associated with a message. """
    return processor.subtitle(msg, **config)

@legacy_condition(six.text_type)
@with_processor()
def msg2long_form(msg, processor, **config):
    """ Return a 'long form' text representation of a message.

    For most message, this will just default to the terse subtitle, but for
    some messages a long paragraph-structured block of text may be returned.
    """
    result = processor.long_form(msg, **config)
    if not result:
        result = processor.subtitle(msg, **config)
    return result

@with_processor()
def msg2lexer(msg, processor, **config):
    """ Return a Pygments lexer able to parse the long_form of this message.
    """


"""
.. _`external_code.py`:
"""

import glob
import logging
import os.path
import shutil
import stat
import sys
import time

# pylint: disable-msg=E0611,F0401
from openmdao.main.datatypes.api import Bool, Dict, Str, FileRef, Float, Int, List

from openmdao.main.api import Component
from openmdao.main.exceptions import RunInterrupted, RunStopped
from openmdao.main.rbac import AccessController, RoleError, rbac, remote_access
from openmdao.main.resource import ResourceAllocationManager as RAM

from openmdao.util.filexfer import filexfer, pack_zipfile, unpack_zipfile
from openmdao.util import shellproc

from distutils.spawn import find_executable

class ExternalCode(Component):
    """
    Run an external code as a component. The component can be configured to
    run the code on a remote server. See :meth:`execute`.

    Default stdin is the 'null' device, default stdout is the console, and
    default stderr is ``error.out``.
    """

    STDOUT   = shellproc.STDOUT
    DEV_NULL = shellproc.DEV_NULL

    # pylint: disable-msg=E1101
    command = List(Str, desc='The command to be executed.')
    env_vars = Dict({}, iotype='in',
                    desc='Environment variables required by the command.')
    resources = Dict({}, iotype='in',
                     desc='Resources required to run this component.')
    poll_delay = Float(0., low=0., units='s', iotype='in',
                       desc='Delay between polling for command completion.'
                            ' A value of zero will use an internally computed'
                            ' default.')
    timeout = Float(0., low=0., iotype='in', units='s',
                    desc='Maximum time to wait for command completion.'
                         ' A value of zero implies an infinite wait.')
    timed_out = Bool(False, iotype='out', desc='True if the command timed-out.')
    return_code = Int(0, iotype='out', desc='Return code from the command.')

    def __init__(self):
        super(ExternalCode, self).__init__()
        self.check_external_outputs=True

        self.stdin  = self.DEV_NULL
        self.stdout = None
        self.stderr = "error.out"

        self._process = None
        self._server = None

    # This gets used by remote server.
    def get_access_controller(self):  #pragma no cover
        """ Return :class:`AccessController` for this object. """
        return _AccessController()

    @rbac(('owner', 'user'))
    def set(self, path, value):
        """
        Don't allow setting of 'command' or 'resources' by a remote client.
        """
        if path in ('command', 'resources', 'get_access_controller') \
           and remote_access():
            self.raise_exception('%r may not be set() remotely' % path,
                                 RuntimeError)
        return super(ExternalCode, self).set(path, value)

    def execute(self):
        """
        Runs the specified command.

            1. Checks that all external input files exist.
            2. Runs the command.
            3. Checks that all external output files exist.

        If a subclass generates outputs (such as postprocessing results),
        then it should set attribute ``check_external_outputs`` False and call
        :meth:`check_files` itself.

        If `resources` have been specified, an appropriate server
        is allocated and the command is run on that server.
        Otherwise the command is run locally.

        When running remotely, the following resources are set:

        ================ =====================================
        Key              Value
        ================ =====================================
        job_name         self.get_pathname()
        ---------------- -------------------------------------
        remote_command   self.command (first item)
        ---------------- -------------------------------------
        args             self.command (2nd through last items)
        ---------------- -------------------------------------
        job_environment  self.env_vars
        ---------------- -------------------------------------
        input_path       self.stdin
        ---------------- -------------------------------------
        output_path      self.stdout
        ---------------- -------------------------------------
        error_path       self.stderr (if != STDOUT)
        ---------------- -------------------------------------
        join_files       If self.stderr == STDOUT
        ---------------- -------------------------------------
        wallclock_time   self.timeout (if non-zero)
        ================ =====================================

        .. note::

            Input files to be sent to the remote server are defined by
            :class:`FileMetadata` entries in the `external_files` list
            with `input` True.  Similarly, output files to be retrieved
            from the remote server are defined by entries with `output`
            True.

        .. warning::

            Any file **not** labeled with `binary` True will undergo
            newline translation if the local and remote machines have
            different newline representations. Newline translation will
            corrupt a file which is binary but hasn't been labeled as
            such.

        """
        self.return_code = -12345678
        self.timed_out = False

        if not self.command:
            self.raise_exception('Empty command list', ValueError)

        self.check_files(inputs=True)

        return_code = None
        error_msg = ''
        try:
            if self.resources:
                return_code, error_msg = self._execute_remote()
            else:
                return_code, error_msg = self._execute_local()

            if return_code is None:
                if self._stop:
                    self.raise_exception('Run stopped', RunStopped)
                else:
                    self.timed_out = True
                    self.raise_exception('Timed out', RunInterrupted)

            elif return_code:
                if isinstance(self.stderr, str):
                    if os.path.exists(self.stderr):
                        stderrfile = open(self.stderr, 'r')
                        error_desc = stderrfile.read()
                        stderrfile.close()
                        err_fragment = "\nError Output:\n%s" % error_desc
                    else:
                        err_fragment = "\n[stderr %r missing]" % self.stderr
                else:
                    err_fragment = error_msg

                self.raise_exception('return_code = %d%s' \
                    % (return_code, err_fragment), RuntimeError)

            if self.check_external_outputs:
                self.check_files(inputs=False)
        finally:
            self.return_code = -999999 if return_code is None else return_code

    def check_files(self, inputs):
        """
        Check that all 'specific' input or output external files exist.
        If an external file path specifies a pattern, it is *not* checked.

        inputs: bool
            If True, check inputs; otherwise outputs.
        """
        # External files.
        for metadata in self.external_files:
            path = metadata.path
            for ch in ('*?['):
                if ch in path:
                    break
            else:
                if inputs:
                    if not metadata.get('input', False):
                        continue
                else:
                    if not metadata.get('output', False):
                        continue
                if not os.path.exists(path):
                    iotype = 'input' if inputs else 'output'
                    self.raise_exception('missing %s file %r' % (iotype, path),
                                         RuntimeError)
        # Stdin, stdout, stderr.
        if inputs and self.stdin and self.stdin != self.DEV_NULL:
            if not os.path.exists(self.stdin):
                self.raise_exception('missing stdin file %r' % self.stdin,
                                     RuntimeError)

        if not inputs and self.stdout and self.stdout != self.DEV_NULL:
            if not os.path.exists(self.stdout):
                self.raise_exception('missing stdout file %r' % self.stdout,
                                     RuntimeError)

        if not inputs and self.stderr \
                      and self.stderr != self.DEV_NULL \
                      and self.stderr != self.STDOUT \
                      and (not self.resources or \
                           not self.resources.get('join_files')):
            if not os.path.exists(self.stderr):
                self.raise_exception('missing stderr file %r' % self.stderr,
                                     RuntimeError)
        # File variables.
        if inputs:
            for pathname, obj in self.items(iotype='in', recurse=True):
                if isinstance(obj, FileRef):
                    path = self.get_metadata(pathname, 'local_path')
                    if path and not os.path.exists(path):
                        self.raise_exception("missing 'in' file %r" % path,
                                             RuntimeError)
        else:
            for pathname, obj in self.items(iotype='out', recurse=True):
                if isinstance(obj, FileRef):
                    if not os.path.exists(obj.path):
                        self.raise_exception("missing 'out' file %r" % obj.path,
                                             RuntimeError)

    def _execute_local(self):
        """ Run command. """
        self._logger.info('executing %s...', self.command)
        start_time = time.time()

        # check to make sure command exists
        if isinstance(self.command, basestring):
            program_to_execute = self.command
        else:
            program_to_execute = self.command[0]
        command_full_path = find_executable( program_to_execute )

        if not command_full_path:
            self.raise_exception("The command to be executed, '%s', cannot be found" % program_to_execute,
                                 ValueError)

        self._process = \
            shellproc.ShellProc(self.command, self.stdin,
                                self.stdout, self.stderr, self.env_vars)
        self._logger.debug('PID = %d', self._process.pid)

        try:
            return_code, error_msg = \
                self._process.wait(self.poll_delay, self.timeout)
        finally:
            self._process.close_files()
            self._process = None

        et = time.time() - start_time
        if et >= 60:  #pragma no cover
            self._logger.info('elapsed time: %.1f sec.', et)

        return (return_code, error_msg)

    def _execute_remote(self):
        """
        Allocate a server based on required resources, send inputs,
        run command, and retrieve results.
        """
        rdesc = self.resources.copy()

        # Allocate server.
        self._server, server_info = RAM.allocate(rdesc)
        if self._server is None:
            self.raise_exception('Server allocation failed :-(', RuntimeError)

        if self._logger.level == logging.NOTSET:
            # By default avoid lots of protocol messages.
            self._server.set_log_level(logging.DEBUG)
        else:
            self._server.set_log_level(self._logger.level)

        return_code = -88888888
        error_msg = ''
        try:
            # Create resource description for command.
            rdesc['job_name'] = self.get_pathname() or self.__class__.__name__
            rdesc['remote_command'] = self.command[0]
            if len(self.command) > 1:
                rdesc['args'] = self.command[1:]
            if self.env_vars:
                rdesc['job_environment'] = self.env_vars
            if not self.stdin:
                self.raise_exception('Remote execution requires stdin of'
                                     ' DEV_NULL or filename, got %r'
                                     % self.stdin, ValueError)
            if self.stdin != self.DEV_NULL:
                rdesc['input_path'] = self.stdin
            if self.stdout:
                rdesc['output_path'] = self.stdout
            else:
                rdesc['output_path'] = '%s.stdout' % self.command[0]
            if self.stderr:
                if self.stderr == self.STDOUT:
                    rdesc['join_files'] = True
                else:
                    rdesc['error_path'] = self.stderr
            else:
                rdesc['error_path'] = '%s.stderr' % self.command[0]
            if self.timeout:
                if 'resource_limits' in rdesc:
                    limits = rdesc['resource_limits'].copy()
                else:
                    limits = {}
                limits['wallclock_time'] = self.timeout
                rdesc['resource_limits'] = limits

            # Send inputs.
            patterns = []
            textfiles = []
            for metadata in self.external_files:
                if metadata.get('input', False):
                    patterns.append(metadata.path)
                    if not metadata.binary:
                        textfiles.append(metadata.path)
            for pathname, obj in self.items(iotype='in', recurse=True):