<commit_before>import unittest
import tempfile
import shutil
import asyncio

from electrum.util import bh2u, bfh, create_and_start_event_loop
from electrum.lnonion import (OnionHopsDataSingle, new_onion_packet,
                              process_onion_packet, _decode_onion_error, decode_onion_error,
                              OnionFailureCode, OnionPacket)
from electrum import bitcoin, lnrouter
from electrum.constants import BitcoinTestnet
from electrum.simple_config import SimpleConfig
from electrum.lnrouter import PathEdge

from . import TestCaseForTestnet
from .test_bitcoin import needs_test_with_all_chacha20_implementations


class Test_LNRouter(TestCaseForTestnet):

    def setUp(self):
        super().setUp()
        self.asyncio_loop, self._stop_loop, self._loop_thread = create_and_start_event_loop()
        self.config = SimpleConfig({'electrum_path': self.electrum_path})

    def tearDown(self):
        self.asyncio_loop.call_soon_threadsafe(self._stop_loop.set_result, 1)
        self._loop_thread.join(timeout=1)
        super().tearDown()

    def test_find_path_for_payment(self):
        class fake_network:
            config = self.config
            asyncio_loop = asyncio.get_event_loop()
            trigger_callback = lambda *args: None
            register_callback = lambda *args: None
            interface = None
        fake_network.channel_db = lnrouter.ChannelDB(fake_network())
        fake_network.channel_db.data_loaded.set()
        cdb = fake_network.channel_db
        path_finder = lnrouter.LNPathFinder(cdb)
        self.assertEqual(cdb.num_channels, 0)
        cdb.add_channel_announcements({'node_id_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'node_id_2': b'\x02cccccccccccccccccccccccccccccccc',
                                     'bitcoin_key_1': b'\<KEY>', 'bitcoin_key_2': b'\x02cccccccccccccccccccccccccccccccc',
                                     'short_channel_id': bfh('0000000000000001'),
                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
                                     'len': 0, 'features': b''}, trusted=True)
        self.assertEqual(cdb.num_channels, 1)
        cdb.add_channel_announcements({'node_id_1': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
                                     'bitcoin_key_1': b'\<KEY>', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
                                     'short_channel_id': bfh('0000000000000002'),
                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
                                     'len': 0, 'features': b''}, trusted=True)
        cdb.add_channel_announcements({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',
                                     'bitcoin_key_1': b'\<KEY>', 'bitcoin_key_2': b'\<KEY>',
                                     'short_channel_id': bfh('0000000000000003'),
                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
                                     'len': 0, 'features': b''}, trusted=True)
        cdb.add_channel_announcements({'node_id_1': b'\x02cccccccccccccccccccccccccccccccc', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd',
                                     'bitcoin_key_1': b'\x02cccccccccccccccccccccccccccccccc', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd',
                                     'short_channel_id': bfh('0000000000000004'),
                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
                                     'len': 0, 'features': b''}, trusted=True)
        cdb.add_channel_announcements({'node_id_1': b'\x02dddddddddddddddddddddddddddddddd', 'node_id_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
                                     'bitcoin_key_1': b'\x02dddddddddddddddddddddddddddddddd', 'bitcoin_key_2': b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
                                     'short_channel_id': bfh('0000000000000005'),
                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
                                     'len': 0, 'features': b''}, trusted=True)
        cdb.add_channel_announcements({'node_id_1': b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'node_id_2': b'\x02dddddddddddddddddddddddddddddddd',
                                     'bitcoin_key_1': b'\<KEY>', 'bitcoin_key_2': b'\x02dddddddddddddddddddddddddddddddd',
                                     'short_channel_id': bfh('0000000000000006'),
                                     'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
                                     'len': 0, 'features': b''}, trusted=True)
        def add_chan_upd(payload):
            cdb.add_channel_update(payload, verify=False)
        add_chan_upd({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000001'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 99, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000002'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000003'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000004'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000005'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 99999999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': bfh('0000000000000006'), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        path = path_finder.find_path_for_payment(
            nodeA=b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
            nodeB=b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
            invoice_amount_msat=100000)
        self.assertEqual([PathEdge(start_node=b'\x02aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', end_node=b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', short_channel_id=bfh('0000000000000003')),
                          PathEdge(start_node=b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', end_node=b'\x02eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', short_channel_id=bfh('0000000000000002')),
                         ], path)
        route = path_finder.create_route_from_path(path)
        self.assertEqual(b'\x02bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', route[0].node_id)
        self.assertEqual(bfh('0000000000000003'),                 route[0].short_channel_id)

        cdb.stop()
        asyncio.run_coroutine_threadsafe(cdb.stopped_event.wait(), self.asyncio_loop).result()

    @needs_test_with_all_chacha20_implementations
    def test_new_onion_packet_legacy(self):
        # test vector from bolt-04
        payment_path_pubkeys = [
            bfh('<KEY>'),
            bfh('0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c'),
            bfh('027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007'),
            bfh('<KEY>'),
            bfh('<KEY>'),
        ]
        session_key = bfh('4141414141414141414141414141414141414141414141414141414141414141')
        associated_data = bfh('4242424242424242424242424242424242424242424242424242424242424242')
        hops_data = [
            OnionHopsDataSingle(is_tlv_payload=False, payload={
                "amt_to_forward": {"amt_to_forward": 0},
                "outgoing_cltv_value": {"outgoing_cltv_value": 0},
                "short_channel_id": {"short_channel_id": bfh('0000000000000000')},
            }),
            OnionHopsDataSingle(is_tlv_payload=False, payload={
                "amt_to_forward": {"amt_to_forward": 1},
                "outgoing_cltv_value": {"outgoing_cltv_value": 1},
                "short_channel_id": {"short_channel_id": bfh('0101010101010101')},
            }),
            OnionHopsDataSingle(is_tlv_payload=False, payload={
                "amt_to_forward": {"amt_to_forward": 2},
                "outgoing_cltv_value": {"outgoing_cltv_value": 2},<commit_msg>lnrouter tests: add another channel to graph<commit_after>import unittest
import tempfile
import shutil
import asyncio

from electrum.util import bh2u, bfh, create_and_start_event_loop
from electrum.lnutil import ShortChannelID
from electrum.lnonion import (OnionHopsDataSingle, new_onion_packet,
                              process_onion_packet, _decode_onion_error, decode_onion_error,
                              OnionFailureCode, OnionPacket)
from electrum import bitcoin, lnrouter
from electrum.constants import BitcoinTestnet
from electrum.simple_config import SimpleConfig
from electrum.lnrouter import PathEdge

from . import TestCaseForTestnet
from .test_bitcoin import needs_test_with_all_chacha20_implementations


def channel(number: int) -> ShortChannelID:
    return ShortChannelID(bfh(format(number, '016x')))


def node(character: str) -> bytes:
    return b'\x02' + f'{character}'.encode() * 32


class Test_LNRouter(TestCaseForTestnet):

    def setUp(self):
        super().setUp()
        self.asyncio_loop, self._stop_loop, self._loop_thread = create_and_start_event_loop()
        self.config = SimpleConfig({'electrum_path': self.electrum_path})

    def tearDown(self):
        self.asyncio_loop.call_soon_threadsafe(self._stop_loop.set_result, 1)
        self._loop_thread.join(timeout=1)
        super().tearDown()

    def prepare_graph(self):
        """
        Network topology with channel ids:
                3
            A  ---  B
            |    2/ |
          6 |   E   | 1
            | /5 \7 |
            D  ---  C
                4
        valid routes from A -> E:
        A -3-> B -2-> E
        A -6-> D -5-> E
        A -6-> D -4-> C -7-> E
        A -3-> B -1-> C -7-> E
        A -6-> D -4-> C -1-> B -2-> E
        A -3-> B -1-> C -4-> D -5-> E
        """
        class fake_network:
            config = self.config
            asyncio_loop = asyncio.get_event_loop()
            trigger_callback = lambda *args: None
            register_callback = lambda *args: None
            interface = None
        fake_network.channel_db = lnrouter.ChannelDB(fake_network())
        fake_network.channel_db.data_loaded.set()
        self.cdb = fake_network.channel_db
        self.path_finder = lnrouter.LNPathFinder(self.cdb)
        self.assertEqual(self.cdb.num_channels, 0)
        self.cdb.add_channel_announcements({
            'node_id_1': node('b'), 'node_id_2': node('c'),
            'bitcoin_key_1': node('b'), 'bitcoin_key_2': node('c'),
            'short_channel_id': channel(1),
            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
            'len': 0, 'features': b''
        }, trusted=True)
        self.assertEqual(self.cdb.num_channels, 1)
        self.cdb.add_channel_announcements({
            'node_id_1': node('b'), 'node_id_2': node('e'),
            'bitcoin_key_1': node('b'), 'bitcoin_key_2': node('e'),
            'short_channel_id': channel(2),
            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
            'len': 0, 'features': b''
        }, trusted=True)
        self.cdb.add_channel_announcements({
            'node_id_1': node('a'), 'node_id_2': node('b'),
            'bitcoin_key_1': node('a'), 'bitcoin_key_2': node('b'),
            'short_channel_id': channel(3),
            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
            'len': 0, 'features': b''
        }, trusted=True)
        self.cdb.add_channel_announcements({
            'node_id_1': node('c'), 'node_id_2': node('d'),
            'bitcoin_key_1': node('c'), 'bitcoin_key_2': node('d'),
            'short_channel_id': channel(4),
            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
            'len': 0, 'features': b''
        }, trusted=True)
        self.cdb.add_channel_announcements({
            'node_id_1': node('d'), 'node_id_2': node('e'),
            'bitcoin_key_1': node('d'), 'bitcoin_key_2': node('e'),
            'short_channel_id': channel(5),
            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
            'len': 0, 'features': b''
        }, trusted=True)
        self.cdb.add_channel_announcements({
            'node_id_1': node('a'), 'node_id_2': node('d'),
            'bitcoin_key_1': node('a'), 'bitcoin_key_2': node('d'),
            'short_channel_id': channel(6),
            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
            'len': 0, 'features': b''
        }, trusted=True)
        self.cdb.add_channel_announcements({
            'node_id_1': node('c'), 'node_id_2': node('e'),
            'bitcoin_key_1': node('c'), 'bitcoin_key_2': node('e'),
            'short_channel_id': channel(7),
            'chain_hash': BitcoinTestnet.rev_genesis_bytes(),
            'len': 0, 'features': b''
        }, trusted=True)
        def add_chan_upd(payload):
            self.cdb.add_channel_update(payload, verify=False)
        add_chan_upd({'short_channel_id': channel(1), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(1), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(2), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 99, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(2), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(3), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(3), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(4), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(4), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(5), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(5), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 999, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(6), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 200, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(6), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(7), 'message_flags': b'\x00', 'channel_flags': b'\x00', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})
        add_chan_upd({'short_channel_id': channel(7), 'message_flags': b'\x00', 'channel_flags': b'\x01', 'cltv_expiry_delta': 10, 'htlc_minimum_msat': 250, 'fee_base_msat': 100, 'fee_proportional_millionths': 150, 'chain_hash': BitcoinTestnet.rev_genesis_bytes(), 'timestamp': 0})

    def test_find_path_for_payment(self):
        self.prepare_graph()
        amount_to_send = 100000

        path = self.path_finder.find_path_for_payment(
            nodeA=node('a'),
            nodeB=node('e'),
            invoice_amount_msat=amount_to_send)
        self.assertEqual([
            PathEdge(start_node=node('a'), end_node=node('b'), short_channel_id=channel(3)),
            PathEdge(start_node=node('b'), end_node=node('e'), short_channel_id=channel(2)),
        ], path)

        route = self.path_finder.create_route_from_path(path)
        self.assertEqual(node('b'), route[0].node_id)
        self.assertEqual(channel(3), route[0].short_channel_id)

        self.cdb.stop()
        asyncio.run_coroutine_threadsafe(self.cdb.stopped_event.wait(), self.asyncio_loop).result()

    @needs_test_with_all_chacha20_implementations
    def test_new_onion_packet_legacy(self):
        # test vector from bolt-04
        payment_path_pubkeys = [
            bfh('<KEY>'),
            bfh('0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c'),
            bfh('027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007'),
            bfh('<KEY>'),
            bfh('<KEY>'),
        ]
        session_key = bfh('4141414141414141414141414141414141414141414141414141414141414141')
        associated_data = bfh('4242424242424242424242424242424242424242424242424242424242424242')
        hops_data = [
            OnionHopsDataSingle(is_tlv_payload=False, payload={
                "amt_to_forward": {"amt_to_forward": 0},
                "outgoing_cltv_value": {"outgoing_cltv_value": 0},
                "short_channel_id": {"short_channel_id": bfh('0000000000000000')},
            }),
            OnionHopsDataSingle(is_tlv_payload=False, payload={
                "amt_to_forward": {"amt_to_forward": 1},
                "outgoing_cltv_value": {"outgoing_cltv_value": 1},
                "short_channel_id": {"short_channel_id": bfh('0101010101010101')},
            }),
            OnionHopsDataSingle(is_tlv_payload=False, payload={
                "amt_to_forward": {"amt_to_forward": 2},
                "outgoing_cltv_value": {"outgoing_cltv_value": 2},
{'fraction_non_alphanumeric': 0.13753896937465615, 'fraction_numerical': 0.06675224646983312, 'mean_word_length': 3.5349272349272347, 'pattern_counts': {'":': 32, '<': 14, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 50, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7367703', 'n_tokens_mistral': 8581, 'n_tokens_neox': 7331, 'n_words': 1126}