<jupyter_start><jupyter_text>Faster calculations with aggregation and simplificationsThis notebook shows how we can get very fast calculations for a specific use case where the foreground is dynamic, but we know that the background doesn't change, and where the LCIA methods are fixed ahead of time. We know that [precalculated aggregated datasets can be fast](https://link.springer.com/article/10.1007/s11367-018-1444-x), and they are easy to generate in Brightway. If we know that the background won't change, we can replace *only the needed background activities* with precalculated activities, and skip loading and processing the whole background system completely.We can also reduce the biosphere matrix to only the flows needed to calculate the preselected LCIA method scores.<jupyter_code>import brightway2 as bw
import numpy as np
import random<jupyter_output><empty_output><jupyter_text>Generate example database `B`In our example, we will use ecoinvent as `A`, and make up a random `B`.<jupyter_code>ECOINVENT_LABEL = "ecoinvent 3.5 cutoff"
assert ECOINVENT_LABEL in bw.databases
some_methods = [bw.methods.random() for _ in range(3)]
all_flows = [o[0] for method in some_methods for o in bw.Method(method).load()]
some_flows = [random.choice(all_flows) for _ in range(10)]
db = iter(bw.Database(ECOINVENT_LABEL))

some_inputs = [next(db) for _ in range(5)]<jupyter_output><empty_output><jupyter_text>We can now build our example database.<jupyter_code>example_data = [{
    'name': 'fake {}'.format(index),
    'code': str(index),
    'exchanges': [{
        'input': ('example', str(index)),
        'amount': 1,
        'type': 'production',
    }]
} for index in range(5)]
# Add links to background database
for _ in range(10):
    random.choice(example_data)['exchanges'].append({
        'input': random.choice(some_inputs),
        'amount': np.random.normal() + 1,
        'type': 'technosphere'
    })
    
    
# Add links to foreground database
for _ in range(8):
    random.choice(example_data)['exchanges'].append({
        'input': ('example', random.choice(example_data)['code']),
        'amount': np.random.normal() + 1,
        'type': 'technosphere'
    })

# Add links to biosphere flows
for _ in range(6):
    random.choice(example_data)['exchanges'].append({
        'input': random.choice(some_flows),
        'amount': np.random.normal(),
        'type': 'biosphere'
    })
bw.Database("example").write({('example', o.pop("code")): o for o in example_data})<jupyter_output>Writing activities to SQLite3 database:
0% [#####] 100% | ETA: 00:00:00
Total time elapsed: 00:00:00<jupyter_text>Check to make sure that all our foreground activities have LCIA scores for our methods:<jupyter_code>lca = bw.LCA({bw.Database("example").random(): 1}, method=some_methods[0])
lca.lci()
lca.lcia()

for act in bw.Database("example"):
    for method in some_methods:
        lca.switch_method(method)
        lca.redo_lcia({act: 1})
        assert lca.score != 0<jupyter_output><empty_output><jupyter_text>Aggregate background activitiesUltimately, we want the [processed](https://docs.brightwaylca.org/intro.htmlintermediate-and-processed-data) [array](https://docs.brightwaylca.org/lca.htmlbuilding-matrices) for this database - but this will be generated automatically if we write a new database.<jupyter_code>background_scores = {}

for act in some_inputs:
    background_scores[act.key] = {}
    for method in some_methods:
        lca.switch_method(method)
        lca.redo_lcia({act: 1})
        background_scores[act.key][method] = lca.score
NEW_NAME = "Aggregated ecoinvent background"
def as_tuple(o):
    if isinstance(o, tuple):
        return o
    elif hasattr(o, "key"):
        return o.key
    else:
        raise ValueError
def extract_relevant_flows(act, lca, flows):
    """Return the life cycle inventory elementary flow amounts for one unit of demand of ``act``.
    
    ``lca`` is a pre-calculated ``LCA`` object in which ``act`` is in the technosphere.
    ``flows`` is an iterable of biosphere flows.
    
    Returns a dictionary like ``{flow_key: amount}``."""
    lca.redo_lci({act: 1})
    flows_present = {o for o in flows if o in lca.biosphere_dict}
    # Get total inventory (for entire supply chain)
    inventory_vector = lca.inventory.sum(axis=1)
    data = {flow: inventory_vector[lca.biosphere_dict[flow]] for flow in flows_present}
    return {k: v for k, v in data.items() if v != 0}
def format_as_brightway_dataset(act, new_db_name, biosphere_exchanges):
    """Minimally format an activity and a dict of biosphere exchange amounts as a Brightway database ready for writing."""
    # Because we have calculated the inventory for 1 unit already, don't need to rescale
    # or deal with other production exchange weirdness. A production exchange will
    # be added automatically
    return {'exchanges': [{
            'input': as_tuple(flow),
            'output': (new_db_name, act[1]),
            'amount': amount,
            'type': "biosphere",
        } for flow, amount in biosphere_exchanges.items()]
    }
new_background_database = {
    (NEW_NAME, act['code']): format_as_brightway_dataset(act, NEW_NAME, extract_relevant_flows(act, lca, some_flows)) 
    for act in some_inputs
}
bw.Database(NEW_NAME).write(new_background_database)<jupyter_output>Writing activities to SQLite3 database:
0% [#####] 100% | ETA: 00:00:00
Total time elapsed: 00:00:00<jupyter_text>Check to make sure our aggregated database gives the right results<jupyter_code>processed_array_filepaths = [bw.Database(NEW_NAME).filepath_processed(), bw.Database("biosphere3").filepath_processed()]
class IndependentLCA(bw.IndepentLCAMixin, bw.LCA):
    pass
bw.Method(method).filepath_processed()
for method in some_methods:
    original_lca = bw.LCA({some_inputs[0]: 1}, method=method)
    original_lca.lci()
    original_lca.lcia()
    
    independent_lca = IndependentLCA(
        {bw.mapping[some_inputs[0]]: 1}, 
        database_filepath=processed_array_filepaths,
        method=[bw.Method(method).filepath_processed()]
    )
    independent_lca.lci()
    independent_lca.lcia()
def map_model_outputs(list_of_keys, offset=100_000_000):
    """Generate a mapping dictionary of ``(key: integer)`` for use in preparation of processed arrays.
    
    Requires Python 3.6 for fancy number stuff :)"""
    return {key: offset + index for index, key in enumerate(sorted(set(list_of_keys)))}
from bw2data.utils import TYPE_DICTIONARY, MAX_INT_32
import numpy as np


def process_list_of_exchanges(data, filepath):
    """Given an iterable of ``(input_key, output_key, type, amount)``, generate a Brightway processed array at the given filepath.
    
    The input and output keys must already be mapped, i.e. they must be integers between 0 and 4294967295. Be careful to ensure they don't overlap with other keys used!"""
    dtype = [
        ('input', np.uint32),
        ('output', np.uint32),
        ('row', np.uint32),
        ('col', np.uint32),
        ('type', np.uint8),
        ('amount', np.float32),
    ]
    arr = np.zeros((len(data),), dtype=dtype)
    
    for index, line in enumerate(data):
        arr[index] = (
            line['input'],
            line['output'],
            MAX_INT_32,
            MAX_INT_32,
            TYPE_DICTIONARY[line["type"]],
            line['amount'],
        )

    arr.sort(order=['input', 'output', 'amount'])
    np.save(filepath, arr, allow_pickle=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11826807430175064, 'fraction_numerical': 0.016437257784311106, 'mean_word_length': 3.93993399339934, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '366286', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2444, 'n_tokens_neox': 2319, 'n_words': 680, 'path': 'notebooks/Faster calculations with aggregation and simplifications.ipynb', 'repo_name': 'elinlucy/brightway2', 'size': '7483'}