<commit_before>"""
lcopt.model
-----------

Module containing the LcoptModel class.

"""

from lcopt.io import *
#from lcopt.ipython_interactive import IFS
from lcopt.interact import FlaskSandbox
from lcopt.bw2_export import Bw2Exporter
from lcopt.analysis import Bw2Analysis
from lcopt.data_store import storage
from .export_disclosure import export_disclosure

from .utils import check_for_config, lcopt_bw2_autosetup, bw2_project_exists, write_search_index, upgrade_old_default, lcopt_bw2_forwast_setup, forwast_autosetup
from .constants import DEFAULT_PROJECT_STEM, FORWAST_PROJECT_NAME, DEFAULT_ECOINVENT_VERSION, DEFAULT_ECOINVENT_SYSTEM_MODEL, LEGACY_SAVE_OPTION
# This is a copy straight from bw2data.query, extracted so as not to cause a dependency.
#from lcopt.bw2query import Query, Dictionaries, Filter
from bw2data.query import Query, Dictionaries, Filter

from functools import partial
from collections import OrderedDict
import numpy as np
import pickle
import random
import pandas as pd
import warnings
from random import randint

from jinja2 import Environment, PackageLoader

import os
import getpass

#From bw2 - edited to reinsert capitalisation of units

UNITS_NORMALIZATION = {
    "a": "year",  # Common in LCA circles; could be confused with acre
    "Bq": "Becquerel",
    "g": "gram",
    "Gj": "gigajoule",
    "h": "hour",
    "ha": "hectare",
    "hr": "hour",
    "kBq": "kilo Becquerel",
    "kg": "kilogram",
    "kgkm": "kilogram kilometer",
    "km": "kilometer",
    "kj": "kilojoule",
    "kWh": "kilowatt hour",
    "l": "litre",
    "lu": "livestock unit",
    "m": "meter",
    "m*year": "meter-year",
    "m2": "square meter",
    "m2*year": "square meter-year",
    "m2a": "square meter-year",
    "m2y": "square meter-year",
    "m3": "cubic meter",
    "m3*year": "cubic meter-year",
    "m3a": "cubic meter-year",
    "m3y": "cubic meter-year",
    "ma": "meter-year",
    "metric ton*km": "ton kilometer",
    "MJ": "megajoule",
    "my": "meter-year",
    "nm3": "cubic meter",
    "p": "unit",
    "personkm": "person kilometer",
    "person*km": "person kilometer",
    "pkm": "person kilometer",
    "tkm": "ton kilometer",
    "vkm": "vehicle kilometer",
    'kg sw': "kilogram separative work unit",
    'km*year': "kilometer-year",
    'metric ton*km': "ton kilometer",
    'person*km': "person kilometer",
    'Wh': 'watt hour',
}


def unnormalise_unit(unit):
    if unit in UNITS_NORMALIZATION.keys():
        return unit
    else:
        
        un_units = list(filter(lambda x: UNITS_NORMALIZATION[x] == unit, UNITS_NORMALIZATION))
        #print (un_units)
        return un_units[0]


class LcoptModel(object):
    """
    This is the base model class.

    To create a new model, enter a name e.g. ``model = LcoptModel('My_Model')``

    To load an existing model use the ``load`` option e.g. ``model = LcoptModel(load = 'My_Model')``

    """

    def __init__(self, name=hex(random.getrandbits(128))[2:-1], load=None, useForwast=False, ecoinvent_version=None, ecoinvent_system_model=None, ei_username = None, ei_password = None, write_config=None):
        super(LcoptModel, self).__init__()

        # name the instance
        self.name = name
        
        # set up the database, parameter dictionaries, the matrix and the names of the exchanges
        self.database = {'items': OrderedDict(), 'name': '{}_Database'.format(self.name)}
        self.external_databases = []
        self.params = OrderedDict()
        self.production_params = OrderedDict()
        self.allocation_params = OrderedDict()
        self.ext_params = []
        self.matrix = None
        self.names = None
        self.parameter_sets = OrderedDict()
        self.model_matrices = OrderedDict()
        self.technosphere_matrices = OrderedDict()
        self.leontif_matrices = OrderedDict()
        self.parameter_map = {}

        self.sandbox_positions = {}

        # If ecoinvent isn't specified in the setup, look for a default in the config and fall back on default set in constants
        if ecoinvent_version is None:
            ecoinvent_version = str(storage.ecoinvent_version)

        if ecoinvent_system_model is None:
            ecoinvent_system_model = storage.ecoinvent_system_model

        ei_name = "Ecoinvent{}_{}_{}".format(*ecoinvent_version.split("."), ecoinvent_system_model) #"Ecoinvent3_3_cutoff"
        
        self.ecoinventName = ei_name # "Ecoinvent3_3_cutoff"
        self.biosphereName = "biosphere3"
        self.ecoinventFilename = ei_name # "ecoinvent3_3"
        self.biosphereFilename = "biosphere3"
        self.forwastName = "forwast"
        self.forwastFilename = "forwast"
        self.useForwast = useForwast
        self.technosphere_databases = []
        #if self.useForwast:
        #    self.technosphere_databases = [self.forwastName]
        #else:
        #    self.technosphere_databases = [self.ecoinventName]

        #self.biosphere_databases = [self.biosphereName]
        self.biosphere_databases = []

        # default settings for bw2 analysis
        self.analysis_settings = {'amount': 1, 
                                  'methods': [('IPCC 2013', 'climate change', 'GWP 100a'), ('USEtox', 'human toxicity', 'total')], 
                                  #'top_processes': 10, 
                                  #'gt_cutoff': 0.01, 
                                  'pie_cutoff': 0.05
                                  }

        self.allow_allocation = False

        # initialise with a blank result set
        self.result_set = None

        # set the save option, this defaults to the config value but should be overwritten on load for existing models
        self.save_option = storage.save_option

        # check if lcopt is set up, and if not, set it up
        self.lcopt_setup(ei_username=ei_username, ei_password=<PASSWORD>, write_config=write_config,
                         ecoinvent_version=ecoinvent_version, ecoinvent_system_model = ecoinvent_system_model)

        if load is not None:
            self.load(load)

        asset_path = storage.search_index_dir #os.path.join(os.path.dirname(os.path.realpath(__file__)), 'assets')
        ecoinventPath = os.path.join(asset_path, self.ecoinventFilename)
        biospherePath = os.path.join(asset_path, self.biosphereFilename)
        forwastPath = os.path.join(asset_path, self.forwastFilename)

        # Try and initialise the external databases if they're not there already
        if self.useForwast:
            if self.forwastName not in [x['name'] for x in self.external_databases]:
                self.import_external_db(forwastPath, 'technosphere')

        else:
            if self.ecoinventName not in [x['name'] for x in self.external_databases]:
                self.import_external_db(ecoinventPath, 'technosphere')

        if self.biosphereName not in [x['name'] for x in self.external_databases]:
            self.import_external_db(biospherePath, 'biosphere')

        # create partial version of io functions
        self.add_to_database = partial(add_to_specified_database, database=self.database)
        self.get_exchange = partial(get_exchange_from_database, database=self.database)
        self.exists_in_database = partial(exists_in_specific_database, database=self.database)
        self.get_name = partial(get_exchange_name_from_database, database=self.database)
        self.get_unit = partial(get_exchange_unit_from_database, database=self.database)

        self.parameter_scan()

    def lcopt_setup(self, ei_username, ei_password, write_config, ecoinvent_version, ecoinvent_system_model):
        if storage.project_type == 'single':

            if self.useForwast:

                forwast_autosetup()

            else:

                self.base_project_name = storage.single_project_name

                #if bw2_project_exists(self.base_project_name):
                lcopt_bw2_autosetup(ei_username = ei_username, ei_password = <PASSWORD>, write_config=write_config, ecoinvent_version=ecoinvent_version, ecoinvent_system_model = ecoinvent_system_model, overwrite=False)

        elif not self.useForwast:

            self.base_project_name = DEFAULT_PROJECT_STEM + self.ecoinventName
            old_default = DEFAULT_PROJECT_STEM[:-1]
            is_default = ecoinvent_version == "3.3" and ecoinvent_system_model == "cutoff"

            if bw2_project_exists(self.base_project_name):
                # make sure the search index file is there too
                write_search_index(self.base_project_name, self.ecoinventName)
            elif is_default and bw2_project_exists(old_default):
                upgrade_old_default()
            else:
                print("Lcopt needs to be set up to integrate with brightway2 - this only needs to be done once per version/system model combo")
                lcopt_bw2_autosetup(ei_username = ei_username, ei_password = <PASSWORD>, write_config=write_config, ecoinvent_version=ecoinvent_version, ecoinvent_system_model = ecoinvent_system_model, overwrite=True)

        else:
            forwast_autosetup()

    def rename(self, newname):
        """change the name of the model (i.e. what the .lcopt file will be saved as)"""
        self.name = newname
        
    #def saveAs(self, filename):
    #    """save the instance as a pickle"""
    #    pickle.dump(self, open("{}.pickle".format(filename), "wb"))
    
    def save(self):
        """save the instance as a .lcopt file"""
        if self.save_option == 'curdir':
            model_path = os.path.join(
                os.getcwd(),
                '{}.lcopt'.format(self.name)
            )
        else: # default to appdir
            model_path = os.path.join(
                storage.model_dir,
                '{}.lcopt'.format(self.name)
            )
        with open(model_path, 'wb') as model_file:
            pickle.dump(self, model_file)

    def load(self, filename):
        """load data from a saved .lcopt file"""
        if filename[-6:] != ".lcopt":
            filename += ".lcopt"
        try:
            savedInstance = pickle.load(open("{}".format(filename), "rb"))
        except FileNotFoundError:
            savedInstance = pickle.load(open(os.path.join(storage.model_dir, "{}".format(filename)), "rb"))
        
        attributes = ['name',
                      'database',
                      'params',
                      'production_params',
                      'allocation_params',
                      'ext_params',
                      'matrix',
                      'names',
                      'parameter_sets',
                      'model_matrices',
                      'technosphere_matrices',
                      'leontif_matrices',
                      'external_databases',
                      'parameter_map',
                      'sandbox_positions',
                      'ecoinventName',
                      'biosphereName',
                      'forwastName',
                      'analysis_settings',
                      'technosphere_databases',
                      'biosphere_databases',
                      'result_set',
                      'evaluated_parameter_sets',
                      'useForwast',
                      'base_project_name',
                      'save_option',
                      'allow_allocation'
                      ]

        for attr in attributes:

            if hasattr(savedInstance, attr):
                setattr(self, attr, getattr(savedInstance, attr))

        # use legacy save option if this is missing from the model
        if not hasattr(savedInstance, 'save_option'):
            setattr(self, 'save_option', LEGACY_SAVE_OPTION)

    def create_product (self, name, location='GLO', unit='kg', **kwargs):

        """
        Create a new product in the model database
        """

        new_product = item_factory(name=name, location=location, unit=unit, type='product', **kwargs)

        if not self.exists_in_database(new_product['code']):
            self.add_to_database(new_product)
            print ('{} added to database'.format(name))
            return self.get_exchange(name)
        else:
            #print('{} already exists in this database'.format(name))
            return False

    def create_process(self, name, exchanges, location='GLO', unit='kg'):
        """
        Create a new process, including all new exchanges (in brightway2's exchange format) in the model database.

        Exchanges must have at least a name, type and unit field
        """

        found_exchanges = []
        for e in exchanges:

            exc_name = e.pop('name', None)
            exc_type = e.pop('type', None)

            this_exchange = self.get_exchange(exc_name)<commit_msg>change ordering of load and setup to fix #56<commit_after>"""
lcopt.model
-----------

Module containing the LcoptModel class.

"""

from lcopt.io import *
#from lcopt.ipython_interactive import IFS
from lcopt.interact import FlaskSandbox
from lcopt.bw2_export import Bw2Exporter
from lcopt.analysis import Bw2Analysis
from lcopt.data_store import storage
from .export_disclosure import export_disclosure

from .utils import check_for_config, lcopt_bw2_autosetup, bw2_project_exists, write_search_index, upgrade_old_default, lcopt_bw2_forwast_setup, forwast_autosetup
from .constants import DEFAULT_PROJECT_STEM, FORWAST_PROJECT_NAME, DEFAULT_ECOINVENT_VERSION, DEFAULT_ECOINVENT_SYSTEM_MODEL, LEGACY_SAVE_OPTION
# This is a copy straight from bw2data.query, extracted so as not to cause a dependency.
#from lcopt.bw2query import Query, Dictionaries, Filter
from bw2data.query import Query, Dictionaries, Filter

from functools import partial
from collections import OrderedDict
import numpy as np
import pickle
import random
import pandas as pd
import warnings
from random import randint

from jinja2 import Environment, PackageLoader

import os
import getpass

#From bw2 - edited to reinsert capitalisation of units

UNITS_NORMALIZATION = {
    "a": "year",  # Common in LCA circles; could be confused with acre
    "Bq": "Becquerel",
    "g": "gram",
    "Gj": "gigajoule",
    "h": "hour",
    "ha": "hectare",
    "hr": "hour",
    "kBq": "kilo Becquerel",
    "kg": "kilogram",
    "kgkm": "kilogram kilometer",
    "km": "kilometer",
    "kj": "kilojoule",
    "kWh": "kilowatt hour",
    "l": "litre",
    "lu": "livestock unit",
    "m": "meter",
    "m*year": "meter-year",
    "m2": "square meter",
    "m2*year": "square meter-year",
    "m2a": "square meter-year",
    "m2y": "square meter-year",
    "m3": "cubic meter",
    "m3*year": "cubic meter-year",
    "m3a": "cubic meter-year",
    "m3y": "cubic meter-year",
    "ma": "meter-year",
    "metric ton*km": "ton kilometer",
    "MJ": "megajoule",
    "my": "meter-year",
    "nm3": "cubic meter",
    "p": "unit",
    "personkm": "person kilometer",
    "person*km": "person kilometer",
    "pkm": "person kilometer",
    "tkm": "ton kilometer",
    "vkm": "vehicle kilometer",
    'kg sw': "kilogram separative work unit",
    'km*year': "kilometer-year",
    'metric ton*km': "ton kilometer",
    'person*km': "person kilometer",
    'Wh': 'watt hour',
}


def unnormalise_unit(unit):
    if unit in UNITS_NORMALIZATION.keys():
        return unit
    else:
        
        un_units = list(filter(lambda x: UNITS_NORMALIZATION[x] == unit, UNITS_NORMALIZATION))
        #print (un_units)
        return un_units[0]


class LcoptModel(object):
    """
    This is the base model class.

    To create a new model, enter a name e.g. ``model = LcoptModel('My_Model')``

    To load an existing model use the ``load`` option e.g. ``model = LcoptModel(load = 'My_Model')``

    """

    def __init__(self, name=hex(random.getrandbits(128))[2:-1], load=None, useForwast=False, ecoinvent_version=None, ecoinvent_system_model=None, ei_username = None, ei_password = None, write_config=None):
        super(LcoptModel, self).__init__()

        # name the instance
        self.name = name
        
        # set up the database, parameter dictionaries, the matrix and the names of the exchanges
        self.database = {'items': OrderedDict(), 'name': '{}_Database'.format(self.name)}
        self.external_databases = []
        self.params = OrderedDict()
        self.production_params = OrderedDict()
        self.allocation_params = OrderedDict()
        self.ext_params = []
        self.matrix = None
        self.names = None
        self.parameter_sets = OrderedDict()
        self.model_matrices = OrderedDict()
        self.technosphere_matrices = OrderedDict()
        self.leontif_matrices = OrderedDict()
        self.parameter_map = {}

        self.sandbox_positions = {}

        # If ecoinvent isn't specified in the setup, look for a default in the config and fall back on default set in constants
        if ecoinvent_version is None:
            self.ecoinvent_version = str(storage.ecoinvent_version)
        else:
            self.ecoinvent_version = ecoinvent_version

        if ecoinvent_system_model is None:
            self.ecoinvent_system_model = storage.ecoinvent_system_model
        else:
            self.ecoinvent_system_model = ecoinvent_system_model

        ei_name = "Ecoinvent{}_{}_{}".format(*self.ecoinvent_version.split("."), self.ecoinvent_system_model) #"Ecoinvent3_3_cutoff"

        print(ei_name)
        
        self.ecoinventName = ei_name # "Ecoinvent3_3_cutoff"
        self.biosphereName = "biosphere3"
        self.ecoinventFilename = ei_name # "ecoinvent3_3"
        self.biosphereFilename = "biosphere3"
        self.forwastName = "forwast"
        self.forwastFilename = "forwast"
        self.useForwast = useForwast
        self.technosphere_databases = []
        #if self.useForwast:
        #    self.technosphere_databases = [self.forwastName]
        #else:
        #    self.technosphere_databases = [self.ecoinventName]

        #self.biosphere_databases = [self.biosphereName]
        self.biosphere_databases = []

        # default settings for bw2 analysis
        self.analysis_settings = {'amount': 1, 
                                  'methods': [('IPCC 2013', 'climate change', 'GWP 100a'), ('USEtox', 'human toxicity', 'total')], 
                                  #'top_processes': 10, 
                                  #'gt_cutoff': 0.01, 
                                  'pie_cutoff': 0.05
                                  }

        self.allow_allocation = False

        # initialise with a blank result set
        self.result_set = None

        # set the save option, this defaults to the config value but should be overwritten on load for existing models
        self.save_option = storage.save_option
        
        if load is not None:
            self.load(load)
            print(self.ecoinventName)

        # check if lcopt is set up, and if not, set it up
        self.lcopt_setup(ei_username=ei_username, ei_password=<PASSWORD>, write_config=write_config,
                         ecoinvent_version=self.ecoinvent_version, ecoinvent_system_model = self.ecoinvent_system_model)

        asset_path = storage.search_index_dir #os.path.join(os.path.dirname(os.path.realpath(__file__)), 'assets')
        ecoinventPath = os.path.join(asset_path, self.ecoinventFilename)
        biospherePath = os.path.join(asset_path, self.biosphereFilename)
        forwastPath = os.path.join(asset_path, self.forwastFilename)

        # Try and initialise the external databases if they're not there already
        if self.useForwast:
            if self.forwastName not in [x['name'] for x in self.external_databases]:
                self.import_external_db(forwastPath, 'technosphere')

        else:
            if self.ecoinventName not in [x['name'] for x in self.external_databases]:
                self.import_external_db(ecoinventPath, 'technosphere')

        if self.biosphereName not in [x['name'] for x in self.external_databases]:
            self.import_external_db(biospherePath, 'biosphere')

        # create partial version of io functions
        self.add_to_database = partial(add_to_specified_database, database=self.database)
        self.get_exchange = partial(get_exchange_from_database, database=self.database)
        self.exists_in_database = partial(exists_in_specific_database, database=self.database)
        self.get_name = partial(get_exchange_name_from_database, database=self.database)
        self.get_unit = partial(get_exchange_unit_from_database, database=self.database)

        self.parameter_scan()

    def lcopt_setup(self, ei_username, ei_password, write_config, ecoinvent_version, ecoinvent_system_model):
        print(ecoinvent_version, ecoinvent_system_model)
        if storage.project_type == 'single':

            if self.useForwast:

                forwast_autosetup()

            else:

                self.base_project_name = storage.single_project_name

                #if bw2_project_exists(self.base_project_name):
                lcopt_bw2_autosetup(ei_username = ei_username, ei_password = <PASSWORD>, write_config=write_config, ecoinvent_version=ecoinvent_version, ecoinvent_system_model = ecoinvent_system_model, overwrite=False)

        elif not self.useForwast:

            self.base_project_name = DEFAULT_PROJECT_STEM + self.ecoinventName
            old_default = DEFAULT_PROJECT_STEM[:-1]
            is_default = ecoinvent_version == "3.3" and ecoinvent_system_model == "cutoff"

            if bw2_project_exists(self.base_project_name):
                # make sure the search index file is there too
                write_search_index(self.base_project_name, self.ecoinventName)
            elif is_default and bw2_project_exists(old_default):
                upgrade_old_default()
            else:
                print("Lcopt needs to be set up to integrate with brightway2 - this only needs to be done once per version/system model combo")
                lcopt_bw2_autosetup(ei_username = ei_username, ei_password = <PASSWORD>, write_config=write_config, ecoinvent_version=ecoinvent_version, ecoinvent_system_model = ecoinvent_system_model, overwrite=True)

        else:
            forwast_autosetup()

    def rename(self, newname):
        """change the name of the model (i.e. what the .lcopt file will be saved as)"""
        self.name = newname
        
    #def saveAs(self, filename):
    #    """save the instance as a pickle"""
    #    pickle.dump(self, open("{}.pickle".format(filename), "wb"))
    
    def save(self):
        """save the instance as a .lcopt file"""
        if self.save_option == 'curdir':
            model_path = os.path.join(
                os.getcwd(),
                '{}.lcopt'.format(self.name)
            )
        else: # default to appdir
            model_path = os.path.join(
                storage.model_dir,
                '{}.lcopt'.format(self.name)
            )
        with open(model_path, 'wb') as model_file:
            pickle.dump(self, model_file)

    def load(self, filename):
        """load data from a saved .lcopt file"""
        if filename[-6:] != ".lcopt":
            filename += ".lcopt"
        try:
            savedInstance = pickle.load(open("{}".format(filename), "rb"))
        except FileNotFoundError:
            savedInstance = pickle.load(open(os.path.join(storage.model_dir, "{}".format(filename)), "rb"))
        
        attributes = ['name',
                      'database',
                      'params',
                      'production_params',
                      'allocation_params',
                      'ext_params',
                      'matrix',
                      'names',
                      'parameter_sets',
                      'model_matrices',
                      'technosphere_matrices',
                      'leontif_matrices',
                      'external_databases',
                      'parameter_map',
                      'sandbox_positions',
                      'ecoinventName',
                      'biosphereName',
                      'forwastName',
                      'analysis_settings',
                      'technosphere_databases',
                      'biosphere_databases',
                      'result_set',
                      'evaluated_parameter_sets',
                      'useForwast',
                      'base_project_name',
                      'save_option',
                      'allow_allocation',
                      'ecoinvent_version',
                      'ecoinvent_system_model',
                      ]

        for attr in attributes:

            if hasattr(savedInstance, attr):
                setattr(self, attr, getattr(savedInstance, attr))
            else:
                print ("can't set {}".format(attr))

        # use legacy save option if this is missing from the model
        if not hasattr(savedInstance, 'save_option'):
            setattr(self, 'save_option', LEGACY_SAVE_OPTION)

        # figure out ecoinvent version and system model if these are missing from the model
        if not hasattr(savedInstance, 'ecoinvent_version') or not hasattr(savedInstance, 'ecoinvent_system_model'):
            
            parts = savedInstance.ecoinventName.split("_")
            main_version = parts[0][-1]
            sub_version = parts[1]
            system_model = parts[2]

            print(parts)

            setattr(self, 'ecoinvent_version', '{}.{}'.format(main_version, sub_version))
            setattr(self, 'ecoinvent_system_model', system_model)



    def create_product (self, name, location='GLO', unit='kg', **kwargs):

        """
        Create a new product in the model database
        """

        new_product = item_factory(name=name, location=location, unit=unit, type='product', **kwargs)

        if not self.exists_in_database(new_product['code']):
            self.add_to_database(new_product)
            print ('{} added to database'.format(name))
            return self.get_exchange(name)
        else:
            #print('{} already exists in this database'.format(name))
            return False

    def create_process(self, name, exchanges, location='GLO', unit='kg'):
        """
        Create a new process, including all new exchanges (in brightway2's exchange format) in the model database.

        Exchanges must have at least a name, type and unit field
        """

        found_exchanges = []
        for e in exchanges:

            exc_name = e.pop('name', None)
            exc_type = e.pop('type', None)

            this_exchange = self.get_exchange(exc_name)
{'fraction_non_alphanumeric': 0.1014917521512043, 'fraction_numerical': 0.004884830721827678, 'mean_word_length': 2.376125840416085, 'pattern_counts': {'":': 74, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2828139', 'n_tokens_mistral': 8156, 'n_tokens_neox': 7764, 'n_words': 1868}