<commit_before>import os
import glob
from collections import OrderedDict

from flask_restplus import Namespace, Resource, abort
import pandas

import cea.databases
import cea.scripts
from cea.utilities.schedule_reader import read_cea_schedule, save_cea_schedule

api = Namespace("Databases", description="Database data for technologies in CEA")
cea_database_path = os.path.dirname(cea.databases.__file__)

# FIXME: Should use inputlocator to find default databases instead? But still need a way to map database names to their locator method for schemas
# { db_type: { db_name: db_prop, ... }, ... }
DATABASES = OrderedDict([
    ("archetypes", {
        "construction_properties": {
            "file_ext": ".xlsx",
            "schema_key": "get_archetypes_properties"
        },
        "schedules": {
            "file_ext": "",  # Folder
            "schema_key": "get_database_standard_schedules_use"
        }
    }),
    ("systems", {
        "air_conditioning_systems": {
            "file_ext": ".xls",
            "schema_key": "get_database_air_conditioning_systems"
        },
        "envelope_systems": {
            "file_ext": ".xls",
            "schema_key": "get_database_envelope_systems"
        },
        "conversion_systems": {
            "file_ext": ".xls",
            "schema_key": "get_database_conversion_systems"
        },
        "distribution_systems": {
            "file_ext": ".xls",
            "schema_key": "get_database_distribution_systems"
        }
    }),
    ("feedstocks", {
        "feedstocks": {
            "file_ext": ".xls",
            "schema_key": "get_database_feedstocks"
        },
    })
])

DATABASES_TYPE_MAP = {db_name: db_type for db_type, db_dict in DATABASES.items() for db_name in db_dict}
DATABASE_NAMES = DATABASES_TYPE_MAP.keys()


def get_regions():
    return [folder for folder in os.listdir(cea_database_path) if folder != "weather"
            and os.path.isdir(os.path.join(cea_database_path, folder))]


# # FIXME: Temp solution for database locators for shipped databases
def get_database_path(region, db_name):
    db_type = DATABASES_TYPE_MAP[db_name]
    file_name = '{}{}'.format(db_name, DATABASES[db_type][db_name]['file_ext'])
    return os.path.join(cea_database_path, region, db_type, file_name)


def get_all_schedules_dict(schedules_folder):
    """Read all schedules in schedules folder to a dict with their names as keys"""
    out = {}
    schedule_files = glob.glob(os.path.join(schedules_folder, "*.csv"))
    for schedule_file in schedule_files:
        archetype = os.path.splitext(os.path.basename(schedule_file))[0]
        schedule_path = os.path.join(schedules_folder, schedule_file)
        out[archetype] = schedule_to_dict(schedule_path)<commit_msg>Update databases.py<commit_after>import os
import glob
from collections import OrderedDict

from flask_restplus import Namespace, Resource, abort
import pandas

import cea.databases
import cea.scripts
from cea.utilities.schedule_reader import read_cea_schedule, save_cea_schedule

api = Namespace("Databases", description="Database data for technologies in CEA")
cea_database_path = os.path.dirname(cea.databases.__file__)

# FIXME: Should use inputlocator to find default databases instead? But still need a way to map database names to their locator method for schemas
# { db_type: { db_name: db_prop, ... }, ... }
DATABASES = OrderedDict([
    ("archetypes", {
        "CONSTRUCTION": {
            "file_ext": ".xlsx",
            "schema_key": "get_archetypes_properties"
        },
        "SCHEDULES": {
            "file_ext": "",  # Folder
            "schema_key": "get_database_standard_schedules_use"
        }
    }),
    ("assemblies", {
        "ASSEMBLIES": {
            "file_ext": ".xls",
            "schema_key": "get_database_assemblies"
        },
    }),
    ("components", {
        "AIR-CONDITIONING": {
            "file_ext": ".xls",
            "schema_key": "get_database_air_conditioning_systems"
        },
        "ENVELOPE": {
            "file_ext": ".xls",
            "schema_key": "get_database_envelope_systems"
        },
        "CONVERSION": {
            "file_ext": ".xls",
            "schema_key": "get_database_conversion_systems"
        },
        "DISTRIBUTION": {
            "file_ext": ".xls",
            "schema_key": "get_database_distribution_systems"
        }
    }),
    ("feedstocks", {
        "FEEDSTOCKS": {
            "file_ext": ".xls",
            "schema_key": "get_database_feedstocks"
        },
    })
])

DATABASES_TYPE_MAP = {db_name: db_type for db_type, db_dict in DATABASES.items() for db_name in db_dict}
DATABASE_NAMES = DATABASES_TYPE_MAP.keys()


def get_regions():
    return [folder for folder in os.listdir(cea_database_path) if folder != "weather"
            and os.path.isdir(os.path.join(cea_database_path, folder))]


# # FIXME: Temp solution for database locators for shipped databases
def get_database_path(region, db_name):
    db_type = DATABASES_TYPE_MAP[db_name]
    file_name = '{}{}'.format(db_name, DATABASES[db_type][db_name]['file_ext'])
    return os.path.join(cea_database_path, region, db_type, file_name)


def get_all_schedules_dict(schedules_folder):
    """Read all schedules in schedules folder to a dict with their names as keys"""
    out = {}
    schedule_files = glob.glob(os.path.join(schedules_folder, "*.csv"))
    for schedule_file in schedule_files:
        archetype = os.path.splitext(os.path.basename(schedule_file))[0]
        schedule_path = os.path.join(schedules_folder, schedule_file)
        out[archetype] = schedule_to_dict(schedule_path)
{'fraction_non_alphanumeric': 0.11641153435386259, 'fraction_numerical': 0.000355998576005696, 'mean_word_length': 3.10445580715851, 'pattern_counts': {'":': 45, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '660887', 'n_tokens_mistral': 1839, 'n_tokens_neox': 1786, 'n_words': 372}