<commit_before>#!/usr/bin/env python3

from demandlib import bdew as bdew, particular_profiles as profiles
from egoio.db_tables.calc_ego_loads import EgoDeuConsumptionArea as orm_loads,\
    EgoDemandPerTransitionPoint as orm_demand
from egoio.tools import db
from sqlalchemy.orm import sessionmaker
from sqlalchemy import func
from workalendar.europe import Germany
import pandas as pd
from datetime import time as settime
from math import sqrt
from data_processing.tools import io, metadata
import time


def demand_per_mv_grid_district():
    year = 2013
    schema = 'calc_ego_loads'
    target_table = 'ego_demand_per_transition_point'
    db_group = 'oeuser'

    columns_names = {'h0': 'residential',
                     'g0': 'retail',
                     'i0': 'industrial',
                     'l0': 'agricultural'}

    inv_columns_names = {v: k for k, v in columns_names.items()}

    # The following dictionary is create by "workalendar"
    # pip3 install workalendar

    cal = Germany()
    holidays = dict(cal.holidays(2010))

    # Alternatively, define holidays manually
    # holidays = {
    #     datetime.date(2010, 5, 24): 'Whit Monday',
    #     datetime.date(2010, 4, 5): 'Easter Monday',
    #     datetime.date(2010, 5, 13): 'Ascension Thursday',
    #     datetime.date(2010, 1, 1): 'New year',
    #     datetime.date(2010, 10, 3): 'Day of German Unity',
    #     datetime.date(2010, 12, 25): 'Christmas Day',
    #     datetime.date(2010, 5, 1): 'Labour Day',
    #     datetime.date(2010, 4, 2): 'Good Friday',
    #     datetime.date(2010, 12, 26): 'Second Christmas Day'}

    # retrieve sectoral demand from oedb

    # get database connection
    conn = io.oedb_session(section='oedb')
    Session = sessionmaker(bind=conn)
    session = Session()

    query_demand = session.query(orm_loads.subst_id,
                                 func.sum(orm_loads.sector_consumption_residential).\
                                 label('residential'),
                                 func.sum(orm_loads.sector_consumption_retail).label('retail'),
                                 func.sum(orm_loads.sector_consumption_industrial).\
                                 label('industrial'),
                                 func.sum(orm_loads.sector_consumption_agricultural).\
                                 label('agricultural')).\
                                 group_by(orm_loads.subst_id)

    annual_demand_df = pd.read_sql_query(
        query_demand.statement, session.bind, index_col='subst_id').fillna(0)

    large_scale_industrial = pd.read_sql_table('large_scale_consumer', conn, schema,
                                               index_col='id')

    # add extra industrial demand ontop of MV industrial demand
    annual_demand_df = pd.concat(
        [annual_demand_df,
         large_scale_industrial.groupby(
             by='subst_id').sum()['consumption']],
        axis=1)
    annual_demand_df['industrial'] = annual_demand_df[
        ['industrial', 'consumption']].sum(axis=1)
    annual_demand_df.drop('consumption', axis=1, inplace=True)
<commit_msg>change input db table and usage of id from subst_id to otg_id<commit_after>#!/usr/bin/env python3

from demandlib import bdew as bdew, particular_profiles as profiles
from egoio.db_tables.calc_ego_loads import EgoDeuConsumption as orm_loads,\
    EgoDemandPerTransitionPoint as orm_demand
from egoio.tools import db
from sqlalchemy.orm import sessionmaker
from sqlalchemy import func
from workalendar.europe import Germany
import pandas as pd
from datetime import time as settime
from math import sqrt
from data_processing.tools import io, metadata
import time


def demand_per_mv_grid_district():
    year = 2013
    schema = 'calc_ego_loads'
    target_table = 'ego_demand_per_transition_point'
    db_group = 'oeuser'

    columns_names = {'h0': 'residential',
                     'g0': 'retail',
                     'i0': 'industrial',
                     'l0': 'agricultural'}

    inv_columns_names = {v: k for k, v in columns_names.items()}

    # The following dictionary is create by "workalendar"
    # pip3 install workalendar

    cal = Germany()
    holidays = dict(cal.holidays(2010))

    # Alternatively, define holidays manually
    # holidays = {
    #     datetime.date(2010, 5, 24): 'Whit Monday',
    #     datetime.date(2010, 4, 5): 'Easter Monday',
    #     datetime.date(2010, 5, 13): 'Ascension Thursday',
    #     datetime.date(2010, 1, 1): 'New year',
    #     datetime.date(2010, 10, 3): 'Day of German Unity',
    #     datetime.date(2010, 12, 25): 'Christmas Day',
    #     datetime.date(2010, 5, 1): 'Labour Day',
    #     datetime.date(2010, 4, 2): 'Good Friday',
    #     datetime.date(2010, 12, 26): 'Second Christmas Day'}

    # retrieve sectoral demand from oedb

    # get database connection
    conn = io.oedb_session(section='oedb')
    Session = sessionmaker(bind=conn)
    session = Session()

    query_demand = session.query(orm_loads.otg_id,
                                 func.sum(orm_loads.sector_consumption_residential).\
                                 label('residential'),
                                 func.sum(orm_loads.sector_consumption_retail).label('retail'),
                                 func.sum(orm_loads.sector_consumption_industrial).\
                                 label('industrial'),
                                 func.sum(orm_loads.sector_consumption_agricultural).\
                                 label('agricultural')).\
                                 group_by(orm_loads.otg_id)

    annual_demand_df = pd.read_sql_query(
        query_demand.statement, session.bind, index_col='otg_id').fillna(0)

    large_scale_industrial = pd.read_sql_table('large_scale_consumer', conn, schema,
                                               index_col='id')

    # add extra industrial demand ontop of MV industrial demand
    annual_demand_df = pd.concat(
        [annual_demand_df,
         large_scale_industrial.groupby(
             by='otg_id').sum()['consumption']],
        axis=1)
    annual_demand_df['industrial'] = annual_demand_df[
        ['industrial', 'consumption']].sum(axis=1)
    annual_demand_df.drop('consumption', axis=1, inplace=True)

{'fraction_non_alphanumeric': 0.10162991371045062, 'fraction_numerical': 0.02524768296580377, 'mean_word_length': 2.427710843373494, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1888434', 'n_tokens_mistral': 2010, 'n_tokens_neox': 1822, 'n_words': 425}