<jupyter_start><jupyter_text>Filter osemosys_global datapackaged based on user-defined geographic scope  Import modules<jupyter_code>import pandas as pd
import os
geographic_scope = ['IND', 'CHN']
model_name = 'geo_filter_test_v4'

if not os.path.exists(os.path.join(os.getcwd(),
                           model_name,
                           'data')):
    os.makedirs(os.path.join(os.getcwd(),
                           model_name,
                           'data'))

for each_csv in (os.listdir(os.path.join(os.getcwd(),
                                         r'osemosys_global_model\data'))):
    
    df = pd.read_csv(os.path.join(os.getcwd(),
                                  r'osemosys_global_model\data',
                                  each_csv)
                    )
    if not df.empty:
        if 'TECHNOLOGY' in df.columns:
            df = df.loc[df['TECHNOLOGY'].str[3:6].isin(geographic_scope) | 
                        df['TECHNOLOGY'].str[6:9].isin(geographic_scope) | 
                        df['TECHNOLOGY'].str[8:11].isin(geographic_scope)]
            
        if 'FUEL' in df.columns:
            df = df.loc[df['FUEL'].str[3:6].isin(geographic_scope) | 
                        df['FUEL'].str[6:9].isin(geographic_scope)]
        
        if each_csv == 'FUEL.csv':
            df = df.loc[df['VALUE'].str[3:6].isin(geographic_scope) | 
                        df['VALUE'].str[6:9].isin(geographic_scope)]
        
        if each_csv == 'TECHNOLOGY.csv':
            df = df.loc[df['VALUE'].str[3:6].isin(geographic_scope) | 
                        df['VALUE'].str[6:9].isin(geographic_scope) | 
                        df['VALUE'].str[8:11].isin(geographic_scope)]
        
    
    df.to_csv(os.path.join(os.getcwd(),
                           model_name,
                           'data',
                           each_csv),
              index = None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13291796469366562, 'fraction_numerical': 0.011941848390446522, 'mean_word_length': 1.486451612903226, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '448217', 'license': 'MIT', 'n_tokens_mistral': 636, 'n_tokens_neox': 601, 'n_words': 74, 'path': 'notebooks/OPG_geographic_filter.ipynb', 'repo_name': 'maartenbrinkerink/osemosys_global', 'size': '1926'}