<commit_before># -*- coding: utf-8 -*-

from collections import OrderedDict
import logging
import time
from datetime import datetime
import io
import tempfile
import os

#TODO: simplejson ou json
import json

import pandas
import sdmx

from dlstats.fetchers._commons import Fetcher, Categories, Providers, Datasets

logger = logging.getLogger(__name__)

DATASETS = {
    'MEI': { 
        'name': 'Main Economics Indicators',
        'doc_href': 'http://www.oecd-ilibrary.org/economics/data/main-economic-indicators_mei-data-en',
    },
    #'EO': { 
    #    'name': 'Economic Outlook',
    #    'doc_href': 'http://www.oecd.org/eco/outlook/',
    #},
}

class OECD(Fetcher):
    
    def __init__(self, db=None, **kwargs):
        super().__init__(provider_name='OECD', db=db, **kwargs)
        self.provider_name = 'OECD'
        self.provider = Providers(name=self.provider_name, 
                                  long_name='Organisation for Economic Co-operation and Development',
                                  region='world',
                                  website='http://www.oecd.org', 
                                  fetcher=self)

    def upsert_dataset(self, dataset_code, datas=None):
        
        start = time.time()
        
        logger.info("upsert dataset[%s] - START" % (dataset_code))
        
        if not DATASETS.get(dataset_code):
            raise Exception("This dataset is unknown" + dataset_code)
        
        dataset = Datasets(provider_name=self.provider_name, 
                           dataset_code=dataset_code, 
                           name=DATASETS[dataset_code]['name'], 
                           doc_href=DATASETS[dataset_code]['doc_href'],
                           fetcher=self)
        
        fetcher_data = OECD_Data(dataset)
        dataset.series.data_iterator = fetcher_data
        dataset.update_database()

        end = time.time() - start
        logger.info("upsert dataset[%s] - END-BEFORE-METAS - time[%.3f seconds]" % (dataset_code, end))

        self.update_metas(dataset_code)
        
        end = time.time() - start
        logger.info("upsert dataset[%s] - END - time[%.3f seconds]" % (dataset_code, end))

    def datasets_list(self):
        return DATASETS.keys()

    def datasets_long_list(self):
        return [(key, dataset['name']) for key, dataset in DATASETS.items()]

    def upsert_all_datasets(self):
        start = time.time()
        logger.info("update fetcher[%s] - START" % (self.provider_name))
        
        for dataset_code in DATASETS.keys():
            self.upsert_dataset(dataset_code) 
        end = time.time() - start
        logger.info("update fetcher[%s] - END - time[%.3f seconds]" % (self.provider_name, end))
        
    def upsert_categories(self):
        
        for dataset_code in DATASETS.keys():
            document = Categories(provider=self.provider_name, 
                                  name=DATASETS[dataset_code]['name'], 
                                  categoryCode=dataset_code,
                                  exposed=True,
                                  fetcher=self)
            
            document.update_database()                            

class OECD_Data():
    
    '''
    init -> load_data_from_sdmx() -> load_codes() -> __next__() 
    '''
    
    def __init__(self, dataset, limited_countries=None, is_autoload=True):
        
        self.dataset = dataset
        
        #TODO: limited countries
        self.limited_countries = limited_countries# or ['FRA']
        
        self.prepared = None
        
        self.codes = OrderedDict()
        
        self.countries = {}

        self.dimension_keys = []

        self.attribute_keys = []
        
        self.fp = None
                
        self.sdmx_client = sdmx.Repository('http://stats.oecd.org/sdmx-json','json', 
                                           '2_1', 
                                           'OECD', 
                                           timeout=60 * 5)
        
        self.codes_loaded = False
        self.datas_loaded = False
        
        if is_autoload:
            self.load_data_from_sdmx()
    
    def load_codes(self):
        
        if self.codes_loaded:
            return
        
        codes = self.sdmx_client.codes(self.dataset.dataset_code)
        
        header = codes.pop('header')
        
        #'2015-10-27T21:30:00.27625Z'
        self.prepared = datetime.strptime(header['prepared'], "%Y-%m-%dT%H:%M:%S.%fZ")<commit_msg>remove pysdmx depandencies<commit_after># -*- coding: utf-8 -*-

from collections import OrderedDict
import logging
import time
from datetime import datetime
import tempfile
import os
from io import StringIO

#TODO: simplejson ou json
import json

import pandas
import requests

from dlstats.fetchers._commons import Fetcher, Categories, Providers, Datasets

logger = logging.getLogger(__name__)

DATASETS = {
    'MEI': { 
        'name': 'Main Economics Indicators',
        'doc_href': 'http://www.oecd-ilibrary.org/economics/data/main-economic-indicators_mei-data-en',
    },
    #'EO': { 
    #    'name': 'Economic Outlook',
    #    'doc_href': 'http://www.oecd.org/eco/outlook/',
    #},
}

class SDMXJson(object):
    
    def __init__(self, sdmx_url=None, agencyID=None, 
                 timeout=20, requests_client=None):
        
        self.sdmx_url = sdmx_url
        self.agencyID = agencyID
        self.timeout = timeout
        self.requests_client = requests_client
        
        self._codes = {}

    def query_rest(self,url):
        """Retrieve SDMX-json messages.

        :param url: The URL of the message.
        :type url: str
        :return: A dictionnary of the SDMX message
        """
        # Fetch data from the provider    
        logger.info('Requesting %s', url)
        client = self.requests_client or requests
        request = client.get(url, timeout=self.timeout)
        return json.load(StringIO(request.text), object_pairs_hook=OrderedDict)
        
    def metadata(self, flowRef):
        
        resource = 'metadata'
        url = '/'.join([self.sdmx_url, resource, flowRef])
        message_dict = self.query_rest(url)
        #message_dict['structure']['dimensions']['observation']
        self._codes['header'] = message_dict.pop('header', None)
        for code in message_dict['structure']['dimensions']['observation']:
            self._codes[code['name']] = [(x['id'],x['name']) for x in code['values']]
        return self._codes

    def raw_data(self, flowRef, key=None, startperiod=None, endperiod=None):

        code_lists = []         
        raw_dates = {}
        raw_values = {}
        raw_attributes = {}
        raw_codes = {}

        if key is None:
            key = 'all'
        resource = 'data'
        if startperiod and endperiod:
            query = '/'.join([resource, flowRef, key
                    + 'all?startperiod=' + startperiod
                    + '&endPeriod=' + endperiod
                    + '&dimensionAtObservation=TIME'])
        else:
            query = '/'.join([resource, flowRef, key,'all'])
        url = '/'.join([self.sdmx_url,query])
        message_dict = self.query_rest(url)
        
        dates = message_dict['structure']['dimensions']['observation'][0]
        dates = [node['name'] for node in dates['values']]
        series = message_dict['dataSets'][0]['series']
        dimensions = message_dict['structure']['dimensions']
        
        for dimension in dimensions['series']:
            dimension['keyPosition']
            dimension['id']
            dimension['name']
            dimension['values']

        for key in series:
            dims = key.split(':')
            code = ''
            for dimension, position in zip(dimensions['series'],dims):
                code = code + '.' + dimension['values'][int(position)]['id']
            code_lists.append((key, code))

        for key,code in code_lists:
            observations = message_dict['dataSets'][0]['series'][key]['observations']
            series_dates = [int(point) for point in list(observations.keys())]
            raw_dates[code] = [dates[position] for position in series_dates]
            raw_values[code] = [observations[key][0] for key in list(observations.keys())]
            raw_attributes[code] = [observations[key][1] for key in list(observations.keys())]
            raw_codes[code] = {}
            for code_,dim in zip(key.split(':'),message_dict['structure']['dimensions']['series']):
                raw_codes[code][dim['name']] = dim['values'][int(code_)]['id']

        return (raw_values, raw_dates, raw_attributes, raw_codes)
    

class OECD(Fetcher):
    
    def __init__(self, db=None, **kwargs):
        super().__init__(provider_name='OECD', db=db, **kwargs)
        self.provider_name = 'OECD'
        self.provider = Providers(name=self.provider_name, 
                                  long_name='Organisation for Economic Co-operation and Development',
                                  region='world',
                                  website='http://www.oecd.org', 
                                  fetcher=self)

    def upsert_dataset(self, dataset_code, datas=None):
        
        start = time.time()
        
        logger.info("upsert dataset[%s] - START" % (dataset_code))
        
        if not DATASETS.get(dataset_code):
            raise Exception("This dataset is unknown" + dataset_code)
        
        dataset = Datasets(provider_name=self.provider_name, 
                           dataset_code=dataset_code, 
                           name=DATASETS[dataset_code]['name'], 
                           doc_href=DATASETS[dataset_code]['doc_href'],
                           fetcher=self)
        
        fetcher_data = OECD_Data(dataset)
        dataset.series.data_iterator = fetcher_data
        dataset.update_database()

        end = time.time() - start
        logger.info("upsert dataset[%s] - END-BEFORE-METAS - time[%.3f seconds]" % (dataset_code, end))

        self.update_metas(dataset_code)
        
        end = time.time() - start
        logger.info("upsert dataset[%s] - END - time[%.3f seconds]" % (dataset_code, end))

    def datasets_list(self):
        return DATASETS.keys()

    def datasets_long_list(self):
        return [(key, dataset['name']) for key, dataset in DATASETS.items()]

    def upsert_all_datasets(self):
        start = time.time()
        logger.info("update fetcher[%s] - START" % (self.provider_name))
        
        for dataset_code in DATASETS.keys():
            self.upsert_dataset(dataset_code) 
        end = time.time() - start
        logger.info("update fetcher[%s] - END - time[%.3f seconds]" % (self.provider_name, end))
        
    def upsert_categories(self):
        
        for dataset_code in DATASETS.keys():
            document = Categories(provider=self.provider_name, 
                                  name=DATASETS[dataset_code]['name'], 
                                  categoryCode=dataset_code,
                                  exposed=True,
                                  fetcher=self)
            
            document.update_database()                            

class OECD_Data():
    
    '''
    init -> load_data_from_sdmx() -> load_codes() -> __next__() 
    '''
    
    def __init__(self, dataset, limited_countries=None, is_autoload=True):
        
        self.dataset = dataset
        
        #TODO: limited countries
        self.limited_countries = limited_countries# or ['FRA']
        
        self.prepared = None
        
        self.codes = OrderedDict()
        
        self.countries = {}

        self.dimension_keys = []

        self.attribute_keys = []
        
        self.fp = None
                
        self.sdmx_client = SDMXJson(sdmx_url='http://stats.oecd.org/sdmx-json', 
                                    agencyID='OECD', 
                                    timeout=60 * 5, 
                                    requests_client=None)
                                    
        
        self.codes_loaded = False
        self.datas_loaded = False
        
        if is_autoload:
            self.load_data_from_sdmx()
    
    def load_codes(self):
        
        if self.codes_loaded:
            return
        
        codes = self.sdmx_client.metadata(self.dataset.dataset_code)
        
        header = codes.pop('header')
        
        #'2015-10-27T21:30:00.27625Z'
        self.prepared = datetime.strptime(header['prepared'], "%Y-%m-%dT%H:%M:%S.%fZ")
{'fraction_non_alphanumeric': 0.11758163184237705, 'fraction_numerical': 0.004846269961070946, 'mean_word_length': 1.8035634743875277, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6853208', 'n_tokens_mistral': 3681, 'n_tokens_neox': 3539, 'n_words': 671}