<commit_before>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
.. module:: eurostat
    :platform: Unix
    :synopsis: Populate a MongoDB database with data from Eurostat

.. :moduleauthor :: Widukind team <<EMAIL>>
"""

import collections
from dlstats.fetchers.skeleton import Skeleton
import lxml.etree
import urllib
from pandas.tseries.offsets import *
import pandas
import datetime
from io import BytesIO, StringIO, TextIOWrapper
import urllib.request
import gzip
import re
import pymongo
import logging
from multiprocessing import Pool
import pysdmx


def _create_series(leaf):
    """Create time series documents in MongoDB. This function is defined outside of Eurostat() so that the multiprocessing module can use :mod:`pickle`.
    :param leaf: A leaf from http://epp.eurostat.ec.europa.eu/NavTree_prod/everybody/BulkDownloadListing/table_of_contents.xml
    :type lxml.etree.ElementTree 
    :returns: A tuple providing additional info. The first member is True if the insertion succeeded. The second member is the flowRef identifying the DataFlow that was pulled."""
    try:
        id_journal = self.db.journal.insert({'method': '_create_series'})
        client = pymongo.MongoClient()
        db = client.eurostat
        key = '....'
        series_ = pysdmx.eurostat.data_extraction(leaf['flowRef'][0],key)
        for series in [series_.time_series[key] for key in series_.time_series.keys()]:<commit_msg>Import order<commit_after>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
.. module:: eurostat
    :platform: Unix
    :synopsis: Populate a MongoDB database with data from Eurostat

.. :moduleauthor :: Widukind team <<EMAIL>>
"""

from dlstats.fetchers.skeleton import Skeleton
import collections
import lxml.etree
import urllib
from pandas.tseries.offsets import *
import pandas
import datetime
from io import BytesIO, StringIO, TextIOWrapper
import urllib.request
import gzip
import re
import pymongo
import logging
from multiprocessing import Pool
import pysdmx


def _create_series(leaf):
    """Create time series documents in MongoDB. This function is defined outside of Eurostat() so that the multiprocessing module can use :mod:`pickle`.
    :param leaf: A leaf from http://epp.eurostat.ec.europa.eu/NavTree_prod/everybody/BulkDownloadListing/table_of_contents.xml
    :type lxml.etree.ElementTree 
    :returns: A tuple providing additional info. The first member is True if the insertion succeeded. The second member is the flowRef identifying the DataFlow that was pulled."""
    try:
        id_journal = self.db.journal.insert({'method': '_create_series'})
        client = pymongo.MongoClient()
        db = client.eurostat
        key = '....'
        series_ = pysdmx.eurostat.data_extraction(leaf['flowRef'][0],key)
        for series in [series_.time_series[key] for key in series_.time_series.keys()]:
{'fraction_non_alphanumeric': 0.09610662925289372, 'fraction_numerical': 0.002104524728165556, 'mean_word_length': 4.02112676056338, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3067503', 'n_tokens_mistral': 866, 'n_tokens_neox': 847, 'n_words': 276}