<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import codecs
import logging
import os

import requests

from udata.models import (
    Dataset, GeoZone, TERRITORY_DATASETS, ResourceBasedTerritoryDataset
)
from udata.commands import submanager
from udata.core.storages import logos, references

log = logging.getLogger(__name__)


m = submanager(
    'territories',
    help='Territories specifics operations',
    description='Handle all territories related operations and maintenance'
)


@m.command
def fetch_territories_logos():
    """Retrieves images of flags or blazons from wikimedia."""
    log.info('As of January 2016, the download is about 1GB. Time to relax.')
    # A bit tricky but otherwise you cannot guess the final file URL.
    DBPEDIA_MEDIA_URL = 'http://commons.wikimedia.org/wiki/Special:FilePath/'
    LOGOS_FOLDER_PATH = logos.root
    if not os.path.exists(LOGOS_FOLDER_PATH):
        os.makedirs(LOGOS_FOLDER_PATH)

    geozones = GeoZone.objects.filter(level='fr/town')
    for geozone in geozones.only('flag', 'blazon'):
        if geozone.flag or geozone.blazon:
            filename = geozone.flag.filename or geozone.blazon.filename
            with open(logos.path(filename), 'wb') as file_destination:
                r = requests.get(DBPEDIA_MEDIA_URL + filename, stream=True)
                for chunk in r.iter_content(chunk_size=1024):
                    file_destination.write(chunk)

    log.info('Done')


@m.command
def collect_references_files():
    """Retrieve locally CSV files in use for dynamic territories' resources."""
    REFERENCES_PATH = references.root
    if not os.path.exists(REFERENCES_PATH):
        os.makedirs(REFERENCES_PATH)
    for territory_class in TERRITORY_DATASETS.values():
        if not issubclass(territory_class, ResourceBasedTerritoryDataset):
            continue
        dataset = Dataset.objects.get(id=territory_class.dataset_id)
        for resource in dataset.resources:
            if resource.id == territory_class.resource_id:
                break
            filename = resource.url.split('/')[-1]
            reference_path = references.path(filename)
            with codecs.open(reference_path, 'w', encoding='utf8') as fd:
                r = requests.get(resource.url, stream=True)
                for chunk in r.iter_content(chunk_size=1024):
                    fd.write(chunk.decode('latin-1'))  # TODO: detect?

    log.info('Done')<commit_msg>Download territories/references files only if missing, refs #284<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import codecs
import logging
import os

import requests

from udata.models import (
    Dataset, GeoZone, TERRITORY_DATASETS, ResourceBasedTerritoryDataset
)
from udata.commands import submanager
from udata.core.storages import logos, references

log = logging.getLogger(__name__)


m = submanager(
    'territories',
    help='Territories specifics operations',
    description='Handle all territories related operations and maintenance'
)


@m.command
def fetch_territories_logos():
    """Retrieves images of flags or blazons from wikimedia."""
    log.info('As of January 2016, the download is about 1GB. Time to relax.')
    # A bit tricky but otherwise you cannot guess the final file URL.
    DBPEDIA_MEDIA_URL = 'http://commons.wikimedia.org/wiki/Special:FilePath/'
    LOGOS_FOLDER_PATH = logos.root
    if not os.path.exists(LOGOS_FOLDER_PATH):
        os.makedirs(LOGOS_FOLDER_PATH)

    geozones = GeoZone.objects.filter(level='fr/town')
    for geozone in geozones.only('flag', 'blazon'):
        if geozone.flag or geozone.blazon:
            filename = geozone.flag.filename or geozone.blazon.filename
            if os.path.exists(logos.path(filename)):
                continue
            with open(logos.path(filename), 'wb') as file_destination:
                r = requests.get(DBPEDIA_MEDIA_URL + filename, stream=True)
                for chunk in r.iter_content(chunk_size=1024):
                    file_destination.write(chunk)

    log.info('Done')


@m.command
def collect_references_files():
    """Retrieve locally CSV files in use for dynamic territories' resources."""
    REFERENCES_PATH = references.root
    if not os.path.exists(REFERENCES_PATH):
        os.makedirs(REFERENCES_PATH)
    for territory_class in TERRITORY_DATASETS.values():
        if not issubclass(territory_class, ResourceBasedTerritoryDataset):
            continue
        dataset = Dataset.objects.get(id=territory_class.dataset_id)
        for resource in dataset.resources:
            if resource.id == territory_class.resource_id:
                break
            filename = resource.url.split('/')[-1]
            reference_path = references.path(filename)
            if os.path.exists(reference_path):
                continue
            with codecs.open(reference_path, 'w', encoding='utf8') as fd:
                r = requests.get(resource.url, stream=True)
                for chunk in r.iter_content(chunk_size=1024):
                    fd.write(chunk.decode('latin-1'))  # TODO: detect?

    log.info('Done')
{'fraction_non_alphanumeric': 0.09558966388187294, 'fraction_numerical': 0.0071886535846123955, 'mean_word_length': 3.0093457943925235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6800827', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1506, 'n_words': 370}