<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

import logging
import xml.etree.ElementTree as et
from flask import render_template_string
from superdesk.utils import config
from superdesk import get_resource_service
from superdesk.metadata.item import ITEM_STATE, CONTENT_STATE
from datetime import datetime
from superdesk.ftp import ftp_connect
from io import BytesIO
from flask import current_app as app


logger = logging.getLogger(__name__)

# This is a list of the names of the forecast locations that we are interested in
met_forecast_areas = {'NSW': ['Sydney', 'Parramatta', 'Campbelltown', 'Penrith', 'Wollongong', 'Newcastle'],
                      'QLD': ['Brisbane', 'Gold Coast Seaway', 'Cairns', 'Gladstone', 'Mount Isa'],
                      'VIC': ['Melbourne', 'Phillip Island', 'Geelong', 'Ballarat', 'Wilsons Promontory'],
                      'SA': ['Adelaide', 'Whyalla', 'Coober Pedy', 'Port Augusta', 'Mount Gambier'],
                      'WA': ['Perth', 'Margaret River', 'Geraldton', 'Broome', 'Kalgoorlie']}

# This is a list of the names ov the available observation stations
met_observation_stations = {'NSW': ['Sydney - Observatory Hill', 'Sydney Olympic Park', 'Parramatta', 'Campbelltown',
                                    'Penrith', 'Albion Park', 'Newcastle University', 'Newcastle Nobbys'],
                            'QLD': ['Brisbane', 'Gold Coast Seaway', 'Cairns', 'Gladstone', 'Mount Isa'],
                            'VIC': ['Melbourne (Olympic Park)', 'Rhyll', 'Geelong Racecourse', 'Ballarat',
                                    'Wilsons Promontory'],
                            'SA': ['Adelaide (West Terrace /  ngayirdapira)', 'Whyalla', 'Coober Pedy Airport',
                                   'Port Augusta', 'Mount Gambier'],
                            'WA': ['Perth', 'Witchcliffe', 'Geraldton Airport', 'Broome', 'Kalgoorlie-Boulder']}

forecast_files = {'NSW': 'IDN11060.xml',
                  'QLD': 'IDQ11295.xml',
                  'SA': 'IDS10044.xml',
                  'VIC': 'IDV10753.xml',
                  'WA': 'IDW14199.xml'}

observation_files = {'NSW': 'IDN60920.xml',
                     'QLD': 'IDQ60920.xml',
                     'SA': 'IDS60920.xml',
                     'VIC': 'IDV60920.xml',
                     'WA': 'IDW60920.xml'}


def forecast_story(item, **kwargs):

    def _get_file(filename):
        raw = BytesIO()
        with ftp_connect({'username': app.config.get('BOM_WEATHER_FTP_USERNAME', ''),
                          'password': app.config.get('BOM_WEATHER_FTP_PASSWORD', ''),
                          'host': 'ftp.bom.gov.au',
                          'path': 'fwo'}) as ftp:
            ftp.retrbinary('RETR ' + filename, raw.write)
        return raw.getvalue()

    def _get_forecast(state):
        return _get_file(forecast_files.get(state))

    def _get_observations(state):
        return _get_file(observation_files.get(state))

    def add_value(period, dict, name, base):

        direction_map = {'N': 'north', 'NNE': 'north-northeast', 'NE': 'northeast', 'ENE': 'east-northeast',
                         'E': 'east', 'ESE': 'east-southeast', 'SE': 'southeast', 'SSE': 'south-southeast',
                         'S': 'south', 'SSW': 'south-southwest', 'SW': 'southwest', 'WSW': 'west-southwest',
                         'W': 'west', 'WNW': 'west-northwest', 'NW': 'northwest', 'NNW': 'north-northwest'}

        try:
            node = period.find('.//*[@type="{}"]'.format(name))
            if not node:
                period.get(name)
            value = node.text
            if 'temperature' in name:
                value = value.replace('.0', '')
            if 'precis' in name:
                value = value.replace('.', '')
            if 'wind_dir' == name:<commit_msg>fix(bom) retry download from BOM (#798)<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

import logging
import xml.etree.ElementTree as et
from flask import render_template_string
from superdesk.utils import config
from superdesk import get_resource_service
from superdesk.metadata.item import ITEM_STATE, CONTENT_STATE
from datetime import datetime
from superdesk.ftp import ftp_connect
from io import BytesIO
from flask import current_app as app


logger = logging.getLogger(__name__)

# This is a list of the names of the forecast locations that we are interested in
met_forecast_areas = {'NSW': ['Sydney', 'Parramatta', 'Campbelltown', 'Penrith', 'Wollongong', 'Newcastle'],
                      'QLD': ['Brisbane', 'Gold Coast Seaway', 'Cairns', 'Gladstone', 'Mount Isa'],
                      'VIC': ['Melbourne', 'Phillip Island', 'Geelong', 'Ballarat', 'Wilsons Promontory'],
                      'SA': ['Adelaide', 'Whyalla', 'Coober Pedy', 'Port Augusta', 'Mount Gambier'],
                      'WA': ['Perth', 'Margaret River', 'Geraldton', 'Broome', 'Kalgoorlie']}

# This is a list of the names ov the available observation stations
met_observation_stations = {'NSW': ['Sydney - Observatory Hill', 'Sydney Olympic Park', 'Parramatta', 'Campbelltown',
                                    'Penrith', 'Albion Park', 'Newcastle University', 'Newcastle Nobbys'],
                            'QLD': ['Brisbane', 'Gold Coast Seaway', 'Cairns', 'Gladstone', 'Mount Isa'],
                            'VIC': ['Melbourne (Olympic Park)', 'Rhyll', 'Geelong Racecourse', 'Ballarat',
                                    'Wilsons Promontory'],
                            'SA': ['Adelaide (West Terrace /  ngayirdapira)', 'Whyalla', 'Coober Pedy Airport',
                                   'Port Augusta', 'Mount Gambier'],
                            'WA': ['Perth', 'Witchcliffe', 'Geraldton Airport', 'Broome', 'Kalgoorlie-Boulder']}

forecast_files = {'NSW': 'IDN11060.xml',
                  'QLD': 'IDQ11295.xml',
                  'SA': 'IDS10044.xml',
                  'VIC': 'IDV10753.xml',
                  'WA': 'IDW14199.xml'}

observation_files = {'NSW': 'IDN60920.xml',
                     'QLD': 'IDQ60920.xml',
                     'SA': 'IDS60920.xml',
                     'VIC': 'IDV60920.xml',
                     'WA': 'IDW60920.xml'}


def _get_file(filename):
    retries = 0
    while True:
        try:
            raw = BytesIO()
            with ftp_connect({'username': app.config.get('BOM_WEATHER_FTP_USERNAME', ''),
                              'password': app.config.get('BOM_WEATHER_FTP_PASSWORD', ''),
                              'host': 'ftp.bom.gov.au',
                              'path': 'fwo'}) as ftp:
                ftp.retrbinary('RETR ' + filename, raw.write)
        except:
            logger.exception('Failed to download on attempt {} file: {}'.format(retries, filename), exc_info=True)
            retries += 1
            if retries < 3:
                continue
            else:
                logger.exception('Retries exceeded downloading {}'.format(filename))
                raise
        break

    return raw.getvalue()


def forecast_story(item, **kwargs):

    def _get_forecast(state):
        return _get_file(forecast_files.get(state))

    def _get_observations(state):
        return _get_file(observation_files.get(state))

    def add_value(period, dict, name, base):

        direction_map = {'N': 'north', 'NNE': 'north-northeast', 'NE': 'northeast', 'ENE': 'east-northeast',
                         'E': 'east', 'ESE': 'east-southeast', 'SE': 'southeast', 'SSE': 'south-southeast',
                         'S': 'south', 'SSW': 'south-southwest', 'SW': 'southwest', 'WSW': 'west-southwest',
                         'W': 'west', 'WNW': 'west-northwest', 'NW': 'northwest', 'NNW': 'north-northwest'}

        try:
            node = period.find('.//*[@type="{}"]'.format(name))
            if not node:
                period.get(name)
            value = node.text
            if 'temperature' in name:
                value = value.replace('.0', '')
            if 'precis' in name:
                value = value.replace('.', '')
            if 'wind_dir' == name:
{'fraction_non_alphanumeric': 0.14410226612434632, 'fraction_numerical': 0.014642649622312608, 'mean_word_length': 2.1921364985163203, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2099026', 'n_tokens_mistral': 2793, 'n_tokens_neox': 2538, 'n_words': 674}