<jupyter_start><jupyter_text>Unwetter Simulator<jupyter_code>import os
os.chdir('..')


f'Working directory: {os.getcwd()}'
from unwetter import db, map
from datetime import datetime
from unwetter import config

config.SEVERITY_FILTER = ['Severe', 'Extreme']
config.STATES_FILTER = ['NW']
config.URGENCY_FILTER = ['Immediate']

severities = {
    'Minor': 'Wetterwarnung',
    'Moderate': 'Markante Wetterwarnung',
    'Severe': '🔴 Amtliche Unwetterwarnung',
    'Extreme': '🔴 Amtliche Extreme Unwetterwarnung',
}
search_start = datetime(2019, 6, 19, 11, 0)
search_end = datetime(2019, 6, 19, 22, 0)

search_filter = {
    '$and': [
        {
            'sent': {
                '$gt': search_start,
            },
        },
        {
            'sent': {
                '$lt': search_end,
            },
        },
    ]
}

events = list(db.collection.find(search_filter).sort([('sent', 1)]))
len(events)
len([e for e in events if e['published']])
for e in events:
    e['published'] = False
def mock_by_ids(ids):
    return [event for event in events if event['id'] in ids]

def mock_publish(ids):
    for event in events:
        if event['id'] in ids:
            event['published'] = True
from unwetter.dwd import special_type

def mock_has_changes(event, old_events):

    if not any(t['published'] for t in old_events):
        extended_references = set()
        extended_references.update(event.get('extended_references', event['references']))
        
        for old_event in old_events:
            if 'extended_references' in old_event:
                extended_references.update(old_event['extended_references'])
            elif 'references' in old_event:
                extended_references.update(old_event['references'])
        
        event['extended_references'] = sorted(extended_references, reverse=True)
        
        old_events = mock_by_ids(extended_references)
        
    event['has_changes'] = [
        {
            'id': old_event['id'],
            'changed': mock_changes(event, old_event),
            'published': old_event['published'],
        }
        for old_event in old_events
    ]

    event['special_type'] = special_type(event, old_events)
    return event
from datetime import datetime, timedelta
from unwetter.generate.blocks import expires, district_list, state_for_cell_id, region_list, dates
from unwetter.generate.helpers import upper_first, local_time

STATES_FILTER = config.STATES_FILTER


def mock_changes_old(event, old_event):
    """
    Generate a list of changes between two events
    :param event:
    :param old_event:
    :return: str
    """
    text = ''

    simple_fields = {
        'severity': 'Warnstufe',
        'event': 'Wetterphänomen',
        'certainty': 'Wahrscheinlichkeit',
    }

    for field in simple_fields:
        if old_event.get(field) != event.get(field):
            if field == 'severity' and event[field] in ['Minor', 'Moderate']:
                text += f'{simple_fields[field]}: Herabstufung auf {severities[event[field]]}\n\n'
            elif field == 'severity':
                text += f'{simple_fields[field]}: {severities[event[field]]} ' \
                        f'(zuvor "{severities[old_event[field]]}")\n\n'
            else:
                text += f'{simple_fields[field]}: {event[field]} ' \
                        f'(zuvor "{old_event.get(field, "Nicht angegeben")}")\n\n'

    # Editorial request to check only, if expires time changed, since every update has new onset-time
    if abs(event['onset'] - event['sent']) > timedelta(minutes=2) and dates(old_event) != dates(event):
        text += f'Gültigkeit: {dates(event)} (zuvor "{dates(old_event)}")\n\n'
    elif expires(old_event) != expires(event):
        text += f'Ende der Gültigkeit: {expires(event)} (zuvor "{expires(old_event)}")\n\n'

    if district_list(old_event) != district_list(event):
        districts_now = {
            district['name'] for district in event['districts']
            if state_for_cell_id(district['warn_cell_id']) in STATES_FILTER
        }
        districts_before = {
            district['name'] for district in old_event['districts']
            if state_for_cell_id(district['warn_cell_id']) in STATES_FILTER
        }

        added = districts_now - districts_before
        removed = districts_before - districts_now

        if added:
            text += f'Neue Kreise/Städte: {", ".join(sorted(added))}\n'

        if removed:
            text += f'Nicht mehr betroffene Kreise/Städte: {", ".join(sorted(removed))}\n'

        if region_list(old_event) != region_list(event):
            text += f'Regionale Zuordnung: {upper_first(region_list(event))} ' \
                    f'(zuvor: "{upper_first(region_list(old_event))}")\n\n'
        else:
            text += f'Regionale Zuordnung unverändert: {upper_first(region_list(event))}\n\n'

    '''
    # Editorial choice --> No relevant information due to relatively small area --> Thus, no update

    elif commune_list(old_event) != commune_list(event):
        text += 'Regionale Zuordnung: Änderung der betroffenen Gemeinden\n\n'
    '''
    
    return text
from datetime import datetime, timedelta
import re

STATES_FILTER = config.STATES_FILTER


def mock_changes(event, old_event):
    """
    Generate a list of changes between two events
    :param event:
    :param old_event:
    :return: bool
    """
    
    if any(event.get(field) != old_event.get(field) for field in ['severity', 'certainty']):
        return True
    
    # Notify about big hail sizes
    if 'Hagel' not in event['parameters']:
        if event['event'] != old_event['event'].replace(' und HAGEL', ''):
            return True
    else:
        hail_re = r'^.*?(\d+).*?cm'
        hail_size_now = int(re.match(hail_re, event['parameters']['Hagel']).group(1))
        hail_size_before = int(re.match(hail_re, old_event['parameters'].get('Hagel', '0 cm')).group(1))
        
        if hail_size_now >= 3 and hail_size_before < 3:
            return True
        else:
            if event['event'].replace(' und HAGEL', '') != old_event['event'].replace(' und HAGEL', ''):
                return True
    
    if abs(event['onset'] - event['sent']) > timedelta(minutes=2) and event['sent'] - event['onset'] < timedelta(minutes=2) and old_event['onset'] != event['onset']:
        return True
    elif old_event['expires'] != event['expires']:
        return True
    
    if len(set(r[0] for r in event['regions']) - set(r[0] for r in old_event['regions'])) > 0:
        return True

    districts_now = {
        district['name'] for district in event['districts']
        if state_for_cell_id(district['warn_cell_id']) in STATES_FILTER
    }
    districts_before = {
        district['name'] for district in old_event['districts']
        if state_for_cell_id(district['warn_cell_id']) in STATES_FILTER
    }
    added = districts_now - districts_before
    
    if len(districts_before) <= 3 and added:
        return True
    
    return False
from unwetter.config import filter_event

def mock_update(new_events):

    filtered = []
    for event in new_events:
        if filter_event(event):
            if event['msg_type'] in ['Alert', 'Cancel']:
                filtered.append(event)

            elif any(t['changed'] and t['published'] for t in event['has_changes']):
                filtered.append(event)
            
            elif event['special_type'] == 'UpdateAlert':
                filtered.append(event)

            elif not any(t['changed'] and t['published'] for t in event['has_changes']):
                continue

            else:
                print(f'Event was not filtered 1: {event["id"]}')

    mock_publish([event['id'] for event in filtered])
    return filtered
from unwetter.generate.blocks import changes

current_sent = events[0]['sent']

bins = []
current_bin = []

for event in events:
    if event['sent'] != current_sent:
        current_sent = event['sent']
        bins.append(current_bin)
        current_bin = []
    
    current_bin.append(event)

bins.append(current_bin)
processed = []

for bin in bins:
    for event in bin:
        if 'references' in event:
            old_events = mock_by_ids(event.get('extended_references', event['references']))
            mock_has_changes(event, old_events)
    
    processed.append(mock_update(bin))
sum(len(bin) for bin in processed)
[print(event['event'], event['sent'] + timedelta(hours=2), event.get('special_type'), event['id']) for bin in processed for event in bin]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13148061653432974, 'fraction_numerical': 0.004203643157403083, 'mean_word_length': 2.7731277533039647, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '11640', 'license': 'MIT', 'n_tokens_mistral': 2742, 'n_tokens_neox': 2630, 'n_words': 618, 'path': 'jupyter_notebooks/Simulator.ipynb', 'repo_name': 'wdr-data/unwetter2', 'size': '8564'}