<jupyter_start><jupyter_text>Month Report<jupyter_code>from nlg.generation import Generation
from nlg.realization import Realization
from inpe_deter.content import content
from datetime import datetime

from db.model import DeforestationDeterINPE, CoronaVirus
from datetime import datetime
structuring_path = 'inpe_deter/data/pt-br/month_grammar/structuring.json'
lexicalization_path = 'inpe_deter/data/pt-br/month_grammar/lexicalization.json'
reference_path = 'inpe_deter/data/pt-br/month_grammar/references.json'
lexicon_path = 'nlg/lexicons/pt-br/'
gen = Generation(structuring_path, lexicalization_path, reference_path, lexicon_path)
messages = [
    {
        'intent': 'CAUSE', 
        'attributes': {
            'area': 835, 'cause': 'DESMATAMENTO_CR', 'year': 2019, 'month': 6, 'location': 'deter-amz'
        }, 
        'str_msg': 'CAUSE(area="835",cause="DESMATAMENTO_CR",location="deter-amz",month="6",year="2019")', 
        'delex_msg': 'CAUSE(area=AREA,cause=CAUSE,location=LOCATION,month=MONTH,year=YEAR)'
    }, 
    {
        'intent': 'TOTAL_DEFORESTATION', 
        'attributes': {'area': 12, 'year': 2019, 'month': 6, 'location': 'deter-amz', 'state': 'PA', 'city': 'Novo Progresso,Itaituba', 'UC': 'FLORESTA NACIONAL DO JAMANXIM'}, 
        'str_msg': 'TOTAL_DEFORESTATION(area="12",city="Novo Progresso,Itaituba",location="deter-amz",month="6",state="PA",UC="FLORESTA NACIONAL DO JAMANXIM",year="2019")', 
        'delex_msg': 'TOTAL_DEFORESTATION(area=AREA,city=CITY,location=LOCATION,month=MONTH,state=STATE,UC=UC,year=YEAR)'
    }, 
    {
        'intent': 'TOTAL_DEFORESTATION', 
        'attributes': {'area': 454, 'year': 2019, 'month': 6, 'location': 'deter-amz', 'state': 'PA'}, 
        'str_msg': 'TOTAL_DEFORESTATION(area="454",location="deter-amz",month="6",state="PA",year="2019")', 
        'delex_msg': 'TOTAL_DEFORESTATION(area=AREA,location=LOCATION,month=MONTH,state=STATE,year=YEAR)'
    }, 
    {
        'intent': 'TOTAL_DEFORESTATION', 
        'attributes': {'area': 914, 'year': 2019, 'month': 6, 'location': 'deter-amz'}, 
        'str_msg': 'TOTAL_DEFORESTATION(area="914",location="deter-amz",month="6",year="2019")', 
        'delex_msg': 'TOTAL_DEFORESTATION(area=AREA,location=LOCATION,month=MONTH,year=YEAR)'
    }, 
    {
        'intent': 'TOTAL_DEFORESTATION', 
        'attributes': {'area': 95, 'year': 2019, 'month': 6, 'location': 'deter-amz', 'state': 'PA', 'city': 'Altamira'}, 
        'str_msg': 'TOTAL_DEFORESTATION(area="95",city="Altamira",location="deter-amz",month="6",state="PA",year="2019")', 
        'delex_msg': 'TOTAL_DEFORESTATION(area=AREA,city=CITY,location=LOCATION,month=MONTH,state=STATE,year=YEAR)'
    }
]

entry, template, paragraphs = gen.generate(messages, strategy='random')

text = []
for p in paragraphs:
    text.append(' '.join(p))
text = '\n\n'.join(text)

print(text)
for date in DeforestationDeterINPE.objects().distinct('date')[:5]:
    month, year = date.month, date.year
    messages, date = content(month, year)
    try:
        entry, template, paragraphs = gen.generate(messages, strategy='random')

        text = []
        for p in paragraphs:
            text.append(' '.join(p))
        text = '\n\n'.join(text)

        print("Portuguese:")
        print(text)
    except:
        print('ERROR')
    print(10 * '*', '\n')<jupyter_output>Portuguese:
Um total de 914 km² de desmatamento na Amazônia Legal foi detectado no mês de junho de 2019 pelo INPE. Com 835 km², a principal causa de devastação foi o desmatamento com solo exposto, que deixa a terra sem nenhuma cobertura florestal.

O estado que mais teve desmatamento foi Pará (454 km²). O município que teve mais desmatamento no mês foi Altamira (Pará), com 95 km² de área desmatada.

Com 12 km², a FLORESTA NACIONAL DO JAMANXIM / Pará foi a Unidade de Conservação mais devastada no mês.
********** 

Portuguese:
Um total de 914 km² de desmatamento na Amazônia legal foi detectado no mês de junho de 2019 pelo INPE. Com 835 km², a principal causa de devastação foi o desmatamento com solo exposto, aquele que deixa a terra sem nenhuma vegetação.

Pará foi o estado mais atingido pelo desmatamento, com 454 km². Altamira foi o município mais afetado (95 km²).

A Unidade de Conservação onde teve mais desmatamento foi a FLORESTA NACIONAL DO JAMANXIM / Pará (12 km²).
********** 

Por[...]<jupyter_text>Daily Report<jupyter_code>from datetime import datetime, timedelta
import inpe_deter.daily_content as daily_content

structuring_path = 'inpe_deter/data/pt-br/daily_grammar/structuring.json'
lexicalization_path = 'inpe_deter/data/pt-br/daily_grammar/lexicalization.json'
reference_path = 'inpe_deter/data/pt-br/daily_grammar/references.json'
lexicon_path = 'nlg/lexicons/pt-br/'
gen = Generation(structuring_path, lexicalization_path, reference_path, lexicon_path)
messages = [
    {
        'intent': 'CAUSE', 
        'attributes': {'year': 2020, 'month': 10, 'location': 'deter-amz', 'state': 'AM', 'city': 'Labrea', 'cause': 'DESMATAMENTO_CR'}, 
        'str_msg': 'CAUSE(cause="DESMATAMENTO_CR",city="Labrea",location="deter-amz",month="10",state="AM",year="2020")', 
        'delex_msg': 'CAUSE(cause=CAUSE,city=CITY,location=LOCATION,month=MONTH,state=STATE,year=YEAR)'
    }, 
    {
        'intent': 'DAILY_ALERT', 
        'attributes': {'year': 2020, 'month': 10, 'day': 17, 'location': 'deter-amz', 'state': 'AM', 'city': 'Labrea', 'area': 6.075527808385738, 'daily_accumulation': 7}, 
        'str_msg': 'DAILY_ALERT(area="6.075527808385738",city="Labrea",daily_accumulation="7",day="17",location="deter-amz",month="10",state="AM",year="2020")', 
        'delex_msg': 'DAILY_ALERT(area=AREA,city=CITY,daily_accumulation=DAILY_ACCUMULATION,day=DAY,location=LOCATION,month=MONTH,state=STATE,year=YEAR)'
    }, 
    {
        'intent': 'TOTAL_DEFORESTATION', 
        'attributes': {'year': 2020, 'month': 10, 'location': 'deter-amz', 'state': 'AM', 'city': 'Labrea', 'area': 36.66000131810721}, 
        'str_msg': 'TOTAL_DEFORESTATION(area="36.66000131810721",city="Labrea",location="deter-amz",month="10",state="AM",year="2020")', 
        'delex_msg': 'TOTAL_DEFORESTATION(area=AREA,city=CITY,location=LOCATION,month=MONTH,state=STATE,year=YEAR)'
    }
]
entry, template, paragraphs = gen.generate(messages, strategy='random')

text = []
for p in paragraphs:
    text.append(' '.join(p))
print('\n'.join(text))
date = datetime.now() - timedelta(days=60)

alerts = []
for last_day in sorted(DeforestationDeterINPE.objects(date__gte=date).distinct('date'))[:5]:
    cities, ucs = {}, {}
    cases = DeforestationDeterINPE.objects(date=last_day)
    for case in cases:
        if case.cause in ['DESMATAMENTO_CR', 'DESMATAMENTO_VEG', 'MINERACAO']:
            if case.UC:
                if (last_day, case.state, case.city, case.UC) not in ucs:
                    ucs[(last_day, case.state, case.city, case.UC)] = 0
                ucs[(last_day, case.state, case.city, case.UC)] += case.uc_area
            else:
                if (last_day, case.state, case.city, None) not in cities:
                    cities[(last_day, case.state, case.city, None)] = 0
                cities[(last_day, case.state, case.city, None)] += case.city_area

    cities = sorted(cities.items(), key=lambda x:x[1], reverse=True)[:3]
    ucs = sorted(ucs.items(), key=lambda x:x[1], reverse=True)[:3]

    for city in cities:
        alerts.append(city[0])
    for uc in ucs:
        alerts.append(uc[0])
for alert in alerts[:5]:
    date, state, city, uc = alert
    messages, _ = daily_content.content(date, state, city, uc)
    print('\n'.join([msg['str_msg'] for msg in messages]))
    print()
    try:
        entry, template, paragraphs = gen.generate(messages, strategy='random')

        text = []
        for p in paragraphs:
            text.append(' '.join(p))
        print('\n'.join(text))
    except:
        print('ERROR')
    print(10 * '*', '\n')<jupyter_output>CAUSE(cause="DESMATAMENTO_CR",city="Porto Velho",location="deter-amz",month="6",state="RO",year="2020")
DAILY_ALERT(area="6.7931018036451825",city="Porto Velho",daily_accumulation="9",day="30",location="deter-amz",month="6",state="RO",year="2020")
TOTAL_DEFORESTATION(area="46.0882973437658",city="Porto Velho",location="deter-amz",month="6",state="RO",year="2020")

No dia 30 de junho de 2020, o INPE registrou alertas de desmatamento somando 6,79 km² em Porto Velho / Rondônia, que acumula 9 dias com alertas no mês. O principal tipo de desmatamento foi o desmatamento com solo exposto, que deixa o solo sem vegetação. No total, 46,09 km² foram desmatados em Porto Velho no mês de junho.
********** 

CAUSE(cause="DESMATAMENTO_CR",city="Sao Felix do Xingu",location="deter-amz",month="6",state="PA",year="2020")
DAILY_ALERT(area="6.161107067840259",city="Sao Felix do Xingu",daily_accumulation="11",day="30",location="deter-amz",month="6",state="PA",year="2020")
TOTAL_DEFORESTATION(area="67.975544[...]
{'fraction_non_alphanumeric': 0.1637969588550984, 'fraction_numerical': 0.03745527728085868, 'mean_word_length': 4.504615384615384, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '284611', 'license': 'MIT', 'n_tokens_mistral': 3518, 'n_tokens_neox': 3174, 'n_words': 640, 'path': 'deter_pt.ipynb', 'repo_name': 'BotsDoBem/DEMO_INPE_COVID', 'size': '8944'}