<jupyter_start><jupyter_text>ABS Monthly Labour Force 6202 Table of Contents1&nbsp;&nbsp;Python set-up2&nbsp;&nbsp;Get data from ABS3&nbsp;&nbsp;Plot3.1&nbsp;&nbsp;Set-up some constants we will use across all plots3.2&nbsp;&nbsp;Headline charts3.3&nbsp;&nbsp;State comparisons3.4&nbsp;&nbsp;National/State Growth3.5&nbsp;&nbsp;Gendered comparisons3.6&nbsp;&nbsp;Victorian/COVID19 comparisons4&nbsp;&nbsp;Finished  Python set-up<jupyter_code># system imports
import sys
import pathlib
import calendar
import datetime

# analytic imports
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt

# local imports
from finalise_plot import finalise_plot
from abs_common import (
    get_fs_constants,
    get_ABS_meta_and_data,
    get_plot_constants,
    get_identifier,
    plot_Mgrowth,
)

# pandas display settings
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999

# plotting stuff
plt.style.use('ggplot')
catalogue_id = '6202'
source, CHART_DIR, META_DATA = get_fs_constants(catalogue_id)<jupyter_output><empty_output><jupyter_text>Get data from ABS<jupyter_code># do the data capture and extraction
abs = get_ABS_meta_and_data(catalogue_id)
if abs is None:
    sys.exit(-1)<jupyter_output>Retrieving zip-file from cache ./ABS_CACHE/-statistics-labour-employment-and-unemployment-labour-force-australia-mar-2021-6202_all_spreadsheets.zip<jupyter_text>Plot  Set-up some constants we will use across all plots<jupyter_code>meta = abs[META_DATA]
RECENT, plot_times, plot_tags = get_plot_constants(meta)

# reference point for COVID comparisons
REFERENCE = pd.Timestamp('2020-02-01')
REF_STRING = f'{calendar.month_abbr[REFERENCE.month]} {REFERENCE.year}'
index_base = 100
plot_times<jupyter_output><empty_output><jupyter_text>Headline charts Note: with COVID, the ABS has stopped publishing the Trend series.<jupyter_code>headlines = {
    # data item                                       # table number
    'Employed total ;  Persons ;':                    '1',
    '> Employed part-time ;  Persons ;':              '1',
    '> Employed full-time ;  Persons ;':              '1',
    'Unemployed total ;  Persons ;':                  '1',
    'Unemployment rate ;  Persons ;':                 '1', 
    'Participation rate ;  Persons ;':                '1',
    'Labour force total ;  Persons ;':                '1',
    'Monthly hours worked in all jobs ;  Persons ;':  '19',
}
series_type = "Seasonally Adjusted"

for headline, table in headlines.items():

    seasadj, units = get_identifier(meta, headline, 
                                    series_type, table)
    df = abs[table]
    
    for start, tag in zip(plot_times, plot_tags):
        
        frame = df[df.index >= start] if start else df

        ax = frame[[seasadj]].plot(c='#dd0000', lw=2)
        ax.legend([series_type], loc='best')
        title = f'Australia: {headline}'
        finalise_plot(ax, title, units, tag, CHART_DIR,
                      rfooter=f'{source} {table}',
                     )<jupyter_output><empty_output><jupyter_text>State comparisons<jupyter_code>states = {
    # state                     # table
    "NSW":                      '4',
    "Vic":                      '5',
    "Qld":                      "6",
    "SA":                       "7",
    "WA":                       "8",
    "Tas":                      "9",
}

plots = [
    'Unemployment rate ;  Persons ;', 
    'Participation rate ;  Persons ;',
]

colours = ['blue', 'darkorchid', 'brown', 'olivedrab', 'red', 
           'darkorange', 'darkgoldenrod', 'teal',]

series_type = "Seasonally Adjusted"

for plot in plots:
    
    collection = []
    tables = []
    df = pd.DataFrame()
    for state, table in states.items():
        id, units = get_identifier(meta, plot, series_type, table)
        collection.append(id)
        tables.append(table)
        df = abs[table] if df.empty else df.join(abs[table])
    
    for start, tag in zip(plot_times, plot_tags):
        frame = df[df.index >= start] if start else df
        ax = frame[collection].plot(lw=2, color=colours)
        ax.legend(states.keys(), loc='best')
        title = f'Australian states: {plot}'
        finalise_plot(ax, title, units, tag, CHART_DIR,
                      rfooter=f'{source}s {",".join(tables)}',
                      lfooter=f'From {series_type.lower()} series')<jupyter_output><empty_output><jupyter_text>National/State Growth<jupyter_code>series_type = "Seasonally Adjusted"
tables = ['12', '12a']
plot_stems = [
    'Employed total ;  Persons ;',
    '> Employed full-time ;  Persons ;',
    '> Employed part-time ;  Persons ;',
    'Unemployed total ;  Persons ;',
    'Labour force total ;  Persons ;',
]

for stem in plot_stems:
    for table in tables:
        
        descriptions = meta[
            (meta['Table'] == table) &
            (meta['Series Type'] == series_type) &
            (meta['Data Item Description'].str.contains(stem))
        ]['Data Item Description']
        
        for desc in descriptions:
            
            # get the relevant ABS series identifier
            series, units = get_identifier(meta, desc, 
                                           series_type, 
                                           table)
            
            # tidy-up the plot title
            desc_title = desc.replace('> ', '')
            desc_title = desc_title.replace(' ;', '')
            desc_title = desc_title.replace('  ', ' ')
            
            # plot
            df = abs[table]
            for plot_from, plot_tag in zip(plot_times, plot_tags):
                title = f'Growth in {desc_title}'
                plot_Mgrowth(df[series], 
                             title, plot_from, 
                             plot_tag, CHART_DIR,
                             lfooter=f'Calculated from {series_type.lower()} series', 
                             rfooter=f'{source} {table}',
                )<jupyter_output><empty_output><jupyter_text>Gendered comparisons<jupyter_code>genders = ['Males', 'Females']
colours = ['cornflowerblue', 'hotpink']

TOKEN = '<PASSWORD>' # note: also appears in plot title
plots = {
    # data description                       # table
    f'Employed total ;  > {TOKEN} ;':       '1',
    f'> Employed part-time ;  > {TOKEN} ;': '1',
    f'> Employed full-time ;  > {TOKEN} ;': '1',
    f'Unemployed total ;  > {TOKEN} ;':     '1',
    f'Labour force total ;  > {TOKEN} ;':     '1',
    f"Unemployment rate ;  > {TOKEN} ;":    '1',
    f"Participation rate ;  > {TOKEN} ;":   '1',
    f'Monthly hours worked in all jobs ;  > {TOKEN} ;':  '19',
}

series_type = "Seasonally Adjusted"

for plot, table in plots.items():
    
    # assemble data
    collection = []
    for gender in genders:
        this_gender = plot.replace(TOKEN, gender)
        ident, units = get_identifier(meta, this_gender, 
                                   series_type, table)
        collection.append(ident)

    # plot
    df = abs[table]
    for start, tag in zip(plot_times, plot_tags):
        frame = df[df.index >= start] if start else df
        ax = frame[collection].plot(lw=2, color=colours)
        ax.legend(genders, loc='best')
        title = f"Australia: {plot}"
        finalise_plot(ax, title, units, tag, CHART_DIR,
                      rfooter=f'{source} {table}',
                      lfooter=f'From {series_type.lower()} series')
        
        # COVID - plot index refereced to Feb 2020
        if '000' in units:
            # index comparison
            selection = frame[collection].copy()
            reference_row = selection.loc[REFERENCE]
            selection = selection.div(reference_row) * index_base
            ax = selection.plot(lw=2, color=colours)
            ax.legend(genders, loc='best')
            title = f"Australia: {plot} (Index)"
            finalise_plot(ax, title, 
                  f'Index ({REF_STRING} = {index_base})', 
                  f'{tag}', CHART_DIR,
                  rfooter=f'{source} {table}',
                  lfooter=f'Calculated from {series_type.lower()} series')<jupyter_output><empty_output><jupyter_text>Victorian/COVID19 comparisons<jupyter_code>table = '12'
plots = [
    'Employed total ;  Persons ;  X ;',
    '> Employed full-time ;  Persons ;  X ;',
    'Unemployed total ;  Persons ;  X ;',
    'Labour force total ;  Persons ;  X ;',
]
df = abs[table]
df = df[df.index >= RECENT]
series_type = "Seasonally Adjusted"

for plot in plots:
    # get data - calculate index
    national, _ = get_identifier(meta, 
                    plot.replace('X', 'Australia'), 
                    series_type, table)
    victoria, _ = get_identifier(meta, 
                    plot.replace('X', '> Victoria'), 
                    series_type, table)
    frame = df[[victoria, national]].copy()
    frame[national] = frame[national] - frame[victoria] # ROA
    reference_row = frame.loc[REFERENCE]
    frame = frame.div(reference_row) * index_base

    # plot 
    ax = frame.plot(lw=2, color=['darkorange', 'blue'])
    ax.legend(['Victoria', 'Rest of Australia'], loc='best')
    title = f"Vic v Aus: {plot.replace('X ;', 'Index')}"
    finalise_plot(ax, title, 
                  f'Index ({REF_STRING} = {index_base})', 
                  'COVID-Index', CHART_DIR,
                  rfooter=f'{source} {table}',
                  lfooter=f'Calculated from {series_type.lower()} series')<jupyter_output><empty_output><jupyter_text>Finished<jupyter_code>print('Finished')<jupyter_output>Finished
{'fraction_non_alphanumeric': 0.12409676405906378, 'fraction_numerical': 0.010472300764477955, 'mean_word_length': 2.295376121463078, 'pattern_counts': {'":': 8, '<': 36, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '457488', 'license': 'MIT', 'n_tokens_mistral': 3128, 'n_tokens_neox': 2950, 'n_words': 665, 'path': 'notebooks/ABS Monthly Labour Force 6202.ipynb', 'repo_name': 'bpalmer4/Charts-based-on-ABS-data', 'size': '9548'}