<commit_before>import logging
import os
import sys

import requests

from django.db import IntegrityError

from .spreadsheets import SpreadsheetMunger

from stagecraft.apps.dashboards.models import Dashboard
from stagecraft.apps.dashboards.models import Module
from stagecraft.apps.dashboards.models import ModuleType
from stagecraft.apps.organisation.models import Node
from stagecraft.apps.datasets.models import DataSet


log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
log.addHandler(handler)


def import_dashboards(summaries, dry_run=False, all_records=False):
    try:
        username = os.environ['GOOGLE_USERNAME']
        password = os.environ['GOOGLE_PASSWORD']
    except KeyError:
        log.fatal("Please supply username (GOOGLE_USERNAME)\
                and password (GOOGLE_PASSWORD) as environment variables")
        sys.exit(1)

    loader = SpreadsheetMunger(positions={
        'names_description': 8,
        'names_name': 11,
        'names_slug': 12,
        'names_notes': 17,
        'names_other_notes': 18,
        'names_tx_id': 19,
    })
    records = loader.load_tx_worksheet(username, password)
    log.debug('Loaded {} records'.format(len(records)))

    for record in records:
        if all_records or not record['high_volume']:
            loader.sanitise_record(record)
            import_dashboard(record, summaries, dry_run)


def import_dashboard(record, summaries, dry_run=False):

    log.debug(record)
    try:
        dashboard = Dashboard.objects.get(slug=record['tx_id'])
        log.debug('Retrieved dashboard: {}'.format(record['tx_id']))
    except Dashboard.DoesNotExist:
        dashboard = Dashboard()
        log.debug('Creating dashboard: {}'.format(record['tx_id']))

    dashboard.title = record['name']
    dashboard.slug = record['tx_id']
    dashboard.description = record['description']
    dashboard.description_extra = record['description_extra']
    dashboard.costs = record['costs']
    dashboard.other_notes = record['other_notes']

    if dashboard.organisation is None:
        if record.get('agency'):<commit_msg>Only set slug on new dashboards<commit_after>import logging
import os
import sys

import requests

from django.db import IntegrityError

from .spreadsheets import SpreadsheetMunger

from stagecraft.apps.dashboards.models import Dashboard
from stagecraft.apps.dashboards.models import Module
from stagecraft.apps.dashboards.models import ModuleType
from stagecraft.apps.organisation.models import Node
from stagecraft.apps.datasets.models import DataSet


log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
log.addHandler(handler)


def import_dashboards(summaries, dry_run=False, all_records=False):
    try:
        username = os.environ['GOOGLE_USERNAME']
        password = <PASSWORD>['<PASSWORD>']
    except KeyError:
        log.fatal("Please supply username (GOOGLE_USERNAME)\
                and password (GOOGLE_PASSWORD) as environment variables")
        sys.exit(1)

    loader = SpreadsheetMunger(positions={
        'names_description': 8,
        'names_name': 11,
        'names_slug': 12,
        'names_notes': 17,
        'names_other_notes': 18,
        'names_tx_id': 19,
    })
    records = loader.load_tx_worksheet(username, password)
    log.debug('Loaded {} records'.format(len(records)))

    for record in records:
        if all_records or not record['high_volume']:
            loader.sanitise_record(record)
            import_dashboard(record, summaries, dry_run)


def import_dashboard(record, summaries, dry_run=False):

    log.debug(record)
    try:
        dashboard = Dashboard.objects.get(slug=record['tx_id'])
        log.debug('Retrieved dashboard: {}'.format(record['tx_id']))
    except Dashboard.DoesNotExist:
        dashboard = Dashboard()
        log.debug('Creating dashboard: {}'.format(record['tx_id']))

    dashboard.title = record['name']
    # Only set slug on new dashboards
    if dashboard.pk is None:
        dashboard.slug = record['tx_id']
    dashboard.description = record['description']
    dashboard.description_extra = record['description_extra']
    dashboard.costs = record['costs']
    dashboard.other_notes = record['other_notes']

    if dashboard.organisation is None:
        if record.get('agency'):
{'fraction_non_alphanumeric': 0.10647938377888537, 'fraction_numerical': 0.005437245129134572, 'mean_word_length': 3.772972972972973, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3000455', 'n_tokens_mistral': 1338, 'n_tokens_neox': 1274, 'n_words': 280}