<commit_before>#!/usr/bin/python
import os
import sys
import datetime
import MySQLdb
import re
import logging
import shutil

import toolkit.diary.models
import toolkit.members.models
import settings
from django.core.exceptions import ValidationError, ObjectDoesNotExist

FORMATS_PATH="./formats"

SITE_ROOT = ".."
MEDIA_PATH = "media"
EVENT_IMAGES_PATH = "diary"
EVENT_THUMB_IMAGES_PATH = "diary_thumbnails"
VOLUNTEER_IMAGE_PATH = "volunteers"
VOLUNTEER_IMAGE_THUMB_PATH = "volunteers_thumbnails"

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

# Set up logging:
consoleHandler = logging.StreamHandler()
consoleHandler.setLevel(logging.DEBUG)
logger.addHandler(consoleHandler)

def titlecase(string):
#   return string.title() # Really doesn't cope with apostrophes.
#   return " ".join([ word.capitalize() for word in  "This is the voice".split() ])
    if isinstance(string, basestring):
        return re.sub("(^|\s)(\S)", lambda match : match.group(1) + match.group(2).upper(), string)
    else:
        return string

def connect():
    conn = MySQLdb.connect (host = "localhost",
                            user = settings.IMPORT_SCRIPT_USER,
                           passwd = "<PASSWORD>",
                           db = settings.IMPORT_SCRIPT_DATABASE,
#                           use_unicode = True,
                           )
    return conn

def decode(string):
    if string is str:
        string = string.strip()
        return string.decode('utf-8')
    else:
        return string

def int_def(string, default):
    try:
        return int(string)
    except ValueError:
        return default

def html_ify(string):
    result = None
    if string is not None:
        result = "<p>" + string.strip().replace("\r\n","<br>") + "</p>"
    return result

def markdown_ify(string):
    return string

event_tot = 0

def import_event_roles(connection, showings, legacy_event_id, role_map):
    cursor = connection.cursor()

    # *cough*
    cursor.execute("SELECT * from roles_merged WHERE event_id = %s", legacy_event_id)

    for ev in cursor:
        # event_id = ev[0]
        col_no = 0
        for role_col in ev[1:]:
            if role_col is not None:
                for rank in range(0,role_col):
                    for s in showings:
                        rota_entry = toolkit.diary.models.RotaEntry()
                        rota_entry.role_id = role_map[col_no]
                        rota_entry.rank = rank + 1

                        rota_entry.showing_id = s.id
                        rota_entry.save()
            col_no += 1

    cursor.close()

def import_event_showings(connection, event, legacy_event_id):
    global event_tot
    # Some slightly funky logic to do the mapping:
    #
    aggregate_hire = False
    cancelled_list = []
    private_list = []

    all_showings = []


    cursor = connection.cursor()
    showing_count = cursor.execute("SELECT datetime, event_id, booked_by, confirmed, cancelled, discounted, outside_hire, private_event FROM diary WHERE event_id = '%s' ORDER BY datetime" % legacy_event_id)
    event_tot += showing_count

    results = cursor.fetchall()

    for r in results:

        s = toolkit.diary.models.Showing()
        all_showings.append(s)
        s.event = event
        s.start = r[0]
        if r[2] is not None and r[2].strip() != '':
            s.booked_by = titlecase(decode(r[2]))
        else:
            s.booked_by = 'unknown'

        s.confirmed = bool(r[3])

        if r[4]:
            cancelled_list.append(s)

        s.discounted = bool(r[5])
        aggregate_hire |= bool(r[6])
        if bool(r[7]):
            private_list.append(s)

        s.full_clean()
        s.save()

    if len(cancelled_list) == showing_count:
        event.cancelled = True
    else:
        for cancelled_showing in cancelled_list:
            cancelled_showing.cancelled = True
            cancelled_showing.save()

    if len(private_list) == showing_count:
        event.private = True
    else:
        for hidden_showing in private_list:
            hidden_showing.hide_in_programme = True
            hidden_showing.save()

    event.outside_hire = aggregate_hire

    event.full_clean()
    event.save()

    cursor.close()

    return all_showings

def import_ideas(connection):
    cursor = connection.cursor()
    results = cursor.execute("SELECT date, ideas FROM ideas ORDER BY date")

    for r in cursor.fetchall():
        i, created = toolkit.diary.models.DiaryIdea.objects.get_or_create(month=r[0])
        i.ideas = decode(r[1])
        i.full_clean()
        i.save()

    cursor.close()

def import_events(connection, role_map):
    cursor = connection.cursor()
    results = cursor.execute("SELECT event_id, event_name, copy, copy_summary, duration, image_credits, terms FROM events ORDER BY event_id")

    count = 0
    tenpc = results / 100
    pc = 1
    logger.info("%d events" % (results))
    for r in cursor.fetchall():
        r = [ decode(item) for item in r ]
        e = toolkit.diary.models.Event()
        e.legacy_id = r[0]

        # Name
        e.name = titlecase(r[1])
        if e.name in (None, u''):
            # Looking at the db, it's safe to skip all of these
            logging.error("Skipping event with no/missing name id %s", e.legacy_id)
            continue
        # Copy
        if r[2] is not None:
            e.copy = markdown_ify(r[2])
        else:
            logger.error("Missing copy for event [%s] %s", r[0], e.name)
            e.copy = ''
        # Copy summary
        e.copy_summary = r[3]

        # Duration:
        if r[4] is not None and r[4] != '':
            durn_hour, durn_min  = r[4].split('/')
            durn_hour = int_def(durn_hour,0)
            durn_min = int_def(durn_min,0)
            e.duration = datetime.time(durn_hour, durn_min)
        else:
            e.duration = datetime.time(0,0)

        # Terms
        e.terms = r[6]
        try:
            e.full_clean()
            e.save()
        except ValidationError as verr:<commit_msg>Regex a go-go to get copy/copy summary more markdown-ified<commit_after>#!/usr/bin/python
import os
import sys
import datetime
import MySQLdb
import re
import logging
import shutil

import toolkit.diary.models
import toolkit.members.models
import settings
from django.core.exceptions import ValidationError, ObjectDoesNotExist

FORMATS_PATH="./formats"

SITE_ROOT = ".."
MEDIA_PATH = "media"
EVENT_IMAGES_PATH = "diary"
EVENT_THUMB_IMAGES_PATH = "diary_thumbnails"
VOLUNTEER_IMAGE_PATH = "volunteers"
VOLUNTEER_IMAGE_THUMB_PATH = "volunteers_thumbnails"

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

# Set up logging:
consoleHandler = logging.StreamHandler()
consoleHandler.setLevel(logging.DEBUG)
logger.addHandler(consoleHandler)


##############################################################################
# Some utility functions

def connect():
    # Open database connection
    conn = MySQLdb.connect (host = "localhost",
                            user = settings.IMPORT_SCRIPT_USER,
                           passwd = "<PASSWORD>",
                           db = settings.IMPORT_SCRIPT_DATABASE,
#                           use_unicode = True,
                           )
    return conn

def titlecase(string):
#   return string.title() # Really doesn't cope with apostrophes.
#   return " ".join([ word.capitalize() for word in  "This is the voice".split() ])
    if isinstance(string, basestring):
        return re.sub("(^|\s)(\S)", lambda match : match.group(1) + match.group(2).upper(), string)
    else:
        return string

def decode(string):
    if string is str:
        string = string.strip()
        return string.decode('utf-8')
    else:
        return string

def int_def(string, default):
    try:
        return int(string)
    except ValueError:
        return default

def html_ify(string):
    result = None
    if string is not None:
        result = "<p>" + string.strip().replace("\r\n","<br>") + "</p>"
    return result

def markdown_ify(string):
    return string

wrap_re = re.compile(r'(.{70,})\n')
lotsofnewlines_re = re.compile(r'\n\n+')
# Catch well-formatted links (ie. beginning http://)
link_re_1 = re.compile(r'(http:\/\/\S{4,})')
# Optimistic stab at spotting other things that are probably links, based on
# a smattering of TLDs:
link_re_2 = re.compile(r'(\s)(www\.[\w.]+\.(com|org|net|uk|de|ly|us|tk)[^\t\n\r\f\v\. ]*)')

def convert_copy_to_markdown(string):
    if isinstance(string, basestring):
        # remove all whitespace from start and end of line:
        result = string.strip()
        # Strip out carriage returns:
        result = result.strip().replace('\r','')
        # Strip out new lines when they occur after 70 other characters (try to fix wrapping)
        result = wrap_re.sub(r'\1 ', result)
        # Replace a sequence of 1+ new lines with a single line break;
        result = lotsofnewlines_re.sub('\n', result)
        # Now replace all single line breaks with double line breaks (which
        # markdown will actually show as line breaks)
        result = result.replace('\n','\n\n')

        # Attempt to magically convert any links to markdown:
        result = link_re_1.sub(r'[\1](\1)', result)
        result = link_re_2.sub(r'\1[\2](http://\2)', result)
        return result
    else:
        return string

##############################################################################
# Diary import

event_tot = 0

def import_event_roles(connection, showings, legacy_event_id, role_map):
    cursor = connection.cursor()

    # *cough*
    cursor.execute("SELECT * from roles_merged WHERE event_id = %s", legacy_event_id)

    for ev in cursor:
        # event_id = ev[0]
        col_no = 0
        for role_col in ev[1:]:
            if role_col is not None:
                for rank in range(0,role_col):
                    for s in showings:
                        rota_entry = toolkit.diary.models.RotaEntry()
                        rota_entry.role_id = role_map[col_no]
                        rota_entry.rank = rank + 1

                        rota_entry.showing_id = s.id
                        rota_entry.save()
            col_no += 1

    cursor.close()

def import_event_showings(connection, event, legacy_event_id):
    global event_tot
    # Some slightly funky logic to do the mapping:
    #
    aggregate_hire = False
    cancelled_list = []
    private_list = []

    all_showings = []


    cursor = connection.cursor()
    showing_count = cursor.execute("SELECT datetime, event_id, booked_by, confirmed, cancelled, discounted, outside_hire, private_event FROM diary WHERE event_id = '%s' ORDER BY datetime" % legacy_event_id)
    event_tot += showing_count

    results = cursor.fetchall()

    for r in results:

        s = toolkit.diary.models.Showing()
        all_showings.append(s)
        s.event = event
        s.start = r[0]
        if r[2] is not None and r[2].strip() != '':
            s.booked_by = titlecase(decode(r[2]))
        else:
            s.booked_by = 'unknown'

        s.confirmed = bool(r[3])

        if r[4]:
            cancelled_list.append(s)

        s.discounted = bool(r[5])
        aggregate_hire |= bool(r[6])
        if bool(r[7]):
            private_list.append(s)

        s.full_clean()
        s.save()

    if len(cancelled_list) == showing_count:
        event.cancelled = True
    else:
        for cancelled_showing in cancelled_list:
            cancelled_showing.cancelled = True
            cancelled_showing.save()

    if len(private_list) == showing_count:
        event.private = True
    else:
        for hidden_showing in private_list:
            hidden_showing.hide_in_programme = True
            hidden_showing.save()

    event.outside_hire = aggregate_hire

    event.full_clean()
    event.save()

    cursor.close()

    return all_showings

def import_ideas(connection):
    cursor = connection.cursor()
    results = cursor.execute("SELECT date, ideas FROM ideas ORDER BY date")

    for r in cursor.fetchall():
        i, created = toolkit.diary.models.DiaryIdea.objects.get_or_create(month=r[0])
        i.ideas = decode(r[1])
        i.full_clean()
        i.save()

    cursor.close()

def import_events(connection, role_map):
    cursor = connection.cursor()
    results = cursor.execute("SELECT event_id, event_name, copy, copy_summary, duration, image_credits, terms FROM events ORDER BY event_id")

    count = 0
    tenpc = results / 100
    pc = 1
    logger.info("%d events" % (results))
    for r in cursor.fetchall():
        r = [ decode(item) for item in r ]
        e = toolkit.diary.models.Event()
        e.legacy_id = r[0]

        # Name
        e.name = titlecase(r[1])

        if e.name in (None, u'', ''):
            # Looking at the db, it's safe to skip all of these
            logging.error("Skipping event with no/missing name id %s", e.legacy_id)
            continue
        else:
            # Special fix for the omnipresent Djs:
            e.name = e.name.replace("Djs", "DJs")

        # Copy
        if r[2] is not None:
            e.copy = convert_copy_to_markdown(r[2])
        else:
            logger.error("Missing copy for event [%s] %s", r[0], e.name)
            e.copy = ''
        # Copy summary
        e.copy_summary = convert_copy_to_markdown(r[3])

        # Duration:
        if r[4] is not None and r[4] != '':
            durn_hour, durn_min  = r[4].split('/')
            durn_hour = int_def(durn_hour,0)
            durn_min = int_def(durn_min,0)
            e.duration = datetime.time(durn_hour, durn_min)
        else:
            e.duration = datetime.time(0,0)

        # Terms
        e.terms = r[6]
        try:
            e.full_clean()
            e.save()
        except ValidationError as verr:
{'fraction_non_alphanumeric': 0.11954006363899335, 'fraction_numerical': 0.0069424356378362745, 'mean_word_length': 2.280123339658444, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4975523', 'n_tokens_mistral': 4371, 'n_tokens_neox': 4245, 'n_words': 1061}