<commit_before>#!/usr/bin/python
import os
import sys
import datetime
import MySQLdb
import re
import logging

import diary.models

FORMATS_PATH="./formats"

MEDIA_PATH = "./media"
EVENT_IMAGES_PATH = "event"
EVENT_THUMB_IMAGES_PATH = "event_thumbnails"

def titlecase(string):
#   return string.title() # Really doesn't cope with apostrophes.
#   return " ".join([ word.capitalize() for word in  "This is the voice".split() ])
    if string is str:
        return re.sub("(^|\s)(\S)", lambda match : match.group(1) + match.group(2).upper(), string)
    else:
        return string

def connect():
    conn = MySQLdb.connect (host = "localhost",
                           user = "cube-import",
                           passwd = "<PASSWORD>",
                           db = "toolkit",
                           use_unicode = True,
                           )
    return conn

def int_def(string, default):
    try:
        return int(string)
    except ValueError:
        return default

def html_ify(string):
    result = None 
    if string is not None:
        result = "<p>" + string.strip().replace("\r\n","<br>") + "</p>" 
    return result

def markdown_ify(string):
    return string

event_tot = 0

def import_event_roles(connection, showings, legacy_event_id, role_map):
    cursor = connection.cursor()

    # *cough*
    cursor.execute("SELECT * from roles_merged WHERE event_id = %s", legacy_event_id)

    for ev in cursor:
        # event_id = ev[0]
        col_no = 0
        for role_col in ev[1:]:
            if role_col is not None:
                for rank in range(0,role_col):
                    for s in showings:
                        rota_entry = diary.models.RotaEntry()
                        rota_entry.role_id = role_map[col_no]
                        rota_entry.rank = rank + 1

                        rota_entry.showing_id = s.id
                        rota_entry.save()
            col_no += 1

    cursor.close()

def import_event_showings(connection, event, legacy_event_id):
    global event_tot
    # Some slightly funky logic to do the mapping:
    #
    aggregate_hire = False 
    cancelled_list = []
    private_list = []

    all_showings = []
    

    cursor = connection.cursor()
    showing_count = cursor.execute("SELECT datetime, event_id, booked_by, confirmed, cancelled, discounted, outside_hire, private_event FROM diary WHERE event_id = '%s' ORDER BY datetime" % legacy_event_id)
    event_tot += showing_count

    results = cursor.fetchall()
    
    for r in results:

        s = diary.models.Showing()
        all_showings.append(s)
        s.event = event
        s.start = r[0]
        if r[2] is not None:
            s.booked_by = titlecase(r[2])
        
        s.confirmed = bool(r[3])

        if r[4]:
            cancelled_list.append(s)

        s.discounted = bool(r[5])
        aggregate_hire |= bool(r[6])
        if bool(r[7]):
            private_list.append(s)

        s.save()

    if len(cancelled_list) == showing_count:
        event.cancelled = True
    else:
        for cancelled_showing in cancelled_list:
            cancelled_showing.cancelled = True
            cancelled_showing.save()

    if len(private_list) == showing_count:
        event.private = True
    else:
        for hidden_showing in private_list:
            hidden_showing.hide_in_programme = True
            hidden_showing.save()

    event.outside_hire = aggregate_hire 

    event.save()

    cursor.close()

    return all_showings

def import_ideas(connection):
    cursor = connection.cursor()
    results = cursor.execute("SELECT date, ideas FROM ideas ORDER BY date")

    for r in cursor.fetchall():
        i, created = diary.models.DiaryIdea.objects.get_or_create(month=r[0])
        i.ideas = r[1]
        i.save()

    cursor.close()

def import_events(connection, role_map):
    cursor = connection.cursor()
    results = cursor.execute("SELECT event_id, event_name, copy, copy_summary, duration, image_credits, terms FROM events ORDER BY event_id")

    count = 0
    tenpc = results / 100
    pc = 1
    logging.info("%d events" % (results))
    for r in cursor.fetchall():
        e = diary.models.Event()

        # Name
        e.name = titlecase(r[1])
        # Copy
        if r[2] is not None:
            e.copy = markdown_ify(r[2])
        else:
            logging.error("Missing copy for event [%s] %s", r[0], e.name)
            e.copy = ''
        # Copy summary
        e.copy_summary = r[3]
 
        # Duration:
        if r[4] is not None and r[4] != '':
            durn_hour, durn_min  = r[4].split('/')
            durn_hour = int_def(durn_hour,0)
            durn_min = int_def(durn_min,0)
            e.duration = datetime.time(durn_hour, durn_min) 

        # Image
        image_name = r[0].replace(" ","_") + ".jpg"
        image_path = os.path.join(MEDIA_PATH, EVENT_IMAGES_PATH, image_name)
        if os.path.exists(image_path):
            e.image = os.path.join(EVENT_IMAGES_PATH, image_name)
        # Thumbnail
        image_path = os.path.join(MEDIA_PATH, EVENT_THUMB_IMAGES_PATH, image_name)
        if os.path.exists(image_path):
            e.image_thumbnail = os.path.join(EVENT_THUMB_IMAGES_PATH, image_name)

        # Image credits
        e.image_credits = titlecase(r[5])
        # Terms
        e.terms = r[6]<commit_msg>Fix fairly stupid typo that meant image credits were getting lost<commit_after>#!/usr/bin/python
import os
import sys
import datetime
import MySQLdb
import re
import logging

import diary.models

FORMATS_PATH="./formats"

MEDIA_PATH = "./media"
EVENT_IMAGES_PATH = "event"
EVENT_THUMB_IMAGES_PATH = "event_thumbnails"

def titlecase(string):
#   return string.title() # Really doesn't cope with apostrophes.
#   return " ".join([ word.capitalize() for word in  "This is the voice".split() ])
    if string is str:
        return re.sub("(^|\s)(\S)", lambda match : match.group(1) + match.group(2).upper(), string)
    else:
        return string

def connect():
    conn = MySQLdb.connect (host = "localhost",
                           user = "cube-import",
                           passwd = "<PASSWORD>",
                           db = "toolkit",
                           use_unicode = True,
                           )
    return conn

def int_def(string, default):
    try:
        return int(string)
    except ValueError:
        return default

def html_ify(string):
    result = None 
    if string is not None:
        result = "<p>" + string.strip().replace("\r\n","<br>") + "</p>" 
    return result

def markdown_ify(string):
    return string

event_tot = 0

def import_event_roles(connection, showings, legacy_event_id, role_map):
    cursor = connection.cursor()

    # *cough*
    cursor.execute("SELECT * from roles_merged WHERE event_id = %s", legacy_event_id)

    for ev in cursor:
        # event_id = ev[0]
        col_no = 0
        for role_col in ev[1:]:
            if role_col is not None:
                for rank in range(0,role_col):
                    for s in showings:
                        rota_entry = diary.models.RotaEntry()
                        rota_entry.role_id = role_map[col_no]
                        rota_entry.rank = rank + 1

                        rota_entry.showing_id = s.id
                        rota_entry.save()
            col_no += 1

    cursor.close()

def import_event_showings(connection, event, legacy_event_id):
    global event_tot
    # Some slightly funky logic to do the mapping:
    #
    aggregate_hire = False 
    cancelled_list = []
    private_list = []

    all_showings = []
    

    cursor = connection.cursor()
    showing_count = cursor.execute("SELECT datetime, event_id, booked_by, confirmed, cancelled, discounted, outside_hire, private_event FROM diary WHERE event_id = '%s' ORDER BY datetime" % legacy_event_id)
    event_tot += showing_count

    results = cursor.fetchall()
    
    for r in results:

        s = diary.models.Showing()
        all_showings.append(s)
        s.event = event
        s.start = r[0]
        if r[2] is not None:
            s.booked_by = titlecase(r[2])
        
        s.confirmed = bool(r[3])

        if r[4]:
            cancelled_list.append(s)

        s.discounted = bool(r[5])
        aggregate_hire |= bool(r[6])
        if bool(r[7]):
            private_list.append(s)

        s.save()

    if len(cancelled_list) == showing_count:
        event.cancelled = True
    else:
        for cancelled_showing in cancelled_list:
            cancelled_showing.cancelled = True
            cancelled_showing.save()

    if len(private_list) == showing_count:
        event.private = True
    else:
        for hidden_showing in private_list:
            hidden_showing.hide_in_programme = True
            hidden_showing.save()

    event.outside_hire = aggregate_hire 

    event.save()

    cursor.close()

    return all_showings

def import_ideas(connection):
    cursor = connection.cursor()
    results = cursor.execute("SELECT date, ideas FROM ideas ORDER BY date")

    for r in cursor.fetchall():
        i, created = diary.models.DiaryIdea.objects.get_or_create(month=r[0])
        i.ideas = r[1]
        i.save()

    cursor.close()

def import_events(connection, role_map):
    cursor = connection.cursor()
    results = cursor.execute("SELECT event_id, event_name, copy, copy_summary, duration, image_credits, terms FROM events ORDER BY event_id")

    count = 0
    tenpc = results / 100
    pc = 1
    logging.info("%d events" % (results))
    for r in cursor.fetchall():
        e = diary.models.Event()

        # Name
        e.name = titlecase(r[1])
        # Copy
        if r[2] is not None:
            e.copy = markdown_ify(r[2])
        else:
            logging.error("Missing copy for event [%s] %s", r[0], e.name)
            e.copy = ''
        # Copy summary
        e.copy_summary = r[3]
 
        # Duration:
        if r[4] is not None and r[4] != '':
            durn_hour, durn_min  = r[4].split('/')
            durn_hour = int_def(durn_hour,0)
            durn_min = int_def(durn_min,0)
            e.duration = datetime.time(durn_hour, durn_min) 

        # Image
        image_name = r[0].replace(" ","_") + ".jpg"
        image_path = os.path.join(MEDIA_PATH, EVENT_IMAGES_PATH, image_name)
        if os.path.exists(image_path):
            e.image = os.path.join(EVENT_IMAGES_PATH, image_name)
        # Thumbnail
        image_path = os.path.join(MEDIA_PATH, EVENT_THUMB_IMAGES_PATH, image_name)
        if os.path.exists(image_path):
            e.image_thumbnail = os.path.join(EVENT_THUMB_IMAGES_PATH, image_name)

        # Image credits
        e.image_credit = titlecase(r[5])
        # Terms
        e.terms = r[6]
{'fraction_non_alphanumeric': 0.10739879013494649, 'fraction_numerical': 0.006886924150767799, 'mean_word_length': 2.0852713178294575, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3242954', 'n_tokens_mistral': 3449, 'n_tokens_neox': 3343, 'n_words': 797}