<commit_before>#!/usr/local/bin/python

import urllib2
from bs4 import BeautifulSoup
from parsingLibrary import loadHtmlTags, parseChannel
from datetime import timedelta
from pymongo import MongoClient
import pytz

client = MongoClient('localhost', 27017)
db = client['freeview']
contentCollection = db['tvContent']
contentCollection.drop()

import re


def parse_title(tv_content_html):
    m = re.search('<div class=\'prog_header\'>(.+?)</div>', tv_content_html['onmouseover'].replace('\\', ''))
    return m.group(1)


def parse_time(tv_content_html, tv_content, after_noon):
    m = re.search('<div class=\'prog_pre_header\'>(.+?)</div>', tv_content_html['onmouseover'].replace('\\', ''))
    start_hour = m.group(1).split('-')[0]
    end_hour = m.group(1).split('-')[1]
    start = str(month) + ' ' + str(day) + ' ' + str(year) + ' ' + start_hour
    end = str(month) + ' ' + str(day) + ' ' + str(year) + ' ' + end_hour
    start_datetime = datetime.strptime(start, '%m %d %Y %I.%M%p')
    end_datetime = datetime.strptime(end, '%m %d %Y %I.%M%p')

    if not after_noon and 'pm' in start_hour and 'am' in end_hour:
        start_datetime = start_datetime - timedelta(days = 1)
    else:
        if after_noon and 'pm' in start_hour and 'am' in end_hour:
            end_datetime = end_datetime + timedelta(days = 1)
        else:
            if after_noon and 'am' in start_hour and 'am' in end_hour:
                start_datetime = start_datetime + timedelta(days = 1)
                end_datetime = end_datetime + timedelta(days = 1)

    tz = pytz.timezone('Europe/London')
    tv_start_uk_time = tz.localize(start_datetime)
    tv_end_uk_time = tz.localize(start_datetime)
    tv_content['start'] = tv_start_uk_time
    tv_content['end'] = tv_end_uk_time

def parse_actors(actors):
    import numpy as np

    array_actors = actors.replace(' and ', ', ').split(', ')
    return np.array(array_actors).tolist()

def parse_content(tv_content_html, title, tv_content):
    m = re.search('<div class=\'prog_pre_content\'>(.+?)</div>', tv_content_html['onmouseover'].replace('\\', ''))
    description_html = m.group(1).strip()

    if ('Episode' in description_html and 'Series' in description_html):
        tv_content['series'] = {}
        series(description_html, title, tv_content['series'])
    else:
        if 'FILM: ' in (title):
            tv_content['film'] = {}
            title_film_details(tv_content['film'], title)
            content_details(description_html, tv_content['film'])
        else:
            tv_content['program'] = {}
            tv_content['program']['title'] = title
            content_details(description_html, tv_content['program'])

def parse_series_details(tv_content_series, tv_content_series_html):<commit_msg>Fixing time start/end<commit_after>#!/usr/local/bin/python

import urllib2
from bs4 import BeautifulSoup
from parsingLibrary import loadHtmlTags, parseChannel
from datetime import timedelta
from pymongo import MongoClient
import pytz

client = MongoClient('localhost', 27017)
db = client['freeview']
contentCollection = db['tvContent']
contentCollection.drop()

import re


def parse_title(tv_content_html):
    m = re.search('<div class=\'prog_header\'>(.+?)</div>', tv_content_html['onmouseover'].replace('\\', ''))
    return m.group(1)


def parse_time(tv_content_html, tv_content, after_noon):
    m = re.search('<div class=\'prog_pre_header\'>(.+?)</div>', tv_content_html['onmouseover'].replace('\\', ''))
    start_hour = m.group(1).split('-')[0]
    end_hour = m.group(1).split('-')[1]
    start = str(month) + ' ' + str(day) + ' ' + str(year) + ' ' + start_hour
    end = str(month) + ' ' + str(day) + ' ' + str(year) + ' ' + end_hour
    start_datetime = datetime.strptime(start, '%m %d %Y %I.%M%p')
    end_datetime = datetime.strptime(end, '%m %d %Y %I.%M%p')

    if not after_noon and 'pm' in start_hour and 'am' in end_hour:
        start_datetime = start_datetime - timedelta(days = 1)
    else:
        if after_noon and 'pm' in start_hour and 'am' in end_hour:
            end_datetime = end_datetime + timedelta(days = 1)
        else:
            if after_noon and 'am' in start_hour and 'am' in end_hour:
                start_datetime = start_datetime + timedelta(days = 1)
                end_datetime = end_datetime + timedelta(days = 1)

    tz = pytz.timezone('Europe/London')
    tv_start_uk_time = tz.localize(start_datetime)
    tv_end_uk_time = tz.localize(end_datetime)
    tv_content['start'] = tv_start_uk_time
    tv_content['end'] = tv_end_uk_time

def parse_actors(actors):
    import numpy as np

    array_actors = actors.replace(' and ', ', ').split(', ')
    return np.array(array_actors).tolist()

def parse_content(tv_content_html, title, tv_content):
    m = re.search('<div class=\'prog_pre_content\'>(.+?)</div>', tv_content_html['onmouseover'].replace('\\', ''))
    description_html = m.group(1).strip()

    if ('Episode' in description_html and 'Series' in description_html):
        tv_content['series'] = {}
        series(description_html, title, tv_content['series'])
    else:
        if 'FILM: ' in (title):
            tv_content['film'] = {}
            title_film_details(tv_content['film'], title)
            content_details(description_html, tv_content['film'])
        else:
            tv_content['program'] = {}
            tv_content['program']['title'] = title
            content_details(description_html, tv_content['program'])

def parse_series_details(tv_content_series, tv_content_series_html):
{'fraction_non_alphanumeric': 0.1503800217155266, 'fraction_numerical': 0.006152732537097358, 'mean_word_length': 3.6289782244556115, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '874033', 'n_tokens_mistral': 1882, 'n_tokens_neox': 1805, 'n_words': 373}