<commit_before>from flask import Flask, render_template, redirect, abort, request, url_for, Response, jsonify
from datetime import datetime
import icalendar
import os
import json
import re

catapp = Flask(__name__)
root = os.path.dirname((os.path.dirname(os.path.realpath(__file__))))


@catapp.route("/videos")
def videos():
    term = _term()
    videos = _read_json(root + '/html/videos.json')
    results = []
    if term != '':
        for v in videos:
            if term in v['title'].lower():
                results.append(v)
                continue
            if term in v['short_description'].lower():
                results.append(v)
                continue
            if 'tags' in v:
                tags = [x['link'] for x in v['tags']]
                if term in tags:
                    results.append(v)
                    continue

    return render_template('videos.html',
        title            = 'Tech videos worth watching', 
        h1               = 'Videos',
        number_of_videos = len(videos),
        term             = term,
        videos           = results,
    )


@catapp.route("/people")
def people():
    term = _term()
    ppl = _read_json(root + '/html/people.json')
    result = {}
    if term != '':
        for nickname in ppl.keys():
            if re.search(term, ppl[nickname]['name'].lower()):
                result[nickname] = ppl[nickname]
            elif re.search(term, ppl[nickname].get('location', '').lower()):
                result[nickname] = ppl[nickname]
            elif re.search(term, ppl[nickname].get('topics', '').lower()):
                result[nickname] = ppl[nickname]
            elif 'tags' in ppl[nickname] and term in ppl[nickname]['tags']:
                result[nickname] = ppl[nickname]

    return render_template('people.html',
        title            = 'People who talk at conferences or in podcasts', 
        h1               = 'People who talk',
        number_of_people = len(ppl.keys()),
        term             = term,
        people           = result,
        people_ids       = sorted(result.keys()),
    )

@catapp.route("/series")
def series():
    data = _read_json(root + '/html/series.json')
    return render_template('series.html',
        h1     = 'Event Series',
        title  = 'Event Series',
        series = data,
    )

@catapp.route("/about")
def about(filename = None):
    cat = _read_json(root + '/html/cat.json')
    return render_template('about.html',
        h1          = 'About Code And Talk',
        title       = 'About Code And Talk',
        stats       = cat['stats'],
    )

@catapp.route("/")
def main():
    cat = _read_json(root + '/html/cat.json')
    return render_template('index.html',
        h1          = 'Presentations from tech events worth watching',
        title       = 'Conferences, Videos, Podcasts, and People',
        stats       = cat['stats'],
    )

### static page for the time of transition
@catapp.route("/<filename>")
def static_file(filename = None):
    #index.html  redirect

    if not filename:
        filename  = 'index.html'
    mime = 'text/html'
    content = _read(root + '/html/' + filename)
    if filename[-4:] == '.css':
        mime = 'text/css'
    elif filename[-5:] == '.json':
        mime = 'application/javascript'
    elif filename[-3:] == '.js':
        mime = 'application/javascript'
    elif filename[-4:] == '.xml':
        mime = 'text/xml'
    elif filename[-4:] == '.ico':
        mime = 'image/x-icon'
    return Response(content, mimetype=mime)

@catapp.route("/v/<event>/<video>")
def video(event = None, video = None):
    path = root + '/html/v/{}/{}'.format(event, video)
    #html_file = path + '.html'
    data = json.loads(open(path + '.json').read())

    #os.path.exists(html_file):
    #   data['description'] = open(html_file).read()
    return render_template('video.html',
        h1          = data['title'],
        title       = data['title'],
        video       = data,
        blasters    = data.get('blasters'),
    )

@catapp.route("/p/<person>")
def person(person = None):
    path = root + '/html/p/{}'.format(person)
    data = json.load(open(path + '.json'))
    return render_template('person.html',
        h1          = data['info']['name'],
        title       = 'Presentations and podcasts by ' + data['info']['name'],
        person      = data,
        id          = person,
    )

@catapp.route("/cal/all.ics")
def calendar():
    cat = _read_json(root + '/html/cat.json')
    now = datetime.now().strftime('%Y-%m-%d')

    future = sorted(list(filter(lambda e: e['start_date'] >= now, cat["events"].values())), key = lambda e: e['start_date'])

    cal = icalendar.Calendar()
    for e in future:
        event = icalendar.Event()
        event['dtstart'] = re.sub(r'-', '', e['start_date']) + 'T080000' # '20170104T080000'
        event['summary'] = e['name']
        event['description'] = e['url']
        event['dtend'] = re.sub(r'-', '', e['end_date']) + 'T080000' # '20170104T080000'
        cal.add_component(event)

    return cal.to_ical().decode('UTF-8')

@catapp.route("/t/<tag>")
@catapp.route("/e/<event>")
@catapp.route("/l/<location>")
@catapp.route("/s/<source>")
@catapp.route("/blaster/<blaster>")
def html(event = None, source = None, tag = None, location = None, blaster = None):
    if blaster:
        return _read(root + '/html/blaster/' + blaster)
    if location:
        return _read(root + '/html/l/' + location)
    if source:
        return _read(root + '/html/s/' + source)
    if event:
        return _read(root + '/html/e/' + event)
    if tag:
        return _read(root + '/html/t/' + tag)

###### Helper functions

def _read(filename):
    try:
        return open(filename).read()
    except Exception:
        return open(root + '/html/404.html').read()
        
def _term():
    term = request.args.get('term', '')
    term = term.lower()
    term = re.sub(r'^\s*(.*?)\s*$', r'\1', term)
    return term

def _read_json(filename):
    catapp.logger.debug("Reading '{}'".format(filename))
    try:
        with open(filename) as fh:
            data = json.loads(fh.read())
    except Exception as e:
        catapp.logger.error("Reading '{}' {}".format(search_file, e))
        data = {}
        pass
    return data

# vim: expandtab
<commit_msg>make calendar work in Google calendar and pass the http://severinghaus.org/projects/icv/ validator<commit_after>from flask import Flask, render_template, redirect, abort, request, url_for, Response, jsonify
from datetime import datetime
import os
import json
import re

catapp = Flask(__name__)
root = os.path.dirname((os.path.dirname(os.path.realpath(__file__))))


@catapp.route("/videos")
def videos():
    term = _term()
    videos = _read_json(root + '/html/videos.json')
    results = []
    if term != '':
        for v in videos:
            if term in v['title'].lower():
                results.append(v)
                continue
            if term in v['short_description'].lower():
                results.append(v)
                continue
            if 'tags' in v:
                tags = [x['link'] for x in v['tags']]
                if term in tags:
                    results.append(v)
                    continue

    return render_template('videos.html',
        title            = 'Tech videos worth watching', 
        h1               = 'Videos',
        number_of_videos = len(videos),
        term             = term,
        videos           = results,
    )


@catapp.route("/people")
def people():
    term = _term()
    ppl = _read_json(root + '/html/people.json')
    result = {}
    if term != '':
        for nickname in ppl.keys():
            if re.search(term, ppl[nickname]['name'].lower()):
                result[nickname] = ppl[nickname]
            elif re.search(term, ppl[nickname].get('location', '').lower()):
                result[nickname] = ppl[nickname]
            elif re.search(term, ppl[nickname].get('topics', '').lower()):
                result[nickname] = ppl[nickname]
            elif 'tags' in ppl[nickname] and term in ppl[nickname]['tags']:
                result[nickname] = ppl[nickname]

    return render_template('people.html',
        title            = 'People who talk at conferences or in podcasts', 
        h1               = 'People who talk',
        number_of_people = len(ppl.keys()),
        term             = term,
        people           = result,
        people_ids       = sorted(result.keys()),
    )

@catapp.route("/series")
def series():
    data = _read_json(root + '/html/series.json')
    return render_template('series.html',
        h1     = 'Event Series',
        title  = 'Event Series',
        series = data,
    )

@catapp.route("/about")
def about(filename = None):
    cat = _read_json(root + '/html/cat.json')
    return render_template('about.html',
        h1          = 'About Code And Talk',
        title       = 'About Code And Talk',
        stats       = cat['stats'],
    )

@catapp.route("/")
def main():
    cat = _read_json(root + '/html/cat.json')
    return render_template('index.html',
        h1          = 'Presentations from tech events worth watching',
        title       = 'Conferences, Videos, Podcasts, and People',
        stats       = cat['stats'],
    )

### static page for the time of transition
@catapp.route("/<filename>")
def static_file(filename = None):
    #index.html  redirect

    if not filename:
        filename  = 'index.html'
    mime = 'text/html'
    content = _read(root + '/html/' + filename)
    if filename[-4:] == '.css':
        mime = 'text/css'
    elif filename[-5:] == '.json':
        mime = 'application/javascript'
    elif filename[-3:] == '.js':
        mime = 'application/javascript'
    elif filename[-4:] == '.xml':
        mime = 'text/xml'
    elif filename[-4:] == '.ico':
        mime = 'image/x-icon'
    return Response(content, mimetype=mime)

@catapp.route("/v/<event>/<video>")
def video(event = None, video = None):
    path = root + '/html/v/{}/{}'.format(event, video)
    #html_file = path + '.html'
    data = json.loads(open(path + '.json').read())

    #os.path.exists(html_file):
    #   data['description'] = open(html_file).read()
    return render_template('video.html',
        h1          = data['title'],
        title       = data['title'],
        video       = data,
        blasters    = data.get('blasters'),
    )

@catapp.route("/p/<person>")
def person(person = None):
    path = root + '/html/p/{}'.format(person)
    data = json.load(open(path + '.json'))
    return render_template('person.html',
        h1          = data['info']['name'],
        title       = 'Presentations and podcasts by ' + data['info']['name'],
        person      = data,
        id          = person,
    )

@catapp.route("/cal/all.ics")
def calendar():
    cat = _read_json(root + '/html/cat.json')
    now = datetime.now().strftime('%Y-%m-%d')

    future = sorted(list(filter(lambda e: e['start_date'] >= now, cat["events"].values())), key = lambda e: e['start_date'])

    cal = ""
    cal += "BEGIN:VCALENDAR\r\n"
    cal += "PRODID:https://codeandtalk.com/cal/all.ics\r\n"
    cal += "VERSION:2.0\r\n"
    #PRODID:-//http://XXX//Event
    #METHOD:PUBLISH

    for e in future:
        cal += "BEGIN:VEVENT\r\n"
        cal += "DTSTAMP:{}T000000Z\r\n".format(re.sub(r'-', '', e['start_date']))
        cal += "DTSTART;VALUE=DATE:{}\r\n".format(re.sub(r'-', '', e['start_date']))
        cal += "DTEND;VALUE=DATE:{}\r\n".format(re.sub(r'-', '', e['end_date']))
        uid = re.sub(r'\W+', '-', e['url'])
        uid = re.sub(r'\W+$', '', uid)
        cal += "UID:{}\r\n".format(uid)
        cal += "SUMMARY:{}\r\n".format(e['name'])
        cal += "DESCRIPTION:{}\r\n".format(e['url'])
        try:
            location = e['city']
            if e['state']:
                location += ", " + e['state']
            location += ", " + e['country']
            cal += "LOCATION:{}\r\n".format(location)
        except Exception:
            pass
            # hide Unicode error from beyondtellerrand-2017
        cal += "END:VEVENT\r\n"

    cal += "END:VCALENDAR\r\n"

    return cal
    #return cal.to_ical().decode('UTF-8')

@catapp.route("/t/<tag>")
@catapp.route("/e/<event>")
@catapp.route("/l/<location>")
@catapp.route("/s/<source>")
@catapp.route("/blaster/<blaster>")
def html(event = None, source = None, tag = None, location = None, blaster = None):
    if blaster:
        return _read(root + '/html/blaster/' + blaster)
    if location:
        return _read(root + '/html/l/' + location)
    if source:
        return _read(root + '/html/s/' + source)
    if event:
        return _read(root + '/html/e/' + event)
    if tag:
        return _read(root + '/html/t/' + tag)

###### Helper functions

def _read(filename):
    try:
        return open(filename).read()
    except Exception:
        return open(root + '/html/404.html').read()
        
def _term():
    term = request.args.get('term', '')
    term = term.lower()
    term = re.sub(r'^\s*(.*?)\s*$', r'\1', term)
    return term

def _read_json(filename):
    catapp.logger.debug("Reading '{}'".format(filename))
    try:
        with open(filename) as fh:
            data = json.loads(fh.read())
    except Exception as e:
        catapp.logger.error("Reading '{}' {}".format(search_file, e))
        data = {}
        pass
    return data

# vim: expandtab

{'fraction_non_alphanumeric': 0.169816937044203, 'fraction_numerical': 0.006399761869325792, 'mean_word_length': 2.1613737944013174, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3452169', 'n_tokens_mistral': 4359, 'n_tokens_neox': 4249, 'n_words': 950}