<commit_before>#!/home/gerard/.virtualenvs/arenavision/bin/python

import requests
from time import sleep
from datetime import datetime, timedelta
from tabulate import tabulate
from lxml import html
import subprocess
import re
import sys
import argparse

BASE_URL = "http://arenavision.in/"

SOP_PORT = "8908"

HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'es,ca;q=0.8,en-US;q=0.6,en;q=0.4,:max-age=0',
    'Host': 'arenavision.in',
    'Referer': 'http://arenavision.in/',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36'
}

DATEFORMAT = "%d/%m/%y %H:%M"

CHAN_EXP = "/(AV[0-9]{1,2})"
SPORT_EXP = "CET ([^:]+):"
DESC_EXP = "CET {sport}: (.*)/{join_channels}"


class Item(object):
    def __init__(self, data):
        self.time, self.sport, self.match, self.category, self.channels = data        
        self.soplinks = []
    
    def __str__(self):
        return str((self.time, self.sport, self.match, self.category, self.channels, self.soplinks))

    def gettime(self):
        return self.time.strftime(DATEFORMAT)

    def header(self):
        return "\n".join([self.sport + " - " + self.category, self.gettime() + " " + self.match])

    def tolist(self):
        return [
            self.gettime(),
            self.sport.decode("utf8"),
            self.category.decode("utf8"),
            self.match.decode("utf8")
        ]

    def matches(self, keywords):
        haystack = str(self).lower()
        return any(keyword for keyword in keywords if keyword in haystack)


def parse_schedule_row(row):
    row = row.encode("utf8").strip()
    try:
        time = row[:14]
        time = datetime.strptime(time, DATEFORMAT)
    except Exception, e:
        return None

    try:                
        sport = re.findall(SPORT_EXP, row)[0]
    except Exception, e:
        return None

    try:    
        channels = re.findall(CHAN_EXP, row)
    except Exception, e:
        return None

    try:
        exp = DESC_EXP.format(sport=sport, join_channels="/".join(channels)) 
        match = re.findall(exp, row)[0]
        category = re.findall("\(([^\(]+)\)", match)[0]
        match = match.replace("({0})".format(category), "").strip()
    except Exception, e:
        return None

    return time, sport, match, category, channels


def get_page(url):
    try:
        req = requests.get(url, headers=HEADERS)
        return req.content
    except requests.exceptions.ConnectionError:
        print "Could not connect with host. Please retry later. Sorry."
        raise KeyboardInterrupt


def get_schedule():
    page = get_page(BASE_URL + "agenda")
    tree = html.fromstring(page)
    items = []

    for match in tree.xpath('//div[contains(@class, "field-item")]/p[2]/text()'):
        data = parse_schedule_row(match)
        if data:
            item = Item(parse_schedule_row(match))
            items.append(item)

    return items


def crawl_sopcast_links(item):
    for chan in item.channels:
        chan = chan.lower().replace("av", "")
        if int(chan) >= 20:
            page = get_page(BASE_URL + "av" + chan)
            tree = html.fromstring(page)
            link = tree.xpath("//a[contains(@href, 'sop://')]/@href")
            if link:
                item.soplinks.append(link[0])


def clear_screen():
    sys.stdout.write("\033[2J\033[;H")


def print_buffering(value):
    sys.stdout.write("\rBuffering... {0}%".format(value))


def show_match_options(match):

    choose = "Choose a sopcast channel to start streaming"

    option = option_chooser(header=match.header(), choose=choose, options=[[link,] for link in match.soplinks])

    start_streaming(match.soplinks[option])
    

def start_streaming(soplink):
    print "Start streaming channel: " + soplink

    vlcrunning = False
    sopprocess = None

    try:
        sopcmd = ["sp-sc-auth", soplink, "3908", SOP_PORT]
        vlccmd = ["vlc", "http://localhost:" + SOP_PORT + "/tv.asf"]

        sopprocess = subprocess.Popen(sopcmd, stdout=subprocess.PIPE)
        print "Wating for stream to buffer"
        print_buffering(0)
        while True:
            line = sopprocess.stdout.readline()
            if "nblockAvailable" in line:
                buff = line.split("nblockAvailable=")[-1]
                print_buffering(buff.strip())
                if not vlcrunning and int(buff) > 30:
                    subprocess.Popen(vlccmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    vlcrunning = True
        sopprocess.terminate()
        print "Bye!"
    except KeyboardInterrupt, e:
        print "\nStop stream. Exit."<commit_msg>adapted for new page layout<commit_after>#!/home/gerard/dev/arenavision/.venv/bin/python

import requests
from time import sleep
from datetime import datetime, timedelta
from tabulate import tabulate
from lxml import html
import subprocess
import re
import sys
import argparse

BASE_URL = "http://arenavision.in/"

SOP_PORT = "8908"

HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'es,ca;q=0.8,en-US;q=0.6,en;q=0.4,:max-age=0',
    'Host': 'arenavision.in',
    'Referer': 'http://arenavision.in/',
    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36'
}

DATEFORMAT = "%d/%m/%Y %H:%M"

CHAN_EXP = "/(AV[0-9]{1,2})"
SPORT_EXP = "CET ([^:]+):"
DESC_EXP = "CET {sport}: (.*)/{join_channels}"


class Item(object):
    def __init__(self, data):
        self.time, self.sport, self.match, self.category, self.channels = data        
        self.soplinks = []
    
    def __str__(self):
        return str((self.time, self.sport, self.match, self.category, self.channels, self.soplinks))

    def gettime(self):
        return self.time.strftime(DATEFORMAT)

    def header(self):
        return "\n".join([self.sport + " - " + self.category, self.gettime() + " " + self.match])

    def tolist(self):
        return [
            self.gettime(),
            self.sport.decode("utf8"),
            self.category.decode("utf8"),
            self.match.decode("utf8")
        ]

    def matches(self, keywords):
        haystack = str(self).lower()
        return any(keyword for keyword in keywords if keyword in haystack)


def get_page(url):
    try:
        req = requests.get(url, headers=HEADERS)
        return req.content
    except requests.exceptions.ConnectionError:
        print "Could not connect with host. Please retry later. Sorry."
        raise KeyboardInterrupt


def parse_schedule_row_node(row):
    try:
        rdate, rtime, rsport, rcat, rmatch, channels = map(lambda x: x.text.strip(), row)
	if rdate and rtime:
            time = datetime.strptime(rdate + " " + rtime.replace(" CET", ""), DATEFORMAT)
	    channels = channels[:channels.find("[")] 
	    channels = [ch.strip() for ch in channels.split("-") if 'S' in ch]
	    if channels:
		return time, rsport, rmatch, rcat, channels
    except Exception as e:
	print e
    return None


def get_schedule():
    page = get_page(BASE_URL + "agenda")
    tree = html.fromstring(page)
    items = []
    for match in tree.xpath('//table//tr[td/@class="auto-style3"]'):
	data = parse_schedule_row_node(match)
        if data:
            item = Item(data)
            items.append(item)

    return items


def crawl_sopcast_links(item):
    for chan in item.channels:
        page = get_page(BASE_URL + "av" + chan.lower())
        tree = html.fromstring(page)
        link = tree.xpath("//a[contains(@href, 'sop://')]/@href")
        if link:
            item.soplinks.append(link[0])


def clear_screen():
    sys.stdout.write("\033[2J\033[;H")


def print_buffering(value):
    sys.stdout.write("\rBuffering... {0}%".format(value))


def show_match_options(match):

    choose = "Choose a sopcast channel to start streaming"

    option = option_chooser(header=match.header(), choose=choose, options=[[link,] for link in match.soplinks])

    start_streaming(match.soplinks[option])
    

def start_streaming(soplink):
    print "Start streaming channel: " + soplink

    vlcrunning = False
    sopprocess = None

    try:
        sopcmd = ["sp-sc-auth", soplink, "3908", SOP_PORT]
        vlccmd = ["cvlc", "http://localhost:" + SOP_PORT + "/tv.asf"]

        sopprocess = subprocess.Popen(sopcmd, stdout=subprocess.PIPE)
        print "Wating for stream to buffer"
        print_buffering(0)
        while True:
            line = sopprocess.stdout.readline()
            if "nblockAvailable" in line:
                buff = line.split("nblockAvailable=")[-1]
                print_buffering(buff.strip())
                if not vlcrunning and int(buff) > 30:
                    subprocess.Popen(vlccmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    vlcrunning = True
        sopprocess.terminate()
        print "Bye!"
    except KeyboardInterrupt, e:
        print "\nStop stream. Exit."
{'fraction_non_alphanumeric': 0.1427783902976847, 'fraction_numerical': 0.015766262403528113, 'mean_word_length': 2.6356713426853706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1741060', 'n_tokens_mistral': 3103, 'n_tokens_neox': 2984, 'n_words': 652}