<commit_before>#GOMTVNET - by Futurefive 2011
#Content scraped from m.gomtv.net
#Thanks to Voinage for his XBMC plugin tutorial and boilerplate code http://wiki.xbmc.org/index.php?title=HOW-TO_write_plugins_for_XBMC
#Thanks to Temhil for his XBMC Eclipse debugging tutorial http://wiki.xbmc.org/index.php?title=HOW-TO_debug_Python_Scripts_with_Eclipse#Add_Pydev_Python_source_code_.28pysrc.29_to_XBMC

from BeautifulSoup import BeautifulSoup
import urllib2, urllib
import sys, os
import re
import simplejson as json
import xbmc, xbmcgui, xbmcplugin
import cookielib

BASE_COOKIE_PATH = os.path.join(xbmc.translatePath( "special://profile/" ), "addon_data", os.path.basename(os.getcwd()), 'cookie.txt')

# plugin modes
MODE_MATCHLIST = 1
MODE_VODLINKS = 2

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0'
REFERER = 'http://m.gomtv.net/'

#Prevent page load from potentially running infinitely
MAX_PAGE_LOADS = 20
 
# plugin handle
handle = int(sys.argv[1])

# build opener with HTTPCookieProcessor
cookie_jar = cookielib.LWPCookieJar(BASE_COOKIE_PATH)
# load cookie if it exists
if not os.path.exists(os.path.dirname(BASE_COOKIE_PATH)):
    os.makedirs(os.path.dirname(BASE_COOKIE_PATH))
if (os.path.isfile(BASE_COOKIE_PATH)):
    cookie_jar.load(BASE_COOKIE_PATH)

opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar))

opener.addheaders = [('User-agent',  USER_AGENT)]
opener.addheaders = [('Referer',  REFERER)]
urllib2.install_opener(opener)


##########################
##       LEAGUES
##########################

def show_leagues():
    url = 'http://m.gomtv.net/league/'
    
    f = opener.open( url )
    html = f.read()
    f.close()
    cookie_jar.save()

    soup = BeautifulSoup(html)
    ul_list = soup('ul', {'class':'list_wrap'})

    for i in ul_list:
        league_name = i('li', {'class':'namelist_long'})[0](text=True)[0]
        league_url = i('li', {'class':'nextbtn'})[0].a['href'].split('=')
        league_id = league_url[len(league_url)-1]
        league_ajax_url = "http://m.gomtv.net/ajax/getVideoList.gom?league="+league_id+"&pid=&mid=&page="
        addDirectoryItem(league_name.strip(), True, {'mode':'1', 'name':league_name, 'url':league_ajax_url})

    xbmcplugin.endOfDirectory(handle=handle, succeeded=True)


##########################
##      MATCHLIST
##########################
#http://m.gomtv.net/ajax/getVideoList.gom?league=22824&pid=&mid=&page=1
#Above AJAX returns 10 items per page, need to make multiple request to show more items
#Only works with POST
def show_matches(base_url, load_all = False):
    #url = 'http://m.gomtv.net/ajax/getVideoList.gom'
    
    #match_params = parameters_string_to_dict("?"+base_url.split('?')[1])
    
    pages_loaded = 0
    empty_page = False #have we hit an empty response
    updateListing = True
    
    if load_all == False:
        updateListing = False
		totalItems = perpage + 1
	else:
		totalItems = MAX_PAGE_LOADS*10
        
    while pages_loaded < MAX_PAGE_LOADS:
        url = base_url+str(pages_loaded+1)
        
        #post_query = {'league':match_params['league'],
         #             'mid':'',
         #             'pid':'',
         #             'page':str(pages_loaded+1)}
        
        print "Opening: "+url.split('?')[0]+"?"+url.split('?')[1]
        #print repr(post_query)
        
        f = opener.open( url.split('?')[0], url.split('?')[1] )
        html = f.read()
        f.close()<commit_msg>Fixed some tab issues<commit_after>#GOMTVNET - by Futurefive 2011
#Content scraped from m.gomtv.net
#Thanks to Voinage for his XBMC plugin tutorial and boilerplate code http://wiki.xbmc.org/index.php?title=HOW-TO_write_plugins_for_XBMC
#Thanks to Temhil for his XBMC Eclipse debugging tutorial http://wiki.xbmc.org/index.php?title=HOW-TO_debug_Python_Scripts_with_Eclipse#Add_Pydev_Python_source_code_.28pysrc.29_to_XBMC

from BeautifulSoup import BeautifulSoup
import urllib2, urllib
import sys, os
import re
import simplejson as json
import xbmc, xbmcgui, xbmcplugin
import cookielib

BASE_COOKIE_PATH = os.path.join(xbmc.translatePath( "special://profile/" ), "addon_data", os.path.basename(os.getcwd()), 'cookie.txt')

# plugin modes
MODE_MATCHLIST = 1
MODE_VODLINKS = 2

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0'
REFERER = 'http://m.gomtv.net/'

#Prevent page load from potentially running infinitely
MAX_PAGE_LOADS = 20
 
# plugin handle
handle = int(sys.argv[1])

# build opener with HTTPCookieProcessor
cookie_jar = cookielib.LWPCookieJar(BASE_COOKIE_PATH)
# load cookie if it exists
if not os.path.exists(os.path.dirname(BASE_COOKIE_PATH)):
    os.makedirs(os.path.dirname(BASE_COOKIE_PATH))
if (os.path.isfile(BASE_COOKIE_PATH)):
    cookie_jar.load(BASE_COOKIE_PATH)

opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar))

opener.addheaders = [('User-agent',  USER_AGENT)]
opener.addheaders = [('Referer',  REFERER)]
urllib2.install_opener(opener)


##########################
##       LEAGUES
##########################

def show_leagues():
    url = 'http://m.gomtv.net/league/'
    
    f = opener.open( url )
    html = f.read()
    f.close()
    cookie_jar.save()

    soup = BeautifulSoup(html)
    ul_list = soup('ul', {'class':'list_wrap'})

    for i in ul_list:
        league_name = i('li', {'class':'namelist_long'})[0](text=True)[0]
        league_url = i('li', {'class':'nextbtn'})[0].a['href'].split('=')
        league_id = league_url[len(league_url)-1]
        league_ajax_url = "http://m.gomtv.net/ajax/getVideoList.gom?league="+league_id+"&pid=&mid=&page="
        addDirectoryItem(league_name.strip(), True, {'mode':'1', 'name':league_name, 'url':league_ajax_url})

    xbmcplugin.endOfDirectory(handle=handle, succeeded=True)


##########################
##      MATCHLIST
##########################
#http://m.gomtv.net/ajax/getVideoList.gom?league=22824&pid=&mid=&page=1
#Above AJAX returns 10 items per page, need to make multiple request to show more items
#Only works with POST
def show_matches(base_url, load_all = False):
    #url = 'http://m.gomtv.net/ajax/getVideoList.gom'
    
    #match_params = parameters_string_to_dict("?"+base_url.split('?')[1])
    
    pages_loaded = 0
    empty_page = False #have we hit an empty response
    updateListing = True
    
    if load_all == False:
        updateListing = False
        totalItems = perpage + 1
    else:
        totalItems = MAX_PAGE_LOADS*10
        
    while pages_loaded < MAX_PAGE_LOADS:
        url = base_url+str(pages_loaded+1)
        
        print "Opening: "+url.split('?')[0]+"?"+url.split('?')[1]
        
        f = opener.open( url.split('?')[0], url.split('?')[1] )
        html = f.read()
        f.close()
{'fraction_non_alphanumeric': 0.16629678946589732, 'fraction_numerical': 0.017310252996005325, 'mean_word_length': 4.014836795252226, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '699876', 'n_tokens_mistral': 2544, 'n_tokens_neox': 2402, 'n_words': 451}