I code a lot of web crawlers and web scrapers and I find myself writing the same functions over and over again. I also find myself having to come to Stack Overflow and find the answer to a question I have had to ask myself a dozen times. Things like how to supply proxies to requests or how to use custom headers with requests or how to set the User-Agent in requests and things of that sort. So I'm writing this module to abstract some of these mundane routines.My concernsIs the code pythonic?Would this be of use to anybody other than me?Are there any bugs?Is it ok to have that many methods in a class?How are my naming conventions?#!/usr/bin/env python'''this module was designed with web scrapers and web crawlers in mind.I find my self writing these functions all the time. I Wrote this modelto save time.'''import requestsimport urlparseimport urllib2import urllibimport reimport osimport jsonfrom fake_useragent import UserAgentclass InvalidURL(Exception):    passclass URL(object):    '''Commomn routines for dealing with URLS.    '''    def __init__(self, url):        '''Setup the initial state        '''        self.raw_url = url        self.url = urlparse.urlparse(url)        self.scheme = self.url.scheme        self.domain = self.url.netloc        self.path = self.url.path        self.params = self.url.params        self.query = self.url.query        self.fragment = self.url.fragment    def __str__(self):        ''' This os called when somthing        asks for a string representation of the        url        '''        return self.raw_url    def valid(self):        Validate the url.        returns True if url is valid        and False if it is not                regex = re.compile(            r'^(?:http|ftp)s?://' # http:// or https://            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'            r'localhost|' #localhost...            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip            r'(?::\d+)?' # optional port            r'(?:/?|[/?]\S+)$', re.IGNORECASE)        match = regex.match(self.raw_url)        if match:            return True    def unquote(self):        unquote('abc%20def') -> 'abc def'.        return urllib2.unquote(self.raw_url)    def quote(self):        quote('abc def') -> 'abc%20def'        Each part of a URL, e.g. the path info, the query, etc., has a        different set of reserved characters that must be quoted.        RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists        the following reserved characters.        reserved    = ; | / | ? | : | @ | & | = | + |                      $ | ,        Each of these characters is reserved in some component of a URL,        but not necessarily in all of them.        By default, the quote function is intended for quoting the path        section of a URL.  Thus, it will not encode '/'.  This character        is reserved, but in typical usage the quote function is being        called on a path where the existing slash characters are used as        reserved characters.                return urllib2.quote(self.raw_url)    def parameters(self):                parse the parameters of the url        and return them as a dict.                return urlparse.parse_qs(self.params)    def secure(self):         Checks if the url uses ssl.         if self.scheme == 'https':            return True    def extention(self):         return the file extention         return os.path.splitext(self.path)[1]    def absolute(self):         Checks if the URL is absolute.         return bool(self.domain)    def relitive(self):         Checks if the url is relitive.         return bool(self.scheme) is False    def encode(self, mapping):        Encode a sequence of two-element tuples or dictionary into a URL query string.        If any values in the query arg are sequences and doseq is true, each        sequence element is converted to a separate parameter.        If the query arg is a sequence of two-element tuples, the order of the        parameters in the output will match the order of parameters in the        input.                query = urllib.urlencode(mapping)        return urlparse.urljoin(self.raw_url, query)class Request(object):    allow_redirects = True    timeout = 5    ramdom_useragent = 0    verify_ssl = False    session = requests.Session()    stream = True    proxies = {}    def __init__(self, url):         Set the inital state         self.agentHeaders = {}        self.url = URL(url)        if not self.url.valid():            raise InvalidURL({} is invalid.format(url))    def stream(self, answer):        self.stream = bool(answer)    def randomUserAgent(self):         Set a random User-Agent         self.setUserAgent(UserAgent().random)    def allowRedirects(self, answer):         Choose whether or not to follow redirects.        self.allow_redirects = bool(answer)    def setUserAgent(self, agent):         Set the User-Agent         self.setHeaders('User-Agent', agent)    def setHeaders(self, key, value):         Set custom headers         self.agentHeaders[key] = value    def verify(self, answer):         Set whether or not to verify SSL certs        self.verify_ssl = bool(answer)    def get(self):        Sends a GET request        return self.session.get(            url=self.url,            headers=self.agentHeaders,            allow_redirects=self.allow_redirects,            timeout=self.timeout,            verify=self.verify_ssl,            stream=self.stream,            proxies=self.proxies            )    def head(self):         Send a head request and return the headers         return self.session.head(            self.url,            headers=self.agentHeaders,            allow_redirects=self.allow_redirects,            timeout=self.timeout,            verify=self.verify_ssl,            proxies=self.proxies            ).headers    def options(self):         Send a options request and return the options         return self.session.options(            self.url,            headers=self.agentHeaders,            allow_redirects=self.allow_redirects,            timeout=self.timeout,            verify=self.verify_ssl,            proxies=self.proxies            ).headers['allow']    def json(self):                Deserialize json data (a ``str`` or ``unicode`` instance        containing a JSON document) to a Python object.                return json.loads(self.text)    def headerValue(self, value):         Get a value from the headers.         return self.headers().get(value)request = Request('https://www.google.com')req =  request.get()
A Swiss Army like module for dealing with HTTP and URLs
python;http;url