<commit_before>import os
import re
import time
import copy
import zlib
import gzip
import ssl
import struct
import socket
import urlparse
import operator
import random
import tempfile
import webbrowser
import threading
import shlex
import subprocess
import datetime
import itertools
from collections import defaultdict
from StringIO import StringIO
from hashlib import sha256
from burst import console
from burst.conf import conf
from burst.cookie import Cookie
from burst.color import *
from burst.exception import *
from burst.utils import make_table, clear_line, chunks, encode, \
                         re_space, smart_split, smart_rsplit, \
                         truncate, stats, parse_qs, play_notifier, \
                         play_updater

re_curl_counter = re.compile(r"""
    (?# ignore patterns with leading unbalanced backslashes )
      (?P<escaped> \x5c ?  )
      (?P<leading_escapes> (?: \x5c\x5c )* )
    \[
      (?P<start> [0-9]+ )
      -
      (?P<stop>  [0-9]+ )
      (?# optionally take a step argument: )
        :?
        (?P<step>  [0-9]* )
    \]  """, re.VERBOSE)

re_curl_characters = re.compile(r"""
    (?# ignore patterns with leading unbalanced backslashes )
      (?P<escaped> \x5c ?  )
      (?P<leading_escapes> (?: \x5c\x5c )* )
    \[
      (?P<start> [^-\n] )
      -
      (?P<stop>  [^-\n] )
      (?# optionally take a step argument: )
        :?
        (?P<step> [0-9]* )
    \]  """, re.VERBOSE)

re_curl_brace_set = re.compile(r"""
    (?# ignore patterns with leading unbalanced backslashes )
      (?P<escaped> \x5c ?  )
      (?P<leading_escapes> (?: \x5c\x5c )* )
    \{
      (?P<first> ( [^\n,]* ,)* )
      (?P<last>    [^\n,]* )
    \} """, re.VERBOSE)

class Request():
  """The Request class is the base of Burst. To create an instance, you have
  two options: either use a socket or a string representing the whole request
  into the constructor or use the 'create' function.

  The two methods __repr__ and __str__ have been defined to provide
  user-friendly interaction within the interpreter.
  """

  def __init__(self, fd, hostname=None, port=None, use_ssl=False):
    """Create a request. fd should be either a socket descriptor
       or a string. In both case, it should contain a full request.
       To generate a request from a URL, see c()"""
    if isinstance(fd, basestring):
      fd = StringIO(fd)
    try:
      banner = read_banner(fd)
      # ASSUMPTION: The request line contains three elements seperated by
      #             a space.
      self.method, url, self.http_version = banner
    except ValueError:
      raise NotConnected(' '.join(banner))
    if self.method.upper() == "CONNECT":
      # ASSUMPTION: CONNECT method needs a hostname and port
      self.hostname, self.port = url.rsplit(":", 1)
      self.port = int(self.port)
      self.url = ""
      self.use_ssl = False
    elif hostname:
      self.hostname = hostname
      self.port = port if port else 80
      self.use_ssl = use_ssl
      self.url = url
    else:
      p_url = urlparse.urlparse(url)
      self.url = urlparse.urlunparse(("", "") + p_url[2:])
      if p_url.scheme:
        self.hostname = p_url.hostname
        if not self.hostname:
          raise BurstException("No hostname: " + str(url))
        if p_url.scheme == 'https':
          self.use_ssl = True
          self.port = int(p_url.port) if p_url.port else 443
        else:
          self.port = int(p_url.port) if p_url.port else 80
          self.use_ssl = use_ssl
    self.raw_headers = read_headers(fd)
    if not hasattr(self, "hostname"): # Last chance, try the Host header
      hosts = self.get_header('Host')
      if not hosts:
        raise BurstException("Unable to find the host for the request")
      else:
        host = hosts[0]
      self.hostname = host.split(":")[0]
      self.port = int(host.split(":")[1]) if ":" in host else 80
      self.use_ssl = False
    self.raw_content = read_content(fd, parse_headers(self.raw_headers), method=self.method)
    if self.raw_content:
      self.content = _clear_content(parse_headers(self.raw_headers), self.raw_content)
    else:
      self.content = ""
    self.response = None

  def expand_curl_ranges(self):
    """Expands curl range sequences in the Request, returning a new RequestSet"""
    matches = {}

    def param_inject(s):
      def _param_inject(generator):
        ## can't come up with a better/faster way to generate unique keys in a thread-safe fashion :-(
        key = sha256('{}_{}'.format(len(matches), str(self))).digest().encode('hex')[:20]
        matches[key] = {
          'at': key,
          'payloads': generator }
        return key

      def _param_inject_counter(m):
        escapes = m.expand(m.group('leading_escapes'))
        if m.group('escaped'):
          ret = escapes + m.string[m.end('leading_escapes') : m.end(0)]
          return _param_inject( [ ret ] )

        values = [int(i or '1') for i in m.group(3, 4)]
        ## NOTE: curl's ranges are inclusive; python's are not, account for that:
        values[1] += 1
        return _param_inject( (escapes + str(i) for i in xrange(*values)) )

      def _param_inject_characters(m):
        escapes = m.expand(m.group('leading_escapes'))
        start = ord(m.group('start'))
        ## NOTE: curl's ranges are inclusive; python's are not, account for that:
        stop  = ord(m.group('stop')) + 1
        if m.group('escaped'):
          ret = escapes + m.string[m.end('leading_escapes') : m.end(0)]
          return _param_inject( [ ret ] )
        step  = int( m.group('step') or '1' )<commit_msg>Use hexdigest() instead of digest()<commit_after>import os
import re
import time
import copy
import zlib
import gzip
import ssl
import struct
import socket
import urlparse
import operator
import random
import tempfile
import webbrowser
import threading
import shlex
import subprocess
import datetime
import itertools
from collections import defaultdict
from StringIO import StringIO
from hashlib import sha256
from burst import console
from burst.conf import conf
from burst.cookie import Cookie
from burst.color import *
from burst.exception import *
from burst.utils import make_table, clear_line, chunks, encode, \
                         re_space, smart_split, smart_rsplit, \
                         truncate, stats, parse_qs, play_notifier, \
                         play_updater

re_curl_counter = re.compile(r"""
    (?# ignore patterns with leading unbalanced backslashes )
      (?P<escaped> \x5c ?  )
      (?P<leading_escapes> (?: \x5c\x5c )* )
    \[
      (?P<start> [0-9]+ )
      -
      (?P<stop>  [0-9]+ )
      (?# optionally take a step argument: )
        :?
        (?P<step>  [0-9]* )
    \]  """, re.VERBOSE)

re_curl_characters = re.compile(r"""
    (?# ignore patterns with leading unbalanced backslashes )
      (?P<escaped> \x5c ?  )
      (?P<leading_escapes> (?: \x5c\x5c )* )
    \[
      (?P<start> [^-\n] )
      -
      (?P<stop>  [^-\n] )
      (?# optionally take a step argument: )
        :?
        (?P<step> [0-9]* )
    \]  """, re.VERBOSE)

re_curl_brace_set = re.compile(r"""
    (?# ignore patterns with leading unbalanced backslashes )
      (?P<escaped> \x5c ?  )
      (?P<leading_escapes> (?: \x5c\x5c )* )
    \{
      (?P<first> ( [^\n,]* ,)* )
      (?P<last>    [^\n,]* )
    \} """, re.VERBOSE)

class Request():
  """The Request class is the base of Burst. To create an instance, you have
  two options: either use a socket or a string representing the whole request
  into the constructor or use the 'create' function.

  The two methods __repr__ and __str__ have been defined to provide
  user-friendly interaction within the interpreter.
  """

  def __init__(self, fd, hostname=None, port=None, use_ssl=False):
    """Create a request. fd should be either a socket descriptor
       or a string. In both case, it should contain a full request.
       To generate a request from a URL, see c()"""
    if isinstance(fd, basestring):
      fd = StringIO(fd)
    try:
      banner = read_banner(fd)
      # ASSUMPTION: The request line contains three elements seperated by
      #             a space.
      self.method, url, self.http_version = banner
    except ValueError:
      raise NotConnected(' '.join(banner))
    if self.method.upper() == "CONNECT":
      # ASSUMPTION: CONNECT method needs a hostname and port
      self.hostname, self.port = url.rsplit(":", 1)
      self.port = int(self.port)
      self.url = ""
      self.use_ssl = False
    elif hostname:
      self.hostname = hostname
      self.port = port if port else 80
      self.use_ssl = use_ssl
      self.url = url
    else:
      p_url = urlparse.urlparse(url)
      self.url = urlparse.urlunparse(("", "") + p_url[2:])
      if p_url.scheme:
        self.hostname = p_url.hostname
        if not self.hostname:
          raise BurstException("No hostname: " + str(url))
        if p_url.scheme == 'https':
          self.use_ssl = True
          self.port = int(p_url.port) if p_url.port else 443
        else:
          self.port = int(p_url.port) if p_url.port else 80
          self.use_ssl = use_ssl
    self.raw_headers = read_headers(fd)
    if not hasattr(self, "hostname"): # Last chance, try the Host header
      hosts = self.get_header('Host')
      if not hosts:
        raise BurstException("Unable to find the host for the request")
      else:
        host = hosts[0]
      self.hostname = host.split(":")[0]
      self.port = int(host.split(":")[1]) if ":" in host else 80
      self.use_ssl = False
    self.raw_content = read_content(fd, parse_headers(self.raw_headers), method=self.method)
    if self.raw_content:
      self.content = _clear_content(parse_headers(self.raw_headers), self.raw_content)
    else:
      self.content = ""
    self.response = None

  def expand_curl_ranges(self):
    """Expands curl range sequences in the Request, returning a new RequestSet"""
    matches = {}

    def param_inject(s):
      def _param_inject(generator):
        ## can't come up with a better/faster way to generate unique keys in a thread-safe fashion :-(
        key = sha256('{}_{}'.format(len(matches), str(self))).hexdigest()[:20]
        matches[key] = {
          'at': key,
          'payloads': generator }
        return key

      def _param_inject_counter(m):
        escapes = m.expand(m.group('leading_escapes'))
        if m.group('escaped'):
          ret = escapes + m.string[m.end('leading_escapes') : m.end(0)]
          return _param_inject( [ ret ] )

        values = [int(i or '1') for i in m.group(3, 4)]
        ## NOTE: curl's ranges are inclusive; python's are not, account for that:
        values[1] += 1
        return _param_inject( (escapes + str(i) for i in xrange(*values)) )

      def _param_inject_characters(m):
        escapes = m.expand(m.group('leading_escapes'))
        start = ord(m.group('start'))
        ## NOTE: curl's ranges are inclusive; python's are not, account for that:
        stop  = ord(m.group('stop')) + 1
        if m.group('escaped'):
          ret = escapes + m.string[m.end('leading_escapes') : m.end(0)]
          return _param_inject( [ ret ] )
        step  = int( m.group('step') or '1' )
{'fraction_non_alphanumeric': 0.12440916793008115, 'fraction_numerical': 0.008561491126371177, 'mean_word_length': 2.526415094339623, 'pattern_counts': {'":': 10, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6458394', 'n_tokens_mistral': 3692, 'n_tokens_neox': 3542, 'n_words': 1024}