<commit_before>import os
import time
import copy
import zlib
import gzip
import ssl
import struct
import socket
import urlparse
import operator
import random
import tempfile
import webbrowser
import threading
import subprocess
import datetime
from collections import defaultdict
from StringIO import StringIO

from burst import console
from burst.conf import conf
from burst.cookie import Cookie
from burst.color import *
from burst.exception import *
from burst.utils import make_table, clear_line, chunks, encode, \
                         re_space, smart_split, smart_rsplit, \
                         truncate, stats, parse_qs

class Request():
  """The Request class is the base of Burst. To create an instance, you have
  two options: either use a socket or a string representing the whole request
  into the constructor or use the 'create' function.

  The two methods __repr__ and __str__ have been defined to provide
  user-friendly interaction within the interpreter.
  """

  def __init__(self, fd, hostname=None, port=None, use_ssl=False):
    """Create a request. fd should be either a socket descriptor
       or a string. In both case, it should contain a full request.
       To generate a request from a URL, see c()"""
    if isinstance(fd, basestring):
      fd = StringIO(fd)
    try:
      banner = read_banner(fd)
      # ASSUMPTION: The request line contains three elements seperated by
      #             a space.
      self.method, url, self.http_version = banner
    except ValueError:
      raise NotConnected(' '.join(banner))
    if self.method.upper() == "CONNECT":
      # ASSUMPTION: CONNECT method needs a hostname and port
      self.hostname, self.port = url.rsplit(":", 1)
      self.port = int(self.port)
      self.url = ""
      self.use_ssl = False
    elif hostname:
      self.hostname = hostname
      self.port = port if port else 80
      self.use_ssl = use_ssl
      self.url = url
    else:
      p_url = urlparse.urlparse(url)
      if not p_url.scheme:
        raise BurstException("No scheme: " + str(url))
      self.url = urlparse.urlunparse(("", "") + p_url[2:])
      self.hostname = p_url.hostname
      if not self.hostname:
        raise BurstException("No hostname: " + str(url))
      if p_url.scheme == 'https':
        self.use_ssl = True
        self.port = int(p_url.port) if p_url.port else 443
      else:
        self.port = int(p_url.port) if p_url.port else 80
        self.use_ssl = use_ssl
    self.raw_headers = read_headers(fd)
    self.raw_content = read_content(fd, parse_headers(self.raw_headers), method=self.method)
    if self.raw_content:
      self.content = _clear_content(parse_headers(self.raw_headers), self.raw_content)
    else:
      self.content = ""
    self.response = None

  @property
  def path(self):
    return urlparse.urlparse(self.url).path

  @property
  def query(self):
    return urlparse.urlparse(self.url).query

  @property
  def cookies(self):
    cookies = []
    for h in self.get_header("Cookie"):
      cookies.extend(Cookie.parse(h))
    return cookies

  @property
  def headers(self):
    return parse_headers(self.raw_headers)

  def has_header(self, name, value=None):
    """Test if the request contains a specific header (case insensitive).
    If value is supplied, it is matched (case insensitive) against the first
    header with the matching name.
    """
    return _has_header(parse_headers(self.raw_headers), name, value)

  def get_header(self, name):
    """Return the headers of the request matching name (case insensitive).
    This method always returns a list.
    """
    return _get_header(parse_headers(self.raw_headers), name)

  def update_content_length(self):
    """Update the Content-Length header according to the content of the
    request"""
    l = str(len(self.raw_content)) if self.raw_content else "0"
    self.remove_header('Content-Length')
    self.add_header('Content-Length', l)

  def add_header(self, name, value):
    new_headers = parse_headers(self.raw_headers)
    new_headers.append((name, value))
    self.raw_headers = build_headers(new_headers)

  def remove_header(self, name):
    """Remove all the headers matching the specified name"""
    new_headers = [ (h, v) for (h, v) in parse_headers(self.raw_headers) if not h.title() == name ]
    self.raw_headers = build_headers(new_headers)

  def bind(self, r):
    """Bind the Request to another one. This method will copy the supplied
    request's cookies and response set-cookies to the Request."""
    r_new = self.copy()
    for c in r.get_header('Cookie'):
      r_new.add_header('Cookie', c)
    if r.response:
      cookies = []
      for c in r.response.get_header('Set-Cookie'):
        cookies.append(Cookie.parse(c, set_cookie=True))
      r_new.add_header('Cookie', "; ".join([str(x) for x in cookies]))
    return r_new

  def __repr__(self):
    return self.repr(width=None)

  def repr(self, width=None, rl=False):
    if width:
      hostname = smart_rsplit(self.hostname, int(0.3 * width), ".")
      path = smart_split(self.path, int(0.6 * width), "/")
    else:
      hostname = self.hostname
      path = self.path
    fields = [info(self.method, rl=rl), hostname, path]
    if self.use_ssl and self.port != 443:
      fields.insert(2, str(self.port))
    elif not self.use_ssl and self.port != 80:
      fields.insert(2, str(self.port))
    if self.use_ssl: fields.append(warning("SSL", rl=rl))
    return ("<" + " ".join(fields) + ">").encode("utf-8")

  def copy(self):
    """Copy a request. The response is not duplicated."""
    r_new = copy.copy(self)
    r_new.response = None
    return r_new

  def __str__(self, headers_only=False):
    s = StringIO()
    if self.method == "CONNECT":
      s.write("{s.method} {s.hostname}:{s.port} {s.http_version}\r\n".format(s=self))
    else:
      s.write("{s.method} {s.url} {s.http_version}\r\n".format(s=self))
    s.write(build_headers(parse_headers(self.raw_headers)))
    s.write("\r\n")
    if not headers_only and self.content:
      s.write(self.content)
    return s.getvalue()

  def __mul__(self, op):
    """Duplicate a request"""
    return RequestSet([self.copy() for i in range(op)])

  def __eq__(self, r):
    """Compare two requests based on the host, port, use of ssl, url, headers
    and content (if present)"""
    if self.hostname != r.hostname or \
       self.port != r.port or \
       self.use_ssl != r.use_ssl or \
       self.url != r.url or \
       self.headers != r.headers:
      return False
    if (self.content or r.content) and self.content != r.content:
      return False
    return True

  def similar(self, r):
    """Compare two requests based on the host, port, use of ssl and path"""
    if self.hostname != r.hostname or \
       self.port != r.port or \<commit_msg>Add commutativity to Request duplication<commit_after>import os
import time
import copy
import zlib
import gzip
import ssl
import struct
import socket
import urlparse
import operator
import random
import tempfile
import webbrowser
import threading
import subprocess
import datetime
from collections import defaultdict
from StringIO import StringIO

from burst import console
from burst.conf import conf
from burst.cookie import Cookie
from burst.color import *
from burst.exception import *
from burst.utils import make_table, clear_line, chunks, encode, \
                         re_space, smart_split, smart_rsplit, \
                         truncate, stats, parse_qs

class Request():
  """The Request class is the base of Burst. To create an instance, you have
  two options: either use a socket or a string representing the whole request
  into the constructor or use the 'create' function.

  The two methods __repr__ and __str__ have been defined to provide
  user-friendly interaction within the interpreter.
  """

  def __init__(self, fd, hostname=None, port=None, use_ssl=False):
    """Create a request. fd should be either a socket descriptor
       or a string. In both case, it should contain a full request.
       To generate a request from a URL, see c()"""
    if isinstance(fd, basestring):
      fd = StringIO(fd)
    try:
      banner = read_banner(fd)
      # ASSUMPTION: The request line contains three elements seperated by
      #             a space.
      self.method, url, self.http_version = banner
    except ValueError:
      raise NotConnected(' '.join(banner))
    if self.method.upper() == "CONNECT":
      # ASSUMPTION: CONNECT method needs a hostname and port
      self.hostname, self.port = url.rsplit(":", 1)
      self.port = int(self.port)
      self.url = ""
      self.use_ssl = False
    elif hostname:
      self.hostname = hostname
      self.port = port if port else 80
      self.use_ssl = use_ssl
      self.url = url
    else:
      p_url = urlparse.urlparse(url)
      if not p_url.scheme:
        raise BurstException("No scheme: " + str(url))
      self.url = urlparse.urlunparse(("", "") + p_url[2:])
      self.hostname = p_url.hostname
      if not self.hostname:
        raise BurstException("No hostname: " + str(url))
      if p_url.scheme == 'https':
        self.use_ssl = True
        self.port = int(p_url.port) if p_url.port else 443
      else:
        self.port = int(p_url.port) if p_url.port else 80
        self.use_ssl = use_ssl
    self.raw_headers = read_headers(fd)
    self.raw_content = read_content(fd, parse_headers(self.raw_headers), method=self.method)
    if self.raw_content:
      self.content = _clear_content(parse_headers(self.raw_headers), self.raw_content)
    else:
      self.content = ""
    self.response = None

  @property
  def path(self):
    return urlparse.urlparse(self.url).path

  @property
  def query(self):
    return urlparse.urlparse(self.url).query

  @property
  def cookies(self):
    cookies = []
    for h in self.get_header("Cookie"):
      cookies.extend(Cookie.parse(h))
    return cookies

  @property
  def headers(self):
    return parse_headers(self.raw_headers)

  def has_header(self, name, value=None):
    """Test if the request contains a specific header (case insensitive).
    If value is supplied, it is matched (case insensitive) against the first
    header with the matching name.
    """
    return _has_header(parse_headers(self.raw_headers), name, value)

  def get_header(self, name):
    """Return the headers of the request matching name (case insensitive).
    This method always returns a list.
    """
    return _get_header(parse_headers(self.raw_headers), name)

  def update_content_length(self):
    """Update the Content-Length header according to the content of the
    request"""
    l = str(len(self.raw_content)) if self.raw_content else "0"
    self.remove_header('Content-Length')
    self.add_header('Content-Length', l)

  def add_header(self, name, value):
    new_headers = parse_headers(self.raw_headers)
    new_headers.append((name, value))
    self.raw_headers = build_headers(new_headers)

  def remove_header(self, name):
    """Remove all the headers matching the specified name"""
    new_headers = [ (h, v) for (h, v) in parse_headers(self.raw_headers) if not h.title() == name ]
    self.raw_headers = build_headers(new_headers)

  def bind(self, r):
    """Bind the Request to another one. This method will copy the supplied
    request's cookies and response set-cookies to the Request."""
    r_new = self.copy()
    for c in r.get_header('Cookie'):
      r_new.add_header('Cookie', c)
    if r.response:
      cookies = []
      for c in r.response.get_header('Set-Cookie'):
        cookies.append(Cookie.parse(c, set_cookie=True))
      r_new.add_header('Cookie', "; ".join([str(x) for x in cookies]))
    return r_new

  def __repr__(self):
    return self.repr(width=None)

  def repr(self, width=None, rl=False):
    if width:
      hostname = smart_rsplit(self.hostname, int(0.3 * width), ".")
      path = smart_split(self.path, int(0.6 * width), "/")
    else:
      hostname = self.hostname
      path = self.path
    fields = [info(self.method, rl=rl), hostname, path]
    if self.use_ssl and self.port != 443:
      fields.insert(2, str(self.port))
    elif not self.use_ssl and self.port != 80:
      fields.insert(2, str(self.port))
    if self.use_ssl: fields.append(warning("SSL", rl=rl))
    return ("<" + " ".join(fields) + ">").encode("utf-8")

  def copy(self):
    """Copy a request. The response is not duplicated."""
    r_new = copy.copy(self)
    r_new.response = None
    return r_new

  def __str__(self, headers_only=False):
    s = StringIO()
    if self.method == "CONNECT":
      s.write("{s.method} {s.hostname}:{s.port} {s.http_version}\r\n".format(s=self))
    else:
      s.write("{s.method} {s.url} {s.http_version}\r\n".format(s=self))
    s.write(build_headers(parse_headers(self.raw_headers)))
    s.write("\r\n")
    if not headers_only and self.content:
      s.write(self.content)
    return s.getvalue()

  def __mul__(self, op):
    """Duplicate a request"""
    return RequestSet([self.copy() for i in range(op)])

  __rmul__ = __mul__

  def __eq__(self, r):
    """Compare two requests based on the host, port, use of ssl, url, headers
    and content (if present)"""
    if self.hostname != r.hostname or \
       self.port != r.port or \
       self.use_ssl != r.use_ssl or \
       self.url != r.url or \
       self.headers != r.headers:
      return False
    if (self.content or r.content) and self.content != r.content:
      return False
    return True

  def similar(self, r):
    """Compare two requests based on the host, port, use of ssl and path"""
    if self.hostname != r.hostname or \
       self.port != r.port or \
{'fraction_non_alphanumeric': 0.11335824723382429, 'fraction_numerical': 0.003224151828240639, 'mean_word_length': 3.048650252150697, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1591761', 'n_tokens_mistral': 4300, 'n_tokens_neox': 4246, 'n_words': 1290}