<commit_before>import Cookie
import formatters
import httplib2
import json
import oauth2 as oauth
import os
import subprocess
import version


class HttpVerb(object):
    def __init__(self, args, logger, verb):
        self.args = args
        self.logger = logger
        self.verb = verb

    def run(self, url, path, pipe=None, headers=None, cookies=None, body=""):
        self.url = url
        self.path = path
        self.pipe = pipe
        self.headers = headers
        self.cookies = cookies

        host = self.url.netloc

        self.http = self.init_httpclient()
        self.http.follow_redirects = False
        httplib2.debuglevel = self.args.debuglevel

        # check for authentication credentials
        if "@" in host:
            split = host.split("@")
            if len(split) > 1:
                host = split[1]
                creds = split[0].split(":")
                self.http.add_credentials(creds[0], creds[1])
            else:
                host = split[0]

        uri = ("{0}://{1}{2}".format(self.url.scheme, host, self.path))

        if not self.args.disable_cookies:
            self.set_request_cookies()

        if not "host" in self.headers:
            self.headers["host"] = host
        if not "accept-encoding" in self.headers:
            self.headers["accept-encoding"] = "gzip, deflate"
        if not "user-agent" in self.headers:
            self.headers["user-agent"] = "httpsh/" + version.VERSION

        self.logger.print_text("Connecting to " + uri)

        response, content = self.http.request(
            uri, method=self.verb, body=body, headers=headers)

        self.handle_response(response, content)

    def init_httpclient(self):
        http = None

        keysfile = os.path.join(os.path.expanduser("~"),
            ".httpshell", self.url.netloc + ".json")

        if os.path.isfile(keysfile):
            try:
                with open(keysfile, "r") as file:
                    keys = json.load(file)
                    consumer = oauth.Consumer(
                        keys["consumer-key"], keys["consumer-secret"])
                    token = oauth.Token(
                        keys["access-token"], keys["access-token-secret"])
                    http = oauth.Client(consumer, token)
            except:
                self.logger.print_error(
                    "Failed reading oauth data from: " + keysfile)
        else:
            http = httplib2.Http()

        return http

    def set_request_cookies(self):
        if self.url.netloc in self.cookies:
            l = []
            cookie = self.cookies[self.url.netloc]
            #  very basic cookie support atm.  no expiry, etc.
            for morsel in cookie.values():
                l.append(morsel.key + "=" + morsel.coded_value)
            self.headers["cookie"] = "; ".join(l)

    def handle_response(self, response, content):
        self.logger.print_response_code(response)
        if self.args.show_headers:
            self.logger.print_headers(self.headers.items(), True)
            self.logger.print_headers(response.items())

        if not self.args.disable_cookies:<commit_msg>Restructure format of OAuth config files<commit_after>import Cookie
import formatters
import httplib2
import json
import oauth2 as oauth
import os
import subprocess
import version


class HttpVerb(object):
    def __init__(self, args, logger, verb):
        self.args = args
        self.logger = logger
        self.verb = verb

    def run(self, url, path, pipe=None, headers=None, cookies=None, body=""):
        self.url = url
        self.path = path
        self.pipe = pipe
        self.headers = headers
        self.cookies = cookies

        host = self.url.netloc

        self.http = self.init_httpclient()
        self.http.follow_redirects = False
        httplib2.debuglevel = self.args.debuglevel

        # check for authentication credentials
        if "@" in host:
            split = host.split("@")
            if len(split) > 1:
                host = split[1]
                creds = split[0].split(":")
                self.http.add_credentials(creds[0], creds[1])
            else:
                host = split[0]

        uri = ("{0}://{1}{2}".format(self.url.scheme, host, self.path))

        if not self.args.disable_cookies:
            self.set_request_cookies()

        if not "host" in self.headers:
            self.headers["host"] = host
        if not "accept-encoding" in self.headers:
            self.headers["accept-encoding"] = "gzip, deflate"
        if not "user-agent" in self.headers:
            self.headers["user-agent"] = "httpsh/" + version.VERSION

        self.logger.print_text("Connecting to " + uri)

        response, content = self.http.request(
            uri, method=self.verb, body=body, headers=headers)

        self.handle_response(response, content)

    def init_httpclient(self):
        http = None

        keysfile = os.path.join(os.path.expanduser("~"),
            ".httpshell", self.url.netloc + ".json")

        if os.path.isfile(keysfile):
            try:
                with open(keysfile, "r") as file:
                    keys = json.load(file)
                    token = None

                    consumer = oauth.Consumer(keys["consumer"]["consumer-key"],
                        keys["consumer"]["consumer-secret"])
                    if "access" in keys:
                        token = oauth.Token(keys["access"]["access-token"],
                            keys["access"]["access-token-secret"])

                    http = oauth.Client(consumer, token)
            except:
                self.logger.print_error(
                    "Failed reading OAuth data from: " + keysfile)
        else:
            http = httplib2.Http()

        return http

    def set_request_cookies(self):
        if self.url.netloc in self.cookies:
            l = []
            cookie = self.cookies[self.url.netloc]
            #  very basic cookie support atm.  no expiry, etc.
            for morsel in cookie.values():
                l.append(morsel.key + "=" + morsel.coded_value)
            self.headers["cookie"] = "; ".join(l)

    def handle_response(self, response, content):
        self.logger.print_response_code(response)
        if self.args.show_headers:
            self.logger.print_headers(self.headers.items(), True)
            self.logger.print_headers(response.items())

        if not self.args.disable_cookies:
{'fraction_non_alphanumeric': 0.1152479530356867, 'fraction_numerical': 0.004016684690251815, 'mean_word_length': 1.8111159357359965, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5179549', 'n_tokens_mistral': 1883, 'n_tokens_neox': 1808, 'n_words': 412}