<commit_before>#!/usr/bin/env python
from __future__ import print_function
from __future__ import unicode_literals
from future import standard_library
standard_library.install_aliases()
from builtins import object
import argparse
import urllib.parse
import urllib.request, urllib.parse, urllib.error
import requests
import sys
import os
import time
import json
import base64
import pprint
import xml.etree.ElementTree
import socket
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

class IgorError(EnvironmentError):
    pass
    
CONFIG = configparser.ConfigParser(
    dict(
        url="http://igor.local:9333/data",
        bearer=None,
        access=None,
        credentials=None,
        certificate=None,
        noverify=None,
        verbose=None,
    ))
CONFIG.add_section('igor')
CONFIG.read(os.path.expanduser('~/.igor/igor.cfg'))
# Override from environment:
for k, _ in CONFIG.items('igor'):
    envKey = 'IGORSERVER_' + k.upper()
    if envKey in os.environ:
        CONFIG.set('igor', k, os.environ[envKey])
        
# DEFAULT_URL="http://framboos.local:9333/data"
# if 'IGORSERVER_URL' in os.environ:
#   DEFAULT_URL = os.environ['IGORSERVER_URL']

VERBOSE=False

class IgorServer(object):
    def __init__(self, url, bearer_token=None, access_token=None, credentials=None, certificate=None, noverify=False, printmessages=False):
        self.baseUrl = url
        if url[-1] != '/':
            url = url + '/'
        self.url = url
        self.bearer_token = bearer_token
        self.access_token = access_token
        self.credentials = credentials
        if certificate:
            certificate = os.path.join(os.path.expanduser('~/.igor'), certificate)
        self.certificate = certificate
        self.noverify = not not noverify
        self.printmessages = printmessages or VERBOSE
        
    def get(self, item, variant=None, format=None, query=None):
        if format == None:
            format = 'application/xml'
        return self._action("GET", item, variant, format=format, query=query)
        
    def delete(self, item, variant=None, format=None, query=None):
        if variant == None:
            variant = "ref"
        if format == None:
            format = "text/plain"
        return self._action("DELETE", item, variant, format=format, query=query)
        
    def put(self, item, data, datatype, variant=None, format=None, query=None):
        if format == None:
            format = 'text/plain'
        return self._action("PUT", item, variant, format=format, data=data, datatype=datatype, query=query)
        
    def post(self, item, data, datatype, variant=None, format=None, query=None):
        if format == None:
            format = 'text/plain'
        return self._action("POST", item, variant, format, data=data, datatype=datatype)
        
    def _action(self, method, item, variant, format=None, data=None, datatype=None, query=None):
        # Convert to unicode for Python 2
        try:
            item = unicode(item)
        except NameError:
            pass
        url = urllib.parse.urljoin(self.url, item)
        if query is None:
            query = {}
        else:
            query = dict(query)
        if variant:
            query['.VARIANT'] = variant
        if self.access_token:
            query['access_token'] = self.access_token
        if query:
            assert not '?' in url
            url = url + '?' + urllib.parse.urlencode(query)
        headers = {}
        if format:
            headers['Accept'] = format
        if datatype:
            headers['Content-Type'] = datatype
        if self.bearer_token and self.credentials:
            raise IgorError("both bearer token and credentials specified")
        if self.bearer_token:
            headers['Authorization'] = 'Bearer %s' % self.bearer_token
        if self.credentials:
            headers['Authorization'] = 'Basic %s' % base64.b64encode(self.credentials.encode('utf-8'))
        if VERBOSE:
            if self.certificate or self.noverify:
                print('certificate=%s, noverify=%s' % (repr(self.certificate), repr(self.noverify)))
        kwargs = {}
        if self.noverify:
            kwargs['verify'] = False
        elif self.certificate:
            kwargs['verify'] = self.certificate
        if VERBOSE:
            print(">>> GET", url, file=sys.stderr)
            print("... Headers", headers, file=sys.stderr)
            if data:
                print("... Data", repr(data), file=sys.stderr)
        try:
            r = requests.request(method, url, headers=headers, data=data, **kwargs)
            # reply, content
        except socket.error as e:
            if e.args[1:]:
                argstr = e.args[1]
            else:
                e = repr(e)
            raise IgorError("%s: %s" % (url, argstr))
        except socket.gaierror:
            raise IgorError("%s: unknown host" %  url)
        except requests.exceptions.RequestException as e:
            raise IgorError(*e.args)
        except socket.timeout:
            raise IgorError("%s: timeout during connect" % url)
        if VERBOSE:
            print("<<< Headers", r.headers, file=sys.stderr)
            print("...", r.text, file=sys.stderr)
        if r.status_code != 200:<commit_msg>Fixed two more issues of mixing unicode and ascii<commit_after>#!/usr/bin/env python
from __future__ import print_function
from __future__ import unicode_literals
from future import standard_library
standard_library.install_aliases()
from builtins import object
import argparse
import urllib.parse
import urllib.request, urllib.parse, urllib.error
import requests
import sys
import os
import time
import json
import base64
import pprint
import xml.etree.ElementTree
import socket
try:
    import configparser
except ImportError:
    import ConfigParser as configparser

class IgorError(EnvironmentError):
    pass
    
CONFIG = configparser.ConfigParser(
    dict(
        url="http://igor.local:9333/data",
        bearer=None,
        access=None,
        credentials=None,
        certificate=None,
        noverify=None,
        verbose=None,
    ))
CONFIG.add_section('igor')
CONFIG.read(os.path.expanduser('~/.igor/igor.cfg'))
# Override from environment:
for k, _ in CONFIG.items('igor'):
    envKey = 'IGORSERVER_' + k.upper()
    if envKey in os.environ:
        CONFIG.set('igor', k, os.environ[envKey])
        
# DEFAULT_URL="http://framboos.local:9333/data"
# if 'IGORSERVER_URL' in os.environ:
#   DEFAULT_URL = os.environ['IGORSERVER_URL']

VERBOSE=False

class IgorServer(object):
    def __init__(self, url, bearer_token=None, access_token=None, credentials=None, certificate=None, noverify=False, printmessages=False):
        self.baseUrl = url
        if url[-1] != '/':
            url = url + '/'
        self.url = url
        self.bearer_token = bearer_token
        self.access_token = access_token
        self.credentials = credentials
        if certificate:
            certificate = os.path.join(os.path.expanduser('~/.igor'), certificate)
        self.certificate = certificate
        self.noverify = not not noverify
        self.printmessages = printmessages or VERBOSE
        
    def get(self, item, variant=None, format=None, query=None):
        if format == None:
            format = 'application/xml'
        return self._action("GET", item, variant, format=format, query=query)
        
    def delete(self, item, variant=None, format=None, query=None):
        if variant == None:
            variant = "ref"
        if format == None:
            format = "text/plain"
        return self._action("DELETE", item, variant, format=format, query=query)
        
    def put(self, item, data, datatype, variant=None, format=None, query=None):
        if format == None:
            format = 'text/plain'
        return self._action("PUT", item, variant, format=format, data=data, datatype=datatype, query=query)
        
    def post(self, item, data, datatype, variant=None, format=None, query=None):
        if format == None:
            format = 'text/plain'
        return self._action("POST", item, variant, format, data=data, datatype=datatype)
        
    def _action(self, method, item, variant, format=None, data=None, datatype=None, query=None):
        # Convert to unicode for Python 2
        try:
            item = unicode(item)
        except NameError:
            pass
        url = urllib.parse.urljoin(str(self.url), str(item))
        if query is None:
            query = {}
        else:
            query = dict(query)
        if variant:
            query['.VARIANT'] = variant
        if self.access_token:
            query['access_token'] = self.access_token
        if query:
            assert not '?' in url
            url = url + '?' + urllib.parse.urlencode(query)
        headers = {}
        if format:
            headers['Accept'] = format
        if datatype:
            headers['Content-Type'] = datatype
        if self.bearer_token and self.credentials:
            raise IgorError("both bearer token and credentials specified")
        if self.bearer_token:
            headers['Authorization'] = 'Bearer %s' % self.bearer_token
        if self.credentials:
            headers['Authorization'] = 'Basic %s' % base64.b64encode(self.credentials.encode('utf-8'))
        if VERBOSE:
            if self.certificate or self.noverify:
                print('certificate=%s, noverify=%s' % (repr(self.certificate), repr(self.noverify)))
        kwargs = {}
        if self.noverify:
            kwargs['verify'] = False
        elif self.certificate:
            kwargs['verify'] = self.certificate
        if VERBOSE:
            print(">>> GET", url, file=sys.stderr)
            print("... Headers", headers, file=sys.stderr)
            if data:
                print("... Data", repr(data), file=sys.stderr)
        if data != None:
            data = data.encode('utf-8')
        try:
            r = requests.request(method, url, headers=headers, data=data, **kwargs)
            # reply, content
        except socket.error as e:
            if e.args[1:]:
                argstr = e.args[1]
            else:
                e = repr(e)
            raise IgorError("%s: %s" % (url, argstr))
        except socket.gaierror:
            raise IgorError("%s: unknown host" %  url)
        except requests.exceptions.RequestException as e:
            raise IgorError(*e.args)
        except socket.timeout:
            raise IgorError("%s: timeout during connect" % url)
        if VERBOSE:
            print("<<< Headers", r.headers, file=sys.stderr)
            print("...", r.text, file=sys.stderr)
        if r.status_code != 200:
{'fraction_non_alphanumeric': 0.11441349110220814, 'fraction_numerical': 0.004192676791204696, 'mean_word_length': 2.294659300184162, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1811370', 'n_tokens_mistral': 3135, 'n_tokens_neox': 3110, 'n_words': 788}