<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

# This file is part of Cockpit.
#
# Copyright (C) 2015 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

# Shared GitHub code. When run as a script, we print out info about
# our GitHub interacition.

import errno
import http.client
import json
import os
import socket
import sys
import time
import urllib.parse

from . import cache

__all__ = (
    'GitHub',
    'Checklist',
    'whitelist',
    'TESTING',
    'NO_TESTING',
    'NOT_TESTED'
)

TESTING = "Testing in progress"
NOT_TESTED = "Not yet tested"
NO_TESTING = "Manual testing required"

OUR_CONTEXTS = [
    "verify/",
    "avocado/",
    "container/",
    "selenium/",

    # generic prefix for external repos
    "cockpit/",
]

ISSUE_TITLE_IMAGE_REFRESH = "Image refresh for {0}"

BASE = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
TOKEN = "~/.config/github-token"

# the user name is accepted if it's found in either list
WHITELIST = os.path.join(BASE, "bots", "whitelist")
WHITELIST_LOCAL = "~/.config/github-whitelist"

def known_context(context):
    for prefix in OUR_CONTEXTS:
        if context.startswith(prefix):
            return True
    return False

def whitelist(filename=WHITELIST):
    # Try to load the whitelists
    whitelist = []
    try:
        with open(filename, "r") as wh:
            whitelist += [x.strip() for x in wh.read().split("\n") if x.strip()]
    except IOError as exc:
        if exc.errno != errno.ENOENT:
            raise

    # The local file may or may not exist
    try:
        path = os.path.expanduser(WHITELIST_LOCAL)
        wh = open(path, "r")
        whitelist += [x.strip() for x in wh.read().split("\n") if x.strip()]
    except IOError as exc:
        if exc.errno != errno.ENOENT:
            raise

    # Remove duplicate entries
    return set(whitelist)

class Logger(object):
    def __init__(self, directory):
        hostname = socket.gethostname().split(".")[0]
        month = time.strftime("%Y%m")
        self.path = os.path.join(directory, "{0}-{1}.log".format(hostname, month))

        if not os.path.exists(directory):
            os.makedirs(directory)

    # Yes, we open the file each time
    def write(self, value):
        with open(self.path, 'a') as f:
            f.write(value)

class GitHub(object):
    def __init__(self, base=None, cacher=None, repo=None):
        if base is None:
            if repo is None:
                repo = os.environ.get("GITHUB_BASE", "cockpit-project/cockpit")
            netloc = os.environ.get("GITHUB_API", "https://api.github.com")
            base = "{0}/repos/{1}/".format(netloc, repo)
        self.url = urllib.parse.urlparse(base)
        self.conn = None
        self.token = None
        self.debug = False
        try:
            gt = open(os.path.expanduser(TOKEN), "r")
            self.token = gt.read().strip()
            gt.close()
        except IOError as exc:
            if exc.errno == errno.ENOENT:
                pass
            else:
                raise
        self.available = self.token and True or False

        # The cache directory is $TEST_DATA/github ~/.cache/github
        if not cacher:
            data = os.environ.get("TEST_DATA",  os.path.expanduser("~/.cache"))
            cacher = cache.Cache(os.path.join(data, "github"))
        self.cache = cacher

        # Create a log for debugging our GitHub access
        self.log = Logger(self.cache.directory)
        self.log.write("")

    def qualify(self, resource):
        return urllib.parse.urljoin(self.url.path, resource)

    def request(self, method, resource, data="", headers=None):
        if headers is None:
            headers = { }
        headers["User-Agent"] = "Cockpit Tests"
        if self.token:
            headers["Authorization"] = "token " + self.token
        connected = False
        while not connected:
            if not self.conn:
                if self.url.scheme == 'http':
                    self.conn = http.client.HTTPConnection(self.url.netloc)
                else:
                    self.conn = http.client.HTTPSConnection(self.url.netloc)
                connected = True
            self.conn.set_debuglevel(self.debug and 1 or 0)
            try:
                self.conn.request(method, self.qualify(resource), data, headers)
                response = self.conn.getresponse()
                break
            # This happens when GitHub disconnects in python3
            except http.client.RemoteDisconnected:
                if connected:
                    raise
                self.conn = None
            # This happens when GitHub disconnects a keep-alive connection
            except http.client.BadStatusLine:
                if connected:
                    raise
                self.conn = None
            # This happens when TLS is the source of a disconnection
            except socket.error as ex:
                if connected or ex.errno != errno.EPIPE:
                    raise
                self.conn = None
        heads = { }<commit_msg>bots: Catch ConnectionResetError when GitHub keepalives dropped<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

# This file is part of Cockpit.
#
# Copyright (C) 2015 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

# Shared GitHub code. When run as a script, we print out info about
# our GitHub interacition.

import errno
import http.client
import json
import os
import socket
import sys
import time
import urllib.parse

from . import cache

__all__ = (
    'GitHub',
    'Checklist',
    'whitelist',
    'TESTING',
    'NO_TESTING',
    'NOT_TESTED'
)

TESTING = "Testing in progress"
NOT_TESTED = "Not yet tested"
NO_TESTING = "Manual testing required"

OUR_CONTEXTS = [
    "verify/",
    "avocado/",
    "container/",
    "selenium/",

    # generic prefix for external repos
    "cockpit/",
]

ISSUE_TITLE_IMAGE_REFRESH = "Image refresh for {0}"

BASE = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
TOKEN = "~/.config/github-token"

# the user name is accepted if it's found in either list
WHITELIST = os.path.join(BASE, "bots", "whitelist")
WHITELIST_LOCAL = "~/.config/github-whitelist"

def known_context(context):
    for prefix in OUR_CONTEXTS:
        if context.startswith(prefix):
            return True
    return False

def whitelist(filename=WHITELIST):
    # Try to load the whitelists
    whitelist = []
    try:
        with open(filename, "r") as wh:
            whitelist += [x.strip() for x in wh.read().split("\n") if x.strip()]
    except IOError as exc:
        if exc.errno != errno.ENOENT:
            raise

    # The local file may or may not exist
    try:
        path = os.path.expanduser(WHITELIST_LOCAL)
        wh = open(path, "r")
        whitelist += [x.strip() for x in wh.read().split("\n") if x.strip()]
    except IOError as exc:
        if exc.errno != errno.ENOENT:
            raise

    # Remove duplicate entries
    return set(whitelist)

class Logger(object):
    def __init__(self, directory):
        hostname = socket.gethostname().split(".")[0]
        month = time.strftime("%Y%m")
        self.path = os.path.join(directory, "{0}-{1}.log".format(hostname, month))

        if not os.path.exists(directory):
            os.makedirs(directory)

    # Yes, we open the file each time
    def write(self, value):
        with open(self.path, 'a') as f:
            f.write(value)

class GitHub(object):
    def __init__(self, base=None, cacher=None, repo=None):
        if base is None:
            if repo is None:
                repo = os.environ.get("GITHUB_BASE", "cockpit-project/cockpit")
            netloc = os.environ.get("GITHUB_API", "https://api.github.com")
            base = "{0}/repos/{1}/".format(netloc, repo)
        self.url = urllib.parse.urlparse(base)
        self.conn = None
        self.token = None
        self.debug = False
        try:
            gt = open(os.path.expanduser(TOKEN), "r")
            self.token = gt.read().strip()
            gt.close()
        except IOError as exc:
            if exc.errno == errno.ENOENT:
                pass
            else:
                raise
        self.available = self.token and True or False

        # The cache directory is $TEST_DATA/github ~/.cache/github
        if not cacher:
            data = os.environ.get("TEST_DATA",  os.path.expanduser("~/.cache"))
            cacher = cache.Cache(os.path.join(data, "github"))
        self.cache = cacher

        # Create a log for debugging our GitHub access
        self.log = Logger(self.cache.directory)
        self.log.write("")

    def qualify(self, resource):
        return urllib.parse.urljoin(self.url.path, resource)

    def request(self, method, resource, data="", headers=None):
        if headers is None:
            headers = { }
        headers["User-Agent"] = "Cockpit Tests"
        if self.token:
            headers["Authorization"] = "token " + self.token
        connected = False
        while not connected:
            if not self.conn:
                if self.url.scheme == 'http':
                    self.conn = http.client.HTTPConnection(self.url.netloc)
                else:
                    self.conn = http.client.HTTPSConnection(self.url.netloc)
                connected = True
            self.conn.set_debuglevel(self.debug and 1 or 0)
            try:
                self.conn.request(method, self.qualify(resource), data, headers)
                response = self.conn.getresponse()
                break
            # This happens when GitHub disconnects in python3
            except http.client.ConnectionResetError:
                if connected:
                    raise
                self.conn = None
            # This happens when GitHub disconnects a keep-alive connection
            except http.client.BadStatusLine:
                if connected:
                    raise
                self.conn = None
            # This happens when TLS is the source of a disconnection
            except socket.error as ex:
                if connected or ex.errno != errno.EPIPE:
                    raise
                self.conn = None
        heads = { }
{'fraction_non_alphanumeric': 0.10723039215686274, 'fraction_numerical': 0.002976190476190476, 'mean_word_length': 2.0987252508814755, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6136633', 'n_tokens_mistral': 3401, 'n_tokens_neox': 3273, 'n_words': 1018}