<commit_before>#!/usr/bin/env python -OO
# This Source Code Form is subject to the terms of the Mozilla
# Public License, v. 2.0. If a copy of the MPL was not distributed
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# CVE -> RHSA Report Generator
#
# Requires Python 2.6 and Beautiful Soup:
#    http://www.crummy.com/software/BeautifulSoup/
#
# Use like: ./rhsa.py < cvelist.txt, where cvelist.txt is a whitespace
# separated list of CVE numbers in the format CVE-YYYY-XXXX.
#
# This will find the CVE on the cve_base_url site and scrape for the
# related RHSA. If it can't find the CVE, chances are it doesn't affect
# Red Hat or Linux. If it can't find an RHSA, then it'll be something
# they don't intend to fix, so output the statement from Red Hat.
# Otherwise, consider resolved and output the link to the RHSA.
# This of course assumes you ARE running the latest CentOS/RHEL release
# versions of the software you're checking the CVEs for.
#
# No guarantees anything this outputs is correct or proper.
#
# vim:ts=4:sw=4:sts=4:ai:si:nu

import sys
if not sys.version_info[:2] <= (2, 6):
    print "I require Python 2.6 or 2.7 to run."
    sys.exit(1)

import re
import urllib2
import sqlite3
import os
import snmp
from BeautifulSoup import BeautifulSoup


class CVEChecker:
    def __init__(self, rhel_version="5", platform="x86_64", host=None):
        if platform not in ['x86_64', 'i386']:
            raise ValueError("Platform must be 'x86_64' or 'i386'.")
        self.rhel_version = rhel_version
        self.platform = platform
        self.host = host
        self.cve_base_url = "https://www.redhat.com/security/data/cve/"
        self.mitre_url = "http://cve.mitre.org/cgi-bin/cvename.cgi?name="
        self.rhsa5_r = re.compile(".*Red Hat Enterprise Linux version 5.*")
        self.rhsa6_r = re.compile(".*Red Hat Enterprise Linux version 6.*")
        self.rhsax_r = re.compile(".*Red Hat Enterprise Linux version [4321].*")
        self.cve_r = re.compile(r"^CVE-\d{4}-\d{4}$")
        self.pkghdr5 = "Red Hat Enterprise Linux (v. 5 server)"
        self.pkghdr6 = "Red Hat Enterprise Linux (v. 6 server)"
        self.curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
        if self.host:
            self.snmpq = snmp.SNMPQueryTool(self.host)
        else:
            self.snmpq = None
        self.soup = None
        self.cve = None

        initdb = False
        if not os.path.exists(os.path.join(self.curdir, 'cache.db')):
            initdb = True

        self.conn = sqlite3.connect(os.path.join(self.curdir, 'cache.db'), check_same_thread=False)

        if initdb:
            self._init_db()

    def _init_db(self):
        cur = self.conn.cursor()
        cur.execute("CREATE TABLE IF NOT EXISTS cache (id INTEGER PRIMARY KEY AUTOINCREMENT, " \
                    + "timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, platform TEXT NOT NULL, " \
                    + "rhelver INTEGER NOT NULL, cve TEXT NOT NULL, result TEXT NOT NULL, " \
                    + "o_ver TEXT DEFAULT NULL)")
        cur.execute("CREATE INDEX IF NOT EXISTS cve_idx ON cache (cve)")
        self.conn.commit()
        cur.close()

    def get_cve_info(self, cve):
        # Fix formatting
        cve = cve.strip()
        cve = cve.replace(',','')
        cve = cve.upper()
        self.cve = cve
        del(cve) # for refactoring

        if not self.cve_r.match(self.cve):
            # Check the CVE is in the correct format
            return {'cve': "{0} -- This is not a CVE reference.".format(self.cve), 'verinfo': None}

        cached_cve = self._cache_retrieve()
        if cached_cve['cve'] is not None:
            if self.host:
                if cached_cve['originalver']:
                    return {'cve': cached_cve['cve'], 'verinfo': self._get_installed_package(cached_cve['originalver'])}
            # Likely a statement, so no verinfo despite wanting us to check a host
            return {'cve': cached_cve['cve'], 'verinfo': cached_cve['originalver']}

        cveurl = self.cve_base_url + self.cve + ".html"  # Not sure if we need .html anymore? They rewrite it anyway.
        try:
            html = urllib2.urlopen(cveurl).read()
        except urllib2.HTTPError:
            # 404 or general screwup, don't cache in case it turns up later
            return {'cve': "{0} -- !!FIX!! Not found on Red Hat's website. " \
                                  + "Might be Windows only or bad CVE reference, see {1}".format(self.cve, self.mitre_url + self.cve), 'verinfo': None}
        except urllib2.URLError:
            return {'cve': "There was a problem with the URL.", 'verinfo': None}

        soup = BeautifulSoup(html)

        if self.rhel_version == "6":
            if soup.find(text=self.rhsa6_r):
                return self.scrape_rhsa(soup, self.rhsa6_r)
            elif soup.find(text=self.rhsa5_r):
                return {'cve': self.cve + " -- Affected RHEL5, not RHEL6. Packages from RHEL6 will not be affected by this issue.", 'verinfo': None}
            elif soup.find(text=self.rhsax_r):
                return {'cve': self.cve + " -- Affected <=RHEL4. Packages from RHEL6 will not be affected by this issue.", 'verinfo': None}
        else:
            if soup.find(text=self.rhsa5_r):<commit_msg>Cache results that don't affect RHEL6<commit_after>#!/usr/bin/env python -OO
# This Source Code Form is subject to the terms of the Mozilla
# Public License, v. 2.0. If a copy of the MPL was not distributed
# with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# CVE -> RHSA Report Generator
#
# Requires Python 2.6 and Beautiful Soup:
#    http://www.crummy.com/software/BeautifulSoup/
#
# Use like: ./rhsa.py < cvelist.txt, where cvelist.txt is a whitespace
# separated list of CVE numbers in the format CVE-YYYY-XXXX.
#
# This will find the CVE on the cve_base_url site and scrape for the
# related RHSA. If it can't find the CVE, chances are it doesn't affect
# Red Hat or Linux. If it can't find an RHSA, then it'll be something
# they don't intend to fix, so output the statement from Red Hat.
# Otherwise, consider resolved and output the link to the RHSA.
# This of course assumes you ARE running the latest CentOS/RHEL release
# versions of the software you're checking the CVEs for.
#
# No guarantees anything this outputs is correct or proper.
#
# vim:ts=4:sw=4:sts=4:ai:si:nu

import sys
if not sys.version_info[:2] <= (2, 6):
    print "I require Python 2.6 or 2.7 to run."
    sys.exit(1)

import re
import urllib2
import sqlite3
import os
import snmp
from BeautifulSoup import BeautifulSoup


class CVEChecker:
    def __init__(self, rhel_version="5", platform="x86_64", host=None):
        if platform not in ['x86_64', 'i386']:
            raise ValueError("Platform must be 'x86_64' or 'i386'.")
        self.rhel_version = rhel_version
        self.platform = platform
        self.host = host
        self.cve_base_url = "https://www.redhat.com/security/data/cve/"
        self.mitre_url = "http://cve.mitre.org/cgi-bin/cvename.cgi?name="
        self.rhsa5_r = re.compile(".*Red Hat Enterprise Linux version 5.*")
        self.rhsa6_r = re.compile(".*Red Hat Enterprise Linux version 6.*")
        self.rhsax_r = re.compile(".*Red Hat Enterprise Linux version [4321].*")
        self.cve_r = re.compile(r"^CVE-\d{4}-\d{4}$")
        self.pkghdr5 = "Red Hat Enterprise Linux (v. 5 server)"
        self.pkghdr6 = "Red Hat Enterprise Linux (v. 6 server)"
        self.curdir = os.path.join(os.getcwd(), os.path.dirname(__file__))
        if self.host:
            self.snmpq = snmp.SNMPQueryTool(self.host)
        else:
            self.snmpq = None
        self.soup = None
        self.cve = None

        initdb = False
        if not os.path.exists(os.path.join(self.curdir, 'cache.db')):
            initdb = True

        self.conn = sqlite3.connect(os.path.join(self.curdir, 'cache.db'), check_same_thread=False)

        if initdb:
            self._init_db()

    def _init_db(self):
        cur = self.conn.cursor()
        cur.execute("CREATE TABLE IF NOT EXISTS cache (id INTEGER PRIMARY KEY AUTOINCREMENT, " \
                    + "timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, platform TEXT NOT NULL, " \
                    + "rhelver INTEGER NOT NULL, cve TEXT NOT NULL, result TEXT NOT NULL, " \
                    + "o_ver TEXT DEFAULT NULL)")
        cur.execute("CREATE INDEX IF NOT EXISTS cve_idx ON cache (cve)")
        self.conn.commit()
        cur.close()

    def get_cve_info(self, cve):
        # Fix formatting
        cve = cve.strip()
        cve = cve.replace(',','')
        cve = cve.upper()
        self.cve = cve
        del(cve) # for refactoring

        if not self.cve_r.match(self.cve):
            # Check the CVE is in the correct format
            return {'cve': "{0} -- This is not a CVE reference.".format(self.cve), 'verinfo': None}

        cached_cve = self._cache_retrieve()
        if cached_cve['cve'] is not None:
            if self.host:
                if cached_cve['originalver']:
                    return {'cve': cached_cve['cve'], 'verinfo': self._get_installed_package(cached_cve['originalver'])}
            # Likely a statement, so no verinfo despite wanting us to check a host
            return {'cve': cached_cve['cve'], 'verinfo': cached_cve['originalver']}

        cveurl = self.cve_base_url + self.cve + ".html"  # Not sure if we need .html anymore? They rewrite it anyway.
        try:
            html = urllib2.urlopen(cveurl).read()
        except urllib2.HTTPError:
            # 404 or general screwup, don't cache in case it turns up later
            return {'cve': "{0} -- !!FIX!! Not found on Red Hat's website. " \
                                  + "Might be Windows only or bad CVE reference, see {1}".format(self.cve, self.mitre_url + self.cve), 'verinfo': None}
        except urllib2.URLError:
            return {'cve': "There was a problem with the URL.", 'verinfo': None}

        soup = BeautifulSoup(html)

        if self.rhel_version == "6":
            if soup.find(text=self.rhsa6_r):
                return self.scrape_rhsa(soup, self.rhsa6_r)
            elif soup.find(text=self.rhsa5_r):
                result = "Affected RHEL5, not RHEL6. Packages from RHEL6 will not be affected by this issue."
                self._cache_store(result)
                return {'cve': self.cve + " -- " + result, 'verinfo': None}
            elif soup.find(text=self.rhsax_r):
                result = "Affected <=RHEL4. Packages from RHEL6 will not be affected by this issue."
                self._cache_store(result)
                return {'cve': self.cve + " -- " + result, 'verinfo': None}
        else:
            if soup.find(text=self.rhsa5_r):
{'fraction_non_alphanumeric': 0.11581973293768547, 'fraction_numerical': 0.013445845697329376, 'mean_word_length': 2.3640049906425453, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '270049', 'n_tokens_mistral': 3609, 'n_tokens_neox': 3432, 'n_words': 1074}