<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import httplib2
import json
import re
import string
import sys
import time

class ArchIssueLink:
    def __init__(self, link, description=None):
        self.link = link
        self.description = description

    def to_JSON(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

class ArchSecurityAdvisoryLink:
    def __init__(self, asaId, link):
        self.asaId = asaId
        self.link = link

    def to_JSON(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

class ArchIssue:

    def __init__(self, packages, disclosureDate, vulnerableVersion, status, btEntries=[], cves=[], links=[], fixedVersion=None, responseTime=None, asas=[]):
        self.status = status
        self.cves = cves
        self.links = links
        self.packages = packages
        self.vulnerableVersion = vulnerableVersion
        self.fixedVersion = fixedVersion
        self.disclosureDate = disclosureDate
        self.asas = asas
        self.responseTime = responseTime
        self.btEntries = btEntries

    def to_JSON(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

class ArchWikiCVEScrapper:
    _URL = 'https://wiki.archlinux.org/api.php?action=query&titles=CVE&prop=revisions&rvprop=content&rvsection=5&format=json&formatversion=2'

    def __init__(self):
        self.issues = []
        self._cveRE = re.compile(r'cve\-\d{4}\-\d{4,}', re.IGNORECASE)
        self._linksRE = re.compile(r'\[\s*(https?://[^\]\s]+)\s*([^\]]*)\]', re.IGNORECASE)
        self._packageRE = re.compile(r'\{\{pkg\|([a-z\d-]+)\}\}', re.IGNORECASE)
        self._versionRE = re.compile(r'^([<>]?=?)?\s*(\d+:)?[.a-zA-Z\d_-]+(-\d+)?$')
        self._responseTimeRE = re.compile(r'^[<~>]?\s*\d+[dmy]$', re.IGNORECASE)
        self._statusRE = re.compile(r'Fixed|Rejected|Invalid|\'\'\'Vulnerable\'\'\'')
        self._btEntryRE = re.compile(r'\{\{bug\|(\d+)\}\}')
        self._asaRE = re.compile(r'\[\s*(https://[^\]\s]+)\s*(ASA-\d{6}-\d{1,})\]')

    def _parseCVEs(self, value):
        cves = self._cveRE.findall(value)
        if cves:
            # remove duplicates
            cves = list(set(cves))<commit_msg>Allow '+' in package names<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import httplib2
import json
import re
import string
import sys
import time

class ArchIssueLink:
    def __init__(self, link, description=None):
        self.link = link
        self.description = description

    def to_JSON(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

class ArchSecurityAdvisoryLink:
    def __init__(self, asaId, link):
        self.asaId = asaId
        self.link = link

    def to_JSON(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

class ArchIssue:

    def __init__(self, packages, disclosureDate, vulnerableVersion, status, btEntries=[], cves=[], links=[], fixedVersion=None, responseTime=None, asas=[]):
        self.status = status
        self.cves = cves
        self.links = links
        self.packages = packages
        self.vulnerableVersion = vulnerableVersion
        self.fixedVersion = fixedVersion
        self.disclosureDate = disclosureDate
        self.asas = asas
        self.responseTime = responseTime
        self.btEntries = btEntries

    def to_JSON(self):
        return json.dumps(self,
                          default=lambda o: o.__dict__,
                          sort_keys=True,
                          indent=4)

class ArchWikiCVEScrapper:
    _URL = 'https://wiki.archlinux.org/api.php?action=query&titles=CVE&prop=revisions&rvprop=content&rvsection=5&format=json&formatversion=2'

    def __init__(self):
        self.issues = []
        self._cveRE = re.compile(r'cve\-\d{4}\-\d{4,}', re.IGNORECASE)
        self._linksRE = re.compile(r'\[\s*(https?://[^\]\s]+)\s*([^\]]*)\]', re.IGNORECASE)
        self._packageRE = re.compile(r'\{\{pkg\|([a-z\d-+]+)\}\}', re.IGNORECASE)
        self._versionRE = re.compile(r'^([<>]?=?)?\s*(\d+:)?[.a-zA-Z\d_-]+(-\d+)?$')
        self._responseTimeRE = re.compile(r'^[<~>]?\s*\d+[dmy]$', re.IGNORECASE)
        self._statusRE = re.compile(r'Fixed|Rejected|Invalid|\'\'\'Vulnerable\'\'\'')
        self._btEntryRE = re.compile(r'\{\{bug\|(\d+)\}\}')
        self._asaRE = re.compile(r'\[\s*(https://[^\]\s]+)\s*(ASA-\d{6}-\d{1,})\]')

    def _parseCVEs(self, value):
        cves = self._cveRE.findall(value)
        if cves:
            # remove duplicates
            cves = list(set(cves))
{'fraction_non_alphanumeric': 0.15894039735099338, 'fraction_numerical': 0.004285157771717959, 'mean_word_length': 2.1483752299202945, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '59313', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1666, 'n_words': 280}