<commit_before>#!/usr/bin/python3

import re
import sys
import requests
from bs4 import BeautifulSoup

##################################################

# ./manage.py shell

# from directory.models import *
# from IanaScraper import IanaScraper

# iana = IanaScraper()
# rec = iana.get_rfc_dicts()
# for e in rec:
#     c, _ = CipherSuite.objects.get_or_create(
#         name=e['name'],
#         hex_byte_1=e['hex1'],
#         hex_byte_2=e['hex2'],
#     )
#     r, _ = Rfc.objects.get_or_create(
#         number=e['rfc'],
#     )
#     c.defining_rfcs.add(r)

##################################################

class ResourceNotFoundException(Exception):
    pass


class IanaScraper:
    def __init__(self, url="https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml"):
        self.url = url


    def get(self, url):
        """Sends a HTTP request to the given URL. If the status code
        of its response is not equal 200, a ResourceNotFoundException
        is raised."""

        response = requests.get(url)
        if response.status_code != 200:
            raise ResourceNotFoundException(
                "Resource not found under the given URL."
            )
        else:
            return response


    def get_url_content(self, url):
        """Tries to fetch a given URL and return the textual contents
        of the respective HTTP response. If retreiving the web page
        fails, an Exception is raised an an empty text returned."""

        try:
            resp = self.get(url)
        except ResourceNotFoundException as e:
            print(e.message)
            return ""
        return resp.text


    def get_rfc_dicts(self):
        """Parses the IANA TLS paramter table for RFC information. List of all
        table contents can be parsed with the filter_dict funciton. Returns a
        list of dicts, where each dict contains the information on one row."""

        page = BeautifulSoup(
            self.get_url_content(self.url),
            'html.parser'
        )
        contents = self.parse_table(page)
        contents = self.filter_dict(contents, 'hex1', r'^0x[0123456789ABCDEF]{2}$')
        contents = self.filter_dict(contents, 'hex2', r'^0x[0123456789ABCDEF]{2}$')
        contents = self.filter_dict(contents, 'rfc', r'\d+')
        return contents


    def filter_dict(self, list_of_dicts, key, regex):<commit_msg>only return cipher suites containing string 'WITH'<commit_after>#!/usr/bin/python3

import re
import sys
import requests
from bs4 import BeautifulSoup

##################################################

# ./manage.py shell

# from directory.models import *
# from IanaScraper import IanaScraper

# iana = IanaScraper()
# rec = iana.get_rfc_dicts()
# for e in rec:
#     c, _ = CipherSuite.objects.get_or_create(
#         name=e['name'],
#         hex_byte_1=e['hex1'],
#         hex_byte_2=e['hex2'],
#     )
#     r, _ = Rfc.objects.get_or_create(
#         number=e['rfc'],
#     )
#     c.defining_rfcs.add(r)

##################################################

class ResourceNotFoundException(Exception):
    pass


class IanaScraper:
    def __init__(self, url="https://www.iana.org/assignments/tls-parameters/tls-parameters.xhtml"):
        self.url = url


    def get(self, url):
        """Sends a HTTP request to the given URL. If the status code
        of its response is not equal 200, a ResourceNotFoundException
        is raised."""

        response = requests.get(url)
        if response.status_code != 200:
            raise ResourceNotFoundException(
                "Resource not found under the given URL."
            )
        else:
            return response


    def get_url_content(self, url):
        """Tries to fetch a given URL and return the textual contents
        of the respective HTTP response. If retreiving the web page
        fails, an Exception is raised an an empty text returned."""

        try:
            resp = self.get(url)
        except ResourceNotFoundException as e:
            print(e.message)
            return ""
        return resp.text


    def get_rfc_dicts(self):
        """Parses the IANA TLS paramter table for RFC information. List of all
        table contents can be parsed with the filter_dict funciton. Returns a
        list of dicts, where each dict contains the information on one row."""

        page = BeautifulSoup(
            self.get_url_content(self.url),
            'html.parser'
        )
        contents = self.parse_table(page)
        contents = self.filter_dict(contents, 'hex1', r'^0x[0123456789ABCDEF]{2}$')
        contents = self.filter_dict(contents, 'hex2', r'^0x[0123456789ABCDEF]{2}$')
        contents = self.filter_dict(contents, 'name', r'.*WITH.*')
        contents = self.filter_dict(contents, 'rfc', r'\d+')
        return contents


    def filter_dict(self, list_of_dicts, key, regex):
{'fraction_non_alphanumeric': 0.1381847914963205, 'fraction_numerical': 0.01553556827473426, 'mean_word_length': 2.5793708851499635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1146164', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1405, 'n_words': 403}