<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-
##################################################################################
# Access information about binary and source packages in multiple
# (independent) apt-repositories utilizing libapt / python-apt/
# apt_pkg without the need to change the local system and it's apt-setup.
#
# Copyright (C) 2018  <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
##################################################################################
import logging
import sys
import configparser
import urllib3
import io
import codecs
import re
import subprocess
import os
import apt_pkg
import tempfile
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
from urllib3.exceptions import MaxRetryError

logger = logging.getLogger(__name__)


def scanRepository(url, suites=None):
    logger.debug("scanRepository('{}', {})".format(url, suites))
    url = url + ("/" if not url.endswith("/") else "")
    res = list()
    if suites:
        for s in suites:
            try:
                res.append(scanReleaseFile(urljoin(url, os.path.join('dists', s, 'Release')), url))
            except Exception:
                logger.warn("Could not resolve suite {} of repository {}".format(s, url))
    else:
        try:
            res.extend(scanReleases(urljoin(url, "dists/"), url))
        except Exception as e:
            logger.warn("Could not resolve repository {}".format(url))
    return res


def scanReleases(url, repoUrl, recursive=True):
    '''
       return suites found at url and all it's relevant subfolders if recursive==True
    '''
    logger.debug("scanReleases('{}', {})".format(url, recursive))
    suites = list()
    ignoreFolders = list(['by-hash'])

    p = urlparse(url)
    if p.scheme == "file":
        index = LocalFilesystemScanner(url)
    else:
        index = HtmlIndexParser(url)

    if index.release:
        suite = scanReleaseFile(index.release, repoUrl)
        if suite:
          suites.append(suite)
          ignoreFolders.extend(suite['components'])

    if recursive:
        for subfolder, suburl in sorted(index.getSubfolders().items()):
            if not subfolder in ignoreFolders:
                suites.extend(scanReleases(suburl, repoUrl))

    return suites


def scanReleaseFile(url, repoUrl):
    logger.debug("scanReleaseFile('{}')".format(url))
    data = getFromURL(url)
    with tempfile.TemporaryFile() as fp:
        fp.write(data)
        fp.seek(0)
        with apt_pkg.TagFile(fp) as tagfile:
            try:
                for section in tagfile:
                    components = section.get('Components').split(" ") if section.get('Components') else list()
                    architectures = section.get('Architectures').split(" ") if section.get('Architectures') else list()
                    md5sum = section.get('MD5Sum').split("\n") if section.get('Md5Sum') else list()
                    files=[re.sub(" +", " ", s.strip()).split(" ")[2] for s in md5sum]
                    hasSources=suiteHasSources(files)
                    return { 
                        'repoUrl': repoUrl,
                        'releaseUrl': url,
                        'suite': section.get('Suite'),
                        'codename': section.get('Codename'),
                        'components': components,<commit_msg>RepositoryScanner ensures that Release-File defines a Suite<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-
##################################################################################
# Access information about binary and source packages in multiple
# (independent) apt-repositories utilizing libapt / python-apt/
# apt_pkg without the need to change the local system and it's apt-setup.
#
# Copyright (C) 2018  <NAME>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
##################################################################################
import logging
import sys
import configparser
import urllib3
import io
import codecs
import re
import subprocess
import os
import apt_pkg
import tempfile
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse
from urllib3.exceptions import MaxRetryError

logger = logging.getLogger(__name__)


def scanRepository(url, suites=None):
    logger.debug("scanRepository('{}', {})".format(url, suites))
    url = url + ("/" if not url.endswith("/") else "")
    res = list()
    if suites:
        for s in suites:
            try:
                res.append(scanReleaseFile(urljoin(url, os.path.join('dists', s, 'Release')), url))
            except Exception:
                logger.warn("Could not resolve suite {} of repository {}".format(s, url))
    else:
        try:
            res.extend(scanReleases(urljoin(url, "dists/"), url))
        except Exception as e:
            logger.warn("Could not resolve repository {}".format(url))
    return res


def scanReleases(url, repoUrl, recursive=True):
    '''
       return suites found at url and all it's relevant subfolders if recursive==True
    '''
    logger.debug("scanReleases('{}', {})".format(url, recursive))
    suites = list()
    ignoreFolders = list(['by-hash'])

    p = urlparse(url)
    if p.scheme == "file":
        index = LocalFilesystemScanner(url)
    else:
        index = HtmlIndexParser(url)

    if index.release:
        suite = scanReleaseFile(index.release, repoUrl)
        if suite:
          suites.append(suite)
          ignoreFolders.extend(suite['components'])

    if recursive:
        for subfolder, suburl in sorted(index.getSubfolders().items()):
            if not subfolder in ignoreFolders:
                suites.extend(scanReleases(suburl, repoUrl))

    return suites


def scanReleaseFile(url, repoUrl):
    logger.debug("scanReleaseFile('{}')".format(url))
    data = getFromURL(url)
    with tempfile.TemporaryFile() as fp:
        fp.write(data)
        fp.seek(0)
        with apt_pkg.TagFile(fp) as tagfile:
            try:
                for section in tagfile:
                    if not section.get('Suite'):
                        continue
                    components = section.get('Components').split(" ") if section.get('Components') else list()
                    architectures = section.get('Architectures').split(" ") if section.get('Architectures') else list()
                    md5sum = section.get('MD5Sum').split("\n") if section.get('Md5Sum') else list()
                    files=[re.sub(" +", " ", s.strip()).split(" ")[2] for s in md5sum]
                    hasSources=suiteHasSources(files)
                    return { 
                        'repoUrl': repoUrl,
                        'releaseUrl': url,
                        'suite': section.get('Suite'),
                        'codename': section.get('Codename'),
                        'components': components,
{'fraction_non_alphanumeric': 0.1442002442002442, 'fraction_numerical': 0.006593406593406593, 'mean_word_length': 2.765977011494253, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5976746', 'n_tokens_mistral': 2220, 'n_tokens_neox': 2119, 'n_words': 711}