<commit_before>import re
import os
import hashlib
import shutil
import string

import bb.utils

import oelite.fetch
import oelite.util
import local
import url
import git
import hg
import svn

FETCHERS = {
    "file"	: local.LocalFetcher,
    "http"	: url.UrlFetcher,
    "https"	: url.UrlFetcher,
    "ftp"	: url.UrlFetcher,
    "git"	: git.GitFetcher,
    "hg"	: hg.HgFetcher,
    "svn"	: svn.SvnFetcher,
}

uri_pattern = re.compile("(?P<scheme>[^:]*)://(?P<location>[^;]+)(;(?P<params>.*))?")

unpack_ext = (
    ("tar_gz",	(".tar.gz", ".tgz", ".tar.Z")), 
    ("tar_bz2",	(".tar.bz2", ".tbz", ".tbz2")),
    ("tar_xz",	(".tar.xz", ".txz")),
    ("tar_lz",	(".tar.lz", ".tlz")),
    ("zip",	(".zip", ".jar")),
    ("gz",	(".gz", ".Z", ".z")),
    ("bz2",	(".bz2",)),
    ("xz",	(".xz",)),
    ("lz",	(".lz",)),
    )

class OEliteUri:

    def __init__(self, uri, d):
        self.uri = uri
        # Note, do not store reference to meta
        self.recipe = "%s:%s_%s"%(d.get("RECIPE_TYPE"),
                                  d.get("PN"), d.get("PV"))
        # Try to avoid readinng recipe meta-data here, as it might
        # change later on, so better to read the meta-data when
        # needed.  This is especially problematic with variables
        # depending on ${EXTRA_ARCH} which might be changed after all
        # parse when resolving task dependencies.
        self.ingredients = d.get("INGREDIENTS")
        self.isubdir = (d.get("INGREDIENTS_SUBDIR") or
                        os.path.basename(d.get("FILE_DIRNAME")))
        self.strict = d.get("SRC_URI_STRICT") or False
        if self.strict == "0":
            self.strict = False
        m = uri_pattern.match(uri)
        if not m:
            raise oelite.fetch.InvalidURI(uri, "not an URI at all")
        self.scheme = m.group("scheme")
        self.location = m.group("location")
        if not self.scheme:
            raise oelite.fetch.InvalidURI(uri, "no URI scheme")
        if not self.location:
            raise oelite.fetch.InvalidURI(uri, "no URI location")
        self.params = {}
        if m.group("params"):
            for param in (m.group("params") or "").split(";"):
                try:
                    name, value = param.split("=")
                except ValueError:
                    raise oelite.fetch.InvalidURI(
                        uri, "bad parameter: %s"%param)
                self.params[name] = value
        try:
            self.isubdir = self.params["isubdir"]
        except KeyError:
            pass
        if not self.scheme in FETCHERS:
            raise oelite.fetch.InvalidURI(
                uri, "unsupported URI scheme: %s"%(self.scheme))
        self.fdepends = []
        self.fetcher = FETCHERS[self.scheme](self, d)
        self.init_unpack_params()
        self.add_unpack_fdepends(d)
        self.init_patch_params(d)
        self.add_patch_fdepends()
        self.mirrors = self.init_mirrors(d.get("MIRRORS"))
        self.premirrors = self.init_mirrors(d.get("PREMIRRORS"))
        self.whitelist = self.init_filter(d.get("URL_WHITELIST"))
        self.blacklist = self.init_filter(d.get("URL_BLACKLIST"))
        return

    def init_mirrors(self, all_mirrors):
        if not all_mirrors:
            return []
        assert isinstance(all_mirrors, basestring)<commit_msg>lib/oelite: Add support for non-value parameters<commit_after>import re
import os
import hashlib
import shutil
import string

import bb.utils

import oelite.fetch
import oelite.util
import local
import url
import git
import hg
import svn

FETCHERS = {
    "file"	: local.LocalFetcher,
    "http"	: url.UrlFetcher,
    "https"	: url.UrlFetcher,
    "ftp"	: url.UrlFetcher,
    "git"	: git.GitFetcher,
    "hg"	: hg.HgFetcher,
    "svn"	: svn.SvnFetcher,
}

uri_pattern = re.compile("(?P<scheme>[^:]*)://(?P<location>[^;]+)(;(?P<params>.*))?")

unpack_ext = (
    ("tar_gz",	(".tar.gz", ".tgz", ".tar.Z")), 
    ("tar_bz2",	(".tar.bz2", ".tbz", ".tbz2")),
    ("tar_xz",	(".tar.xz", ".txz")),
    ("tar_lz",	(".tar.lz", ".tlz")),
    ("zip",	(".zip", ".jar")),
    ("gz",	(".gz", ".Z", ".z")),
    ("bz2",	(".bz2",)),
    ("xz",	(".xz",)),
    ("lz",	(".lz",)),
    )

class OEliteUri:

    def __init__(self, uri, d):
        self.uri = uri
        # Note, do not store reference to meta
        self.recipe = "%s:%s_%s"%(d.get("RECIPE_TYPE"),
                                  d.get("PN"), d.get("PV"))
        # Try to avoid readinng recipe meta-data here, as it might
        # change later on, so better to read the meta-data when
        # needed.  This is especially problematic with variables
        # depending on ${EXTRA_ARCH} which might be changed after all
        # parse when resolving task dependencies.
        self.ingredients = d.get("INGREDIENTS")
        self.isubdir = (d.get("INGREDIENTS_SUBDIR") or
                        os.path.basename(d.get("FILE_DIRNAME")))
        self.strict = d.get("SRC_URI_STRICT") or False
        if self.strict == "0":
            self.strict = False
        m = uri_pattern.match(uri)
        if not m:
            raise oelite.fetch.InvalidURI(uri, "not an URI at all")
        self.scheme = m.group("scheme")
        self.location = m.group("location")
        if not self.scheme:
            raise oelite.fetch.InvalidURI(uri, "no URI scheme")
        if not self.location:
            raise oelite.fetch.InvalidURI(uri, "no URI location")
        self.params = {}
        if m.group("params"):
            for param in (m.group("params") or "").split(";"):
                try:
                    name, value = param.split("=")
                except ValueError:
                    name = param
                    value = "1"
                self.params[name] = value
        try:
            self.isubdir = self.params["isubdir"]
        except KeyError:
            pass
        if not self.scheme in FETCHERS:
            raise oelite.fetch.InvalidURI(
                uri, "unsupported URI scheme: %s"%(self.scheme))
        self.fdepends = []
        self.fetcher = FETCHERS[self.scheme](self, d)
        self.init_unpack_params()
        self.add_unpack_fdepends(d)
        self.init_patch_params(d)
        self.add_patch_fdepends()
        self.mirrors = self.init_mirrors(d.get("MIRRORS"))
        self.premirrors = self.init_mirrors(d.get("PREMIRRORS"))
        self.whitelist = self.init_filter(d.get("URL_WHITELIST"))
        self.blacklist = self.init_filter(d.get("URL_BLACKLIST"))
        return

    def init_mirrors(self, all_mirrors):
        if not all_mirrors:
            return []
        assert isinstance(all_mirrors, basestring)
{'fraction_non_alphanumeric': 0.15443724574355883, 'fraction_numerical': 0.0019587162874792827, 'mean_word_length': 2.1311320754716983, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1042569', 'n_tokens_mistral': 2294, 'n_tokens_neox': 2231, 'n_words': 494}