<commit_before># Copyright (c) 2014-present PlatformIO <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import inspect
import io
import json
import os
import re
import tarfile

from platformio import util
from platformio.clients.http import fetch_remote_content
from platformio.compat import get_object_members, string_types
from platformio.package.exception import ManifestParserError, UnknownManifestError
from platformio.project.helpers import is_platformio_project

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse


class ManifestFileType(object):
    PLATFORM_JSON = "platform.json"
    LIBRARY_JSON = "library.json"
    LIBRARY_PROPERTIES = "library.properties"
    MODULE_JSON = "module.json"
    PACKAGE_JSON = "package.json"

    @classmethod
    def items(cls):
        return get_object_members(cls)

    @classmethod
    def from_uri(cls, uri):
        for t in sorted(cls.items().values()):
            if uri.endswith(t):
                return t
        return None

    @classmethod
    def from_dir(cls, path):
        for t in sorted(cls.items().values()):
            if os.path.isfile(os.path.join(path, t)):
                return t
        return None


class ManifestParserFactory(object):
    @staticmethod
    def read_manifest_contents(path):
        last_err = None
        for encoding in ("utf-8", "latin-1"):
            try:
                with io.open(path, encoding=encoding) as fp:
                    return fp.read()
            except UnicodeDecodeError as e:
                last_err = e
        raise last_err  # pylint: disable=raising-bad-type

    @classmethod
    def new_from_file(cls, path, remote_url=False):
        if not path or not os.path.isfile(path):
            raise UnknownManifestError("Manifest file does not exist %s" % path)
        type_from_uri = ManifestFileType.from_uri(path)
        if not type_from_uri:
            raise UnknownManifestError("Unknown manifest file type %s" % path)
        return ManifestParserFactory.new(
            cls.read_manifest_contents(path), type_from_uri, remote_url
        )

    @classmethod
    def new_from_dir(cls, path, remote_url=None):
        assert os.path.isdir(path), "Invalid directory %s" % path

        type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None
        if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)):
            return ManifestParserFactory.new(
                cls.read_manifest_contents(os.path.join(path, type_from_uri)),
                type_from_uri,
                remote_url=remote_url,
                package_dir=path,
            )

        type_from_dir = ManifestFileType.from_dir(path)
        if not type_from_dir:
            raise UnknownManifestError(
                "Unknown manifest file type in %s directory" % path
            )
        return ManifestParserFactory.new(
            cls.read_manifest_contents(os.path.join(path, type_from_dir)),
            type_from_dir,
            remote_url=remote_url,
            package_dir=path,
        )

    @staticmethod
    def new_from_url(remote_url):
        content = fetch_remote_content(remote_url)
        return ManifestParserFactory.new(
            content,
            ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON,
            remote_url,
        )

    @staticmethod
    def new_from_archive(path):
        assert path.endswith("tar.gz")
        with tarfile.open(path, mode="r:gz") as tf:
            for t in sorted(ManifestFileType.items().values()):
                try:
                    return ManifestParserFactory.new(
                        tf.extractfile(t).read().decode(), t
                    )
                except KeyError:
                    pass
        raise UnknownManifestError("Unknown manifest file type in %s archive" % path)

    @staticmethod
    def new(  # pylint: disable=redefined-builtin
        contents, type, remote_url=None, package_dir=None
    ):
        for _, cls in globals().items():
            if (
                inspect.isclass(cls)
                and issubclass(cls, BaseManifestParser)
                and cls != BaseManifestParser
                and cls.manifest_type == type
            ):
                return cls(contents, remote_url, package_dir)
        raise UnknownManifestError("Unknown manifest file type %s" % type)


class BaseManifestParser(object):
    def __init__(self, contents, remote_url=None, package_dir=None):
        self.remote_url = remote_url
        self.package_dir = package_dir
        try:
            self._data = self.parse(contents)
        except Exception as e:
            raise ManifestParserError("Could not parse manifest -> %s" % e)

        self._data = self.normalize_repository(self._data)
        self._data = self.parse_examples(self._data)

        # remove None fields
        for key in list(self._data.keys()):<commit_msg>Improve initiating manifest parser from a package archive<commit_after># Copyright (c) 2014-present PlatformIO <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import inspect
import io
import json
import os
import re
import tarfile

from platformio import util
from platformio.clients.http import fetch_remote_content
from platformio.compat import get_object_members, string_types
from platformio.package.exception import ManifestParserError, UnknownManifestError
from platformio.project.helpers import is_platformio_project

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse


class ManifestFileType(object):
    PLATFORM_JSON = "platform.json"
    LIBRARY_JSON = "library.json"
    LIBRARY_PROPERTIES = "library.properties"
    MODULE_JSON = "module.json"
    PACKAGE_JSON = "package.json"

    @classmethod
    def items(cls):
        return get_object_members(cls)

    @classmethod
    def from_uri(cls, uri):
        for t in sorted(cls.items().values()):
            if uri.endswith(t):
                return t
        return None

    @classmethod
    def from_dir(cls, path):
        for t in sorted(cls.items().values()):
            if os.path.isfile(os.path.join(path, t)):
                return t
        return None


class ManifestParserFactory(object):
    @staticmethod
    def read_manifest_contents(path):
        last_err = None
        for encoding in ("utf-8", "latin-1"):
            try:
                with io.open(path, encoding=encoding) as fp:
                    return fp.read()
            except UnicodeDecodeError as e:
                last_err = e
        raise last_err  # pylint: disable=raising-bad-type

    @classmethod
    def new_from_file(cls, path, remote_url=False):
        if not path or not os.path.isfile(path):
            raise UnknownManifestError("Manifest file does not exist %s" % path)
        type_from_uri = ManifestFileType.from_uri(path)
        if not type_from_uri:
            raise UnknownManifestError("Unknown manifest file type %s" % path)
        return ManifestParserFactory.new(
            cls.read_manifest_contents(path), type_from_uri, remote_url
        )

    @classmethod
    def new_from_dir(cls, path, remote_url=None):
        assert os.path.isdir(path), "Invalid directory %s" % path

        type_from_uri = ManifestFileType.from_uri(remote_url) if remote_url else None
        if type_from_uri and os.path.isfile(os.path.join(path, type_from_uri)):
            return ManifestParserFactory.new(
                cls.read_manifest_contents(os.path.join(path, type_from_uri)),
                type_from_uri,
                remote_url=remote_url,
                package_dir=path,
            )

        type_from_dir = ManifestFileType.from_dir(path)
        if not type_from_dir:
            raise UnknownManifestError(
                "Unknown manifest file type in %s directory" % path
            )
        return ManifestParserFactory.new(
            cls.read_manifest_contents(os.path.join(path, type_from_dir)),
            type_from_dir,
            remote_url=remote_url,
            package_dir=path,
        )

    @staticmethod
    def new_from_url(remote_url):
        content = fetch_remote_content(remote_url)
        return ManifestParserFactory.new(
            content,
            ManifestFileType.from_uri(remote_url) or ManifestFileType.LIBRARY_JSON,
            remote_url,
        )

    @staticmethod
    def new_from_archive(path):
        assert path.endswith("tar.gz")
        with tarfile.open(path, mode="r:gz") as tf:
            for t in sorted(ManifestFileType.items().values()):
                for member in (t, "./" + t):
                    try:
                        return ManifestParserFactory.new(
                            tf.extractfile(member).read().decode(), t
                        )
                    except KeyError:
                        pass
        raise UnknownManifestError("Unknown manifest file type in %s archive" % path)

    @staticmethod
    def new(  # pylint: disable=redefined-builtin
        contents, type, remote_url=None, package_dir=None
    ):
        for _, cls in globals().items():
            if (
                inspect.isclass(cls)
                and issubclass(cls, BaseManifestParser)
                and cls != BaseManifestParser
                and cls.manifest_type == type
            ):
                return cls(contents, remote_url, package_dir)
        raise UnknownManifestError("Unknown manifest file type %s" % type)


class BaseManifestParser(object):
    def __init__(self, contents, remote_url=None, package_dir=None):
        self.remote_url = remote_url
        self.package_dir = package_dir
        try:
            self._data = self.parse(contents)
        except Exception as e:
            raise ManifestParserError("Could not parse manifest -> %s" % e)

        self._data = self.normalize_repository(self._data)
        self._data = self.parse_examples(self._data)

        # remove None fields
        for key in list(self._data.keys()):
{'fraction_non_alphanumeric': 0.07985169108337854, 'fraction_numerical': 0.0018086453246518358, 'mean_word_length': 2.368565336582394, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2633065', 'n_tokens_mistral': 3117, 'n_tokens_neox': 3010, 'n_words': 823}