<commit_before>#!/usr/bin/env python3
#
# verify.py - part of the FDroid server tools
# Copyright (C) 2013, <NAME>, <EMAIL>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import glob
import json
import logging
import requests
from argparse import ArgumentParser
from collections import OrderedDict

from . import _
from . import common
from . import net
from . import update
from .exception import FDroidException

options = None
config = None


class hashabledict(OrderedDict):
    def __key(self):
        return tuple((k, self[k]) for k in sorted(self))

    def __hash__(self):
        return hash(self.__key())

    def __eq__(self, other):
        return self.__key() == other.__key()

    def __lt__(self, other):
        return self.__key() < other.__key()

    def __qt__(self, other):
        return self.__key() > other.__key()


class Decoder(json.JSONDecoder):
    def __init__(self, **kwargs):
        json.JSONDecoder.__init__(self, **kwargs)
        self.parse_array = self.JSONArray
        # Use the python implemenation of the scanner
        self.scan_once = json.scanner.py_make_scanner(self)

    def JSONArray(self, s_and_end, scan_once, **kwargs):
        values, end = json.decoder.JSONArray(s_and_end, scan_once, **kwargs)
        return set(values), end


def write_json_report(url, remote_apk, unsigned_apk, compare_result):
    """write out the results of the verify run to JSON

    This builds up reports on the repeated runs of `fdroid verify` on
    a set of apps.  It uses the timestamps on the compared files to
    ensure that there is only one report per file, even when run
    repeatedly.

    """

    jsonfile = unsigned_apk + '.json'
    if os.path.exists(jsonfile):
        with open(jsonfile) as fp:
            data = json.load(fp, object_pairs_hook=OrderedDict)
    else:
        data = OrderedDict()
    output = hashabledict()
    output['url'] = url
    for key, filename in (('local', unsigned_apk), ('remote', remote_apk)):
        d = hashabledict()
        output[key] = d
        d['file'] = filename
        d['sha256'] = update.sha256sum(filename)
        d['timestamp'] = os.stat(filename).st_ctime
        d['packageName'], d['versionCode'], d['versionName'] = common.get_apk_id(filename)
    if compare_result:
        output['verified'] = False
        output['result'] = compare_result
    else:
        output['verified'] = True<commit_msg>verify: include diffoscope setup info in output JSON<commit_after>#!/usr/bin/env python3
#
# verify.py - part of the FDroid server tools
# Copyright (C) 2013, <NAME>, <EMAIL>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import glob
import json
import logging
import requests
from argparse import ArgumentParser
from collections import OrderedDict

from . import _
from . import common
from . import net
from . import update
from .exception import FDroidException

options = None
config = None


class hashabledict(OrderedDict):
    def __key(self):
        return tuple((k, self[k]) for k in sorted(self))

    def __hash__(self):
        try:
            return hash(self.__key())
        except TypeError as e:
            print(self.__key())
            raise e

    def __eq__(self, other):
        return self.__key() == other.__key()

    def __lt__(self, other):
        return self.__key() < other.__key()

    def __qt__(self, other):
        return self.__key() > other.__key()


class Decoder(json.JSONDecoder):
    def __init__(self, **kwargs):
        json.JSONDecoder.__init__(self, **kwargs)
        self.parse_array = self.JSONArray
        # Use the python implemenation of the scanner
        self.scan_once = json.scanner.py_make_scanner(self)

    def JSONArray(self, s_and_end, scan_once, **kwargs):
        values, end = json.decoder.JSONArray(s_and_end, scan_once, **kwargs)
        return set(values), end


def _add_diffoscope_info(d):
    """Add diffoscope setup metadata to provided dict under 'diffoscope' key

    The imports are broken out at stages since various versions of
    diffoscope support various parts of these.

    """
    try:
        import diffoscope
        d['diffoscope'] = hashabledict()
        d['diffoscope']['VERSION'] = diffoscope.VERSION

        from diffoscope.comparators import ComparatorManager
        ComparatorManager().reload()

        from diffoscope.tools import tool_check_installed, tool_required
        external_tools = sorted(tool_required.all)
        external_tools = [
            tool
            for tool in external_tools
            if not tool_check_installed(tool)
        ]
        d['diffoscope']['External-Tools-Required'] = tuple(external_tools)

        from diffoscope.tools import OS_NAMES, get_current_os
        from diffoscope.external_tools import EXTERNAL_TOOLS
        current_os = get_current_os()
        os_list = [current_os] if (current_os in OS_NAMES) else iter(OS_NAMES)
        for os_ in os_list:
            tools = set()
            for x in external_tools:
                try:
                    tools.add(EXTERNAL_TOOLS[x][os_])
                except KeyError:
                    pass
            d['diffoscope']['Available-in-{}-packages'.format(OS_NAMES[os_])] = tuple(sorted(tools))

        from diffoscope.tools import python_module_missing
        d['diffoscope']['Missing-Python-Modules'] = tuple(sorted(python_module_missing.modules))
    except ImportError:
        pass


def write_json_report(url, remote_apk, unsigned_apk, compare_result):
    """write out the results of the verify run to JSON

    This builds up reports on the repeated runs of `fdroid verify` on
    a set of apps.  It uses the timestamps on the compared files to
    ensure that there is only one report per file, even when run
    repeatedly.

    """

    jsonfile = unsigned_apk + '.json'
    if os.path.exists(jsonfile):
        with open(jsonfile) as fp:
            data = json.load(fp, object_pairs_hook=OrderedDict)
    else:
        data = OrderedDict()
    output = hashabledict()
    _add_diffoscope_info(output)
    output['url'] = url
    for key, filename in (('local', unsigned_apk), ('remote', remote_apk)):
        d = hashabledict()
        output[key] = d
        d['file'] = filename
        d['sha256'] = update.sha256sum(filename)
        d['timestamp'] = os.stat(filename).st_ctime
        d['packageName'], d['versionCode'], d['versionName'] = common.get_apk_id(filename)
    if compare_result:
        output['verified'] = False
        output['result'] = compare_result
    else:
        output['verified'] = True
{'fraction_non_alphanumeric': 0.09330940784414253, 'fraction_numerical': 0.0030761343245321714, 'mean_word_length': 3.022164948453608, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3269572', 'n_tokens_mistral': 2263, 'n_tokens_neox': 2174, 'n_words': 763}