<commit_before>#!/usr/bin/env python3
#
# update.py - part of the FDroid server tools
# Copyright (C) 2016, Blue Jay Wireless
# Copyright (C) 2014-2016, <NAME> <<EMAIL>>
# Copyright (C) 2010-2015, <NAME> <<EMAIL>>
# Copyright (C) 2013-2014, <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import shutil
import glob
import re
import socket
import zipfile
import hashlib
import pickle
import time
from datetime import datetime
from argparse import ArgumentParser

import collections
from binascii import hexlify

from PIL import Image, PngImagePlugin
import logging

from . import _
from . import common
from . import index
from . import metadata
from .common import SdkToolsPopen
from .exception import BuildException, FDroidException

METADATA_VERSION = 19

# less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
UNSET_VERSION_CODE = -0x100000000

APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
APK_PERMISSION_PAT = \
    re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")

screen_densities = ['640', '480', '320', '240', '160', '120']
screen_resolutions = {
    "xxxhdpi": '640',
    "xxhdpi": '480',
    "xhdpi": '320',
    "hdpi": '240',
    "mdpi": '160',
    "ldpi": '120',
    "undefined": '-1',
    "anydpi": '65534',
    "nodpi": '65535'
}

all_screen_densities = ['0'] + screen_densities

UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])

ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
                   'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')

BLANK_PNG_INFO = PngImagePlugin.PngInfo()


def dpi_to_px(density):
    return (int(density) * 48) / 160


def px_to_dpi(px):
    return (int(px) * 160) / 48


def get_icon_dir(repodir, density):
    if density == '0':
        return os.path.join(repodir, "icons")
    return os.path.join(repodir, "icons-%s" % density)


def get_icon_dirs(repodir):
    for density in screen_densities:
        yield get_icon_dir(repodir, density)


def get_all_icon_dirs(repodir):
    for density in all_screen_densities:
        yield get_icon_dir(repodir, density)


def update_wiki(apps, sortedids, apks):
    """Update the wiki

    :param apps: fully populated list of all applications
    :param apks: all apks, except...
    """
    logging.info("Updating wiki")
    wikicat = 'Apps'
    wikiredircat = 'App Redirects'
    import mwclient
    site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
                         path=config['wiki_path'])
    site.login(config['wiki_user'], config['wiki_password'])
    generated_pages = {}
    generated_redirects = {}

    for appid in sortedids:
        app = metadata.App(apps[appid])

        wikidata = ''
        if app.Disabled:
            wikidata += '{{Disabled|' + app.Disabled + '}}\n'
        if app.AntiFeatures:
            for af in sorted(app.AntiFeatures):
                wikidata += '{{AntiFeature|' + af + '}}\n'
        if app.RequiresRoot:
            requiresroot = 'Yes'
        else:
            requiresroot = 'No'
        wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
            appid,
            app.Name,
            app.added.strftime('%Y-%m-%d') if app.added else '',
            app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
            app.SourceCode,
            app.IssueTracker,
            app.WebSite,
            app.Changelog,
            app.Donate,
            app.FlattrID,
            app.LiberapayID,
            app.Bitcoin,
            app.Litecoin,
            app.License,
            requiresroot,
            app.AuthorName,
            app.AuthorEmail)

        if app.Provides:
            wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))

        wikidata += app.Summary
        wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"

        wikidata += "=Description=\n"
        wikidata += metadata.description_wiki(app.Description) + "\n"

        wikidata += "=Maintainer Notes=\n"
        if app.MaintainerNotes:
            wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
        wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)

        # Get a list of all packages for this application...
        apklist = []
        gotcurrentver = False
        cantupdate = False
        buildfails = False
        for apk in apks:
            if apk['packageName'] == appid:
                if str(apk['versionCode']) == app.CurrentVersionCode:
                    gotcurrentver = True
                apklist.append(apk)
        # Include ones we can't build, as a special case...
        for build in app.builds:<commit_msg>wiki: include per-app link to all related activity on gitlab.com<commit_after>#!/usr/bin/env python3
#
# update.py - part of the FDroid server tools
# Copyright (C) 2016, Blue Jay Wireless
# Copyright (C) 2014-2016, <NAME> <<EMAIL>>
# Copyright (C) 2010-2015, <NAME> <<EMAIL>>
# Copyright (C) 2013-2014, <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import shutil
import glob
import re
import socket
import zipfile
import hashlib
import pickle
import time
from datetime import datetime
from argparse import ArgumentParser

import collections
from binascii import hexlify

from PIL import Image, PngImagePlugin
import logging

from . import _
from . import common
from . import index
from . import metadata
from .common import SdkToolsPopen
from .exception import BuildException, FDroidException

METADATA_VERSION = 19

# less than the valid range of versionCode, i.e. Java's Integer.MIN_VALUE
UNSET_VERSION_CODE = -0x100000000

APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
APK_PERMISSION_PAT = \
    re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")

screen_densities = ['640', '480', '320', '240', '160', '120']
screen_resolutions = {
    "xxxhdpi": '640',
    "xxhdpi": '480',
    "xhdpi": '320',
    "hdpi": '240',
    "mdpi": '160',
    "ldpi": '120',
    "undefined": '-1',
    "anydpi": '65534',
    "nodpi": '65535'
}

all_screen_densities = ['0'] + screen_densities

UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])

ALLOWED_EXTENSIONS = ('png', 'jpg', 'jpeg')
GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
                   'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')

BLANK_PNG_INFO = PngImagePlugin.PngInfo()


def dpi_to_px(density):
    return (int(density) * 48) / 160


def px_to_dpi(px):
    return (int(px) * 160) / 48


def get_icon_dir(repodir, density):
    if density == '0':
        return os.path.join(repodir, "icons")
    return os.path.join(repodir, "icons-%s" % density)


def get_icon_dirs(repodir):
    for density in screen_densities:
        yield get_icon_dir(repodir, density)


def get_all_icon_dirs(repodir):
    for density in all_screen_densities:
        yield get_icon_dir(repodir, density)


def update_wiki(apps, sortedids, apks):
    """Update the wiki

    :param apps: fully populated list of all applications
    :param apks: all apks, except...
    """
    logging.info("Updating wiki")
    wikicat = 'Apps'
    wikiredircat = 'App Redirects'
    import mwclient
    site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
                         path=config['wiki_path'])
    site.login(config['wiki_user'], config['wiki_password'])
    generated_pages = {}
    generated_redirects = {}

    for appid in sortedids:
        app = metadata.App(apps[appid])

        wikidata = ''
        if app.Disabled:
            wikidata += '{{Disabled|' + app.Disabled + '}}\n'
        if app.AntiFeatures:
            for af in sorted(app.AntiFeatures):
                wikidata += '{{AntiFeature|' + af + '}}\n'
        if app.RequiresRoot:
            requiresroot = 'Yes'
        else:
            requiresroot = 'No'
        wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s|activity=%s}}\n' % (
            appid,
            app.Name,
            app.added.strftime('%Y-%m-%d') if app.added else '',
            app.lastUpdated.strftime('%Y-%m-%d') if app.lastUpdated else '',
            app.SourceCode,
            app.IssueTracker,
            app.WebSite,
            app.Changelog,
            app.Donate,
            app.FlattrID,
            app.LiberapayID,
            app.Bitcoin,
            app.Litecoin,
            app.License,
            requiresroot,
            app.AuthorName,
            app.AuthorEmail,
            'https://gitlab.com/search?group_id=28397&scope=issues&search=' + appid,
        )

        if app.Provides:
            wikidata += "This app provides: %s" % ', '.join(app.Summary.split(','))

        wikidata += app.Summary
        wikidata += " - [https://f-droid.org/repository/browse/?fdid=" + appid + " view in repository]\n\n"

        wikidata += "=Description=\n"
        wikidata += metadata.description_wiki(app.Description) + "\n"

        wikidata += "=Maintainer Notes=\n"
        if app.MaintainerNotes:
            wikidata += metadata.description_wiki(app.MaintainerNotes) + "\n"
        wikidata += "\nMetadata: [https://gitlab.com/fdroid/fdroiddata/blob/master/metadata/{0}.txt current] [https://gitlab.com/fdroid/fdroiddata/commits/master/metadata/{0}.txt history]\n".format(appid)

        # Get a list of all packages for this application...
        apklist = []
        gotcurrentver = False
        cantupdate = False
        buildfails = False
        for apk in apks:
            if apk['packageName'] == appid:
                if str(apk['versionCode']) == app.CurrentVersionCode:
                    gotcurrentver = True
                apklist.append(apk)
        # Include ones we can't build, as a special case...
        for build in app.builds:
{'fraction_non_alphanumeric': 0.14974101396954953, 'fraction_numerical': 0.018442944592685608, 'mean_word_length': 3.10006435006435, 'pattern_counts': {'":': 18, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 7, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5287440', 'n_tokens_mistral': 4487, 'n_tokens_neox': 4157, 'n_words': 994}