<commit_before># -*- coding: utf-8 -*-
#
# metadata.py - part of the FDroid server tools
# Copyright (C) 2013, <NAME>, <EMAIL>
# Copyright (C) 2013-2014 <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json
import os
import re
import glob
import cgi
import textwrap

try:
    from cStringIO import StringIO
except:
    from StringIO import StringIO

import yaml
# use libyaml if it is available
try:
    from yaml import CLoader
    YamlLoader = CLoader
except ImportError:
    from yaml import Loader
    YamlLoader = Loader

# use the C implementation when available
import xml.etree.cElementTree as ElementTree

import common

srclibs = None


class MetaDataException(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return self.value

# To filter which ones should be written to the metadata files if
# present
app_fields = set([
    'Disabled',
    'AntiFeatures',
    'Provides',
    'Categories',
    'License',
    'Web Site',
    'Source Code',
    'Issue Tracker',
    'Changelog',
    'Donate',
    'FlattrID',
    'Bitcoin',
    'Litecoin',
    'Name',
    'Auto Name',
    'Summary',
    'Description',
    'Requires Root',
    'Repo Type',
    'Repo',
    'Binaries',
    'Maintainer Notes',
    'Archive Policy',
    'Auto Update Mode',
    'Update Check Mode',
    'Update Check Ignore',
    'Vercode Operation',
    'Update Check Name',
    'Update Check Data',
    'Current Version',
    'Current Version Code',
    'No Source Since',

    'comments',  # For formats that don't do inline comments
    'builds',    # For formats that do builds as a list
])


class App():

    def __init__(self):
        self.Disabled = None
        self.AntiFeatures = []
        self.Provides = None
        self.Categories = ['None']
        self.License = 'Unknown'
        self.WebSite = ''
        self.SourceCode = ''
        self.IssueTracker = ''
        self.Changelog = ''
        self.Donate = None
        self.FlattrID = None
        self.Bitcoin = None
        self.Litecoin = None
        self.Name = None
        self.AutoName = ''
        self.Summary = ''
        self.Description = ''
        self.RequiresRoot = False
        self.RepoType = ''
        self.Repo = ''
        self.Binaries = None
        self.MaintainerNotes = ''
        self.ArchivePolicy = None
        self.AutoUpdateMode = 'None'
        self.UpdateCheckMode = 'None'
        self.UpdateCheckIgnore = None
        self.VercodeOperation = None
        self.UpdateCheckName = None
        self.UpdateCheckData = None
        self.CurrentVersion = ''
        self.CurrentVersionCode = '0'
        self.NoSourceSince = ''

        self.id = None
        self.metadatapath = None
        self.builds = []
        self.comments = {}
        self.added = None
        self.lastupdated = None
        self._modified = set()

    # Translates human-readable field names to attribute names, e.g.
    # 'Auto Name' to 'AutoName'
    @classmethod
    def field_to_attr(cls, f):
        return f.replace(' ', '')

    # Translates attribute names to human-readable field names, e.g.
    # 'AutoName' to 'Auto Name'
    @classmethod
    def attr_to_field(cls, k):
        if k in app_fields:
            return k
        f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
        return f

    # Constructs an old-fashioned dict with the human-readable field
    # names. Should only be used for tests.
    def field_dict(self):
        d = {}
        for k, v in self.__dict__.iteritems():
            if k == 'builds':
                d['builds'] = []
                for build in v:
                    d['builds'].append(build.__dict__)
            else:
                k = App.attr_to_field(k)
                d[k] = v
        return d

    # Gets the value associated to a field name, e.g. 'Auto Name'
    def get_field(self, f):
        if f not in app_fields:
            raise MetaDataException('Unrecognised app field: ' + f)
        k = App.field_to_attr(f)
        return getattr(self, k)

    # Sets the value associated to a field name, e.g. 'Auto Name'
    def set_field(self, f, v):
        if f not in app_fields:
            raise MetaDataException('Unrecognised app field: ' + f)
        k = App.field_to_attr(f)
        self.__dict__[k] = v
        self._modified.add(k)

    # Appends to the value associated to a field name, e.g. 'Auto Name'
    def append_field(self, f, v):
        if f not in app_fields:
            raise MetaDataException('Unrecognised app field: ' + f)
        k = App.field_to_attr(f)
        if k not in self.__dict__:
            self.__dict__[k] = [v]
        else:
            self.__dict__[k].append(v)

    # Like dict.update(), but using human-readable field names
    def update_fields(self, d):<commit_msg>Fix tests by not checking private attributes<commit_after># -*- coding: utf-8 -*-
#
# metadata.py - part of the FDroid server tools
# Copyright (C) 2013, <NAME>, <EMAIL>
# Copyright (C) 2013-2014 <NAME> <<EMAIL>>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import json
import os
import re
import glob
import cgi
import textwrap

try:
    from cStringIO import StringIO
except:
    from StringIO import StringIO

import yaml
# use libyaml if it is available
try:
    from yaml import CLoader
    YamlLoader = CLoader
except ImportError:
    from yaml import Loader
    YamlLoader = Loader

# use the C implementation when available
import xml.etree.cElementTree as ElementTree

import common

srclibs = None


class MetaDataException(Exception):

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return self.value

# To filter which ones should be written to the metadata files if
# present
app_fields = set([
    'Disabled',
    'AntiFeatures',
    'Provides',
    'Categories',
    'License',
    'Web Site',
    'Source Code',
    'Issue Tracker',
    'Changelog',
    'Donate',
    'FlattrID',
    'Bitcoin',
    'Litecoin',
    'Name',
    'Auto Name',
    'Summary',
    'Description',
    'Requires Root',
    'Repo Type',
    'Repo',
    'Binaries',
    'Maintainer Notes',
    'Archive Policy',
    'Auto Update Mode',
    'Update Check Mode',
    'Update Check Ignore',
    'Vercode Operation',
    'Update Check Name',
    'Update Check Data',
    'Current Version',
    'Current Version Code',
    'No Source Since',

    'comments',  # For formats that don't do inline comments
    'builds',    # For formats that do builds as a list
])


class App():

    def __init__(self):
        self.Disabled = None
        self.AntiFeatures = []
        self.Provides = None
        self.Categories = ['None']
        self.License = 'Unknown'
        self.WebSite = ''
        self.SourceCode = ''
        self.IssueTracker = ''
        self.Changelog = ''
        self.Donate = None
        self.FlattrID = None
        self.Bitcoin = None
        self.Litecoin = None
        self.Name = None
        self.AutoName = ''
        self.Summary = ''
        self.Description = ''
        self.RequiresRoot = False
        self.RepoType = ''
        self.Repo = ''
        self.Binaries = None
        self.MaintainerNotes = ''
        self.ArchivePolicy = None
        self.AutoUpdateMode = 'None'
        self.UpdateCheckMode = 'None'
        self.UpdateCheckIgnore = None
        self.VercodeOperation = None
        self.UpdateCheckName = None
        self.UpdateCheckData = None
        self.CurrentVersion = ''
        self.CurrentVersionCode = '0'
        self.NoSourceSince = ''

        self.id = None
        self.metadatapath = None
        self.builds = []
        self.comments = {}
        self.added = None
        self.lastupdated = None
        self._modified = set()

    # Translates human-readable field names to attribute names, e.g.
    # 'Auto Name' to 'AutoName'
    @classmethod
    def field_to_attr(cls, f):
        return f.replace(' ', '')

    # Translates attribute names to human-readable field names, e.g.
    # 'AutoName' to 'Auto Name'
    @classmethod
    def attr_to_field(cls, k):
        if k in app_fields:
            return k
        f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
        return f

    # Constructs an old-fashioned dict with the human-readable field
    # names. Should only be used for tests.
    def field_dict(self):
        d = {}
        for k, v in self.__dict__.iteritems():
            if k == 'builds':
                d['builds'] = []
                for build in v:
                    b = {k: v for k, v in build.__dict__.iteritems() if not k.startswith('_')}
                    d['builds'].append(b)
            elif not k.startswith('_'):
                f = App.attr_to_field(k)
                d[f] = v
        return d

    # Gets the value associated to a field name, e.g. 'Auto Name'
    def get_field(self, f):
        if f not in app_fields:
            raise MetaDataException('Unrecognised app field: ' + f)
        k = App.field_to_attr(f)
        return getattr(self, k)

    # Sets the value associated to a field name, e.g. 'Auto Name'
    def set_field(self, f, v):
        if f not in app_fields:
            raise MetaDataException('Unrecognised app field: ' + f)
        k = App.field_to_attr(f)
        self.__dict__[k] = v
        self._modified.add(k)

    # Appends to the value associated to a field name, e.g. 'Auto Name'
    def append_field(self, f, v):
        if f not in app_fields:
            raise MetaDataException('Unrecognised app field: ' + f)
        k = App.field_to_attr(f)
        if k not in self.__dict__:
            self.__dict__[k] = [v]
        else:
            self.__dict__[k].append(v)

    # Like dict.update(), but using human-readable field names
    def update_fields(self, d):
{'fraction_non_alphanumeric': 0.1007561264462057, 'fraction_numerical': 0.0030973854422884214, 'mean_word_length': 2.2137002341920375, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '779093', 'n_tokens_mistral': 3317, 'n_tokens_neox': 3231, 'n_words': 1096}