<commit_before>#!/usr/bin/env python

"""
A simple Python module for validating BagIt profiles. See https://github.com/ruebot/bagit-profiles
for more information.

This module is intended for use with https://github.com/edsu/bagit but does not extend it.

Usage:

import bagit
import bagit_profile

# Instantiate an existing Bag using https://github.com/edsu/bagit.
bag = bagit.Bag('mydir')

# Instantiate a profile, supplying its URI.
my_profile = bagit_profile.Profile('http://example.com/bagitprofile.json')

# Validate 'Serialization' and 'Accept-Serialization'. This must be done 
# before .validate(bag) is called. 'mydir' is the path to the Bag.
if my_profile.validate_serialization('mydir'):
    print "Serialization validates"
else:
    print "Serialization does not validate"

# Validate the rest of the profile.
if my_profile.validate(bag):
    print "Validates"
else:
    print "Does not validate"

"""
import os
import sys
#import requests
import mimetypes
import logging
import optparse
import json
import urllib

# Define an exceptin class for use within this module.
class ProfileValidationError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

# Define the Profile class.
class Profile(object):
    def __init__(self, url):
        self.url = url
        self.get_profile()

    def get_profile(self):
        try:
            #r = requests.get(self.url)
            #profile = r.json
            f = urllib.urlopen(self.url)
            profile = f.read()
            profile = json.loads(profile)
            self.profile = profile
        except:
            print "Cannot retrieve profile from ", self.url
            logging.error("Cannot retrieve profile from " + self.url)
            # This is a fatal error.
            sys.exit(1) 

        self.validate_bagit_profile_info(profile)
        return self.profile

    # Call all the validate functions other than validate_bagit_profile_info(),
    # which we've already called. 'Serialization' and 'Accept-Serialization'
    #  are validated in validate_serialization().
    def validate(self, bag):
        valid = True
        try:
            self.validate_bag_info(bag)
        except ProfileValidationError as e:
            print "Error in bag-info.txt: ", e.value
            valid = False
        try:
            self.validate_manifests_required(bag)
        except ProfileValidationError as e:
            print "Required manifests not found: ", e.value
            valid = False
        try:
            self.validate_tag_manifests_required(bag)
        except ProfileValidationError as e:
            print "Required tag manifests not found: ", e.value
            valid = False
        try:
            self.validate_tag_files_required(bag)
        except ProfileValidationError as e:
            print "Required tag files not found: ", e.value
            valid = False
        try:
            self.validate_allow_fetch(bag)
        except ProfileValidationError as e:
            print "Required manifests not found: ", e.value
            valid = False
        try:
            self.validate_accept_bagit_version(bag)
        except ProfileValidationError as e:
            print "Required BagIt version not found: ", e.value
            valid = False
        return valid

    # Check self.profile['bag-profile-info'] to see if "Source-Organization", 
    # "External-Description", "Version" and "BagIt-Profile-Identifier" are present. 
    def validate_bagit_profile_info(self, profile):
        if 'Source-Organization' not in profile['BagIt-Profile-Info']:
            raise ProfileValidationError("Required 'Source-Organization' tag is not in 'BagIt-Profile-Info'.")
            logging.error(profile + "Required 'Source-Organization' tag is not in 'BagIt-Profile-Info'." + '\n')
            return False
        if 'Version' not in profile['BagIt-Profile-Info']:
            # raise ProfileValidationError("Required 'Version' tag is not in 'BagIt-Profile-Info'.")
            logging.error(profile + "Required 'Version' tag is not in 'BagIt-Profile-Info'." + '\n')
            return False
        if 'BagIt-Profile-Identifier' not in profile['BagIt-Profile-Info']:
            raise ProfileValidationError("Required 'BagIt-Profile-Identifier' tag is not in 'BagIt-Profile-Info'.")
            logging.error(profile + "Required 'BagIt-Profile-Identifier' tag is not in 'BagIt-Profile-Info'." + '\n')
            return False
        return True

    # Validate tags in self.profile['Bag-Info']. 
    def validate_bag_info(self, bag):
        # First, check to see if bag-info.txt exists.
        path_to_baginfotxt = os.path.join(bag.path, 'bag-info.txt')
        if not os.path.exists(path_to_baginfotxt):
            raise ProfileValidationError("bag-info.txt is not present.")
            logging.error(bag + "bag-info.txt is not present." + '\n')
            return False
        # Then check for the required 'BagIt-Profile-Identifier' tag and ensure it has the same value
        # as self.url.
        if 'BagIt-Profile-Identifier' not in bag.info:
            raise ProfileValidationError("Required 'BagIt-Profile-Identifier' tag is not in bag-info.txt.")
            logging.error(bag + "Required 'BagIt-Profile-Identifier' tag is not in bag-info.txt." + '\n')
            return False
        else:
            if bag.info['BagIt-Profile-Identifier'] != self.url:
                raise ProfileValidationError("'BagIt-Profile-Identifier' tag does not contain this profile's URI.")
                logging.error(bag + "'BagIt-Profile-Identifier' tag does not contain this profile's URI." + '\n')
        # Then, iterate through self.profile['Bag-Info'] and if a key has a dict containing a 'required' key that is
        # True, check to see if that key exists in bag.info. 
        for tag in self.profile['Bag-Info']:
            config = self.profile['Bag-Info'][tag]
            if 'required' in config and config['required'] is True: 
                if tag not in bag.info:
                    raise ProfileValidationError("Required tag '%s' is not present in bag-info.txt." + tag)
                    logging.error(bag + "Required tag '%s' is not present in bag-info.txt." + tag + '\n')
                # If the tag is in bag-info.txt, check to see if the value is constrained.
                else:
                    if 'values' in config: 
                        if bag.info[tag] not in config['values']:
                            raise ProfileValidationError("Required tag '%s' is present in bag-info.txt but does not have an allowed value ('%s')." % (tag, bag.info[tag]))
                            logging.error(bag + "Required tag '%s' is present in bag-info.txt but does not have an allowed value ('%s')." + tag + bag.info[tag] + '\n')

        return True

    # For each member of self.profile['manifests_required'], throw an exception if 
    # the manifest file is not present.
    def validate_manifests_required(self, bag):
        for manifest_type in self.profile['Manifests-Required']:
            path_to_manifest = os.path.join(bag.path, 'manifest-' + manifest_type + '.txt')
            if not os.path.exists(path_to_manifest):
                raise ProfileValidationError("Required manifest type '%s' is not present in Bag." + manifest_type)
                logging.error(bag + "Required manifest type '%s' is not present in Bag." +  manifest_type + '\n')
        return True

    # For each member of self.profile['tag_manifests_required'], throw an exception if 
    # the tag manifest file is not present.
    def validate_tag_manifests_required(self, bag):
        # Tag manifests are optional, so we return True if none are defined in the profile.
        if not self.profile.has_key('Tag-Manifests-Required'):
            return True
        for tag_manifest_type in self.profile['Tag-Manifests-Required']:
            path_to_tag_manifest = os.path.join(bag.path, 'tagmanifest-' + tag_manifest_type + '.txt')
            if not os.path.exists(path_to_tag_manifest):
                raise ProfileValidationError("Required tag manifest type '%s' is not present in Bag." % tag_manifest_type)
                logging.error(bag + "Required tag manifest type '%s' is not present in Bag." %  tag_manifest_type + '\n')
        return True

    # For each member of self.profile['Tag-Files-Required'], throw an exception if 
    # the path does not exist.
    def validate_tag_files_required(self, bag):
        # Tag files are optional, so we return True if none are defined in the profile.
        if not self.profile.has_key('Tag-Files-Required'):
            return True
        for tag_file in self.profile['Tag-Files-Required']:
            path_to_tag_file = os.path.join(bag.path, tag_file)
            if not os.path.exists(path_to_tag_file):
                raise ProfileValidationError("Required tag file '%s' is not present in Bag." % path_to_tag_file)
                logging.error(bag + "Required tag file '%s' is not present in Bag." %  path_to_tag_file + '\n')
        return True

    # Check to see if this constraint is False, and if it is, then check to see
    # if the fetch.txt file exists. If it does, throw an exception.
    def validate_allow_fetch(self, bag):
        if self.profile['Allow-Fetch.txt'] is False: 
            path_to_fetchtxt = os.path.join(bag.path, 'fetch.txt')
            if os.path.exists(path_to_fetchtxt):
                raise ProfileValidationError("Fetch.txt is present but is not allowed.")
                logging.error(bag + "Fetch.txt is present but is not allowed." + '\n')
        return True

    # Check the Bag's version, and if it's not in the list of allowed versions,<commit_msg>Fixed some problems with variable interpolation in error messages.<commit_after>#!/usr/bin/env python

"""
A simple Python module for validating BagIt profiles. See https://github.com/ruebot/bagit-profiles
for more information.

This module is intended for use with https://github.com/edsu/bagit but does not extend it.

Usage:

import bagit
import bagit_profile

# Instantiate an existing Bag using https://github.com/edsu/bagit.
bag = bagit.Bag('mydir')

# Instantiate a profile, supplying its URI.
my_profile = bagit_profile.Profile('http://example.com/bagitprofile.json')

# Validate 'Serialization' and 'Accept-Serialization'. This must be done 
# before .validate(bag) is called. 'mydir' is the path to the Bag.
if my_profile.validate_serialization('mydir'):
    print "Serialization validates"
else:
    print "Serialization does not validate"

# Validate the rest of the profile.
if my_profile.validate(bag):
    print "Validates"
else:
    print "Does not validate"

"""
import os
import sys
#import requests
import mimetypes
import logging
import optparse
import json
import urllib

# Define an exceptin class for use within this module.
class ProfileValidationError(Exception):
    def __init__(self, value):
        self.value = value
    def __str__(self):
        return repr(self.value)

# Define the Profile class.
class Profile(object):
    def __init__(self, url):
        self.url = url
        self.get_profile()

    def get_profile(self):
        try:
            #r = requests.get(self.url)
            #profile = r.json
            f = urllib.urlopen(self.url)
            profile = f.read()
            profile = json.loads(profile)
            self.profile = profile
        except:
            print "Cannot retrieve profile from ", self.url
            logging.error("Cannot retrieve profile from " + self.url)
            # This is a fatal error.
            sys.exit(1) 

        self.validate_bagit_profile_info(profile)
        return self.profile

    # Call all the validate functions other than validate_bagit_profile_info(),
    # which we've already called. 'Serialization' and 'Accept-Serialization'
    #  are validated in validate_serialization().
    def validate(self, bag):
        valid = True
        try:
            self.validate_bag_info(bag)
        except ProfileValidationError as e:
            print "Error in bag-info.txt: ", e.value
            valid = False
        try:
            self.validate_manifests_required(bag)
        except ProfileValidationError as e:
            print "Required manifests not found: ", e.value
            valid = False
        try:
            self.validate_tag_manifests_required(bag)
        except ProfileValidationError as e:
            print "Required tag manifests not found: ", e.value
            valid = False
        try:
            self.validate_tag_files_required(bag)
        except ProfileValidationError as e:
            print "Required tag files not found: ", e.value
            valid = False
        try:
            self.validate_allow_fetch(bag)
        except ProfileValidationError as e:
            print "Required manifests not found: ", e.value
            valid = False
        try:
            self.validate_accept_bagit_version(bag)
        except ProfileValidationError as e:
            print "Required BagIt version not found: ", e.value
            valid = False
        return valid

    # Check self.profile['bag-profile-info'] to see if "Source-Organization", 
    # "External-Description", "Version" and "BagIt-Profile-Identifier" are present. 
    def validate_bagit_profile_info(self, profile):
        if 'Source-Organization' not in profile['BagIt-Profile-Info']:
            raise ProfileValidationError("Required 'Source-Organization' tag is not in 'BagIt-Profile-Info'.")
            logging.error(profile + "Required 'Source-Organization' tag is not in 'BagIt-Profile-Info'." + '\n')
            return False
        if 'Version' not in profile['BagIt-Profile-Info']:
            # raise ProfileValidationError("Required 'Version' tag is not in 'BagIt-Profile-Info'.")
            logging.error(profile + "Required 'Version' tag is not in 'BagIt-Profile-Info'." + '\n')
            return False
        if 'BagIt-Profile-Identifier' not in profile['BagIt-Profile-Info']:
            raise ProfileValidationError("Required 'BagIt-Profile-Identifier' tag is not in 'BagIt-Profile-Info'.")
            logging.error(profile + "Required 'BagIt-Profile-Identifier' tag is not in 'BagIt-Profile-Info'." + '\n')
            return False
        return True

    # Validate tags in self.profile['Bag-Info']. 
    def validate_bag_info(self, bag):
        # First, check to see if bag-info.txt exists.
        path_to_baginfotxt = os.path.join(bag.path, 'bag-info.txt')
        if not os.path.exists(path_to_baginfotxt):
            raise ProfileValidationError("bag-info.txt is not present.")
            logging.error(bag + "bag-info.txt is not present." + '\n')
            return False
        # Then check for the required 'BagIt-Profile-Identifier' tag and ensure it has the same value
        # as self.url.
        if 'BagIt-Profile-Identifier' not in bag.info:
            raise ProfileValidationError("Required 'BagIt-Profile-Identifier' tag is not in bag-info.txt.")
            logging.error(bag + "Required 'BagIt-Profile-Identifier' tag is not in bag-info.txt." + '\n')
            return False
        else:
            if bag.info['BagIt-Profile-Identifier'] != self.url:
                raise ProfileValidationError("'BagIt-Profile-Identifier' tag does not contain this profile's URI.")
                logging.error(bag + "'BagIt-Profile-Identifier' tag does not contain this profile's URI." + '\n')
        # Then, iterate through self.profile['Bag-Info'] and if a key has a dict containing a 'required' key that is
        # True, check to see if that key exists in bag.info. 
        for tag in self.profile['Bag-Info']:
            config = self.profile['Bag-Info'][tag]
            if 'required' in config and config['required'] is True: 
                if tag not in bag.info:
                    raise ProfileValidationError("Required tag '%s' is not present in bag-info.txt." % (tag))
                    logging.error(bag + "Required tag '%s' is not present in bag-info.txt." % (tag) + '\n')
                # If the tag is in bag-info.txt, check to see if the value is constrained.
                else:
                    if 'values' in config: 
                        if bag.info[tag] not in config['values']:
                            raise ProfileValidationError("Required tag '%s' is present in bag-info.txt but does not have an allowed value ('%s')." % (tag, bag.info[tag]))
                            logging.error(bag + "Required tag '%s' is present in bag-info.txt but does not have an allowed value ('%s')." % (tag, bag.info[tag]) + '\n')

        return True

    # For each member of self.profile['manifests_required'], throw an exception if 
    # the manifest file is not present.
    def validate_manifests_required(self, bag):
        for manifest_type in self.profile['Manifests-Required']:
            path_to_manifest = os.path.join(bag.path, 'manifest-' + manifest_type + '.txt')
            if not os.path.exists(path_to_manifest):
                raise ProfileValidationError("Required manifest type '%s' is not present in Bag." % (manifest_type))
                logging.error(bag + "Required manifest type '%s' is not present in Bag." %  (manifest_type) + '\n')
        return True

    # For each member of self.profile['tag_manifests_required'], throw an exception if 
    # the tag manifest file is not present.
    def validate_tag_manifests_required(self, bag):
        # Tag manifests are optional, so we return True if none are defined in the profile.
        if not self.profile.has_key('Tag-Manifests-Required'):
            return True
        for tag_manifest_type in self.profile['Tag-Manifests-Required']:
            path_to_tag_manifest = os.path.join(bag.path, 'tagmanifest-' + tag_manifest_type + '.txt')
            if not os.path.exists(path_to_tag_manifest):
                raise ProfileValidationError("Required tag manifest type '%s' is not present in Bag." % (tag_manifest_type))
                logging.error(bag + "Required tag manifest type '%s' is not present in Bag." % (tag_manifest_type) + '\n')
        return True

    # For each member of self.profile['Tag-Files-Required'], throw an exception if 
    # the path does not exist.
    def validate_tag_files_required(self, bag):
        # Tag files are optional, so we return True if none are defined in the profile.
        if not self.profile.has_key('Tag-Files-Required'):
            return True
        for tag_file in self.profile['Tag-Files-Required']:
            path_to_tag_file = os.path.join(bag.path, tag_file)
            if not os.path.exists(path_to_tag_file):
                raise ProfileValidationError("Required tag file '%s' is not present in Bag." % (path_to_tag_file))
                logging.error(bag + "Required tag file '%s' is not present in Bag." % (path_to_tag_file) + '\n')
        return True

    # Check to see if this constraint is False, and if it is, then check to see
    # if the fetch.txt file exists. If it does, throw an exception.
    def validate_allow_fetch(self, bag):
        if self.profile['Allow-Fetch.txt'] is False: 
            path_to_fetchtxt = os.path.join(bag.path, 'fetch.txt')
            if os.path.exists(path_to_fetchtxt):
                raise ProfileValidationError("Fetch.txt is present but is not allowed.")
                logging.error(bag + "Fetch.txt is present but is not allowed." + '\n')
        return True

    # Check the Bag's version, and if it's not in the list of allowed versions,
{'fraction_non_alphanumeric': 0.0980802614112121, 'fraction_numerical': 0.00010211375472276115, 'mean_word_length': 2.445382585751979, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2434590', 'n_tokens_mistral': 5391, 'n_tokens_neox': 5253, 'n_words': 1773}