<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created by <NAME>
This file is part of the CernVM File System auxiliary tools.
"""

import os
import urlparse
import tempfile
import requests
from datetime import datetime
import dateutil.parser
from dateutil.tz import tzutc

import _common
import cvmfs
from manifest import Manifest
from catalog import Catalog

class RepositoryNotFound(Exception):
    def __init__(self, repo_path):
        self.path = repo_path

    def __str__(self):
        return self.path + " not found"

class UnknownRepositoryType(Exception):
    def __init__(self, repo_fqrn, repo_type):
        self.fqrn = repo_fqrn
        self.type = repo_type

    def __str__(self):
        return self.fqrn + " (" + self.type + ")"

class ConfigurationNotFound(Exception):
    def __init__(self, repo, config_field):
        self.repo         = repo
        self.config_field = config_field

    def __str__(self):
        return repr(self.repo) + " " + self.config_field

class FileNotFoundInRepository(Exception):
    def __init__(self, repo, file_name):
        self.repo      = repo
        self.file_name = file_name

    def __str__(self):
        return repr(self.file_name)

class CannotReplicate(Exception):
    def __init__(self, repo):
        self.repo = repo

    def __str__(self):
        return repr(self.repo)



class Repository:
    """ Abstract Wrapper around a CVMFS Repository representation """
    def __init__(self):
        self._opened_catalogs = {}
        self._read_manifest()
        self._try_to_get_last_replication_timestamp()
        self._try_to_get_replication_state()


    def _read_manifest(self):
        try:
            with self.retrieve_file(_common._MANIFEST_NAME) as manifest_file:
                self.manifest = Manifest(manifest_file)
            self.fqrn = self.manifest.repository_name
        except FileNotFoundInRepository, e:
            raise RepositoryNotFound(self._storage_location)


    def __read_timestamp(self, timestamp_string):
        return dateutil.parser.parse(timestamp_string)


    def _try_to_get_last_replication_timestamp(self):
        try:
            with self.retrieve_file(_common._LAST_REPLICATION_NAME) as rf:
                timestamp = rf.readline()
                self.last_replication = self.__read_timestamp(timestamp)
            if not self.has_repository_type():
                self.type = 'stratum1'
        except FileNotFoundInRepository, e:
            self.last_replication = datetime.fromtimestamp(0, tz=tzutc())<commit_msg>make a whole repository iteratable<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created by <NAME>
This file is part of the CernVM File System auxiliary tools.
"""

import os
import urlparse
import tempfile
import requests
import collections
from datetime import datetime
import dateutil.parser
from dateutil.tz import tzutc

import _common
import cvmfs
from manifest import Manifest
from catalog import Catalog

class RepositoryNotFound(Exception):
    def __init__(self, repo_path):
        self.path = repo_path

    def __str__(self):
        return self.path + " not found"

class UnknownRepositoryType(Exception):
    def __init__(self, repo_fqrn, repo_type):
        self.fqrn = repo_fqrn
        self.type = repo_type

    def __str__(self):
        return self.fqrn + " (" + self.type + ")"

class ConfigurationNotFound(Exception):
    def __init__(self, repo, config_field):
        self.repo         = repo
        self.config_field = config_field

    def __str__(self):
        return repr(self.repo) + " " + self.config_field

class FileNotFoundInRepository(Exception):
    def __init__(self, repo, file_name):
        self.repo      = repo
        self.file_name = file_name

    def __str__(self):
        return repr(self.file_name)

class CannotReplicate(Exception):
    def __init__(self, repo):
        self.repo = repo

    def __str__(self):
        return repr(self.repo)


class RepositoryIterator:
    """ Iterates through all directory entries in a whole Repository """

    def __init__(self, repository):
        self.repository      = repository
        self.catalog_backlog = collections.deque()
        self.current_catalog = None
        self._push(repository.retrieve_root_catalog())
        self.current_catalog_iterator = self._get_next_catalog_iterator()


    def __iter__(self):
        return self


    def next(self):
        full_path, dirent = self._get_next_dirent()
        if dirent.is_nested_catalog_mountpoint():
            self._queue_next_catalog(self.current_catalog, full_path)
            return self.next() # same directory entry is also in nested catalog
        return full_path, dirent


    def _get_next_dirent(self):
        try:
            return self.current_catalog_iterator.next()
        except StopIteration, e:
            self.current_catalog_iterator = self._get_next_catalog_iterator()
            return self.current_catalog_iterator.next()


    def _get_next_catalog_iterator(self):
        if not self._has_more():
            raise StopIteration()
        self.current_catalog = self._pop()
        return self.current_catalog.__iter__()


    def _queue_next_catalog(self, catalog, catalog_mountpoint):
        nested_ref = catalog.find_nested_for_path(catalog_mountpoint)
        catalog = nested_ref.fetch_from(self.repository)
        self._push(catalog)


    def _has_more(self):
        return len(self.catalog_backlog) > 0


    def _push(self, path):
        self.catalog_backlog.append(path)


    def _pop(self):
        return self.catalog_backlog.popleft()


    def _recursion_step(self):
        path, dirent = self._pop()
        if dirent.is_directory():
            new_dirents = self.catalog.list_directory_split_md5(dirent.md5path_1, \
                                                                dirent.md5path_2)
            for new_dirent in new_dirents:
                self._push((path + "/" + new_dirent.name, new_dirent))
        return path, dirent


class Repository:
    """ Abstract Wrapper around a CVMFS Repository representation """
    def __init__(self):
        self._opened_catalogs = {}
        self._read_manifest()
        self._try_to_get_last_replication_timestamp()
        self._try_to_get_replication_state()


    def __iter__(self):
        return RepositoryIterator(self)


    def _read_manifest(self):
        try:
            with self.retrieve_file(_common._MANIFEST_NAME) as manifest_file:
                self.manifest = Manifest(manifest_file)
            self.fqrn = self.manifest.repository_name
        except FileNotFoundInRepository, e:
            raise RepositoryNotFound(self._storage_location)


    def __read_timestamp(self, timestamp_string):
        return dateutil.parser.parse(timestamp_string)


    def _try_to_get_last_replication_timestamp(self):
        try:
            with self.retrieve_file(_common._LAST_REPLICATION_NAME) as rf:
                timestamp = rf.readline()
                self.last_replication = self.__read_timestamp(timestamp)
            if not self.has_repository_type():
                self.type = 'stratum1'
        except FileNotFoundInRepository, e:
            self.last_replication = datetime.fromtimestamp(0, tz=tzutc())
{'fraction_non_alphanumeric': 0.08293285733900865, 'fraction_numerical': 0.0016476726623644103, 'mean_word_length': 2.727737973387922, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4595417', 'n_tokens_mistral': 2144, 'n_tokens_neox': 2049, 'n_words': 471}