<commit_before>// ==========================================================================
//                 SeqAn - The Library for Sequence Analysis
// ==========================================================================
// Copyright (c) 2006-2010, <NAME>, FU Berlin
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of <NAME> or the FU Berlin nor the names of
//       its contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL <NAME> OR THE FU BERLIN BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
// DAMAGE.
//
// ==========================================================================

#ifndef SEQAN_HEADER_STRING_EXTERNAL_H
#define SEQAN_HEADER_STRING_EXTERNAL_H

/* IOREV
 * _nottested_
 * _doc_
 *
 *
 * mostly documented (doc for some functions missing)
 * not tested in any test case nor used in any app right now
 * -> needs testing, especially different iterators
 */



//////////////////////////////////////////////////////////////////////////////

namespace SEQAN_NAMESPACE_MAIN
{

/**
.Spec.External String:
..cat:Strings
..general:Class.String
..summary:String that is stored in external memory.
..signature:String<TValue, External<> >
..signature:String<TValue, External<TConfig> >
..param.TValue:The value type, that is the type of the items/characters stored in the string.
...remarks:Use @Metafunction.Value@ to get the value type for a given class.
..param.TConfig:A structure to configure the external string.
...type:Tag.ExternalConfig
...type:Tag.ExternalConfigLarge
...type:Tag.ExternalConfigSize
...default:@Tag.ExternalConfig@
..remarks:The External String enables to access sequences larger than the available internal memory (RAM)
by using external memory (e.g. Hard disk, Network storage, ...) via a @Class.File@ object.
Sequences of nearly arbitrary size can be accessed even larger than the logically addressable memory,
i.e. they can in particular contain more than 2^32 elements on a 32bit system (see Tag.ExternalConfigLarge).
See the @Memfunc.External String#String@ constructor for more details.
..remarks:This String also supports fast appending and removing of values at the end (see @Spec.Block String@, @Function.appendValue@)
..remarks:The External String implements a LRU mechanism to swap out pages.
The External String's @Metafunction.Iterator@ detects a forward or backward iteration and asynchronously prefetches pages that
certainly will be accessed and automatically swaps out pages that certainly won't be accessed any more in the iteration
process.
..remarks:The String is implemented like a virtual memory manager.
It divides its character sequence into pages of a fixed length (e.g. 1MB) and maintains a
page table with information for each page (e.g. resides in memory or was swapped out, is dirty and needs to be saved, ...).
Besides the page table the String also contains a size-limited list of page frames. A page frame is reserved internal
memory for a page. When accessing values of a page that is stored in external memory, the page is loaded to a page frame
first. In case that there is no page frame free, another page is swapped out before to free a page frame.
..include:seqan/file.h
*/

/**
.Tag.ExternalConfig:
..cat:Strings
..summary:Standard configuration for the @Spec.External String@.
..signature:String<TValue, External< ExternalConfig<> > >
..signature:String<TValue, External< ExternalConfig<TFile[, pageSize[, frames]]> > >
..param.TFile:Type of file the External String will be based on.
...type:Class.File
..param.pageSize:A positive integer that specifies the number of values in one page.
...remarks:This should be a power of 2, to speed up transfer and calculations.
...default:2^20
..param.frames:A positive integer that specifies the maximum number of pages that should reside in internal memory.
...remarks:To enable prefetching and automatic swap-out, $frames$ should be greater than 1.
...default:2
..remarks:When using this configuration, the @Metafunction.Size@ type of the @Spec.External String@ is $unsigned int$.
Thus, with this configuration at most 4.294.967.296 values can be stored in an @Spec.External String@ on a 32bit system.
For a larger size type, use @Tag.ExternalConfigLarge@.
..include:seqan/file.h
*/
    // standard external string
    // size is uint32
    template < typename TFile_ = File<>,				// default file type
               unsigned PAGE_SIZE_ = 4 * 1024 * 1024,	// 1MTypes per default
			   unsigned FRAMES_ = 2 >					// simultanous frames
    struct ExternalConfig {
//IOREV _bug_ doc says default page size is 2^20, but it is 2^22
        typedef TFile_ TFile;
        typedef unsigned TSize;
        enum { PAGE_SIZE = PAGE_SIZE_ };
        enum { FRAMES = FRAMES_ };
    };

/**
.Tag.ExternalConfigLarge:
..cat:Strings
..summary:Large size type configuration for the @Spec.External String@.
..signature:String<TValue, External< ExternalConfigLarge<> > >
..signature:String<TValue, External< ExternalConfigLarge<TFile[, pageSize[, frames]]> > >
..param.TFile:Type of file the External String will be based on.
...type:Class.File
..param.pageSize:A positive integer that specifies the number of values in one page.
...remarks:This should be a power of 2, to speed up transfer and calculations.
...default:2^20
..param.frames:A positive integer that specifies the maximum number of pages that should reside in internal memory.
...remarks:To enable prefetching and automatic swap-out, $frames$ should be greater than 1.
...default:2
..remarks:When using this configuration, the @Metafunction.Size@ type of the @Spec.External String@ is the @Metafunction.Size@
type of $TFile$. Normally this is a 64bit integer. For a smaller size type, use @Tag.ExternalConfig@.
..remarks:Some data structures store size type values (e.g. suffix arrays in @Class.Index@). To save memory,
you should think of using @Tag.ExternalConfig@.
..include:seqan/file.h
*/
    // the same as ExternalConfig
    // but size type is size type of TFile_ (i.e. uint64)
    //
    // ATTENTION:
    // pipes use the size type 
    // uint64 blows up your suffix arrays, lcp-tables, ...
    template < typename TFile_ = File<>,				// default file type
               unsigned PAGE_SIZE_ = 1 * 1024 * 1024,	// 1MTypes per default
			   unsigned FRAMES_ = 2 >					// simultanous frames
    struct ExternalConfigLarge {
//IOREV contains warning in code comments, need to investigate
        typedef TFile_ TFile;
        typedef typename Size<TFile_>::Type TSize;
        enum { PAGE_SIZE = PAGE_SIZE_ };
        enum { FRAMES = FRAMES_ };
    };

/**
.Tag.ExternalConfigSize:
..cat:Strings
..summary:Arbitrary size type configuration for the @Spec.External String@.
..signature:String<TValue, External< ExternalConfigSize< TSize > > >
..signature:String<TValue, External< ExternalConfigSize< TSize, TFile[, pageSize[, frames]]> > >
..param.TSize:Size type the External String will return via @Metafunction.Size@.
..param.TFile:Type of file the External String will be based on.
...type:Class.File
..param.pageSize:A positive integer that specifies the number of values in one page.
...remarks:This should be a power of 2, to speed up transfer and calculations.
...default:2^20
..param.frames:A positive integer that specifies the maximum number of pages that should reside in internal memory.
...remarks:To enable prefetching and automatic swap-out, $frames$ should be greater than 1.
...default:2
..remarks:When using this configuration, the @Metafunction.Size@ type of the @Spec.External String@ is the @Metafunction.Size@
type of $TFile$. Normally this is a 64bit integer. For a smaller size type, use @Tag.ExternalConfig@.
..remarks:Some data structures store size type values (e.g. suffix arrays in @Class.Index@). To save memory,
you should think of using @Tag.ExternalConfig@.
..include:seqan/file.h
*/
    // custom size type
    template < typename TSize_,
		       typename TFile_ = File<>,				// default file type
               unsigned PAGE_SIZE_ = 1 * 1024 * 1024,	// 1MTypes per default
			   unsigned FRAMES_ = 2 >					// simultanous frames
    struct ExternalConfigSize {
//IOREV
		typedef TSize_ TSize;
        typedef TFile_ TFile;
        enum { PAGE_SIZE = PAGE_SIZE_ };
        enum { FRAMES = FRAMES_ };
    };

    template < typename TConfig = ExternalConfig<> >
    struct External {};
//IOREV


    //////////////////////////////////////////////////////////////////////////////
	// random External String iterator
	template < typename TExtString ><commit_msg>[FIX] made default size type of External String larger<commit_after>// ==========================================================================
//                 SeqAn - The Library for Sequence Analysis
// ==========================================================================
// Copyright (c) 2006-2010, <NAME>, FU Berlin
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of <NAME> or the FU Berlin nor the names of
//       its contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
// ARE DISCLAIMED. IN NO EVENT SHALL <NAME> OR THE FU BERLIN BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
// DAMAGE.
//
// ==========================================================================

#ifndef SEQAN_HEADER_STRING_EXTERNAL_H
#define SEQAN_HEADER_STRING_EXTERNAL_H

/* IOREV
 * _nottested_
 * _doc_
 *
 *
 * mostly documented (doc for some functions missing)
 * not tested in any test case nor used in any app right now
 * -> needs testing, especially different iterators
 */



//////////////////////////////////////////////////////////////////////////////

namespace SEQAN_NAMESPACE_MAIN
{

/**
.Spec.External String:
..cat:Strings
..general:Class.String
..summary:String that is stored in external memory.
..signature:String<TValue, External<> >
..signature:String<TValue, External<TConfig> >
..param.TValue:The value type, that is the type of the items/characters stored in the string.
...remarks:Use @Metafunction.Value@ to get the value type for a given class.
..param.TConfig:A structure to configure the external string.
...type:Tag.ExternalConfig
...type:Tag.ExternalConfigLarge
...type:Tag.ExternalConfigSize
...default:@Tag.ExternalConfigLarge@
..remarks:The External String enables to access sequences larger than the available internal memory (RAM)
by using external memory (e.g. Hard disk, Network storage, ...) via a @Class.File@ object.
Sequences of nearly arbitrary size can be accessed even larger than the logically addressable memory,
i.e. they can in particular contain more than 2^32 elements on a 32bit system (see Tag.ExternalConfigLarge).
See the @Memfunc.External String#String@ constructor for more details.
..remarks:This String also supports fast appending and removing of values at the end (see @Spec.Block String@, @Function.appendValue@)
..remarks:The External String implements a LRU mechanism to swap out pages.
The External String's @Metafunction.Iterator@ detects a forward or backward iteration and asynchronously prefetches pages that
certainly will be accessed and automatically swaps out pages that certainly won't be accessed any more in the iteration
process.
..remarks:The String is implemented like a virtual memory manager.
It divides its character sequence into pages of a fixed length (e.g. 1MB) and maintains a
page table with information for each page (e.g. resides in memory or was swapped out, is dirty and needs to be saved, ...).
Besides the page table the String also contains a size-limited list of page frames. A page frame is reserved internal
memory for a page. When accessing values of a page that is stored in external memory, the page is loaded to a page frame
first. In case that there is no page frame free, another page is swapped out before to free a page frame.
..include:seqan/file.h
*/

/**
.Tag.ExternalConfig:
..cat:Strings
..summary:Standard configuration for the @Spec.External String@.
..signature:String<TValue, External< ExternalConfig<> > >
..signature:String<TValue, External< ExternalConfig<TFile[, pageSize[, frames]]> > >
..param.TFile:Type of file the External String will be based on.
...type:Class.File
..param.pageSize:A positive integer that specifies the number of values in one page.
...remarks:This should be a power of 2, to speed up transfer and calculations.
...default:2^20
..param.frames:A positive integer that specifies the maximum number of pages that should reside in internal memory.
...remarks:To enable prefetching and automatic swap-out, $frames$ should be greater than 1.
...default:2
..remarks:When using this configuration, the @Metafunction.Size@ type of the @Spec.External String@ is $unsigned int$.
Thus, with this configuration at most 4.294.967.296 values can be stored in an @Spec.External String@ on a 32bit system.
For a larger size type, use @Tag.ExternalConfigLarge@.
..include:seqan/file.h
*/
    // standard external string
    // size is uint32
    template < typename TFile_ = File<>,				// default file type
               unsigned PAGE_SIZE_ = 4 * 1024 * 1024,	// 1MTypes per default
			   unsigned FRAMES_ = 2 >					// simultanous frames
    struct ExternalConfig {
//IOREV _bug_ doc says default page size is 2^20, but it is 2^22
        typedef TFile_ TFile;
        typedef unsigned TSize;
        enum { PAGE_SIZE = PAGE_SIZE_ };
        enum { FRAMES = FRAMES_ };
    };

/**
.Tag.ExternalConfigLarge:
..cat:Strings
..summary:Large size type configuration for the @Spec.External String@.
..signature:String<TValue, External< ExternalConfigLarge<> > >
..signature:String<TValue, External< ExternalConfigLarge<TFile[, pageSize[, frames]]> > >
..param.TFile:Type of file the External String will be based on.
...type:Class.File
..param.pageSize:A positive integer that specifies the number of values in one page.
...remarks:This should be a power of 2, to speed up transfer and calculations.
...default:2^20
..param.frames:A positive integer that specifies the maximum number of pages that should reside in internal memory.
...remarks:To enable prefetching and automatic swap-out, $frames$ should be greater than 1.
...default:2
..remarks:When using this configuration, the @Metafunction.Size@ type of the @Spec.External String@ is the @Metafunction.Size@
type of $TFile$. Normally this is a 64bit integer. For a smaller size type, use @Tag.ExternalConfig@.
..remarks:Some data structures store size type values (e.g. suffix arrays in @Class.Index@). To save memory,
you should think of using @Tag.ExternalConfig@.
..include:seqan/file.h
*/
    // the same as ExternalConfig
    // but size type is size type of TFile_ (i.e. uint64)
    //
    // ATTENTION:
    // pipes use the size type 
    // uint64 blows up your suffix arrays, lcp-tables, ...
    template < typename TFile_ = File<>,				// default file type
               unsigned PAGE_SIZE_ = 4 * 1024 * 1024,	// 1MTypes per default
			   unsigned FRAMES_ = 2 >					// simultanous frames
    struct ExternalConfigLarge {
//IOREV contains warning in code comments, need to investigate
        typedef TFile_ TFile;
        typedef typename Size<TFile_>::Type TSize;
        enum { PAGE_SIZE = PAGE_SIZE_ };
        enum { FRAMES = FRAMES_ };
    };

/**
.Tag.ExternalConfigSize:
..cat:Strings
..summary:Arbitrary size type configuration for the @Spec.External String@.
..signature:String<TValue, External< ExternalConfigSize< TSize > > >
..signature:String<TValue, External< ExternalConfigSize< TSize, TFile[, pageSize[, frames]]> > >
..param.TSize:Size type the External String will return via @Metafunction.Size@.
..param.TFile:Type of file the External String will be based on.
...type:Class.File
..param.pageSize:A positive integer that specifies the number of values in one page.
...remarks:This should be a power of 2, to speed up transfer and calculations.
...default:2^20
..param.frames:A positive integer that specifies the maximum number of pages that should reside in internal memory.
...remarks:To enable prefetching and automatic swap-out, $frames$ should be greater than 1.
...default:2
..remarks:When using this configuration, the @Metafunction.Size@ type of the @Spec.External String@ is the @Metafunction.Size@
type of $TFile$. Normally this is a 64bit integer. For a smaller size type, use @Tag.ExternalConfig@.
..remarks:Some data structures store size type values (e.g. suffix arrays in @Class.Index@). To save memory,
you should think of using @Tag.ExternalConfig@.
..include:seqan/file.h
*/
    // custom size type
    template < typename TSize_,
		       typename TFile_ = File<>,				// default file type
               unsigned PAGE_SIZE_ = 1 * 1024 * 1024,	// 1MTypes per default
			   unsigned FRAMES_ = 2 >					// simultanous frames
    struct ExternalConfigSize {
//IOREV
		typedef TSize_ TSize;
        typedef TFile_ TFile;
        enum { PAGE_SIZE = PAGE_SIZE_ };
        enum { FRAMES = FRAMES_ };
    };

    template < typename TConfig = ExternalConfigLarge<> >
    struct External {};
//IOREV


    //////////////////////////////////////////////////////////////////////////////
	// random External String iterator
	template < typename TExtString >
{'fraction_non_alphanumeric': 0.12352851245986853, 'fraction_numerical': 0.009478672985781991, 'mean_word_length': 3.4368075966538547, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1071235', 'n_tokens_mistral': 5462, 'n_tokens_neox': 4965, 'n_words': 2294}