<commit_before>﻿using BitSharp.Common;
using BitSharp.Common.ExtensionMethods;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BitSharp.Data;
using System.IO;
using BitSharp.Storage;
using System.Globalization;
using System.Collections;
using NLog;

namespace BitSharp.Blockchain
{
    public class BlockchainCalculator
    {
        private readonly Logger logger;
        private readonly CancellationToken shutdownToken;
        private readonly IBlockchainRules rules;
        private readonly BlockHeaderCache blockHeaderCache;
        private readonly BlockTxHashesCache blockTxHashesCache;
        private readonly TransactionCache transactionCache;
        private readonly BlockView blockView;
        private readonly SpentTransactionsCache spentTransactionsCache;
        private readonly SpentOutputsCache spentOutputsCache;

        public BlockchainCalculator(CancellationToken shutdownToken, Logger logger, IBlockchainRules rules, BlockHeaderCache blockHeaderCache, BlockTxHashesCache blockTxHashesCache, TransactionCache transactionCache, BlockView blockView, SpentTransactionsCache spentTransactionsCache, SpentOutputsCache spentOutputsCache)
        {
            this.logger = logger;
            this.shutdownToken = shutdownToken;
            this.rules = rules;
            this.blockHeaderCache = blockHeaderCache;
            this.blockTxHashesCache = blockTxHashesCache;
            this.transactionCache = transactionCache;
            this.blockView = blockView;
            this.spentTransactionsCache = spentTransactionsCache;
            this.spentOutputsCache = spentOutputsCache;
        }

        public void CalculateBlockchainFromExisting(ChainStateBuilder chainStateBuilder, Func<Chain> getTargetChain, CancellationToken cancelToken, Action<TimeSpan> onProgress = null)
        {
            //chainStateBuilder.Stats.totalStopwatch.Start();
            //chainStateBuilder.Stats.currentRateStopwatch.Start();

            // calculate the new blockchain along the target path
            chainStateBuilder.IsConsistent = true;
            foreach (var pathElement in BlockLookAhead(chainStateBuilder.Chain.NavigateTowards(getTargetChain), lookAhead: 1))
            {
                chainStateBuilder.IsConsistent = false;
                var startTime = DateTime.UtcNow;

                // cooperative loop
                if (this.shutdownToken.IsCancellationRequested)
                    break;
                if (cancelToken.IsCancellationRequested)
                    break;

                // get block and metadata for next link in blockchain
                var direction = pathElement.Item1;
                var chainedBlock = pathElement.Item2;
                var block = pathElement.Item3;

                if (direction < 0)
                {
                    RollbackUtxo(chainStateBuilder, block);

                    chainStateBuilder.Chain.RemoveBlock(chainedBlock);
                }
                else if (direction > 0)
                {
                    // add the block to the current chain
                    chainStateBuilder.Chain.AddBlock(chainedBlock);

                    // validate the block
                    chainStateBuilder.Stats.validateStopwatch.Start();
                    new MethodTimer(false).Time("ValidateBlock", () =>
                        this.rules.ValidateBlock(block, chainStateBuilder));
                    chainStateBuilder.Stats.validateStopwatch.Stop();

                    // calculate the new block utxo, double spends will be checked for
                    long txCount = 0, inputCount = 0;
                    new MethodTimer(false).Time("CalculateUtxo", () =>
                        CalculateUtxo(chainedBlock, block, chainStateBuilder.Utxo, out txCount, out inputCount));

                    // collect rollback informatino and store it
                    chainStateBuilder.Utxo.SaveRollbackInformation(chainedBlock.Height, block.Hash, this.spentTransactionsCache, this.spentOutputsCache);

                    //TODO remove the block and its transactions immediately after processing
                    //TODO this is for startup mode where blocks are not saved but kept in memory just long enough to update the UTXO
                    //TODO when fully caught up, rollback blocks can be reacquired backwards from that point, to the pruning limit
                    //TODO reacquiring is not required, but if a rollback does occur then the information will need to be downloaded then
                    if (true)
                    {
                        foreach (var tx in block.Transactions)
                            this.transactionCache.TryRemove(tx.Hash);
                        this.blockTxHashesCache.TryRemove(block.Hash);
                    }

                    // flush utxo progress
                    //chainStateBuilder.Utxo.Flush();<commit_msg>now that the builder chain gets pruned, stop deleting blocks during calculation<commit_after>﻿using BitSharp.Common;
using BitSharp.Common.ExtensionMethods;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BitSharp.Data;
using System.IO;
using BitSharp.Storage;
using System.Globalization;
using System.Collections;
using NLog;

namespace BitSharp.Blockchain
{
    public class BlockchainCalculator
    {
        private readonly Logger logger;
        private readonly CancellationToken shutdownToken;
        private readonly IBlockchainRules rules;
        private readonly BlockHeaderCache blockHeaderCache;
        private readonly BlockTxHashesCache blockTxHashesCache;
        private readonly TransactionCache transactionCache;
        private readonly BlockView blockView;
        private readonly SpentTransactionsCache spentTransactionsCache;
        private readonly SpentOutputsCache spentOutputsCache;

        public BlockchainCalculator(CancellationToken shutdownToken, Logger logger, IBlockchainRules rules, BlockHeaderCache blockHeaderCache, BlockTxHashesCache blockTxHashesCache, TransactionCache transactionCache, BlockView blockView, SpentTransactionsCache spentTransactionsCache, SpentOutputsCache spentOutputsCache)
        {
            this.logger = logger;
            this.shutdownToken = shutdownToken;
            this.rules = rules;
            this.blockHeaderCache = blockHeaderCache;
            this.blockTxHashesCache = blockTxHashesCache;
            this.transactionCache = transactionCache;
            this.blockView = blockView;
            this.spentTransactionsCache = spentTransactionsCache;
            this.spentOutputsCache = spentOutputsCache;
        }

        public void CalculateBlockchainFromExisting(ChainStateBuilder chainStateBuilder, Func<Chain> getTargetChain, CancellationToken cancelToken, Action<TimeSpan> onProgress = null)
        {
            //chainStateBuilder.Stats.totalStopwatch.Start();
            //chainStateBuilder.Stats.currentRateStopwatch.Start();

            // calculate the new blockchain along the target path
            chainStateBuilder.IsConsistent = true;
            foreach (var pathElement in BlockLookAhead(chainStateBuilder.Chain.NavigateTowards(getTargetChain), lookAhead: 1))
            {
                chainStateBuilder.IsConsistent = false;
                var startTime = DateTime.UtcNow;

                // cooperative loop
                if (this.shutdownToken.IsCancellationRequested)
                    break;
                if (cancelToken.IsCancellationRequested)
                    break;

                // get block and metadata for next link in blockchain
                var direction = pathElement.Item1;
                var chainedBlock = pathElement.Item2;
                var block = pathElement.Item3;

                if (direction < 0)
                {
                    RollbackUtxo(chainStateBuilder, block);

                    chainStateBuilder.Chain.RemoveBlock(chainedBlock);
                }
                else if (direction > 0)
                {
                    // add the block to the current chain
                    chainStateBuilder.Chain.AddBlock(chainedBlock);

                    // validate the block
                    chainStateBuilder.Stats.validateStopwatch.Start();
                    new MethodTimer(false).Time("ValidateBlock", () =>
                        this.rules.ValidateBlock(block, chainStateBuilder));
                    chainStateBuilder.Stats.validateStopwatch.Stop();

                    // calculate the new block utxo, double spends will be checked for
                    long txCount = 0, inputCount = 0;
                    new MethodTimer(false).Time("CalculateUtxo", () =>
                        CalculateUtxo(chainedBlock, block, chainStateBuilder.Utxo, out txCount, out inputCount));

                    // collect rollback informatino and store it
                    chainStateBuilder.Utxo.SaveRollbackInformation(chainedBlock.Height, block.Hash, this.spentTransactionsCache, this.spentOutputsCache);

                    // flush utxo progress
                    //chainStateBuilder.Utxo.Flush();
{'fraction_non_alphanumeric': 0.06023206751054852, 'fraction_numerical': 0.0016877637130801688, 'mean_word_length': 2.252487135506003, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '922254', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2132, 'n_words': 567}