<commit_before>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using AutoDependencyDetector.Data;
using AutoDependencyDetector.Exceptions;
using Newtonsoft.Json;

namespace AutoDependencyDetector.Logic
{
    /// <summary>
    /// Does the entire pipeline of operations
    /// </summary>
    public class ProcessPipeline
    {
        private readonly DependencyDetector _detector;
        public ILogger Logger { get; private set; }


        private Options _options;

        private Config _config;

        public ProcessPipeline( ILogger logger, DependencyDetector detector )
        {
            _detector = detector;
            Logger = logger;
        }

        public void ExecutePipeline( Options options, Config config)
        {

            _options = options;

            Logger.Info( "Analyzing {0}{1}", options.InputDirectory, options.RecurseInput ? " and children" : "" );

            _verifyDir( options.InputDirectory, "Input directory" );
            _verifyDir( options.DependencyDirectory, "Dependency directory" );



            // --------- Obtain configuration
            _config = config;

            // --------- For the configured amount of sweeps repeat the dependency lookup
            for ( var i = 0; i < _config.HowManyIterations; ++i )
            {
                Logger.Info( "Starting dependency search sweep {0}/{1}", i+1, _config.HowManyIterations );

                // --------- Get all directories
                var remainingDependencies = handleDependenciesForDirectories( options );
                if ( remainingDependencies == 0 )
                {
                    break;
                }
            }

          
        }

        private int handleDependenciesForDirectories( Options options )
        {
            var directories = _forEveryDirectory( options.InputDirectory, options.RecurseInput );

            var remainingDependencies = 0;

            // --------- Read all dependencies
            foreach ( var directory in directories )
            {
                remainingDependencies +=_handleDependenciesForDirectory( directory );
            }

            return remainingDependencies;
        }

        private int _handleDependenciesForDirectory( string directory )
        {
            var remainingDependencies = 0;
            Logger.Info( "Analyzing dependencies in {0}", directory );

            var inputs = _gatherInputFiles( directory );
            Logger.Info( "Found {0} input files", inputs.Count );
            foreach ( var input in inputs )
            {
                remainingDependencies += _handleDependenciesForFile( input );
            }

            return remainingDependencies;
        }

        private int _handleDependenciesForFile( string file )
        {
            Logger.Info( "Looking for missing dependencies for {0}", file );

            var bitness = new BitnessDetector().BitnessOf( file );

            var missingDependencies = _detector.GetMissingDependencies( file, bitness );

            if ( missingDependencies.Count == 0 )
            {
                Logger.Info( "No missing dependencies found" );
                return 0;
            }

            Logger.Info( "Found {0} missing dependencies. Resolving", missingDependencies.Count );
<commit_msg>Renamed variable. Added check for incomplete dependency lookup.<commit_after>﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using AutoDependencyDetector.Data;
using AutoDependencyDetector.Exceptions;
using Newtonsoft.Json;

namespace AutoDependencyDetector.Logic
{
    /// <summary>
    /// Does the entire pipeline of operations
    /// </summary>
    public class ProcessPipeline
    {
        private readonly DependencyDetector _detector;
        public ILogger Logger { get; private set; }


        private Options _options;

        private Config _config;

        public ProcessPipeline( ILogger logger, DependencyDetector detector )
        {
            _detector = detector;
            Logger = logger;
        }

        public void ExecutePipeline( Options options, Config config)
        {

            _options = options;

            Logger.Info( "Analyzing {0}{1}", options.InputDirectory, options.RecurseInput ? " and children" : "" );

            _verifyDir( options.InputDirectory, "Input directory" );
            _verifyDir( options.DependencyDirectory, "Dependency directory" );



            // --------- Obtain configuration
            _config = config;

            // --------- For the configured amount of sweeps repeat the dependency lookup
            _repeatDependencyResovling( options );

        }

        private void _repeatDependencyResovling( Options options )
        {
            var locatedDependencies = 0;
            for ( var i = 0; i < _config.HowManyIterations; ++i )
            {
                Logger.Info( "Starting dependency search sweep {0}/{1}", i + 1, _config.HowManyIterations );

                // --------- Get all directories
                locatedDependencies = handleDependenciesForDirectories( options );
                if ( locatedDependencies == 0 )
                {
                    break;
                }
            }

            if ( locatedDependencies != 0 )
            {
                throw new ProcessPipelineException( $"After {_config.HowManyIterations} there might still be missing dependencies. Increase the HowManyIterations to ensure all dependencies are located" );
            }
        }

        private int handleDependenciesForDirectories( Options options )
        {
            var directories = _forEveryDirectory( options.InputDirectory, options.RecurseInput );

            var locatedDependencies = 0;

            // --------- Read all dependencies
            foreach ( var directory in directories )
            {
                locatedDependencies +=_handleDependenciesForDirectory( directory );
            }

            return locatedDependencies;
        }

        private int _handleDependenciesForDirectory( string directory )
        {
            var locatedDependencies = 0;
            Logger.Info( "Analyzing dependencies in {0}", directory );

            var inputs = _gatherInputFiles( directory );
            Logger.Info( "Found {0} input files", inputs.Count );
            foreach ( var input in inputs )
            {
                locatedDependencies += _handleDependenciesForFile( input );
            }

            return locatedDependencies;
        }

        private int _handleDependenciesForFile( string file )
        {
            Logger.Info( "Looking for missing dependencies for {0}", file );

            var bitness = new BitnessDetector().BitnessOf( file );

            var missingDependencies = _detector.GetMissingDependencies( file, bitness );

            if ( missingDependencies.Count == 0 )
            {
                Logger.Info( "No missing dependencies found" );
                return 0;
            }

            Logger.Info( "Found {0} missing dependencies. Resolving", missingDependencies.Count );

{'fraction_non_alphanumeric': 0.0843806104129264, 'fraction_numerical': 0.004419279105095981, 'mean_word_length': 1.8829617834394905, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4618675', 'n_tokens_mistral': 1726, 'n_tokens_neox': 1579, 'n_words': 482}