<commit_before>﻿// Copyright 2005-2015 <NAME> & Contributors. All rights reserved. See License.md in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq;
using CSharpx;

namespace CommandLine.Core
{
    static class TokenPartitioner
    {
        public static
            TokenPartitions Partition(
                IEnumerable<Token> tokens,
                Func<string, Maybe<TypeDescriptor>> typeLookup)
        {
            var tokenList = tokens.Memorize();
            var switches = Switch.Partition(tokenList, typeLookup).Memorize();
            var scalars = Scalar.Partition(tokenList, typeLookup).Memorize();
            var sequences = Sequence.Partition(tokenList, typeLookup).Memorize();
            var nonOptions = tokenList
                .Where(t => !switches.Contains(t))
                .Where(t => !scalars.Contains(t))
                .Where(t => !sequences.Contains(t)).Memorize();
            var values = nonOptions.Where(v => v.IsValue()).Memorize();
            var errors = nonOptions.Except(values).Memorize();

            return TokenPartitions.Create(
                    KeyValuePairHelper.ForSwitch(switches)
                        .Concat(KeyValuePairHelper.ForScalar(scalars))
                        .Concat(KeyValuePairHelper.ForSequence(sequences)),
                values.Select(t => t.Text),
                errors);
        }
    }
}<commit_msg>Used Tuple instead of TokenPartitions in TokenPartitioner<commit_after>﻿// Copyright 2005-2015 <NAME> & Contributors. All rights reserved. See License.md in the project root for license information.

using System;
using System.Collections.Generic;
using System.Linq;
using CSharpx;

namespace CommandLine.Core
{
    static class TokenPartitioner
    {
        public static
            Tuple<
                IEnumerable<KeyValuePair<string, IEnumerable<string>>>,
                IEnumerable<string>,
                IEnumerable<Token>
            > Partition(
                IEnumerable<Token> tokens,
                Func<string, Maybe<TypeDescriptor>> typeLookup)
        {
            var tokenList = tokens.Memorize();
            var switches = Switch.Partition(tokenList, typeLookup).Memorize();
            var scalars = Scalar.Partition(tokenList, typeLookup).Memorize();
            var sequences = Sequence.Partition(tokenList, typeLookup).Memorize();
            var nonOptions = tokenList
                .Where(t => !switches.Contains(t))
                .Where(t => !scalars.Contains(t))
                .Where(t => !sequences.Contains(t)).Memorize();
            var values = nonOptions.Where(v => v.IsValue()).Memorize();
            var errors = nonOptions.Except(values).Memorize();

            return Tuple.Create(
                    KeyValuePairHelper.ForSwitch(switches)
                        .Concat(KeyValuePairHelper.ForScalar(scalars))
                        .Concat(KeyValuePairHelper.ForSequence(sequences)),
                values.Select(t => t.Text),
                errors);
        }
    }
}
{'fraction_non_alphanumeric': 0.0999682640431609, 'fraction_numerical': 0.0050777530942557915, 'mean_word_length': 1.951310861423221, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2965737', 'n_tokens_mistral': 840, 'n_tokens_neox': 780, 'n_words': 166}