<commit_before>﻿using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wyam.Common.Configuration;
using Wyam.Common.Documents;
using Wyam.Common.Execution;
using Wyam.Common.Meta;
using Wyam.Common.Modules;

namespace Wyam.Core.Modules.Metadata
{
    /// <summary>
    /// Add Metadata to the Documents that describes the position of
    /// the Document in a tree structure. 
    /// </summary>
    public class Tree : IModule
    {
        private DocumentConfig isRoot;
        private DocumentConfig getPath;
        private DocumentConfig getOrder;

        private string parentMetadata = "Parent";
        private string childrenMetadata = "Children";
        private string previousSilblingMetadata = "PreviosSilbling";
        private string nextSilblingMetadata = "NextSilbling";

        private string nextNodeMetadata = "Next";
        private string previousNodeMetadata = "Previous";


        public Tree()
        {
            isRoot = (ctx, doc) => false;
            getPath = (doc, ctx) => doc.Source.Segments;
        }

        /// <summary>
        /// Specifying for each Document if it is a root of a Tree. This Results in splitting the generated tree in multiple smaller ones.
        /// </summary>
        /// <param name="config">A Predicate (must return <c>bool</c>) that specifies if the current Document is treated as the root of a new tree.</param>
        /// <returns></returns>
        public Tree WithRoots(DocumentConfig config)
        {
            isRoot = config;
            return this;
        }
        /// <summary>
        /// Specifies the order of the Children in the Tree
        /// </summary>
        /// <param name="config">The DocumentConfig must return an <c>int</c>.</param>
        /// <returns></returns>
        public Tree WithOrder(DocumentConfig config)
        {
            getOrder = config;
            return this;
        }

        /// <summary>
        /// Defines the structure of the Tree
        /// </summary>
        /// <param name="config">Must return <c>string[]</c></param>
        /// <returns></returns>
        public Tree WithPath(DocumentConfig config)
        {
            getPath = config;
            return this;
        }


        public Tree WithMetadataNames(string parentMetadata = "Parent", string childrenMetadata = "Children", string previousSilblingMetadata = "PreviosSilbling", string nextSilblingMetadata = "NextSilbling", string nextNodeMetadata = "Next", string previousNodeMetadata = "Previous")
        {
            this.parentMetadata = parentMetadata;
            this.childrenMetadata = childrenMetadata;
            this.previousSilblingMetadata = previousSilblingMetadata;
            this.nextSilblingMetadata = nextSilblingMetadata;
            this.nextNodeMetadata = nextNodeMetadata;
            this.previousNodeMetadata = previousNodeMetadata;
            return this;
        }



        public IEnumerable<IDocument> Execute(IReadOnlyList<IDocument> inputs, IExecutionContext context)
        {

<commit_msg>Description of the Metadata keys<commit_after>﻿using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Wyam.Common.Configuration;
using Wyam.Common.Documents;
using Wyam.Common.Execution;
using Wyam.Common.Meta;
using Wyam.Common.Modules;

namespace Wyam.Core.Modules.Metadata
{
    /// <summary>
    /// Add Metadata to the Documents that describes the position of
    /// the Document in a tree structure. 
    /// Following keys will be set:
    /// * Parent (<see cref="IDocument"/>) The parent of this node. <c>null</c> if it is a root
    /// * Children (<see cref="ReadOnlyCollection{IDocument}"/>) All children of this node.
    /// * PreviosSilbling (<see cref="IDocument"/>) The previous sibling, that is the previous node in the Children collection of the parent. <c>null</c> if this is the first node in the collection or parent is null.
    /// * NextSilbling (<see cref="IDocument"/>) The next sibling, that is the next node in the Children collection of the parent. <c>null</c> if this is the last node in the collection or parent is null. 
    /// * Next (<see cref="IDocument"/>) The next node using a Depth-First-Search. <c>null</c> if this was the last node.
    /// * Previous (<see cref="IDocument"/>) The previous node using a Depth-First-Search. <c>null</c> if this was the first node.
    /// </summary>
    public class Tree : IModule
    {
        private DocumentConfig isRoot;
        private DocumentConfig getPath;
        private DocumentConfig getOrder;

        private string parentMetadata = "Parent";
        private string childrenMetadata = "Children";
        private string previousSilblingMetadata = "PreviosSilbling";
        private string nextSilblingMetadata = "NextSilbling";

        private string nextNodeMetadata = "Next";
        private string previousNodeMetadata = "Previous";


        public Tree()
        {
            isRoot = (ctx, doc) => false;
            getPath = (doc, ctx) => doc.Source.Segments;
        }

        /// <summary>
        /// Specifying for each Document if it is a root of a Tree. This Results in splitting the generated tree in multiple smaller ones.
        /// </summary>
        /// <param name="config">A Predicate (must return <c>bool</c>) that specifies if the current Document is treated as the root of a new tree.</param>
        /// <returns></returns>
        public Tree WithRoots(DocumentConfig config)
        {
            isRoot = config;
            return this;
        }
        /// <summary>
        /// Specifies the order of the Children in the Tree
        /// </summary>
        /// <param name="config">The DocumentConfig must return an <c>int</c>.</param>
        /// <returns></returns>
        public Tree WithOrder(DocumentConfig config)
        {
            getOrder = config;
            return this;
        }

        /// <summary>
        /// Defines the structure of the Tree
        /// </summary>
        /// <param name="config">Must return <c>string[]</c></param>
        /// <returns></returns>
        public Tree WithPath(DocumentConfig config)
        {
            getPath = config;
            return this;
        }

        /// <summary>
        /// Changes the standard Metadata keys.
        /// </summary>
        /// <param name="parentMetadata"></param>
        /// <param name="childrenMetadata"></param>
        /// <param name="previousSilblingMetadata"></param>
        /// <param name="nextSilblingMetadata"></param>
        /// <param name="nextNodeMetadata"></param>
        /// <param name="previousNodeMetadata"></param>
        /// <returns></returns>
        public Tree WithMetadataNames(string parentMetadata = "Parent", string childrenMetadata = "Children", string previousSilblingMetadata = "PreviosSilbling", string nextSilblingMetadata = "NextSilbling", string nextNodeMetadata = "Next", string previousNodeMetadata = "Previous")
        {
            this.parentMetadata = parentMetadata;
            this.childrenMetadata = childrenMetadata;
            this.previousSilblingMetadata = previousSilblingMetadata;
            this.nextSilblingMetadata = nextSilblingMetadata;
            this.nextNodeMetadata = nextNodeMetadata;
            this.previousNodeMetadata = previousNodeMetadata;
            return this;
        }



        public IEnumerable<IDocument> Execute(IReadOnlyList<IDocument> inputs, IExecutionContext context)
        {


{'fraction_non_alphanumeric': 0.10600706713780919, 'fraction_numerical': 0.0, 'mean_word_length': 2.4055258467023175, 'pattern_counts': {'":': 0, '<': 91, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3817281', 'n_tokens_mistral': 1904, 'n_tokens_neox': 1876, 'n_words': 611}