<commit_before>//
//  Document.cs
//  s.im.pl serialization
//
//  Generated by DotNetTranslator on 02/09/11.
//  Copyright 2011 Interface Ecology Lab. 
//

using System;
using System.Collections.Generic;
using Ecologylab.Semantics.Collecting;
using Ecologylab.Semantics.MetadataNS.Builtins.Declarations;
using Ecologylab.Semantics.MetaMetadataNS;
using Ecologylab.Semantics.Services;
using Simpl.Fundamental.Net;
using Simpl.Serialization.Attributes;
using Ecologylab.Semantics.MetadataNS.Scalar;
using Ecologylab.Semantics.MetadataNS;
using System.IO;

namespace Ecologylab.Semantics.MetadataNS.Builtins 
{
    [SimplInherit]
    public class Document : DocumentDeclaration
    {

        [MmName("generation")]
        [SimplScalar]
        private MetadataInteger generation;

        [MmName("favicon")]
        [SimplScalar]
        private MetadataParsedURL favicon;

        [SimplScalar]
        private DownloadStatus downloadStatus = DownloadStatus.UNPROCESSED;

        private DocumentClosure documentClosure;

        public Document()
        { }

        public Document(MetaMetadataCompositeField metaMetadata) : base(metaMetadata) { }

        public MetadataInteger Generation
        {
            get{return generation;}
            set 
            { 
                if (this.generation != value)   
                this.generation = value;
                 //this.RaisePropertyChanged(() => this.Generation); }
            }
        }

        public MetadataParsedURL Favicon
        {
            get{return favicon;}
            set
            {
                if (this.favicon != value)
                {
                    this.favicon = value; //this.RaisePropertyChanged(() => this.Favicon);
                }
            }
        }

        #region Runtime Properties

        public SemanticsSessionScope SemanticsSessionScope { get; set; }

        #endregion

        public override String ToString()
        {
            return (Title != null) ? Title.Value : base.ToString();
        }

        public void AddAdditionalLocation(MetadataParsedURL additionalLocation)
        {
            if (AdditionalLocations == null)
                AdditionalLocations = new List<MetadataParsedURL>();
            AdditionalLocations.Add(additionalLocation);
        }

        ///<summary>
	    /// Get the old location from this.
	    /// Set the location of this to the newLocation.
        /// Add a mapping in the GlobalCollection from newLocation to this.
	    /// Add the old location for this as an additionalLocation for this.
	    ///</summary>
	    public void ChangeLocation(ParsedUri newLocation)
	    {
		    if (newLocation != null)
		    {
                MetadataParsedURL origLocation = Location;
			    if (!origLocation.Value.Equals(newLocation))
			    {
				    Location = new MetadataParsedURL(newLocation);
                    if (SemanticsSessionScope != null)
                        SemanticsSessionScope.GlobalDocumentCollection.AddDocument(this, newLocation);
                    AddAdditionalLocation(origLocation);
			    }
		    }
	    }


        public void InheritValues(Document oldDocument)
        {
            oldDocument.SemanticsSessionScope.GlobalDocumentCollection.Remap(oldDocument, this);

            if (Location == null)
            {
                Location = oldDocument.Location;
                oldDocument.Location = null;
            }

            this.SemanticsSessionScope = oldDocument.SemanticsSessionScope;

            // TODO semantic inlinks

            List<Metadata> oldMixins = oldDocument.Mixins;
            if (oldMixins != null)
                foreach (Metadata oldMixin in oldMixins)
                    AddMixin(oldMixin);

            List<MetadataParsedURL> oldAdditionalLocations = oldDocument.AdditionalLocations;
            if (oldAdditionalLocations != null)
                foreach (MetadataParsedURL otherLocation in oldAdditionalLocations)
                    AddAdditionalLocation(otherLocation);
        }

        public void DownloadAndParseDone()
        {
            // hook for subclasses to perform actions after parsing done.
            // currently, for CompoundDocument to create and visualize clippings.
            // as clipping creation and visualization goes to semantic actions, do we still need this ??
        }

//        public ParsedUri DownloadLocation
//        {
//            get
//            {
//                ParsedUri result	    = Location.Value;
//		        ParsedUri localLocation	= LocalLocation.Value;
//		        if (localLocation != null)
//		        {
//			        FileInfo localFile	= localLocation.File;
//			        if (localFile.Exists)
//				        result	= localLocation;
//		        }
//		        return result;
//            }
//        }

        ///<summary> 
        /// @return A closure for this, or null, if this is not fit to be parsed.
        ///</summary>
        public DocumentClosure GetOrConstructClosure(MetadataServicesClient client, SemanticsGlobalCollection<Document> downloadedDocumentCollection)
	    {
		    DocumentClosure result	= this.documentClosure;
		    if (result == null)
		    {
				result	= this.documentClosure;
				if (result == null)
				{

                    result = ConstructClosure(client, downloadedDocumentCollection);
					this.documentClosure = result;
				}
		    }
		    return result;
	    }

        ///<summary> 
        /// @return A closure for this, or null, if this is not fit to be parsed.
        ///</summary>
        public DocumentClosure GetOrConstructClosure()
        {
            DocumentClosure result = this.documentClosure;
            if (result == null /*&& !isRecycled() && getLocation() != null*/)
            {
                result = this.documentClosure;
                if (result == null)
                {
                    //					if (semanticInlinks == null)
                    //						semanticInlinks	= new SemanticInLinks();

                    result = ConstructClosure();
                    this.documentClosure = result;
                }
            }
            return result == null/* || result.downloadStatus == DownloadStatus.RECYCLED*/ ? null : result;
        }


        public DocumentClosure ConstructClosure(MetadataServicesClient client, SemanticsGlobalCollection<Document> downloadedDocumentCollection)
        {
            return new DocumentClosure(SemanticsSessionScope , this)
                {MetadataServicesClient = client,
                GlobalDocumentCollection = downloadedDocumentCollection};
        }

        public DocumentClosure ConstructClosure()
        {
            return new DocumentClosure(SemanticsSessionScope, this);
        }

        public virtual void AddClipping(IClipping<Metadata> clipping)
        {
        }
    }   
}
<commit_msg>don't serialize download status for documents.<commit_after>//
//  Document.cs
//  s.im.pl serialization
//
//  Generated by DotNetTranslator on 02/09/11.
//  Copyright 2011 Interface Ecology Lab. 
//

using System;
using System.Collections.Generic;
using Ecologylab.Semantics.Collecting;
using Ecologylab.Semantics.MetadataNS.Builtins.Declarations;
using Ecologylab.Semantics.MetaMetadataNS;
using Ecologylab.Semantics.Services;
using Simpl.Fundamental.Net;
using Simpl.Serialization.Attributes;
using Ecologylab.Semantics.MetadataNS.Scalar;
using Ecologylab.Semantics.MetadataNS;
using System.IO;

namespace Ecologylab.Semantics.MetadataNS.Builtins 
{
    [SimplInherit]
    public class Document : DocumentDeclaration
    {

        [MmName("generation")]
        [SimplScalar]
        private MetadataInteger generation;

        [MmName("favicon")]
        [SimplScalar]
        private MetadataParsedURL favicon;

        private DownloadStatus downloadStatus = DownloadStatus.UNPROCESSED;

        private DocumentClosure documentClosure;

        public Document()
        { }

        public Document(MetaMetadataCompositeField metaMetadata) : base(metaMetadata) { }

        public MetadataInteger Generation
        {
            get{return generation;}
            set 
            { 
                if (this.generation != value)   
                this.generation = value;
                 //this.RaisePropertyChanged(() => this.Generation); }
            }
        }

        public MetadataParsedURL Favicon
        {
            get{return favicon;}
            set
            {
                if (this.favicon != value)
                {
                    this.favicon = value; //this.RaisePropertyChanged(() => this.Favicon);
                }
            }
        }

        #region Runtime Properties

        public SemanticsSessionScope SemanticsSessionScope { get; set; }

        #endregion

        public override String ToString()
        {
            return (Title != null) ? Title.Value : base.ToString();
        }

        public void AddAdditionalLocation(MetadataParsedURL additionalLocation)
        {
            if (AdditionalLocations == null)
                AdditionalLocations = new List<MetadataParsedURL>();
            AdditionalLocations.Add(additionalLocation);
        }

        ///<summary>
	    /// Get the old location from this.
	    /// Set the location of this to the newLocation.
        /// Add a mapping in the GlobalCollection from newLocation to this.
	    /// Add the old location for this as an additionalLocation for this.
	    ///</summary>
	    public void ChangeLocation(ParsedUri newLocation)
	    {
		    if (newLocation != null)
		    {
                MetadataParsedURL origLocation = Location;
			    if (!origLocation.Value.Equals(newLocation))
			    {
				    Location = new MetadataParsedURL(newLocation);
                    if (SemanticsSessionScope != null)
                        SemanticsSessionScope.GlobalDocumentCollection.AddDocument(this, newLocation);
                    AddAdditionalLocation(origLocation);
			    }
		    }
	    }


        public void InheritValues(Document oldDocument)
        {
            oldDocument.SemanticsSessionScope.GlobalDocumentCollection.Remap(oldDocument, this);

            if (Location == null)
            {
                Location = oldDocument.Location;
                oldDocument.Location = null;
            }

            this.SemanticsSessionScope = oldDocument.SemanticsSessionScope;

            // TODO semantic inlinks

            List<Metadata> oldMixins = oldDocument.Mixins;
            if (oldMixins != null)
                foreach (Metadata oldMixin in oldMixins)
                    AddMixin(oldMixin);

            List<MetadataParsedURL> oldAdditionalLocations = oldDocument.AdditionalLocations;
            if (oldAdditionalLocations != null)
                foreach (MetadataParsedURL otherLocation in oldAdditionalLocations)
                    AddAdditionalLocation(otherLocation);
        }

        public void DownloadAndParseDone()
        {
            // hook for subclasses to perform actions after parsing done.
            // currently, for CompoundDocument to create and visualize clippings.
            // as clipping creation and visualization goes to semantic actions, do we still need this ??
        }

//        public ParsedUri DownloadLocation
//        {
//            get
//            {
//                ParsedUri result	    = Location.Value;
//		        ParsedUri localLocation	= LocalLocation.Value;
//		        if (localLocation != null)
//		        {
//			        FileInfo localFile	= localLocation.File;
//			        if (localFile.Exists)
//				        result	= localLocation;
//		        }
//		        return result;
//            }
//        }

        ///<summary> 
        /// @return A closure for this, or null, if this is not fit to be parsed.
        ///</summary>
        public DocumentClosure GetOrConstructClosure(MetadataServicesClient client, SemanticsGlobalCollection<Document> downloadedDocumentCollection)
	    {
		    DocumentClosure result	= this.documentClosure;
		    if (result == null)
		    {
				result	= this.documentClosure;
				if (result == null)
				{

                    result = ConstructClosure(client, downloadedDocumentCollection);
					this.documentClosure = result;
				}
		    }
		    return result;
	    }

        ///<summary> 
        /// @return A closure for this, or null, if this is not fit to be parsed.
        ///</summary>
        public DocumentClosure GetOrConstructClosure()
        {
            DocumentClosure result = this.documentClosure;
            if (result == null /*&& !isRecycled() && getLocation() != null*/)
            {
                result = this.documentClosure;
                if (result == null)
                {
                    //					if (semanticInlinks == null)
                    //						semanticInlinks	= new SemanticInLinks();

                    result = ConstructClosure();
                    this.documentClosure = result;
                }
            }
            return result == null/* || result.downloadStatus == DownloadStatus.RECYCLED*/ ? null : result;
        }


        public DocumentClosure ConstructClosure(MetadataServicesClient client, SemanticsGlobalCollection<Document> downloadedDocumentCollection)
        {
            return new DocumentClosure(SemanticsSessionScope , this)
                {MetadataServicesClient = client,
                GlobalDocumentCollection = downloadedDocumentCollection};
        }

        public DocumentClosure ConstructClosure()
        {
            return new DocumentClosure(SemanticsSessionScope, this);
        }

        public virtual void AddClipping(IClipping<Metadata> clipping)
        {
        }
    }   
}

{'fraction_non_alphanumeric': 0.0793425899688297, 'fraction_numerical': 0.0014168319637291016, 'mean_word_length': 1.692542437535762, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7482074', 'n_tokens_mistral': 3802, 'n_tokens_neox': 3347, 'n_words': 796}