<commit_before>// Copyright 2016 Documize Inc. <<EMAIL>>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <<EMAIL>>.
//
// https://documize.com

package document

import (
	"github.com/documize/community/core/uniqueid"
	"github.com/documize/community/domain"
	"github.com/documize/community/model/category"
	"github.com/documize/community/model/doc"
	"github.com/documize/community/model/page"
	"github.com/documize/community/model/workflow"
	"github.com/pkg/errors"
)

// FilterCategoryProtected removes documents that cannot be seen by user due to
// document cateogory viewing permissions.
func FilterCategoryProtected(docs []doc.Document, cats []category.Category, members []category.Member, viewDrafts bool) (filtered []doc.Document) {
	filtered = []doc.Document{}

	for _, doc := range docs {
		hasCategory := false
		canSeeCategory := false
		skip := false

		// drafts included if user can see them
		if doc.Lifecycle == workflow.LifecycleDraft && !viewDrafts {
			skip = true
		}

		// archived never included
		if doc.Lifecycle == workflow.LifecycleArchived {
			skip = true
		}

	OUTER:

		for _, m := range members {
			if m.DocumentID == doc.RefID {
				hasCategory = true
				for _, cat := range cats {
					if cat.RefID == m.CategoryID {
						canSeeCategory = true
						continue OUTER
					}
				}
			}
		}

		if !skip && (!hasCategory || canSeeCategory) {
			filtered = append(filtered, doc)
		}
	}

	return
}

// CopyDocument clones an existing document
func CopyDocument(ctx domain.RequestContext, s domain.Store, documentID string) (newDocumentID string, err error) {
	doc, err := s.Document.Get(ctx, documentID)
	if err != nil {
		err = errors.Wrap(err, "unable to fetch existing document")
		return
	}

	newDocumentID = uniqueid.Generate()
	doc.RefID = newDocumentID
	doc.ID = 0
	doc.Versioned = false
	doc.VersionID = ""
	doc.GroupID = ""
	doc.Template = false

	// Duplicate pages and associated meta
	pages, err := s.Page.GetPages(ctx, documentID)
	if err != nil {
		err = errors.Wrap(err, "unable to get existing pages")
		return
	}

	var pageModel []page.NewPage

	for _, p := range pages {
		p.DocumentID = newDocumentID
		p.ID = 0

		meta, err2 := s.Page.GetPageMeta(ctx, p.RefID)
		if err2 != nil {
			err = errors.Wrap(err, "unable to get existing pages meta")
			return
		}

		pageID := uniqueid.Generate()
		p.RefID = pageID
		meta.PageID = pageID
		meta.DocumentID = newDocumentID

		m := page.NewPage{}
		m.Page = p
		m.Meta = meta

		pageModel = append(pageModel, m)
	}

	// Duplicate attachments
	attachments, _ := s.Attachment.GetAttachments(ctx, documentID)
	for i, a := range attachments {
		a.DocumentID = newDocumentID
		a.RefID = uniqueid.Generate()
		a.ID = 0
		attachments[i] = a
	}

	// Now create the template: document, attachments, pages and their meta
	err = s.Document.Add(ctx, doc)
	if err != nil {
		err = errors.Wrap(err, "unable to add copied document")
		return
	}

	for _, a := range attachments {
		err = s.Attachment.Add(ctx, a)
		if err != nil {
			err = errors.Wrap(err, "unable to add copied attachment")
			return
		}
	}

	for _, m := range pageModel {
		err = s.Page.Add(ctx, m)
		if err != nil {
			err = errors.Wrap(err, "unable to add copied page")
			return
		}
	}

	return
}

// FilterLastVersion returns the latest version of each document
// by removing all previous versions.
// If a document is not versioned, it is returned as-is.
func FilterLastVersion(docs []doc.Document) (filtered []doc.Document) {
	filtered = []doc.Document{}
	prev := make(map[string]bool)

	for _, doc := range docs {
		add := false

		if doc.GroupID == "" {
			add = true
		} else {
			if _, isExisting := prev[doc.GroupID]; !isExisting {
				add = true
				prev[doc.GroupID] = true
			} else {
				add = false
			}
		}

		if add {
			filtered = append(filtered, doc)
		}
	}<commit_msg>Copt document categories when copying document<commit_after>// Copyright 2016 Documize Inc. <<EMAIL>>. All rights reserved.
//
// This software (Documize Community Edition) is licensed under
// GNU AGPL v3 http://www.gnu.org/licenses/agpl-3.0.en.html
//
// You can operate outside the AGPL restrictions by purchasing
// Documize Enterprise Edition and obtaining a commercial license
// by contacting <<EMAIL>>.
//
// https://documize.com

package document

import (
	"github.com/documize/community/core/uniqueid"
	"github.com/documize/community/domain"
	"github.com/documize/community/model/category"
	"github.com/documize/community/model/doc"
	"github.com/documize/community/model/page"
	"github.com/documize/community/model/workflow"
	"github.com/pkg/errors"
)

// FilterCategoryProtected removes documents that cannot be seen by user due to
// document cateogory viewing permissions.
func FilterCategoryProtected(docs []doc.Document, cats []category.Category, members []category.Member, viewDrafts bool) (filtered []doc.Document) {
	filtered = []doc.Document{}

	for _, doc := range docs {
		hasCategory := false
		canSeeCategory := false
		skip := false

		// drafts included if user can see them
		if doc.Lifecycle == workflow.LifecycleDraft && !viewDrafts {
			skip = true
		}

		// archived never included
		if doc.Lifecycle == workflow.LifecycleArchived {
			skip = true
		}

	OUTER:

		for _, m := range members {
			if m.DocumentID == doc.RefID {
				hasCategory = true
				for _, cat := range cats {
					if cat.RefID == m.CategoryID {
						canSeeCategory = true
						continue OUTER
					}
				}
			}
		}

		if !skip && (!hasCategory || canSeeCategory) {
			filtered = append(filtered, doc)
		}
	}

	return
}

// CopyDocument clones an existing document
func CopyDocument(ctx domain.RequestContext, s domain.Store, documentID string) (newDocumentID string, err error) {
	doc, err := s.Document.Get(ctx, documentID)
	if err != nil {
		err = errors.Wrap(err, "unable to fetch existing document")
		return
	}

	newDocumentID = uniqueid.Generate()
	doc.RefID = newDocumentID
	doc.ID = 0
	doc.Versioned = false
	doc.VersionID = ""
	doc.GroupID = ""
	doc.Template = false

	// Duplicate pages and associated meta
	pages, err := s.Page.GetPages(ctx, documentID)
	if err != nil {
		err = errors.Wrap(err, "unable to get existing pages")
		return
	}

	var pageModel []page.NewPage

	for _, p := range pages {
		p.DocumentID = newDocumentID
		p.ID = 0

		meta, err2 := s.Page.GetPageMeta(ctx, p.RefID)
		if err2 != nil {
			err = errors.Wrap(err, "unable to get existing pages meta")
			return
		}

		pageID := uniqueid.Generate()
		p.RefID = pageID
		meta.PageID = pageID
		meta.DocumentID = newDocumentID

		m := page.NewPage{}
		m.Page = p
		m.Meta = meta

		pageModel = append(pageModel, m)
	}

	// Duplicate attachments
	attachments, _ := s.Attachment.GetAttachments(ctx, documentID)
	for i, a := range attachments {
		a.DocumentID = newDocumentID
		a.RefID = uniqueid.Generate()
		a.ID = 0
		attachments[i] = a
	}

	// Now create the template: document, attachments, pages and their meta
	err = s.Document.Add(ctx, doc)
	if err != nil {
		err = errors.Wrap(err, "unable to add copied document")
		return
	}

	for _, a := range attachments {
		err = s.Attachment.Add(ctx, a)
		if err != nil {
			err = errors.Wrap(err, "unable to add copied attachment")
			return
		}
	}

	for _, m := range pageModel {
		err = s.Page.Add(ctx, m)
		if err != nil {
			err = errors.Wrap(err, "unable to add copied page")
			return
		}
	}

	cats, err := s.Category.GetDocumentCategoryMembership(ctx, documentID)
	if err != nil {
		err = errors.Wrap(err, "unable to add copied page")
		return
	}

	for ci := range cats {
		cm := category.Member{}
		cm.DocumentID = newDocumentID
		cm.CategoryID = cats[ci].RefID
		cm.OrgID = ctx.OrgID
		cm.RefID = uniqueid.Generate()
		s.Category.AssociateDocument(ctx, cm)
		if err != nil {
			err = errors.Wrap(err, "unable to add copied page")
			return
		}
	}

	return
}

// FilterLastVersion returns the latest version of each document
// by removing all previous versions.
// If a document is not versioned, it is returned as-is.
func FilterLastVersion(docs []doc.Document) (filtered []doc.Document) {
	filtered = []doc.Document{}
	prev := make(map[string]bool)

	for _, doc := range docs {
		add := false

		if doc.GroupID == "" {
			add = true
		} else {
			if _, isExisting := prev[doc.GroupID]; !isExisting {
				add = true
				prev[doc.GroupID] = true
			} else {
				add = false
			}
		}

		if add {
			filtered = append(filtered, doc)
		}
	}
{'fraction_non_alphanumeric': 0.12204133471885463, 'fraction_numerical': 0.0027710426047800486, 'mean_word_length': 3.233626588465298, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1433405', 'n_tokens_mistral': 3211, 'n_tokens_neox': 2847, 'n_words': 819}