<commit_before>package move

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
	"github.com/cozy/cozy-stack/pkg/crypto"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/cozy-stack/pkg/vfs"
	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/cozy/echo"
)

const (
	// ExportFilesDir is the directory for storing the files in the export
	// archive.
	ExportFilesDir = "My Cozy/Files"
	// ExportMetasDir is the directory for storing the metadata in the export
	// archive.
	ExportMetasDir = "My Cozy/Metadata"
)

// ExportDoc is a documents storing the metadata of an export.
type ExportDoc struct {
	DocID     string `json:"_id,omitempty"`
	DocRev    string `json:"_rev,omitempty"`
	Domain    string `json:"domain"`
	PartsSize int64  `json:"parts_size,omitempty"`

	PartsCursors     []string      `json:"parts_cursors,omitempty"`
	WithDoctypes     []string      `json:"with_doctypes,omitempty"`
	WithoutFiles     bool          `json:"without_files,omitempty"`
	State            string        `json:"state"`
	CreatedAt        time.Time     `json:"created_at"`
	ExpiresAt        time.Time     `json:"expires_at"`
	TotalSize        int64         `json:"total_size,omitempty"`
	CreationDuration time.Duration `json:"creation_duration,omitempty"`
	Error            string        `json:"error,omitempty"`
}

// PartsSize is the default size of a file bucket, to split the index into
// equal-sized parts.
const PartsSize = 100 * 1024 * 1024 // 100 MB

var (
	// ErrExportNotFound is used when a export document could not be found
	ErrExportNotFound = echo.NewHTTPError(http.StatusNotFound, "exports: not found")
	// ErrExportExpired is used when the export document has expired along with
	// its associated data.
	ErrExportExpired = echo.NewHTTPError(http.StatusNotFound, "exports: has expired")
	// ErrMACInvalid is used when the given MAC is not valid.
	ErrMACInvalid = echo.NewHTTPError(http.StatusUnauthorized, "exports: invalid mac")
	// ErrExportConflict is used when an export is already being perfomed.
	ErrExportConflict = echo.NewHTTPError(http.StatusConflict, "export: an archive is already being created")
	// ErrExportDoesNotContainIndex is used when we could not find the index data
	// in the archive.
	ErrExportDoesNotContainIndex = echo.NewHTTPError(http.StatusBadRequest, "export: archive does not contain index data")
	// ErrExportInvalidCursor is used when the given index cursor is invalid
	ErrExportInvalidCursor = echo.NewHTTPError(http.StatusBadRequest, "export: cursor is invalid")
)

const (
	// ExportStateExporting is the state used when the export document is being
	// created.
	ExportStateExporting = "exporting"
	// ExportStateDone is used when the export document is finished, without
	// error.
	ExportStateDone = "done"
	// ExportStateError is used when the export document is finshed with error.
	ExportStateError = "error"
)

// DocType implements the couchdb.Doc interface
func (e *ExportDoc) DocType() string { return consts.Exports }

// ID implements the couchdb.Doc interface
func (e *ExportDoc) ID() string { return e.DocID }

// Rev implements the couchdb.Doc interface
func (e *ExportDoc) Rev() string { return e.DocRev }

// SetID implements the couchdb.Doc interface
func (e *ExportDoc) SetID(id string) { e.DocID = id }

// SetRev implements the couchdb.Doc interface
func (e *ExportDoc) SetRev(rev string) { e.DocRev = rev }

// Clone implements the couchdb.Doc interface
func (e *ExportDoc) Clone() couchdb.Doc {
	clone := *e

	clone.PartsCursors = make([]string, len(e.PartsCursors))
	copy(clone.PartsCursors, e.PartsCursors)

	return &clone
}

// Links implements the jsonapi.Object interface
func (e *ExportDoc) Links() *jsonapi.LinksList { return nil }

// Relationships implements the jsonapi.Object interface
func (e *ExportDoc) Relationships() jsonapi.RelationshipMap { return nil }

// Included implements the jsonapi.Object interface
func (e *ExportDoc) Included() []jsonapi.Object { return nil }

// HasExpired returns whether or not the export document has expired.
func (e *ExportDoc) HasExpired() bool {
	return time.Until(e.ExpiresAt) <= 0
}

var _ jsonapi.Object = &ExportDoc{}

// GenerateAuthMessage generates a MAC authentificating the access to the
// export data.
func (e *ExportDoc) GenerateAuthMessage(i *instance.Instance) []byte {
	mac, err := crypto.EncodeAuthMessage(archiveMACConfig, i.SessionSecret, []byte(e.ID()), nil)
	if err != nil {<commit_msg>Fix clone for move.ExportDoc<commit_after>package move

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
	"github.com/cozy/cozy-stack/pkg/crypto"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/realtime"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/cozy-stack/pkg/vfs"
	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/cozy/echo"
)

const (
	// ExportFilesDir is the directory for storing the files in the export
	// archive.
	ExportFilesDir = "My Cozy/Files"
	// ExportMetasDir is the directory for storing the metadata in the export
	// archive.
	ExportMetasDir = "My Cozy/Metadata"
)

// ExportDoc is a documents storing the metadata of an export.
type ExportDoc struct {
	DocID     string `json:"_id,omitempty"`
	DocRev    string `json:"_rev,omitempty"`
	Domain    string `json:"domain"`
	PartsSize int64  `json:"parts_size,omitempty"`

	PartsCursors     []string      `json:"parts_cursors,omitempty"`
	WithDoctypes     []string      `json:"with_doctypes,omitempty"`
	WithoutFiles     bool          `json:"without_files,omitempty"`
	State            string        `json:"state"`
	CreatedAt        time.Time     `json:"created_at"`
	ExpiresAt        time.Time     `json:"expires_at"`
	TotalSize        int64         `json:"total_size,omitempty"`
	CreationDuration time.Duration `json:"creation_duration,omitempty"`
	Error            string        `json:"error,omitempty"`
}

// PartsSize is the default size of a file bucket, to split the index into
// equal-sized parts.
const PartsSize = 100 * 1024 * 1024 // 100 MB

var (
	// ErrExportNotFound is used when a export document could not be found
	ErrExportNotFound = echo.NewHTTPError(http.StatusNotFound, "exports: not found")
	// ErrExportExpired is used when the export document has expired along with
	// its associated data.
	ErrExportExpired = echo.NewHTTPError(http.StatusNotFound, "exports: has expired")
	// ErrMACInvalid is used when the given MAC is not valid.
	ErrMACInvalid = echo.NewHTTPError(http.StatusUnauthorized, "exports: invalid mac")
	// ErrExportConflict is used when an export is already being perfomed.
	ErrExportConflict = echo.NewHTTPError(http.StatusConflict, "export: an archive is already being created")
	// ErrExportDoesNotContainIndex is used when we could not find the index data
	// in the archive.
	ErrExportDoesNotContainIndex = echo.NewHTTPError(http.StatusBadRequest, "export: archive does not contain index data")
	// ErrExportInvalidCursor is used when the given index cursor is invalid
	ErrExportInvalidCursor = echo.NewHTTPError(http.StatusBadRequest, "export: cursor is invalid")
)

const (
	// ExportStateExporting is the state used when the export document is being
	// created.
	ExportStateExporting = "exporting"
	// ExportStateDone is used when the export document is finished, without
	// error.
	ExportStateDone = "done"
	// ExportStateError is used when the export document is finshed with error.
	ExportStateError = "error"
)

// DocType implements the couchdb.Doc interface
func (e *ExportDoc) DocType() string { return consts.Exports }

// ID implements the couchdb.Doc interface
func (e *ExportDoc) ID() string { return e.DocID }

// Rev implements the couchdb.Doc interface
func (e *ExportDoc) Rev() string { return e.DocRev }

// SetID implements the couchdb.Doc interface
func (e *ExportDoc) SetID(id string) { e.DocID = id }

// SetRev implements the couchdb.Doc interface
func (e *ExportDoc) SetRev(rev string) { e.DocRev = rev }

// Clone implements the couchdb.Doc interface
func (e *ExportDoc) Clone() couchdb.Doc {
	clone := *e

	clone.PartsCursors = make([]string, len(e.PartsCursors))
	copy(clone.PartsCursors, e.PartsCursors)

	clone.WithDoctypes = make([]string, len(e.WithDoctypes))
	copy(clone.WithDoctypes, e.WithDoctypes)

	return &clone
}

// Links implements the jsonapi.Object interface
func (e *ExportDoc) Links() *jsonapi.LinksList { return nil }

// Relationships implements the jsonapi.Object interface
func (e *ExportDoc) Relationships() jsonapi.RelationshipMap { return nil }

// Included implements the jsonapi.Object interface
func (e *ExportDoc) Included() []jsonapi.Object { return nil }

// HasExpired returns whether or not the export document has expired.
func (e *ExportDoc) HasExpired() bool {
	return time.Until(e.ExpiresAt) <= 0
}

var _ jsonapi.Object = &ExportDoc{}

// GenerateAuthMessage generates a MAC authentificating the access to the
// export data.
func (e *ExportDoc) GenerateAuthMessage(i *instance.Instance) []byte {
	mac, err := crypto.EncodeAuthMessage(archiveMACConfig, i.SessionSecret, []byte(e.ID()), nil)
	if err != nil {
{'fraction_non_alphanumeric': 0.10855603894758649, 'fraction_numerical': 0.0039361922519163045, 'mean_word_length': 4.130180658873539, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3485478', 'n_tokens_mistral': 3110, 'n_tokens_neox': 3052, 'n_words': 898}