<commit_before>package uc_team_migration

import (
	"encoding/json"
	"errors"
	"github.com/tidwall/gjson"
	"github.com/watermint/toolbox/app"
	"github.com/watermint/toolbox/app/app_report"
	"github.com/watermint/toolbox/domain/infra/api_context"
	"github.com/watermint/toolbox/domain/infra/api_parser"
	"github.com/watermint/toolbox/domain/infra/api_util"
	"github.com/watermint/toolbox/domain/model/mo_file"
	"github.com/watermint/toolbox/domain/model/mo_group"
	"github.com/watermint/toolbox/domain/model/mo_group_member"
	"github.com/watermint/toolbox/domain/model/mo_path"
	"github.com/watermint/toolbox/domain/model/mo_profile"
	"github.com/watermint/toolbox/domain/model/mo_sharedfolder"
	"github.com/watermint/toolbox/domain/model/mo_sharedfolder_member"
	"github.com/watermint/toolbox/domain/model/mo_teamfolder"
	"github.com/watermint/toolbox/domain/service/sv_device"
	"github.com/watermint/toolbox/domain/service/sv_file"
	"github.com/watermint/toolbox/domain/service/sv_group"
	"github.com/watermint/toolbox/domain/service/sv_group_member"
	"github.com/watermint/toolbox/domain/service/sv_member"
	"github.com/watermint/toolbox/domain/service/sv_namespace"
	"github.com/watermint/toolbox/domain/service/sv_profile"
	"github.com/watermint/toolbox/domain/service/sv_sharedfolder"
	"github.com/watermint/toolbox/domain/service/sv_sharedfolder_member"
	"github.com/watermint/toolbox/domain/service/sv_team"
	"github.com/watermint/toolbox/domain/service/sv_teamfolder"
	"github.com/watermint/toolbox/domain/usecase/uc_teamfolder_mirror"
	"go.uber.org/zap"
	"io/ioutil"
	"path/filepath"
	"strings"
)

type Migration interface {
	// Define scope
	Scope(opts ...ScopeOpt) (ctx Context, err error)

	// Resume from preserved state
	Resume(opts ...ResumeOpt) (ctx Context, err error)

	// Preflight check (inspect, preserve)
	Preflight(ctx Context) (err error)

	// Mirror team folders
	Content(ctx Context) (err error)

	// Migration process (inspect, preserve, bridge, transfer, permission, clean up)
	Migrate(ctx Context) (err error)

	// Inspect team status.
	// Ensure both team allow externally sharing shared folders.
	Inspect(ctx Context) (err error)

	// Preserve members, groups, and sharing status.
	Preserve(ctx Context) (err error)

	// Bridge shared folders.
	// Share all shared folders to destination admin.
	Bridge(ctx Context) (err error)

	// Transfer members.
	// Convert accounts into Basic, and invite from destination team.
	Transfer(ctx Context) (err error)

	// Mirror permissions.
	// Create groups, invite members to shared folders or nested folders,
	// leave destination admin from bridged shared folders.
	Permissions(ctx Context) (err error)

	// Cleanup
	Cleanup(ctx Context) (err error)
}

type ResumeOpt func(opt *resumeOpts) *resumeOpts
type resumeOpts struct {
	storagePath string
	ec          *app.ExecContext
}

func ResumeFromPath(path string) ResumeOpt {
	return func(opt *resumeOpts) *resumeOpts {
		opt.storagePath = path
		return opt
	}
}
func ResumeExecContext(ec *app.ExecContext) ResumeOpt {
	return func(opt *resumeOpts) *resumeOpts {
		opt.ec = ec
		return opt
	}
}

func New(ctxExec *app.ExecContext, ctxFileSrc, ctxMgtSrc, ctxFileDst, ctxMgtDst api_context.Context, report app_report.Report) Migration {
	return &migrationImpl{
		ctxExec:          ctxExec,
		ctxFileSrc:       ctxFileSrc,
		ctxMgtSrc:        ctxMgtSrc,
		ctxFileDst:       ctxFileDst,
		ctxMgtDst:        ctxMgtDst,
		teamFolderMirror: uc_teamfolder_mirror.New(ctxFileSrc, ctxMgtSrc, ctxFileDst, ctxMgtDst, report),
		report:           report,
	}
}

type migrationImpl struct {
	ctxExec          *app.ExecContext
	ctxFileSrc       api_context.Context
	ctxFileDst       api_context.Context
	ctxMgtSrc        api_context.Context
	ctxMgtDst        api_context.Context
	teamFolderMirror uc_teamfolder_mirror.TeamFolder
	report           app_report.Report
}

func (z *migrationImpl) Resume(opts ...ResumeOpt) (ctx Context, err error) {
	ro := &resumeOpts{}
	for _, o := range opts {
		o(ro)
	}
	ctxImpl := &contextImpl{}

	{
		b, err := ioutil.ReadFile(filepath.Join(ro.storagePath, "context.json"))
		if err != nil {
			z.ctxExec.Log().Error("unable to read stored context", zap.Error(err))
			return nil, err
		}
		err = json.Unmarshal(b, ctxImpl)
		if err != nil {
			z.ctxExec.Log().Error("unable to unmarshal context", zap.Error(err))
			return nil, err
		}
	}

	{
		b, err := ioutil.ReadFile(filepath.Join(ro.storagePath, "namespace_member.json"))
		if err != nil {
			z.ctxExec.Log().Error("unable to read stored context", zap.Error(err))
			return nil, err
		}
		j := gjson.ParseBytes(b)
		ctxImpl.MapNamespaceMember = make(map[string]map[string]mo_sharedfolder_member.Member)
		if j.Exists() && j.IsObject() {
			for k, ja := range j.Map() {
				if ja.IsObject() {
					var members map[string]mo_sharedfolder_member.Member
					members = make(map[string]mo_sharedfolder_member.Member)
					for _, je := range ja.Map() {
						member := &mo_sharedfolder_member.Metadata{}
						if err := api_parser.ParseModel(member, je.Get("Raw")); err != nil {
							z.log().Error("Unable to parse", zap.Error(err), zap.String("entry", je.Raw))
							return nil, err
						}
						if u, e := member.User(); e {
							members[u.Email] = u
						}
						if g, e := member.Group(); e {
							members[g.GroupId] = g
						}
						if i, e := member.Invitee(); e {
							members[i.InviteeEmail] = i
						}
					}
					ctxImpl.MapNamespaceMember[k] = members
				}
			}
		}
	}

	{
		tb, err := ioutil.ReadFile(filepath.Join(ro.storagePath, "teamfolder_content.json"))
		if err != nil {
			z.ctxExec.Log().Error("unable to read stored context", zap.Error(err))
			return nil, err
		}
		tmc, err := uc_teamfolder_mirror.UnmarshalContext(tb)
		if err != nil {
			z.ctxExec.Log().Error("unable to read stored context", zap.Error(err))
			return nil, err
		}
		ctxImpl.ctxTeamFolder = tmc
	}

	ctxImpl.init(ro.ec)
	z.ctxExec.Log().Info("Context restored", zap.String("path", ro.storagePath))
	return ctxImpl, nil
}

func (z *migrationImpl) log() *zap.Logger {
	return z.ctxExec.Log()
}

func (z *migrationImpl) Scope(opts ...ScopeOpt) (ctx Context, err error) {
	so := &scopeOpts{
		membersSpecifiedEmail:    make([]string, 0),
		teamFoldersSpecifiedName: make([]string, 0),
	}
	for _, o := range opts {
		o(so)
	}

	z.log().Info("Define scope")

	// Prepare migration context
	ctx = newContext(z.ctxExec)
	ctx.SetGroupsOnlyRelated(so.groupsOnlyRelated)
	ctx.SetKeepDesktopSessions(so.keepDesktopSessions)
	ctx.SetDontTransferFolderOwnership(so.dontTransferFolderOwnership)

	// validation
	if so.membersAllExceptAdmin && len(so.membersSpecifiedEmail) > 0 {
		z.log().Warn("Conflicted option `membersAllExceptAdmin` and `membersSpecifiedEmail`")
		return nil, errors.New("conflicted option")
	}
	if !so.membersAllExceptAdmin && len(so.membersSpecifiedEmail) < 1 {
		z.log().Warn("Please specify `memberAllExceptAdmin` or `membersSpecifiedEmail`")
		return nil, errors.New("not enough options")
	}
	if so.teamFoldersAll && len(so.teamFoldersSpecifiedName) > 0 {
		z.log().Warn("Conflicted option `teamFoldersAll` and `teamFoldersSpecifiedName`")
		return nil, errors.New("conflicted option")
	}
	if !so.teamFoldersAll && len(so.teamFoldersSpecifiedName) < 1 {
		z.log().Warn("Please specify `teamFoldersAll` or `teamFoldersSpecifiedName`")
		return nil, errors.New("not enough options")
	}

	// Identify admins
	identifyAdmins := func() error {
		adminSrc, err := sv_profile.NewTeam(z.ctxMgtSrc).Admin()
		if err != nil {
			return err
		}
		adminDst, err := sv_profile.NewTeam(z.ctxMgtDst).Admin()
		if err != nil {
			return err
		}
		z.log().Debug("Admins identified",
			zap.String("srcId", adminSrc.TeamMemberId),
			zap.String("srcEmail", adminSrc.Email),
			zap.String("dstId", adminDst.TeamMemberId),
			zap.String("dstEmail", adminDst.Email),
		)
		ctx.SetAdmins(adminSrc, adminDst)
		return nil
	}
	if err = identifyAdmins(); err != nil {
		return nil, err
	}

	// Define scope of members
	z.log().Info("Define scope: members")
	allMembers, err := sv_member.New(z.ctxMgtSrc).List()
	if err != nil {
		return nil, err
	}
	if so.membersAllExceptAdmin {
		for _, member := range allMembers {
			if ctx.AdminSrc().TeamMemberId != member.TeamMemberId {
				ctx.AddMember(member.Profile())
			} else {
				z.log().Debug("Skip admin", zap.String("teamMemberId", member.TeamMemberId), zap.String("email", member.Email))
			}
		}
	} else if len(so.membersSpecifiedEmail) > 0 {
		err = nil
		for _, email := range so.membersSpecifiedEmail {
			found := false
			emailLower := strings.ToLower(email)
			for _, member := range allMembers {
				if strings.ToLower(member.Email) == emailLower {
					ctx.AddMember(member.Profile())
					found = true
					break
				}
			}
			if !found {
				z.log().Warn("Member not found for email address", zap.String("email", email))
				err = errors.New("member not found")
			}
		}
		if err != nil {
			return nil, err
		}
	}
	if len(ctx.Members()) < 1 {
		z.log().Warn("No members found")
		return nil, errors.New("no member to migrate")
	}
	z.log().Debug("Members to migrate", zap.Int("count", len(ctx.Members())))

	// Define scope of team folders
	z.log().Info("Define scope: team folders")
	allFolders, err := sv_teamfolder.New(z.ctxFileSrc).List()
	if err != nil {
		return nil, err
	}
	if so.teamFoldersAll {
		for _, folder := range allFolders {
			ctx.AddTeamFolder(folder)
		}
	} else if len(so.teamFoldersSpecifiedName) > 0 {
		err = nil
		for _, name := range so.teamFoldersSpecifiedName {
			found := false
			nameLower := strings.ToLower(name)
			for _, folder := range allFolders {
				if strings.ToLower(folder.Name) == nameLower {
					ctx.AddTeamFolder(folder)
					found = true
					break
				}
			}
			if !found {
				z.log().Warn("Team folder not found for name", zap.String("name", name))
				err = errors.New("team folder not found")
			}
		}
		if err != nil {
			return nil, err
		}
	}
	z.log().Debug("Team folders to migrate", zap.Int("count", len(ctx.TeamFolders())))

	// Team folder mirror
	z.log().Info("Define scope: mirroring content of team folders")
	prepTeamFolderMirror := func() error {
		names := make([]string, 0)
		for _, f := range ctx.TeamFolders() {<commit_msg>#95 : skip mirror non active team folders<commit_after>package uc_team_migration

import (
	"encoding/json"
	"errors"
	"github.com/tidwall/gjson"
	"github.com/watermint/toolbox/app"
	"github.com/watermint/toolbox/app/app_report"
	"github.com/watermint/toolbox/domain/infra/api_context"
	"github.com/watermint/toolbox/domain/infra/api_parser"
	"github.com/watermint/toolbox/domain/infra/api_util"
	"github.com/watermint/toolbox/domain/model/mo_file"
	"github.com/watermint/toolbox/domain/model/mo_group"
	"github.com/watermint/toolbox/domain/model/mo_group_member"
	"github.com/watermint/toolbox/domain/model/mo_path"
	"github.com/watermint/toolbox/domain/model/mo_profile"
	"github.com/watermint/toolbox/domain/model/mo_sharedfolder"
	"github.com/watermint/toolbox/domain/model/mo_sharedfolder_member"
	"github.com/watermint/toolbox/domain/model/mo_teamfolder"
	"github.com/watermint/toolbox/domain/service/sv_device"
	"github.com/watermint/toolbox/domain/service/sv_file"
	"github.com/watermint/toolbox/domain/service/sv_group"
	"github.com/watermint/toolbox/domain/service/sv_group_member"
	"github.com/watermint/toolbox/domain/service/sv_member"
	"github.com/watermint/toolbox/domain/service/sv_namespace"
	"github.com/watermint/toolbox/domain/service/sv_profile"
	"github.com/watermint/toolbox/domain/service/sv_sharedfolder"
	"github.com/watermint/toolbox/domain/service/sv_sharedfolder_member"
	"github.com/watermint/toolbox/domain/service/sv_team"
	"github.com/watermint/toolbox/domain/service/sv_teamfolder"
	"github.com/watermint/toolbox/domain/usecase/uc_teamfolder_mirror"
	"go.uber.org/zap"
	"io/ioutil"
	"path/filepath"
	"strings"
)

type Migration interface {
	// Define scope
	Scope(opts ...ScopeOpt) (ctx Context, err error)

	// Resume from preserved state
	Resume(opts ...ResumeOpt) (ctx Context, err error)

	// Preflight check (inspect, preserve)
	Preflight(ctx Context) (err error)

	// Mirror team folders
	Content(ctx Context) (err error)

	// Migration process (inspect, preserve, bridge, transfer, permission, clean up)
	Migrate(ctx Context) (err error)

	// Inspect team status.
	// Ensure both team allow externally sharing shared folders.
	Inspect(ctx Context) (err error)

	// Preserve members, groups, and sharing status.
	Preserve(ctx Context) (err error)

	// Bridge shared folders.
	// Share all shared folders to destination admin.
	Bridge(ctx Context) (err error)

	// Transfer members.
	// Convert accounts into Basic, and invite from destination team.
	Transfer(ctx Context) (err error)

	// Mirror permissions.
	// Create groups, invite members to shared folders or nested folders,
	// leave destination admin from bridged shared folders.
	Permissions(ctx Context) (err error)

	// Cleanup
	Cleanup(ctx Context) (err error)
}

type ResumeOpt func(opt *resumeOpts) *resumeOpts
type resumeOpts struct {
	storagePath string
	ec          *app.ExecContext
}

func ResumeFromPath(path string) ResumeOpt {
	return func(opt *resumeOpts) *resumeOpts {
		opt.storagePath = path
		return opt
	}
}
func ResumeExecContext(ec *app.ExecContext) ResumeOpt {
	return func(opt *resumeOpts) *resumeOpts {
		opt.ec = ec
		return opt
	}
}

func New(ctxExec *app.ExecContext, ctxFileSrc, ctxMgtSrc, ctxFileDst, ctxMgtDst api_context.Context, report app_report.Report) Migration {
	return &migrationImpl{
		ctxExec:          ctxExec,
		ctxFileSrc:       ctxFileSrc,
		ctxMgtSrc:        ctxMgtSrc,
		ctxFileDst:       ctxFileDst,
		ctxMgtDst:        ctxMgtDst,
		teamFolderMirror: uc_teamfolder_mirror.New(ctxFileSrc, ctxMgtSrc, ctxFileDst, ctxMgtDst, report),
		report:           report,
	}
}

type migrationImpl struct {
	ctxExec          *app.ExecContext
	ctxFileSrc       api_context.Context
	ctxFileDst       api_context.Context
	ctxMgtSrc        api_context.Context
	ctxMgtDst        api_context.Context
	teamFolderMirror uc_teamfolder_mirror.TeamFolder
	report           app_report.Report
}

func (z *migrationImpl) Resume(opts ...ResumeOpt) (ctx Context, err error) {
	ro := &resumeOpts{}
	for _, o := range opts {
		o(ro)
	}
	ctxImpl := &contextImpl{}

	{
		b, err := ioutil.ReadFile(filepath.Join(ro.storagePath, "context.json"))
		if err != nil {
			z.ctxExec.Log().Error("unable to read stored context", zap.Error(err))
			return nil, err
		}
		err = json.Unmarshal(b, ctxImpl)
		if err != nil {
			z.ctxExec.Log().Error("unable to unmarshal context", zap.Error(err))
			return nil, err
		}
	}

	{
		b, err := ioutil.ReadFile(filepath.Join(ro.storagePath, "namespace_member.json"))
		if err != nil {
			z.ctxExec.Log().Error("unable to read stored context", zap.Error(err))
			return nil, err
		}
		j := gjson.ParseBytes(b)
		ctxImpl.MapNamespaceMember = make(map[string]map[string]mo_sharedfolder_member.Member)
		if j.Exists() && j.IsObject() {
			for k, ja := range j.Map() {
				if ja.IsObject() {
					var members map[string]mo_sharedfolder_member.Member
					members = make(map[string]mo_sharedfolder_member.Member)
					for _, je := range ja.Map() {
						member := &mo_sharedfolder_member.Metadata{}
						if err := api_parser.ParseModel(member, je.Get("Raw")); err != nil {
							z.log().Error("Unable to parse", zap.Error(err), zap.String("entry", je.Raw))
							return nil, err
						}
						if u, e := member.User(); e {
							members[u.Email] = u
						}
						if g, e := member.Group(); e {
							members[g.GroupId] = g
						}
						if i, e := member.Invitee(); e {
							members[i.InviteeEmail] = i
						}
					}
					ctxImpl.MapNamespaceMember[k] = members
				}
			}
		}
	}

	{
		tb, err := ioutil.ReadFile(filepath.Join(ro.storagePath, "teamfolder_content.json"))
		if err != nil {
			z.ctxExec.Log().Error("unable to read stored context", zap.Error(err))
			return nil, err
		}
		tmc, err := uc_teamfolder_mirror.UnmarshalContext(tb)
		if err != nil {
			z.ctxExec.Log().Error("unable to read stored context", zap.Error(err))
			return nil, err
		}
		ctxImpl.ctxTeamFolder = tmc
	}

	ctxImpl.init(ro.ec)
	z.ctxExec.Log().Info("Context restored", zap.String("path", ro.storagePath))
	return ctxImpl, nil
}

func (z *migrationImpl) log() *zap.Logger {
	return z.ctxExec.Log()
}

func (z *migrationImpl) Scope(opts ...ScopeOpt) (ctx Context, err error) {
	so := &scopeOpts{
		membersSpecifiedEmail:    make([]string, 0),
		teamFoldersSpecifiedName: make([]string, 0),
	}
	for _, o := range opts {
		o(so)
	}

	z.log().Info("Define scope")

	// Prepare migration context
	ctx = newContext(z.ctxExec)
	ctx.SetGroupsOnlyRelated(so.groupsOnlyRelated)
	ctx.SetKeepDesktopSessions(so.keepDesktopSessions)
	ctx.SetDontTransferFolderOwnership(so.dontTransferFolderOwnership)

	// validation
	if so.membersAllExceptAdmin && len(so.membersSpecifiedEmail) > 0 {
		z.log().Warn("Conflicted option `membersAllExceptAdmin` and `membersSpecifiedEmail`")
		return nil, errors.New("conflicted option")
	}
	if !so.membersAllExceptAdmin && len(so.membersSpecifiedEmail) < 1 {
		z.log().Warn("Please specify `memberAllExceptAdmin` or `membersSpecifiedEmail`")
		return nil, errors.New("not enough options")
	}
	if so.teamFoldersAll && len(so.teamFoldersSpecifiedName) > 0 {
		z.log().Warn("Conflicted option `teamFoldersAll` and `teamFoldersSpecifiedName`")
		return nil, errors.New("conflicted option")
	}
	if !so.teamFoldersAll && len(so.teamFoldersSpecifiedName) < 1 {
		z.log().Warn("Please specify `teamFoldersAll` or `teamFoldersSpecifiedName`")
		return nil, errors.New("not enough options")
	}

	// Identify admins
	identifyAdmins := func() error {
		adminSrc, err := sv_profile.NewTeam(z.ctxMgtSrc).Admin()
		if err != nil {
			return err
		}
		adminDst, err := sv_profile.NewTeam(z.ctxMgtDst).Admin()
		if err != nil {
			return err
		}
		z.log().Debug("Admins identified",
			zap.String("srcId", adminSrc.TeamMemberId),
			zap.String("srcEmail", adminSrc.Email),
			zap.String("dstId", adminDst.TeamMemberId),
			zap.String("dstEmail", adminDst.Email),
		)
		ctx.SetAdmins(adminSrc, adminDst)
		return nil
	}
	if err = identifyAdmins(); err != nil {
		return nil, err
	}

	// Define scope of members
	z.log().Info("Define scope: members")
	allMembers, err := sv_member.New(z.ctxMgtSrc).List()
	if err != nil {
		return nil, err
	}
	if so.membersAllExceptAdmin {
		for _, member := range allMembers {
			if ctx.AdminSrc().TeamMemberId != member.TeamMemberId {
				ctx.AddMember(member.Profile())
			} else {
				z.log().Debug("Skip admin", zap.String("teamMemberId", member.TeamMemberId), zap.String("email", member.Email))
			}
		}
	} else if len(so.membersSpecifiedEmail) > 0 {
		err = nil
		for _, email := range so.membersSpecifiedEmail {
			found := false
			emailLower := strings.ToLower(email)
			for _, member := range allMembers {
				if strings.ToLower(member.Email) == emailLower {
					ctx.AddMember(member.Profile())
					found = true
					break
				}
			}
			if !found {
				z.log().Warn("Member not found for email address", zap.String("email", email))
				err = errors.New("member not found")
			}
		}
		if err != nil {
			return nil, err
		}
	}
	if len(ctx.Members()) < 1 {
		z.log().Warn("No members found")
		return nil, errors.New("no member to migrate")
	}
	z.log().Debug("Members to migrate", zap.Int("count", len(ctx.Members())))

	// Define scope of team folders
	z.log().Info("Define scope: team folders")
	allFolders, err := sv_teamfolder.New(z.ctxFileSrc).List()
	if err != nil {
		return nil, err
	}
	if so.teamFoldersAll {
		for _, folder := range allFolders {
			if folder.Status == "active" {
				ctx.AddTeamFolder(folder)
			} else {
				z.log().Warn("Skip mirroring non active team folder", zap.String("name", folder.Name))
			}
		}
	} else if len(so.teamFoldersSpecifiedName) > 0 {
		err = nil
		for _, name := range so.teamFoldersSpecifiedName {
			found := false
			nameLower := strings.ToLower(name)
			for _, folder := range allFolders {
				if strings.ToLower(folder.Name) == nameLower {
					ctx.AddTeamFolder(folder)
					found = true
					break
				}
			}
			if !found {
				z.log().Warn("Team folder not found for name", zap.String("name", name))
				err = errors.New("team folder not found")
			}
		}
		if err != nil {
			return nil, err
		}
	}
	z.log().Debug("Team folders to migrate", zap.Int("count", len(ctx.TeamFolders())))

	// Team folder mirror
	z.log().Info("Define scope: mirroring content of team folders")
	prepTeamFolderMirror := func() error {
		names := make([]string, 0)
		for _, f := range ctx.TeamFolders() {
{'fraction_non_alphanumeric': 0.13469269143216933, 'fraction_numerical': 0.0010655301012253596, 'mean_word_length': 3.846948356807512, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4033401', 'n_tokens_mistral': 7939, 'n_tokens_neox': 7240, 'n_words': 1512}