func removeDuplicatesString(elements []string) (result []string) {
	encountered := map[string]bool{}
	for _, element := range elements {
		if !encountered[element] {
			encountered[element] = true
			result = append(result, element)
		}
	}
	return result
}
removeDuplicatesString deduplicates array of strings
package file
import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"github.com/aliyun/aliyun_assist_client/agent/log"
	"github.com/aliyun/aliyun_assist_client/agent/inventory/model"
)
type filterObj struct {
	Path         string
	Pattern      []string
	Recursive    bool
	DirScanLimit *int
}
type fileInfoObject struct {
	fi   os.FileInfo
	path string
}
var getFullPath func(path string, mapping func(string) string) (string, error)
const FileCountLimit = 500
const FileCountLimitExceeded = "File Count Limit Exceeded"
const DirScanLimit = 5000
const DirScanLimitExceeded = "Directory Scan Limit Exceeded"
var DirScanLimitError = errors.New(DirScanLimitExceeded)
var FileCountLimitError = errors.New(FileCountLimitExceeded)
var readDirFunc = readDir
var getFilesFunc = getFiles
var existsPath = exists
var filepathWalk = filepath.Walk
var getMetaDataFunc = getMetaData
func readDir(dirname string) ([]os.FileInfo, error) {
	return ioutil.ReadDir(dirname)
}
func removeDuplicatesFileData(elements []model.FileData) (result []model.FileData) {
	encountered := map[model.FileData]bool{}
	for v := range elements {
		if !encountered[elements[v]] {
			encountered[elements[v]] = true
			result = append(result, elements[v])
		}
	}
	return result
}