<commit_before>package lib 

import (
    "log"
    "os"
    "os/user"
    "time"
    "fmt"
    "io/ioutil"
    "strings"
    "strconv"
    "path/filepath"
    "math/rand"
    "testing"
    oss "github.com/aliyun/aliyun-oss-go-sdk/oss"

    . "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { 
    TestingT(t) 
}

type OssutilCommandSuite struct{}

var _ = Suite(&OssutilCommandSuite{})

var (
    // Update before running test
    endpoint         = "<testEndpoint>"
    accessKeyID      = "<testAccessKeyID>"
    accessKeySecret  = "<testAccessKeySecret>"
    stsToken         = "<testSTSToken>"
)

var (
    logPath             = "ossutil_test_" + time.Now().Format("20060102_150405") + ".log"
    configFile          = "ossutil_test.boto"
    testLogFile, _      = os.OpenFile(logPath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664)
    testLogger          = log.New(testLogFile, "", log.Ldate|log.Ltime|log.Lshortfile)
    resultPath          = "ossutil_test.result"
    testResultFile, _   = os.OpenFile(resultPath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664)
    uploadFileName      = "ossutil_test.upload_file"
    downloadFileName    = "ossutil_test.download_file"
    downloadDir         = "ossutil_test.download_dir"
    inputFileName       = "ossutil_test.input_file"
    content             = "abc"
    cm                  = CommandManager{}
    out                 = os.Stdout
    errout              = os.Stderr
    sleepTime           = time.Second
)

var (
    bucketNamePrefix    = "ossutil-test-"
    bucketNameExist     = "nodelete-ossutil-test-normalcase1"
    bucketNameDest      = "nodelete-ossutil-test-dest"  // bucket not change acl
    bucketNameCP        = "nodelete-ossutil-test-cp"
    bucketNameBCP       = "nodelete-ossutil-test-bcp"
    bucketNameSetMeta   = "nodelete-ossutil-test-setmeta"
    bucketNameSetMeta1  = "nodelete-ossutil-test-setmeta1"
    bucketNameSetACL    = "nodelete-ossutil-test-setacl"
    bucketNameSetACL1   = "nodelete-ossutil-test-setacl1"
    bucketNameMB        = "nodelete-ossutil-test-mb"    // bucket with at most one object 
    bucketNameList      = "nodelete-ossutil-test-ls" 
    bucketNameNotExist  = bucketNamePrefix + "notexistbucket"  // bucket not exist
)

// Run once when the suite starts running
func (s *OssutilCommandSuite) SetUpSuite(c *C) {
    os.Stdout = testLogFile 
    os.Stderr = testLogFile 
    testLogger.Println("test command started")
    SetUpCredential()
    cm.Init()
    s.configNonInteractive(c)
    s.createFile(uploadFileName, content, c)
    s.SetUpBucketEnv(c)
}

func SetUpCredential() {
    if endpoint == "<testEndpoint>" {
        endpoint = os.Getenv("OSS_TEST_ENDPOINT") 
    }
    if strings.HasPrefix(endpoint, "https://") {
        endpoint = endpoint[8:]
    }
    if strings.HasPrefix(endpoint, "http://") {
        endpoint = endpoint[7:]
    }
    if accessKeyID == "<testAccessKeyID>" {
        accessKeyID = os.Getenv("OSS_TEST_ACCESS_KEY_ID")
    }
    if accessKeySecret == "<testAccessKeySecret>" {
        accessKeySecret = os.Getenv("OSS_TEST_ACCESS_KEY_SECRET")
    }
    if ue := os.Getenv("OSS_TEST_UPDATE_ENDPOINT"); ue != "" {
        vUpdateEndpoint = ue
    }
    if ub := os.Getenv("OSS_TEST_UPDATE_BUCKET"); ub != "" {
        vUpdateBucket = ub
    }
    if strings.HasPrefix(vUpdateEndpoint, "https://") {
        vUpdateEndpoint = vUpdateEndpoint[8:]
    }
    if strings.HasPrefix(vUpdateEndpoint, "http://") {
        vUpdateEndpoint = vUpdateEndpoint[7:]
    }
}

func (s *OssutilCommandSuite) SetUpBucketEnv(c *C) {
    s.removeBuckets(bucketNamePrefix, c)
    for _, bucket := range []string{bucketNameExist, bucketNameDest, bucketNameCP, bucketNameBCP, bucketNameSetMeta, bucketNameSetMeta1, bucketNameSetACL, bucketNameSetACL1, bucketNameMB, bucketNameList} { 
        s.putBucket(bucket, c)
    }
    time.Sleep(3*sleepTime)
    for _, bucket := range []string{bucketNameExist, bucketNameDest, bucketNameCP, bucketNameBCP, bucketNameSetMeta, bucketNameSetMeta1, bucketNameSetACL, bucketNameSetACL1, bucketNameMB, bucketNameList} { 
        s.removeObjects(bucket, "", true, true, c)
        time.Sleep(7*time.Second)
    }
}

// Run before each test or benchmark starts running
func (s *OssutilCommandSuite) TearDownSuite(c *C) {
    testLogger.Println("test command completed")
    _ = os.Remove(configFile)
    _ = os.Remove(resultPath)
    _ = os.Remove(uploadFileName)
    _ = os.Remove(downloadFileName)
    _ = os.RemoveAll(downloadDir)
    _ = os.RemoveAll(DefaultOutputDir)
    os.Stdout = out
    os.Stderr = errout
}

// Run after each test or benchmark runs
func (s *OssutilCommandSuite) SetUpTest(c *C) {
    configFile = "ossutil_test.boto"
}

// Run once after all tests or benchmarks have finished running
func (s *OssutilCommandSuite) TearDownTest(c *C) {
}

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

func randStr(n int) string {
    b := make([]rune, n)
    for i := range b {
        b[i] = letters[rand.Intn(len(letters))]
    }
    return string(b)
}

func (s *OssutilCommandSuite) configNonInteractive(c *C) {
    command := "config" 
    var args []string
    options := OptionMapType{
        "endpoint": &endpoint,
        "accessKeyID": &accessKeyID,
        "accessKeySecret": &accessKeySecret,
        "configFile": &configFile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(showElapse, Equals, false)
    c.Assert(err, IsNil)

    opts, err := LoadConfig(configFile) 
    c.Assert(err, IsNil)
    c.Assert(len(opts), Equals, 4)
    c.Assert(opts[OptionLanguage], Equals, DefaultLanguage)
    c.Assert(opts[OptionEndpoint], Equals, endpoint)
    c.Assert(opts[OptionAccessKeyID], Equals, accessKeyID)
    c.Assert(opts[OptionAccessKeySecret], Equals, accessKeySecret)
}

func (s *OssutilCommandSuite) createFile(fileName, content string, c *C) {
    fout, err := os.Create(fileName)
    defer fout.Close()
    c.Assert(err, IsNil)
    _, err = fout.WriteString(content)
    c.Assert(err, IsNil)
}

func (s *OssutilCommandSuite) readFile(fileName string, c *C) (string) {
    f, err := ioutil.ReadFile(fileName)
    c.Assert(err, IsNil)
    return string(f)
}

func (s *OssutilCommandSuite) removeBuckets(prefix string, c *C) {
    buckets := s.listBuckets(false, c)
    for _, bucket := range buckets {
        if strings.HasPrefix(bucket, prefix) {
            s.removeBucket(bucket, true, c)
        }
    }
}

func (s *OssutilCommandSuite) rawList(args []string, cmdline string) (bool, error) {
    array := strings.Split(cmdline, " ")
    if len(array) < 2 {
        return false, fmt.Errorf("ls test wrong cmdline given")
    }

    parameter := strings.Split(array[1], "-")
    if len(parameter) < 2 {
        return false, fmt.Errorf("ls test wrong cmdline given")
    }

    command := array[0]
    sf := strings.Contains(parameter[1], "s")
    d := strings.Contains(parameter[1], "d")
    m := strings.Contains(parameter[1], "m")
    a := strings.Contains(parameter[1], "a")

    str := ""
    options := OptionMapType{
        "endpoint":        &str,
        "accessKeyID":     &str,
        "accessKeySecret": &str,
        "stsToken":        &str,
        "configFile":      &configFile,
        "shortFormat":     &sf,
        "directory":       &d,
        "multipart":       &m,
        "allType":         &a,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    return showElapse, err
}

func (s *OssutilCommandSuite) listBuckets(shortFormat bool, c *C) []string {
    var args []string
    out := os.Stdout
    testResultFile, _ = os.OpenFile(resultPath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664)
    os.Stdout = testResultFile
    showElapse, err := s.rawList(args, "ls -s")
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)
    os.Stdout = out

    // get result
    buckets := s.getBucketResults(c)
    _ = os.Remove(resultPath)
    return buckets
}

func (s *OssutilCommandSuite) getBucketResults(c *C) ([]string) {
    result := s.getResult(c)
    c.Assert(len(result) >= 1, Equals, true)
    buckets := []string{}
    for _, str := range result {
        pos := strings.Index(str, SchemePrefix)
        if pos != -1 {
            buckets = append(buckets, str[pos + len(SchemePrefix):])
        }
    }
    return buckets 
}

func (s *OssutilCommandSuite) getResult(c *C) ([]string) {
    return s.getFileResult(resultPath, c)
}

func (s *OssutilCommandSuite) getFileResult(fileName string, c *C) ([]string) {
    str := s.readFile(fileName, c)
    sli := strings.Split(str, "\n")
    result := []string{}
    for _, str := range sli {
        if str != ""{
            result = append(result, str)
        }
    }
    return result 
}

func (s *OssutilCommandSuite) getReportResult(fileName string, c *C) ([]string) {
    result := s.getFileResult(fileName, c)
    c.Assert(len(result) >= 1, Equals, true)
    c.Assert(strings.HasPrefix(result[0], "#"), Equals, true)
    result = result[1:]
    for _, r := range result {
        c.Assert(strings.HasPrefix(r, "[Error]"), Equals, true)
    }
    return result 
}

func (s *OssutilCommandSuite) removeBucket(bucket string, clearObjects bool, c *C) {
    var showElapse bool
    var err error
    if !clearObjects {
        args := []string{CloudURLToString(bucket, "")}
        showElapse, err = s.rawRemove(args, false, true, true)
    } else {
        showElapse, err = s.removeWrapper("rm -arbf", bucket, "", c)
    }
    if err != nil {
        verr := err.(BucketError).err
        c.Assert(verr.(oss.ServiceError).Code == "NoSuchBucket" || verr.(oss.ServiceError).Code == "BucketNotEmpty", Equals, true)
        c.Assert(showElapse, Equals, false)
    } else {
        c.Assert(showElapse, Equals, true)
    }<commit_msg>modify case removeBucket<commit_after>package lib 

import (
    "log"
    "os"
    "os/user"
    "time"
    "fmt"
    "io/ioutil"
    "strings"
    "strconv"
    "path/filepath"
    "math/rand"
    "testing"
    oss "github.com/aliyun/aliyun-oss-go-sdk/oss"

    . "gopkg.in/check.v1"
)

// Hook up gocheck into the "go test" runner.
func Test(t *testing.T) { 
    TestingT(t) 
}

type OssutilCommandSuite struct{}

var _ = Suite(&OssutilCommandSuite{})

var (
    // Update before running test
    endpoint         = "<testEndpoint>"
    accessKeyID      = "<testAccessKeyID>"
    accessKeySecret  = "<testAccessKeySecret>"
    stsToken         = "<testSTSToken>"
)

var (
    logPath             = "ossutil_test_" + time.Now().Format("20060102_150405") + ".log"
    configFile          = "ossutil_test.boto"
    testLogFile, _      = os.OpenFile(logPath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664)
    testLogger          = log.New(testLogFile, "", log.Ldate|log.Ltime|log.Lshortfile)
    resultPath          = "ossutil_test.result"
    testResultFile, _   = os.OpenFile(resultPath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664)
    uploadFileName      = "ossutil_test.upload_file"
    downloadFileName    = "ossutil_test.download_file"
    downloadDir         = "ossutil_test.download_dir"
    inputFileName       = "ossutil_test.input_file"
    content             = "abc"
    cm                  = CommandManager{}
    out                 = os.Stdout
    errout              = os.Stderr
    sleepTime           = time.Second
)

var (
    bucketNamePrefix    = "ossutil-test-"
    bucketNameExist     = "nodelete-ossutil-test-normalcase1"
    bucketNameDest      = "nodelete-ossutil-test-dest"  // bucket not change acl
    bucketNameCP        = "nodelete-ossutil-test-cp"
    bucketNameBCP       = "nodelete-ossutil-test-bcp"
    bucketNameSetMeta   = "nodelete-ossutil-test-setmeta"
    bucketNameSetMeta1  = "nodelete-ossutil-test-setmeta1"
    bucketNameSetACL    = "nodelete-ossutil-test-setacl"
    bucketNameSetACL1   = "nodelete-ossutil-test-setacl1"
    bucketNameMB        = "nodelete-ossutil-test-mb"    // bucket with at most one object 
    bucketNameList      = "nodelete-ossutil-test-ls" 
    bucketNameNotExist  = bucketNamePrefix + "notexistbucket"  // bucket not exist
)

// Run once when the suite starts running
func (s *OssutilCommandSuite) SetUpSuite(c *C) {
    os.Stdout = testLogFile 
    os.Stderr = testLogFile 
    testLogger.Println("test command started")
    SetUpCredential()
    cm.Init()
    s.configNonInteractive(c)
    s.createFile(uploadFileName, content, c)
    s.SetUpBucketEnv(c)
}

func SetUpCredential() {
    if endpoint == "<testEndpoint>" {
        endpoint = os.Getenv("OSS_TEST_ENDPOINT") 
    }
    if strings.HasPrefix(endpoint, "https://") {
        endpoint = endpoint[8:]
    }
    if strings.HasPrefix(endpoint, "http://") {
        endpoint = endpoint[7:]
    }
    if accessKeyID == "<testAccessKeyID>" {
        accessKeyID = os.Getenv("OSS_TEST_ACCESS_KEY_ID")
    }
    if accessKeySecret == "<testAccessKeySecret>" {
        accessKeySecret = os.Getenv("OSS_TEST_ACCESS_KEY_SECRET")
    }
    if ue := os.Getenv("OSS_TEST_UPDATE_ENDPOINT"); ue != "" {
        vUpdateEndpoint = ue
    }
    if ub := os.Getenv("OSS_TEST_UPDATE_BUCKET"); ub != "" {
        vUpdateBucket = ub
    }
    if strings.HasPrefix(vUpdateEndpoint, "https://") {
        vUpdateEndpoint = vUpdateEndpoint[8:]
    }
    if strings.HasPrefix(vUpdateEndpoint, "http://") {
        vUpdateEndpoint = vUpdateEndpoint[7:]
    }
}

func (s *OssutilCommandSuite) SetUpBucketEnv(c *C) {
    s.removeBuckets(bucketNamePrefix, c)
    for _, bucket := range []string{bucketNameExist, bucketNameDest, bucketNameCP, bucketNameBCP, bucketNameSetMeta, bucketNameSetMeta1, bucketNameSetACL, bucketNameSetACL1, bucketNameMB, bucketNameList} { 
        s.putBucket(bucket, c)
    }
    time.Sleep(3*sleepTime)
    for _, bucket := range []string{bucketNameExist, bucketNameDest, bucketNameCP, bucketNameBCP, bucketNameSetMeta, bucketNameSetMeta1, bucketNameSetACL, bucketNameSetACL1, bucketNameMB, bucketNameList} { 
        s.removeObjects(bucket, "", true, true, c)
        time.Sleep(7*time.Second)
    }
}

// Run before each test or benchmark starts running
func (s *OssutilCommandSuite) TearDownSuite(c *C) {
    testLogger.Println("test command completed")
    _ = os.Remove(configFile)
    _ = os.Remove(resultPath)
    _ = os.Remove(uploadFileName)
    _ = os.Remove(downloadFileName)
    _ = os.RemoveAll(downloadDir)
    _ = os.RemoveAll(DefaultOutputDir)
    os.Stdout = out
    os.Stderr = errout
}

// Run after each test or benchmark runs
func (s *OssutilCommandSuite) SetUpTest(c *C) {
    configFile = "ossutil_test.boto"
}

// Run once after all tests or benchmarks have finished running
func (s *OssutilCommandSuite) TearDownTest(c *C) {
}

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

func randStr(n int) string {
    b := make([]rune, n)
    for i := range b {
        b[i] = letters[rand.Intn(len(letters))]
    }
    return string(b)
}

func (s *OssutilCommandSuite) configNonInteractive(c *C) {
    command := "config" 
    var args []string
    options := OptionMapType{
        "endpoint": &endpoint,
        "accessKeyID": &accessKeyID,
        "accessKeySecret": &accessKeySecret,
        "configFile": &configFile,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    c.Assert(showElapse, Equals, false)
    c.Assert(err, IsNil)

    opts, err := LoadConfig(configFile) 
    c.Assert(err, IsNil)
    c.Assert(len(opts), Equals, 4)
    c.Assert(opts[OptionLanguage], Equals, DefaultLanguage)
    c.Assert(opts[OptionEndpoint], Equals, endpoint)
    c.Assert(opts[OptionAccessKeyID], Equals, accessKeyID)
    c.Assert(opts[OptionAccessKeySecret], Equals, accessKeySecret)
}

func (s *OssutilCommandSuite) createFile(fileName, content string, c *C) {
    fout, err := os.Create(fileName)
    defer fout.Close()
    c.Assert(err, IsNil)
    _, err = fout.WriteString(content)
    c.Assert(err, IsNil)
}

func (s *OssutilCommandSuite) readFile(fileName string, c *C) (string) {
    f, err := ioutil.ReadFile(fileName)
    c.Assert(err, IsNil)
    return string(f)
}

func (s *OssutilCommandSuite) removeBuckets(prefix string, c *C) {
    buckets := s.listBuckets(false, c)
    for _, bucket := range buckets {
        if strings.HasPrefix(bucket, prefix) {
            s.removeBucket(bucket, true, c)
        }
    }
}

func (s *OssutilCommandSuite) rawList(args []string, cmdline string) (bool, error) {
    array := strings.Split(cmdline, " ")
    if len(array) < 2 {
        return false, fmt.Errorf("ls test wrong cmdline given")
    }

    parameter := strings.Split(array[1], "-")
    if len(parameter) < 2 {
        return false, fmt.Errorf("ls test wrong cmdline given")
    }

    command := array[0]
    sf := strings.Contains(parameter[1], "s")
    d := strings.Contains(parameter[1], "d")
    m := strings.Contains(parameter[1], "m")
    a := strings.Contains(parameter[1], "a")

    str := ""
    options := OptionMapType{
        "endpoint":        &str,
        "accessKeyID":     &str,
        "accessKeySecret": &str,
        "stsToken":        &str,
        "configFile":      &configFile,
        "shortFormat":     &sf,
        "directory":       &d,
        "multipart":       &m,
        "allType":         &a,
    }
    showElapse, err := cm.RunCommand(command, args, options)
    return showElapse, err
}

func (s *OssutilCommandSuite) listBuckets(shortFormat bool, c *C) []string {
    var args []string
    out := os.Stdout
    testResultFile, _ = os.OpenFile(resultPath, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0664)
    os.Stdout = testResultFile
    showElapse, err := s.rawList(args, "ls -s")
    c.Assert(err, IsNil)
    c.Assert(showElapse, Equals, true)
    os.Stdout = out

    // get result
    buckets := s.getBucketResults(c)
    _ = os.Remove(resultPath)
    return buckets
}

func (s *OssutilCommandSuite) getBucketResults(c *C) ([]string) {
    result := s.getResult(c)
    c.Assert(len(result) >= 1, Equals, true)
    buckets := []string{}
    for _, str := range result {
        pos := strings.Index(str, SchemePrefix)
        if pos != -1 {
            buckets = append(buckets, str[pos + len(SchemePrefix):])
        }
    }
    return buckets 
}

func (s *OssutilCommandSuite) getResult(c *C) ([]string) {
    return s.getFileResult(resultPath, c)
}

func (s *OssutilCommandSuite) getFileResult(fileName string, c *C) ([]string) {
    str := s.readFile(fileName, c)
    sli := strings.Split(str, "\n")
    result := []string{}
    for _, str := range sli {
        if str != ""{
            result = append(result, str)
        }
    }
    return result 
}

func (s *OssutilCommandSuite) getReportResult(fileName string, c *C) ([]string) {
    result := s.getFileResult(fileName, c)
    c.Assert(len(result) >= 1, Equals, true)
    c.Assert(strings.HasPrefix(result[0], "#"), Equals, true)
    result = result[1:]
    for _, r := range result {
        c.Assert(strings.HasPrefix(r, "[Error]"), Equals, true)
    }
    return result 
}

func (s *OssutilCommandSuite) removeBucket(bucket string, clearObjects bool, c *C) {
    args := []string{CloudURLToString(bucket, "")}
    showElapse, err := s.rawRemove(args, clearObjects, true, true)
    if err != nil {
        error := err.(BucketError).err
        c.Assert(error.(oss.ServiceError).Code == "NoSuchBucket" || error.(oss.ServiceError).Code == "BucketNotEmpty", Equals, true)
        c.Assert(showElapse, Equals, false)
    } else {
        c.Assert(showElapse, Equals, true)
    }
{'fraction_non_alphanumeric': 0.131466735165682, 'fraction_numerical': 0.005753917287438993, 'mean_word_length': 2.915124698310539, 'pattern_counts': {'":': 26, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2086942', 'n_tokens_mistral': 6713, 'n_tokens_neox': 6571, 'n_words': 1341}