<commit_before>/*
 * Copyright 2016 ThoughtWorks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package agent

import (
	"bytes"
	"encoding/json"
	"github.com/bmatcuk/doublestar"
	"github.com/gocd-contrib/gocd-golang-agent/protocol"
	"github.com/gocd-contrib/gocd-golang-agent/stream"
	"io"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

var (
	DefaultSecretMask           = "********"
	DefaultCancelCommandTimeout = 25 * time.Second
	CancelCommandTimeout        = DefaultCancelCommandTimeout
	CancelBuildTimeout          = 30 * time.Second
	BuildDebugToConsoleLog      = true
)

type BuildSession struct {
	send                  chan *protocol.Message
	console               io.WriteCloser
	artifacts             *Artifacts
	command               *protocol.BuildCommand
	artifactUploadBaseURL *url.URL

	envs    map[string]string
	cancel  chan bool
	done    chan bool
	echo    *stream.SubstituteWriter
	secrets *stream.SubstituteWriter

	buildId     string
	buildStatus string
}

func MakeBuildSession(buildId string,
	command *protocol.BuildCommand,
	console io.WriteCloser,
	artifacts *Artifacts,
	artifactUploadBaseURL *url.URL,
	send chan *protocol.Message) *BuildSession {

	secrets := stream.NewSubstituteWriter(console)
	return &BuildSession{
		buildId:               buildId,
		buildStatus:           protocol.BuildPassed,
		console:               console,
		artifacts:             artifacts,
		artifactUploadBaseURL: artifactUploadBaseURL,
		command:               command,
		send:                  send,
		envs:                  make(map[string]string),
		cancel:                make(chan bool),
		done:                  make(chan bool),
		secrets:               secrets,
		echo:                  stream.NewSubstituteWriter(secrets),
	}
}

func (s *BuildSession) Close() error {
	return closeAndWait(s.cancel, s.done, CancelBuildTimeout)
}

func (s *BuildSession) isCanceled() bool {
	if s.buildStatus == protocol.BuildCanceled {
		return true
	}
	if isClosedChan(s.cancel) {
		s.buildStatus = protocol.BuildCanceled
		return true
	} else {
		return false
	}
}

func (s *BuildSession) Run() error {
	defer func() {
		s.console.Close()
		s.send <- protocol.CompletedMessage(s.Report(""))
		LogInfo("Build completed")
	}()
	LogInfo("Build started")
	return s.ProcessCommand()
}

func (s *BuildSession) ProcessCommand() error {
	defer func() {
		close(s.done)
	}()

	return s.process(s.command)
}

func (s *BuildSession) process(cmd *protocol.BuildCommand) (err error) {
	defer s.onCancel(cmd)

	if s.isCanceled() {
		s.debugLog("build canceled, ignore %v", cmd.Name)
		return nil
	}

	if !cmd.RunIfAny() && !cmd.RunIfMatch(s.buildStatus) {
		s.debugLog("ignore %v: build[%v] != runIf[%v]", cmd.Name, s.buildStatus, cmd.RunIfConfig)
		//skip, no failure
		return nil
	}
	s.debugLog("process: %v", cmd.Name)
	if cmd.Test != nil {
		s.debugLog("test: %+v", cmd.Test)
		_, testErr := s.processTestCommand(cmd.Test.Command)
		if s.isCanceled() {
			s.debugLog("test is canceled due to build is canceled")
			return nil
		}
		success := testErr == nil
		if success != cmd.Test.Expectation {
			s.debugLog("test failed: %v, ignore command", testErr)
			return nil
		} else {
			s.debugLog("test matches expectation, continue")
		}
	}

	switch cmd.Name {
	case protocol.CommandCompose:
		return s.processCompose(cmd)
	case protocol.CommandExport:
		s.processExport(cmd)
	case protocol.CommandEcho:
		s.processEcho(cmd)
	case protocol.CommandSecret:
		s.processSecret(cmd)
	case protocol.CommandReportCurrentStatus, protocol.CommandReportCompleting:
		jobState := cmd.Args["status"]
		s.debugLog("report %v", jobState)
		s.send <- protocol.ReportMessage(cmd.Name, s.Report(jobState))
	case protocol.CommandTest:
		err = s.processTest(cmd)
	case protocol.CommandExec:
		err = s.processExec(cmd, s.secrets)
	case protocol.CommandMkdirs:
		err = s.processMkdirs(cmd)
	case protocol.CommandCleandir:
		err = s.processCleandir(cmd)
	case protocol.CommandUploadArtifact:
		err = s.processUploadArtifact(cmd)
	case protocol.CommandDownloadFile:
		err = s.processDownload(cmd)
	case protocol.CommandDownloadDir:
		err = s.processDownload(cmd)
	case protocol.CommandFail:
		err = Err(cmd.Args["0"])
	default:
		s.warn("Golang Agent does not support build comamnd '%v' yet, related GoCD feature will not be supported. More details: https://github.com/gocd-contrib/gocd-golang-agent", cmd.Name)
	}

	if s.isCanceled() {
		LogInfo("build canceled")
		s.buildStatus = protocol.BuildCanceled
	} else if err != nil {
		s.buildStatus = protocol.BuildFailed
		errMsg := Sprintf("ERROR: %v\n", err)
		LogInfo(errMsg)
		s.ConsoleLog(errMsg)
	}

	return
}

func (s *BuildSession) onCancel(cmd *protocol.BuildCommand) {
	if cmd.OnCancel == nil || !s.isCanceled() {
		return
	}
	cancel := &BuildSession{
		buildId:               s.buildId,
		console:               s.console,
		artifacts:             s.artifacts,
		artifactUploadBaseURL: s.artifactUploadBaseURL,
		send:        s.send,
		envs:        s.envs,
		secrets:     s.secrets,
		echo:        s.echo,
		command:     cmd.OnCancel,
		buildStatus: protocol.BuildPassed,
		cancel:      make(chan bool),
		done:        make(chan bool),
	}
	go func() {
		cancel.ProcessCommand()
	}()
	select {
	case <-cancel.done:
	case <-time.After(CancelCommandTimeout):
		s.warn("Kill cancel task because it did not finish in %v.", CancelCommandTimeout)<commit_msg>move test command execution into a method<commit_after>/*
 * Copyright 2016 ThoughtWorks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package agent

import (
	"bytes"
	"encoding/json"
	"github.com/bmatcuk/doublestar"
	"github.com/gocd-contrib/gocd-golang-agent/protocol"
	"github.com/gocd-contrib/gocd-golang-agent/stream"
	"io"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

var (
	DefaultSecretMask           = "********"
	DefaultCancelCommandTimeout = 25 * time.Second
	CancelCommandTimeout        = DefaultCancelCommandTimeout
	CancelBuildTimeout          = 30 * time.Second
	BuildDebugToConsoleLog      = true
)

type BuildSession struct {
	send                  chan *protocol.Message
	console               io.WriteCloser
	artifacts             *Artifacts
	command               *protocol.BuildCommand
	artifactUploadBaseURL *url.URL

	envs    map[string]string
	cancel  chan bool
	done    chan bool
	echo    *stream.SubstituteWriter
	secrets *stream.SubstituteWriter

	buildId     string
	buildStatus string
}

func MakeBuildSession(buildId string,
	command *protocol.BuildCommand,
	console io.WriteCloser,
	artifacts *Artifacts,
	artifactUploadBaseURL *url.URL,
	send chan *protocol.Message) *BuildSession {

	secrets := stream.NewSubstituteWriter(console)
	return &BuildSession{
		buildId:               buildId,
		buildStatus:           protocol.BuildPassed,
		console:               console,
		artifacts:             artifacts,
		artifactUploadBaseURL: artifactUploadBaseURL,
		command:               command,
		send:                  send,
		envs:                  make(map[string]string),
		cancel:                make(chan bool),
		done:                  make(chan bool),
		secrets:               secrets,
		echo:                  stream.NewSubstituteWriter(secrets),
	}
}

func (s *BuildSession) Close() error {
	return closeAndWait(s.cancel, s.done, CancelBuildTimeout)
}

func (s *BuildSession) isCanceled() bool {
	if s.buildStatus == protocol.BuildCanceled {
		return true
	}
	if isClosedChan(s.cancel) {
		s.buildStatus = protocol.BuildCanceled
		return true
	} else {
		return false
	}
}

func (s *BuildSession) Run() error {
	defer func() {
		s.console.Close()
		s.send <- protocol.CompletedMessage(s.Report(""))
		LogInfo("Build completed")
	}()
	LogInfo("Build started")
	return s.ProcessCommand()
}

func (s *BuildSession) ProcessCommand() error {
	defer func() {
		close(s.done)
	}()

	return s.process(s.command)
}

func (s *BuildSession) process(cmd *protocol.BuildCommand) (err error) {
	defer s.onCancel(cmd)

	if s.isCanceled() {
		s.debugLog("build canceled, ignore %v", cmd.Name)
		return nil
	}

	if !cmd.RunIfAny() && !cmd.RunIfMatch(s.buildStatus) {
		s.debugLog("ignore %v: build[%v] != runIf[%v]", cmd.Name, s.buildStatus, cmd.RunIfConfig)
		//skip, no failure
		return nil
	}
	s.debugLog("process: %v", cmd.Name)
	if s.testFailed(cmd.Test) {
		return nil
	}

	switch cmd.Name {
	case protocol.CommandCompose:
		return s.processCompose(cmd)
	case protocol.CommandExport:
		s.processExport(cmd)
	case protocol.CommandEcho:
		s.processEcho(cmd)
	case protocol.CommandSecret:
		s.processSecret(cmd)
	case protocol.CommandReportCurrentStatus, protocol.CommandReportCompleting:
		jobState := cmd.Args["status"]
		s.debugLog("report %v", jobState)
		s.send <- protocol.ReportMessage(cmd.Name, s.Report(jobState))
	case protocol.CommandTest:
		err = s.processTest(cmd)
	case protocol.CommandExec:
		err = s.processExec(cmd, s.secrets)
	case protocol.CommandMkdirs:
		err = s.processMkdirs(cmd)
	case protocol.CommandCleandir:
		err = s.processCleandir(cmd)
	case protocol.CommandUploadArtifact:
		err = s.processUploadArtifact(cmd)
	case protocol.CommandDownloadFile:
		err = s.processDownload(cmd)
	case protocol.CommandDownloadDir:
		err = s.processDownload(cmd)
	case protocol.CommandFail:
		err = Err(cmd.Args["0"])
	default:
		s.warn("Golang Agent does not support build comamnd '%v' yet, related GoCD feature will not be supported. More details: https://github.com/gocd-contrib/gocd-golang-agent", cmd.Name)
	}

	if s.isCanceled() {
		LogInfo("build canceled")
		s.buildStatus = protocol.BuildCanceled
	} else if err != nil {
		s.buildStatus = protocol.BuildFailed
		errMsg := Sprintf("ERROR: %v\n", err)
		LogInfo(errMsg)
		s.ConsoleLog(errMsg)
	}

	return
}

func (s *BuildSession) testFailed(test *protocol.BuildCommandTest) bool {
	if test == nil {
		return false
	}
	s.debugLog("test: %+v", test)
	_, err := s.processTestCommand(test.Command)
	if s.isCanceled() {
		s.debugLog("test is canceled due to build is canceled")
		return true
	}
	success := err == nil
	if success != test.Expectation {
		s.debugLog("test failed with err: %v", err)
		return true
	} else {
		s.debugLog("test matches expectation")
		return false
	}
}

func (s *BuildSession) onCancel(cmd *protocol.BuildCommand) {
	if cmd.OnCancel == nil || !s.isCanceled() {
		return
	}
	cancel := &BuildSession{
		buildId:               s.buildId,
		console:               s.console,
		artifacts:             s.artifacts,
		artifactUploadBaseURL: s.artifactUploadBaseURL,
		send:        s.send,
		envs:        s.envs,
		secrets:     s.secrets,
		echo:        s.echo,
		command:     cmd.OnCancel,
		buildStatus: protocol.BuildPassed,
		cancel:      make(chan bool),
		done:        make(chan bool),
	}
	go func() {
		cancel.ProcessCommand()
	}()
	select {
	case <-cancel.done:
	case <-time.After(CancelCommandTimeout):
		s.warn("Kill cancel task because it did not finish in %v.", CancelCommandTimeout)
{'fraction_non_alphanumeric': 0.11358704723752296, 'fraction_numerical': 0.002169921548990152, 'mean_word_length': 3.3558705925118137, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1268411', 'n_tokens_mistral': 4157, 'n_tokens_neox': 3891, 'n_words': 960}