func assertExecutable(fPath string) error {
	info, err := os.Stat(fPath)
	if err != nil {
		return fmt.Errorf("Could not stat file: %s with error: %w", fPath, err)
	}
	if (info.Mode()&os.ModePerm)&os.FileMode(executePerm) == 0 {
		return fmt.Errorf("File: %s, is not executable.", fPath)
	}
	return nil
}
assertExecutable checks for executable permissions
package kf
import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"container/ring"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
	"unsafe"
	"github.com/l3af-project/l3afd/config"
	"github.com/l3af-project/l3afd/models"
	"github.com/l3af-project/l3afd/stats"
	"github.com/cilium/ebpf"
	ps "github.com/mitchellh/go-ps"
	"github.com/rs/zerolog/log"
	"golang.org/x/sys/unix"
)
var (
	execCommand           = exec.Command
	copyBufPool sync.Pool = sync.Pool{New: func() interface{} { return make([]byte, 32*1024) }}
)
const executePerm uint32 = 0111
const bpfStatus string = "RUNNING"
type BPF struct {
	Program        models.BPFProgram
	Cmd            *exec.Cmd
	FilePath       string                     
	RestartCount   int                        
	LogDir         string                     
	PrevMapName    string                     
	ProgID         int                        
	BpfMaps        map[string]BPFMap          
	MetricsBpfMaps map[string]*MetricsBPFMap  
	Ctx            context.Context
	Done           chan bool `json:"-"`
	DataCenter     string
}
func NewBpfProgram(ctx context.Context, program models.BPFProgram, logDir, dataCenter string) *BPF {
	bpf := &BPF{
		Program:        program,
		RestartCount:   0,
		Cmd:            nil,
		FilePath:       "",
		LogDir:         logDir,
		BpfMaps:        make(map[string]BPFMap, 0),
		MetricsBpfMaps: make(map[string]*MetricsBPFMap, 0),
		Ctx:            ctx,
		Done:           nil,
		DataCenter:     dataCenter,
	}
	return bpf
}
func LoadRootProgram(ifaceName string, direction string, eBPFType string, conf *config.Config) (*BPF, error) {
	log.Info().Msgf("LoadRootProgram iface %s direction %s ebpfType %s", ifaceName, direction, eBPFType)
	var rootProgBPF *BPF
	switch eBPFType {
	case models.XDPType:
		rootProgBPF = &BPF{
			Program: models.BPFProgram{
				Name:          conf.XDPRootProgramName,
				Artifact:      conf.XDPRootProgramArtifact,
				MapName:       conf.XDPRootProgramMapName,
				Version:       conf.XDPRootProgramVersion,
				IsUserProgram: conf.XDPRootProgramIsUserProgram,
				CmdStart:      conf.XDPRootProgramCommand,
				CmdStop:       conf.XDPRootProgramCommand,
				CmdStatus:     "",
				AdminStatus:   models.Enabled,
				SeqID:         0,
			},
			RestartCount: 0,
			Cmd:          nil,
			FilePath:     "",
			LogDir:       "",
			PrevMapName:  "",
		}
	case models.TCType:
		rootProgBPF = &BPF{
			Program: models.BPFProgram{
				Name:          conf.TCRootProgramName,
				Artifact:      conf.TCRootProgramArtifact,
				Version:       conf.TCRootProgramVersion,
				IsUserProgram: conf.TCRootProgramIsUserProgram,
				CmdStart:      conf.TCRootProgramCommand,
				CmdStop:       conf.TCRootProgramCommand,
				CmdStatus:     "",
				AdminStatus:   models.Enabled,
			},
			RestartCount: 0,
			Cmd:          nil,
			FilePath:     "",
			LogDir:       "",
			PrevMapName:  "",
		}
		if direction == models.IngressType {
			rootProgBPF.Program.MapName = conf.TCRootProgramIngressMapName
		} else if direction == models.EgressType {
			rootProgBPF.Program.MapName = conf.TCRootProgramEgressMapName
		}
	default:
		return nil, fmt.Errorf("unknown direction %s for root program in iface %s", direction, ifaceName)
	}
	rootProgBPF.Program.AddStartArgs(models.L3afDNFArgs{Key: "cmd", Value: models.StartType})
	rootProgBPF.Program.AddStopArgs(models.L3afDNFArgs{Key: "cmd", Value: models.StopType})
	if err := rootProgBPF.VerifyAndGetArtifacts(conf); err != nil {
		log.Error().Err(err).Msg("failed to get root artifacts")
		return nil, err
	}
	if fileExists(rootProgBPF.Program.MapName) {
		log.Warn().Msgf("previous instance of root program %s is running, stopping it ", rootProgBPF.Program.Name)
		if err := rootProgBPF.Stop(ifaceName, direction, conf.BpfChainingEnabled); err != nil {
			return nil, fmt.Errorf("failed to stop root program on iface %s name %s direction %s", ifaceName, rootProgBPF.Program.Name, direction)
		}
	}
	if err := rootProgBPF.Start(ifaceName, direction, conf.BpfChainingEnabled); err != nil {
		return nil, fmt.Errorf("failed to start root program on interface %s", ifaceName)
	}
	return rootProgBPF, nil
}
func LinuxDistribution() (string, error) {
	linuxDistrib := execCommand("lsb_release", "-cs")
	var out bytes.Buffer
	linuxDistrib.Stdout = &out
	if err := linuxDistrib.Run(); err != nil {
		return "", fmt.Errorf("l3afd/nf : Failed to run command with error: %w", err)
	}
	return strings.TrimSpace(string(out.Bytes())), nil
}
func StopExternalRunningProcess(processName string) error {
	if len(processName) < 1 {
		return fmt.Errorf("process name can not be empty")
	}
	psName := processName
	if len(processName) > 15 {
		psName = processName[:15]
	}
	myPid := os.Getpid()
	processList, err := ps.Processes()
	if err != nil {
		return fmt.Errorf("failed to fetch processes list")
	}
	log.Info().Msgf("Searching for process %s and not ppid %d", processName, myPid)
	for _, process := range processList {
		if strings.Contains(process.Executable(), psName) {
			if process.PPid() != myPid {
				log.Warn().Msgf("found process id %d name %s ppid %d, stopping it", process.Pid(), process.Executable(), process.PPid())
				err := syscall.Kill(process.Pid(), syscall.SIGTERM)
				if err != nil {
					return fmt.Errorf("external BPFProgram stop failed with error: %w", err)
				}
			}
		}
	}
	return nil
}
func (b *BPF) Stop(ifaceName, direction string, chain bool) error {
	if b.Program.IsUserProgram && b.Cmd == nil {
		return fmt.Errorf("BPFProgram is not running %s", b.Program.Name)
	}
	log.Info().Msgf("Stopping BPF Program - %s", b.Program.Name)
	for key, val := range b.BpfMaps {
		log.Debug().Msgf("removing BPF maps %s value map %#v", key, val)
		delete(b.BpfMaps, key)
	}
	for key, val := range b.MetricsBpfMaps {
		log.Debug().Msgf("removing metric bpf maps %s value %#v", key, val)
		delete(b.MetricsBpfMaps, key)
	}
	if len(b.Program.CmdConfig) > 0 && len(b.Program.ConfigFilePath) > 0 {
		log.Info().Msgf("Stopping KF configs %s ", b.Program.Name)
		b.Done <- true
	}
	b.ProgID = 0
	stats.Incr(stats.NFStopCount, b.Program.Name, direction)
	stats.Set(0.0, stats.NFRunning, b.Program.Name, direction)
	if len(b.Program.CmdStop) < 1 {
		if err := syscall.Kill(b.Cmd.Process.Pid, syscall.SIGTERM); err != nil {
			return fmt.Errorf("BPFProgram %s syscall.SIGTERM failed with error: %w", b.Program.Name, err)
		}
		if b.Cmd != nil {
			if err := b.Cmd.Wait(); err != nil {
				log.Error().Err(err).Msgf("cmd wait at stopping bpf program %s errored", b.Program.Name)
			}
			b.Cmd = nil
		}
		if err := b.VerifyPinnedMapVanish(chain); err != nil {
			log.Error().Err(err).Msgf("stop user program - failed to remove pinned file %s", b.Program.Name)
			return fmt.Errorf("stop user program - failed to remove pinned file %s", b.Program.Name)
		}
		if err := b.VerifyMetricsMapsVanish(); err != nil {
			log.Error().Err(err).Msgf("stop user program - failed to remove metric map references %s", b.Program.Name)
			return fmt.Errorf("stop user program - failed to remove metric map references %s", b.Program.Name)
		}
		return nil
	}
	cmd := filepath.Join(b.FilePath, b.Program.CmdStop)
	if err := assertExecutable(cmd); err != nil {
		return fmt.Errorf("no executable permissions on %s - error %w", b.Program.CmdStop, err)
	}
	args := make([]string, 0, len(b.Program.StopArgs)<<1)
	args = append(args, "--iface="+ifaceName)      
	args = append(args, "--direction="+direction)  
	for _, val := range b.Program.StopArgs {
		args = append(args, "--"+val.Key+"="+val.Value)
	}
	log.Info().Msgf("bpf program stop command : %s %v", cmd, args)
	prog := execCommand(cmd, args...)
	if err := prog.Run(); err != nil {
		log.Warn().Err(err).Msgf("l3afd/nf : Failed to stop the program %s", b.Program.CmdStop)
	}
	b.Cmd = nil
	if err := b.VerifyPinnedMapVanish(chain); err != nil {
		log.Error().Err(err).Msgf("failed to remove pinned file %s", b.Program.Name)
		return fmt.Errorf("failed to remove pinned file %s", b.Program.Name)
	}
	if err := b.VerifyMetricsMapsVanish(); err != nil {
		log.Error().Err(err).Msgf("failed to remove metric map references %s", b.Program.Name)
		return fmt.Errorf("failed to remove metric map references %s", b.Program.Name)
	}
	return nil
}
func (b *BPF) Start(ifaceName, direction string, chain bool) error {
	if b.FilePath == "" {
		return errors.New("no program binary path found")
	}
	if err := StopExternalRunningProcess(b.Program.CmdStart); err != nil {
		return fmt.Errorf("failed to stop external instance of the program %s with error : %w", b.Program.CmdStart, err)
	}
	cmd := filepath.Join(b.FilePath, b.Program.CmdStart)
	if err := assertExecutable(cmd); err != nil {
		return fmt.Errorf("no executable permissions on %s - error %w", b.Program.CmdStart, err)
	}
	if len(b.PrevMapName) > 0 {
		if err := b.RemovePrevProgFD(); err != nil {
			log.Error().Err(err).Msgf("ProgramMap %s entry removal failed", b.PrevMapName)
		}
	}
	args := make([]string, 0, len(b.Program.StartArgs)<<1)
	args = append(args, "--iface="+ifaceName)      
	args = append(args, "--direction="+direction)  
	if chain {
		if len(b.PrevMapName) > 1 {
			args = append(args, "--map-name="+b.PrevMapName)
		}
	}
	if len(b.LogDir) > 1 {
		args = append(args, "--log-dir="+b.LogDir)
	}
	if len(b.Program.RulesFile) > 1 && len(b.Program.Rules) > 1 {
		fileName, err := b.createUpdateRulesFile(direction)
		if err == nil {
			args = append(args, "--rules-file="+fileName)
		}
	}
	for _, val := range b.Program.StartArgs {
		args = append(args, "--"+val.Key+"="+val.Value)
	}
	log.Info().Msgf("BPF Program start command : %s %v", cmd, args)
	b.Cmd = execCommand(cmd, args...)
	if err := b.Cmd.Start(); err != nil {
		log.Info().Err(err).Msgf("user mode BPF program failed - %s", b.Program.Name)
		return fmt.Errorf("failed to start : %s %v", cmd, args)
	}
	if !b.Program.IsUserProgram {
		log.Info().Msgf("no user mode BPF program - %s No Pid", b.Program.Name)
		if err := b.Cmd.Wait(); err != nil {
			return fmt.Errorf("cmd wait at starting of bpf program returned with error %w", err)
		}
		b.Cmd = nil
		if err := b.VerifyPinnedMapExists(chain); err != nil {
			return fmt.Errorf("no userprogram and failed to find pinned file %s, %w", b.Program.MapName, err)
		}
		return nil
	}
	isRunning, err := b.isRunning()
	if isRunning == false {
		log.Error().Err(err).Msg("eBPF program failed to start")
		return fmt.Errorf("bpf program %s failed to start %w", b.Program.Name, err)
	}
	if err := b.VerifyPinnedMapExists(chain); err != nil {
		return fmt.Errorf("failed to find pinned file %s  %w", b.Program.MapName, err)
	}
	if len(b.Program.MapArgs) > 0 {
		if err := b.Update(ifaceName, direction); err != nil {
			log.Error().Err(err).Msg("failed to update network functions BPF maps")
			return fmt.Errorf("failed to update network functions BPF maps %w", err)
		}
	}
	if len(b.PrevMapName) > 0 {
		for i := 0; i < 10; i++ {
			b.ProgID, err = b.GetProgID()
			if err == nil {
				break
			}
			log.Warn().Msg("failed to fetch the program ID, retrying after a second ... ")
			time.Sleep(1 * time.Second)
		}
		if err != nil {
			log.Error().Err(err).Msg("failed to fetch network functions program FD")
			return fmt.Errorf("failed to fetch network functions program FD %w", err)
		}
	}
	if len(b.Program.CmdConfig) > 0 && len(b.Program.ConfigFilePath) > 0 {
		log.Info().Msgf("KP specific config monitoring - %s", b.Program.ConfigFilePath)
		b.Done = make(chan bool)
		go b.RunKFConfigs()
	}
	if err := b.SetPrLimits(); err != nil {
		log.Warn().Err(err).Msg("failed to set resource limits")
	}
	stats.Incr(stats.NFStartCount, b.Program.Name, direction)
	stats.Set(float64(time.Now().Unix()), stats.NFStartTime, b.Program.Name, direction)
	log.Info().Msgf("BPF program - %s started Process id %d Program ID %d", b.Program.Name, b.Cmd.Process.Pid, b.ProgID)
	return nil
}
func (b *BPF) Update(ifaceName, direction string) error {
	for _, val := range b.Program.MapArgs {
		log.Info().Msgf("Update map args key %s val %s", val.Key, val.Value)
		bpfMap, ok := b.BpfMaps[val.Key]
		if !ok {
			if err := b.AddBPFMap(val.Key); err != nil {
				return err
			}
			bpfMap, _ = b.BpfMaps[val.Key]
		}
		bpfMap.Update(val.Value)
	}
	stats.Incr(stats.NFUpdateCount, b.Program.Name, direction)
	return nil
}
func (b *BPF) isRunning() (bool, error) {
	if len(b.Program.CmdStatus) > 1 {
		cmd := filepath.Join(b.FilePath, b.Program.CmdStatus)
		if err := assertExecutable(cmd); err != nil {
			return false, fmt.Errorf("Failed to execute %s with error: %w", b.Program.CmdStatus, err)
		}
		args := make([]string, 0, len(b.Program.StatusArgs)<<1)
		for _, val := range b.Program.StatusArgs {
			args = append(args, "--"+val.Key)
			if len(val.Value) > 0 {
				args = append(args, "="+val.Value)
			}
		}
		prog := execCommand(cmd, args...)
		var out bytes.Buffer
		prog.Stdout = &out
		prog.Stderr = &out
		if err := prog.Run(); err != nil {
			log.Warn().Err(err).Msgf("l3afd/nf : Failed to execute %s", b.Program.CmdStatus)
		}
		outStr, errStr := string(out.Bytes()), string(out.Bytes())
		if strings.EqualFold(outStr, bpfStatus) {
			return true, nil
		}
		return false, fmt.Errorf("l3afd/nf : BPF Program not running %s", errStr)
	}
	if !b.Program.IsUserProgram {
		return true, nil
	}
	if err := b.VerifyProcessObject(); err != nil {
		return false, errors.New("No process id found")
	}
	procState, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/stat", b.Cmd.Process.Pid))
	if err != nil {
		return false, fmt.Errorf("BPF Program not running %s because of error: %w", b.Program.Name, err)
	}
	var u1, u2, state string
	_, err = fmt.Sscanf(string(procState), "%s %s %s", &u1, &u2, &state)
	if err != nil {
		return false, fmt.Errorf("Failed to scan proc state with error: %w", err)
	}
	if state == "Z" {
		return false, fmt.Errorf("Process %d in Zombie state", b.Cmd.Process.Pid)
	}
	return true, nil
}
func (b *BPF) SetPrLimits() error {
	var rlimit unix.Rlimit
	if b.Cmd == nil {
		return errors.New("No Process to set limits")
	}
	if b.Program.Memory != 0 {
		rlimit.Cur = uint64(b.Program.Memory)
		rlimit.Max = uint64(b.Program.Memory)
		if err := prLimit(b.Cmd.Process.Pid, unix.RLIMIT_AS, &rlimit); err != nil {
			log.Error().Err(err).Msgf("Failed to set Memory limits - %s", b.Program.Name)
		}
	}
	if b.Program.CPU != 0 {
		rlimit.Cur = uint64(b.Program.CPU)
		rlimit.Max = uint64(b.Program.CPU)
		if err := prLimit(b.Cmd.Process.Pid, unix.RLIMIT_CPU, &rlimit); err != nil {
			log.Error().Err(err).Msgf("Failed to set CPU limits - %s", b.Program.Name)
		}
	}
	return nil
}
func (b *BPF) VerifyAndGetArtifacts(conf *config.Config) error {
	fPath := filepath.Join(conf.BPFDir, b.Program.Name, b.Program.Version, strings.Split(b.Program.Artifact, ".")[0])
	if _, err := os.Stat(fPath); os.IsNotExist(err) {
		return b.GetArtifacts(conf)
	}
	b.FilePath = fPath
	return nil
}
func (b *BPF) GetArtifacts(conf *config.Config) error {
	var fPath = ""
	kfRepoURL, err := url.Parse(conf.KFRepoURL)
	if err != nil {
		return fmt.Errorf("unknown KF repo url format: %w", err)
	}
	linuxDist, err := LinuxDistribution()
	if err != nil {
		return fmt.Errorf("failed to find KF repo download path: %w", err)
	}
	kfRepoURL.Path = path.Join(kfRepoURL.Path, b.Program.Name, b.Program.Version, linuxDist, b.Program.Artifact)
	log.Info().Msgf("Downloading - %s", kfRepoURL)
	timeOut := time.Duration(conf.HttpClientTimeout) * time.Second
	var netTransport = &http.Transport{
		ResponseHeaderTimeout: timeOut,
	}
	client := http.Client{Transport: netTransport, Timeout: timeOut}
	resp, err := client.Get(kfRepoURL.String())
	if err != nil {
		return fmt.Errorf("Download failed: %w", err)
	}
	defer resp.Body.Close()
	buf := &bytes.Buffer{}
	buf.ReadFrom(resp.Body)
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("Get request returned unexpected status code: %d (%s), %d was expected\n\tResponse Body: %s\n", resp.StatusCode, http.StatusText(resp.StatusCode), http.StatusOK, buf.Bytes())
	}
	archive, err := gzip.NewReader(buf)
	if err != nil {
		return fmt.Errorf("Failed to create Gzip reader: %w", err)
	}
	defer archive.Close()
	tarReader := tar.NewReader(archive)
	tempDir := filepath.Join(conf.BPFDir, b.Program.Name, b.Program.Version)
	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return fmt.Errorf("Untar failed: %w", err)
		}
		fPath = filepath.Join(tempDir, header.Name)
		info := header.FileInfo()
		if info.IsDir() {
			if err = os.MkdirAll(fPath, info.Mode()); err != nil {
				return fmt.Errorf("Untar failed to create directories: %w", err)
			}
			continue
		}
		file, err := os.OpenFile(fPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
		if err != nil {
			return fmt.Errorf("Untar failed to create file: %w", err)
		}
		defer file.Close()
		buf := copyBufPool.Get().([]byte)
		_, err = io.CopyBuffer(file, tarReader, buf)
		if err != nil {
			return fmt.Errorf("GetArtifacts failed to copy files: %w", err)
		}
		copyBufPool.Put(buf)
	}
	newDir := strings.Split(b.Program.Artifact, ".")
	b.FilePath = filepath.Join(tempDir, newDir[0])
	return nil
}
func prLimit(pid int, limit uintptr, rlimit *unix.Rlimit) error {
	_, _, errno := unix.RawSyscall6(unix.SYS_PRLIMIT64,
		uintptr(pid),
		limit,
		uintptr(unsafe.Pointer(rlimit)),
		0, 0, 0)
	if errno != 0 {
		log.Error().Msgf("Failed to set prlimit for process %d and errorno %d", pid, errno)
		return errors.New("Failed to set prlimit")
	}
	return nil
}