<commit_before>package cdsclient

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/venom"
	"github.com/sguiheux/go-coverage"
)

// shrinkQueue is used to shrink the polled queue 200% of the channel capacity (l)
// it returns as reference date the date of the last element in the shrinkked queue
func shrinkQueue(queue *sdk.WorkflowQueue, nbJobsToKeep int) time.Time {
	if len(*queue) == 0 {
		return time.Time{}
	}

	if nbJobsToKeep < 1 {
		nbJobsToKeep = 1
	}

	// nbJobsToKeep is by default the concurrent max worker provisionning.
	// we keep 2x this number
	nbJobsToKeep = nbJobsToKeep * 2

	queue.Sort()

	if len(*queue) > nbJobsToKeep {
		newQueue := (*queue)[:nbJobsToKeep]
		*queue = newQueue
	}
	t0 := time.Now()
	for _, q := range *queue {
		if q.Queued.Before(t0) {
			t0 = q.Queued
		}
	}
	return t0
}

func (c *client) QueuePolling(ctx context.Context, jobs chan<- sdk.WorkflowNodeJobRun, errs chan<- error, delay time.Duration, modelType string, ratioService *int) error {
	jobsTicker := time.NewTicker(delay)

	// This goroutine call the SSE route
	chanMessageReceived := make(chan sdk.WebsocketEvent, 10)
	chanMessageToSend := make(chan sdk.WebsocketFilter, 10)
	sdk.GoRoutine(ctx, "RequestWebsocket", func(ctx context.Context) {
		for ctx.Err() == nil {
			if err := c.RequestWebsocket(ctx, "/ws", chanMessageToSend, chanMessageReceived); err != nil {
				log.Println("QueuePolling", err)
			}
			time.Sleep(1 * time.Second)
		}
	})
	chanMessageToSend <- sdk.WebsocketFilter{
		Queue: true,
		Type:  sdk.WebsocketFilterTypeQueue,
	}

	for {
		select {
		case <-ctx.Done():
			jobsTicker.Stop()
			if jobs != nil {
				close(jobs)
			}
			return ctx.Err()
		case wsEvent := <-chanMessageReceived:
			if jobs == nil {
				continue
			}
			if wsEvent.Event.EventType == "sdk.EventRunWorkflowJob" && wsEvent.Event.Status == sdk.StatusWaiting {
				var jobEvent sdk.EventRunWorkflowJob
				if err := json.Unmarshal(wsEvent.Event.Payload, &jobEvent); err != nil {
					errs <- fmt.Errorf("unable to unmarshal job %v: %v", wsEvent.Event.Payload, err)
					continue
				}
				job, err := c.QueueJobInfo(ctx, jobEvent.ID)
				// Do not log the error if the job does not exist
				if sdk.ErrorIs(err, sdk.ErrWorkflowNodeRunJobNotFound) {
					continue
				}

				if err != nil {
					errs <- fmt.Errorf("unable to get job %v info: %v", job.ID, err)<commit_msg>fix(hatchery): avoid panic on queuePolling (#5302)<commit_after>package cdsclient

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/venom"
	"github.com/sguiheux/go-coverage"
)

// shrinkQueue is used to shrink the polled queue 200% of the channel capacity (l)
// it returns as reference date the date of the last element in the shrinkked queue
func shrinkQueue(queue *sdk.WorkflowQueue, nbJobsToKeep int) time.Time {
	if len(*queue) == 0 {
		return time.Time{}
	}

	if nbJobsToKeep < 1 {
		nbJobsToKeep = 1
	}

	// nbJobsToKeep is by default the concurrent max worker provisionning.
	// we keep 2x this number
	nbJobsToKeep = nbJobsToKeep * 2

	queue.Sort()

	if len(*queue) > nbJobsToKeep {
		newQueue := (*queue)[:nbJobsToKeep]
		*queue = newQueue
	}
	t0 := time.Now()
	for _, q := range *queue {
		if q.Queued.Before(t0) {
			t0 = q.Queued
		}
	}
	return t0
}

func (c *client) QueuePolling(ctx context.Context, jobs chan<- sdk.WorkflowNodeJobRun, errs chan<- error, delay time.Duration, modelType string, ratioService *int) error {
	jobsTicker := time.NewTicker(delay)

	// This goroutine call the SSE route
	chanMessageReceived := make(chan sdk.WebsocketEvent, 10)
	chanMessageToSend := make(chan sdk.WebsocketFilter, 10)
	sdk.GoRoutine(ctx, "RequestWebsocket", func(ctx context.Context) {
		for ctx.Err() == nil {
			if err := c.RequestWebsocket(ctx, "/ws", chanMessageToSend, chanMessageReceived); err != nil {
				log.Println("QueuePolling", err)
			}
			time.Sleep(1 * time.Second)
		}
	})
	chanMessageToSend <- sdk.WebsocketFilter{
		Queue: true,
		Type:  sdk.WebsocketFilterTypeQueue,
	}

	for {
		select {
		case <-ctx.Done():
			jobsTicker.Stop()
			if jobs != nil {
				close(jobs)
			}
			return ctx.Err()
		case wsEvent := <-chanMessageReceived:
			if jobs == nil {
				continue
			}
			if wsEvent.Event.EventType == "sdk.EventRunWorkflowJob" && wsEvent.Event.Status == sdk.StatusWaiting {
				var jobEvent sdk.EventRunWorkflowJob
				if err := json.Unmarshal(wsEvent.Event.Payload, &jobEvent); err != nil {
					errs <- fmt.Errorf("unable to unmarshal job %v: %v", wsEvent.Event.Payload, err)
					continue
				}
				job, err := c.QueueJobInfo(ctx, jobEvent.ID)
				// Do not log the error if the job does not exist
				if sdk.ErrorIs(err, sdk.ErrWorkflowNodeRunJobNotFound) {
					continue
				}

				if err != nil {
					errs <- fmt.Errorf("unable to get job %v info: %v", jobEvent.ID, err)
{'fraction_non_alphanumeric': 0.1312621359223301, 'fraction_numerical': 0.008155339805825243, 'mean_word_length': 3.5026223776223775, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '98087', 'n_tokens_mistral': 2169, 'n_tokens_neox': 1904, 'n_words': 478}