<commit_before>/****************************************************************************
 * This file is part of Builder.
 *
 * Copyright (C) 2015 <NAME>
 *
 * Author(s):
 *    <NAME> <<EMAIL>>
 *
 * $BEGIN_LICENSE:AGPL3+$
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * $END_LICENSE$
 ***************************************************************************/

package master

import (
	"github.com/hawaii-desktop/builder"
	"github.com/hawaii-desktop/builder/database"
	"github.com/hawaii-desktop/builder/logging"
	"github.com/hawaii-desktop/builder/webserver"
	"sync"
	"time"
)

// Master.
type Master struct {
	// Database.
	db *database.Database
	// Web socket hub.
	hub *webserver.WebSocketHub
	// Buffered channel that we can send jobs on.
	buildJobQueue chan *Job
	// Buffered channel that holds the job channels from each slave.
	slaveQueue chan chan *Job
	// Broadcast queue for the web socket.
	webSocketQueue chan interface{}
	// List of jobs to be processed.
	jobs []*Job
	// Protects jobs list.
	jobsMutex sync.Mutex
	// Current statistics.
	stats statistics
	// Mutext that protects statistics.
	sMutex sync.Mutex
}

// Generic request received from the Web user interface.
type request struct {
	Type int `json:"type"`
}

// Generic message sent to the Web user interface.
type message struct {
	Type int         `json:"type"`
	Data interface{} `json:"data,omitifempty"`
}

// Message types
const (
	WEB_SOCKET_STATISTICS = iota
	WEB_SOCKET_QUEUED_JOBS
	WEB_SOCKET_DISPATCHED_JOBS
	WEB_SOCKET_COMPLETED_JOBS
	WEB_SOCKET_FAILED_JOBS
)

// Statistics to show on the Web user interface.
type statistics struct {
	Queued     int `json:"queued"`
	Dispatched int `json:"dispatched"`
	Completed  int `json:"completed"`
	Failed     int `json:"failed"`
	Staging    int `json:"staging"`
	Main       int `json:"main"`
}

// Update function.
type statisticsUpdateFunc func(s *statistics)

// Jobs list to show on the Web user interface.
type jobsList struct {
	Id           uint64    `json:"id"`
	Target       string    `json:"target"`
	Architecture string    `json:"arch"`
	Started      time.Time `json:"started"`
	Finished     time.Time `json:"finished"`
}

// Create a new master.
// This also create or open the database.
func NewMaster(hub *webserver.WebSocketHub) (*Master, error) {
	db, err := database.NewDatabase(Config.Server.Database)
	if err != nil {
		return nil, err
	}

	return &Master{
		db:             db,
		hub:            hub,
		buildJobQueue:  make(chan *Job, Config.Build.MaxJobs),
		slaveQueue:     make(chan chan *Job, Config.Build.MaxSlaves),
		webSocketQueue: make(chan interface{}),
		jobs:           make([]*Job, 0, Config.Build.MaxJobs),
		stats:          statistics{0, 0, 0, 0, 0, 0},
	}, nil
}

// Close the database.
func (m *Master) Close() {
	m.db.Close()
	m.db = nil
}

// Dispatch jobs.
func (m *Master) Dispatch() {
	for {
		select {
		case j := <-m.buildJobQueue:
			logging.Tracef("About to dispatch job #%d...\n", j.Id)
			go func() {
				// Update job
				j.Started = time.Now()
				j.Status = builder.JOB_STATUS_WAITING

				// Save on the database
				m.saveDatabaseJob(j)

				// Update Web socket clients
				m.updateStatistics()
				m.updateAllJobs()

				// Dispatch
				slave := <-m.slaveQueue
				logging.Tracef("Dispatching job #%d...\n", j.Id)
				slave <- j
			}()
		}
	}
}

// Queue events to the Web socket.
func (m *Master) DeliverWebSocketEvents() {
	for {
		select {
		case e := <-m.webSocketQueue:
			if e != nil {
				m.hub.Broadcast(e)
			}
		}
	}
}
<commit_msg>Fix typo in json information<commit_after>/****************************************************************************
 * This file is part of Builder.
 *
 * Copyright (C) 2015 <NAME>
 *
 * Author(s):
 *    <NAME> <<EMAIL>>
 *
 * $BEGIN_LICENSE:AGPL3+$
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * $END_LICENSE$
 ***************************************************************************/

package master

import (
	"github.com/hawaii-desktop/builder"
	"github.com/hawaii-desktop/builder/database"
	"github.com/hawaii-desktop/builder/logging"
	"github.com/hawaii-desktop/builder/webserver"
	"sync"
	"time"
)

// Master.
type Master struct {
	// Database.
	db *database.Database
	// Web socket hub.
	hub *webserver.WebSocketHub
	// Buffered channel that we can send jobs on.
	buildJobQueue chan *Job
	// Buffered channel that holds the job channels from each slave.
	slaveQueue chan chan *Job
	// Broadcast queue for the web socket.
	webSocketQueue chan interface{}
	// List of jobs to be processed.
	jobs []*Job
	// Protects jobs list.
	jobsMutex sync.Mutex
	// Current statistics.
	stats statistics
	// Mutext that protects statistics.
	sMutex sync.Mutex
}

// Generic request received from the Web user interface.
type request struct {
	Type int `json:"type"`
}

// Generic message sent to the Web user interface.
type message struct {
	Type int         `json:"type"`
	Data interface{} `json:"data,omitempty"`
}

// Message types
const (
	WEB_SOCKET_STATISTICS = iota
	WEB_SOCKET_QUEUED_JOBS
	WEB_SOCKET_DISPATCHED_JOBS
	WEB_SOCKET_COMPLETED_JOBS
	WEB_SOCKET_FAILED_JOBS
)

// Statistics to show on the Web user interface.
type statistics struct {
	Queued     int `json:"queued"`
	Dispatched int `json:"dispatched"`
	Completed  int `json:"completed"`
	Failed     int `json:"failed"`
	Staging    int `json:"staging"`
	Main       int `json:"main"`
}

// Update function.
type statisticsUpdateFunc func(s *statistics)

// Jobs list to show on the Web user interface.
type jobsList struct {
	Id           uint64    `json:"id"`
	Target       string    `json:"target"`
	Architecture string    `json:"arch"`
	Started      time.Time `json:"started"`
	Finished     time.Time `json:"finished"`
}

// Create a new master.
// This also create or open the database.
func NewMaster(hub *webserver.WebSocketHub) (*Master, error) {
	db, err := database.NewDatabase(Config.Server.Database)
	if err != nil {
		return nil, err
	}

	return &Master{
		db:             db,
		hub:            hub,
		buildJobQueue:  make(chan *Job, Config.Build.MaxJobs),
		slaveQueue:     make(chan chan *Job, Config.Build.MaxSlaves),
		webSocketQueue: make(chan interface{}),
		jobs:           make([]*Job, 0, Config.Build.MaxJobs),
		stats:          statistics{0, 0, 0, 0, 0, 0},
	}, nil
}

// Close the database.
func (m *Master) Close() {
	m.db.Close()
	m.db = nil
}

// Dispatch jobs.
func (m *Master) Dispatch() {
	for {
		select {
		case j := <-m.buildJobQueue:
			logging.Tracef("About to dispatch job #%d...\n", j.Id)
			go func() {
				// Update job
				j.Started = time.Now()
				j.Status = builder.JOB_STATUS_WAITING

				// Save on the database
				m.saveDatabaseJob(j)

				// Update Web socket clients
				m.updateStatistics()
				m.updateAllJobs()

				// Dispatch
				slave := <-m.slaveQueue
				logging.Tracef("Dispatching job #%d...\n", j.Id)
				slave <- j
			}()
		}
	}
}

// Queue events to the Web socket.
func (m *Master) DeliverWebSocketEvents() {
	for {
		select {
		case e := <-m.webSocketQueue:
			if e != nil {
				m.hub.Broadcast(e)
			}
		}
	}
}

{'fraction_non_alphanumeric': 0.1471612051374385, 'fraction_numerical': 0.0036010082823190494, 'mean_word_length': 3.2531904032669727, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '572362', 'n_tokens_mistral': 2853, 'n_tokens_neox': 2623, 'n_words': 858}