<commit_before>// Copyright (C) 2012,2013 <NAME> <<EMAIL>>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
// Boston, MA 02111-1307, USA.

const GLib = imports.gi.GLib;
const Gio = imports.gi.Gio;
const Lang = imports.lang;
const Format = imports.format;

const GSystem = imports.gi.GSystem;

const Task = imports.task;
const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const JsonUtil = imports.jsonutil;
const Snapshot = imports.snapshot;
const Config = imports.config;

const loop = GLib.MainLoop.new(null, true);

var AutoBuilderIface = <interface name="org.gnome.OSTreeBuild.AutoBuilder">
<method name="queueBuild">
    <arg type="as" direction="in" />
</method>
<method name="queueResolve">
    <arg type="as" direction="in" />
</method>
<property name="Status" type="s" access="read" />
</interface>;

const AutoBuilder = new Lang.Class({
    Name: 'AutoBuilder',
    
    _init: function() {
	this._resolve_proc = null;
	this._build_proc = null;

	this.config = Config.get();
	this.workdir = Gio.File.new_for_path(this.config.getGlobal('workdir'));
	this.prefix = this.config.getPrefix();
	this._snapshot_dir = this.workdir.get_child('snapshots');
	this._status_path = this.workdir.get_child('autobuilder-' + this.prefix + '.json');

	this._build_needed = true;
	this._full_resolve_needed = true;
	this._queued_force_builds = [];
	this._queued_force_resolve = [];
	this._autoupdate_self = true;
	this._resolve_timeout = 0;
	this._source_snapshot_path = null;
	this._prev_source_snapshot_path = null;
	
	this._impl = Gio.DBusExportedObject.wrapJSObject(AutoBuilderIface, this);
	this._impl.export(Gio.DBus.session, '/org/gnome/OSTreeBuild/AutoBuilder');

	let snapshotdir = this.workdir.get_child('snapshots');
	GSystem.file_ensure_directory(snapshotdir, true, null);
	this._src_db = new JsonDB.JsonDB(snapshotdir, this.prefix + '-src-snapshot');

	let taskdir = new Task.TaskDir(this.workdir.get_child('tasks'));
	this._resolve_taskset = taskdir.get(this.prefix + '-resolve');
	this._build_taskset = taskdir.get(this.prefix + '-build');

	this._source_snapshot_path = this._src_db.getLatestPath();

	this._status_path = this.workdir.get_child('autobuilder-' + this.prefix + '.json');

	this._resolve_timeout = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT,
							 60 * 10, Lang.bind(this, this._fetchAll));
	this._fetchAll();
	if (this._source_snapshot_path != null)
	    this._run_build();

	this._updateStatus();
    },

    _updateStatus: function() {
	let newStatus = "";
	if (this._resolve_proc == null && this._build_proc == null) {
	    newStatus = "idle";
	} else {
	    if (this._resolve_proc != null)
		newStatus += "[resolving] ";
	    if (this._build_proc != null)
		newStatus += "[building] ";
	}
	if (newStatus != this._status) {
	    this._status = newStatus;
	    print(this._status);
	    this._impl.emit_property_changed('Status', new GLib.Variant("s", this._status));
	}

	this._writeStatusFile();
    },

    get Status() {
	return this._status;
    },

    queueBuild: function(components) {
	this._queued_force_builds.push.apply(this._queued_force_builds, components);
	this._build_needed = true;
	print("queued builds: " + this._queued_force_builds);
	if (this._build_proc == null)
	    this._run_build();
    },

    queueResolve: function(components) {
	this._queued_force_resolve.push.apply(this._queued_force_resolve, components);
	print("queued resolves: " + this._queued_force_resolve);
	if (this._resolve_proc == null)
	    this._fetch();
    },
    
    _fetchAll: function() {
	this._full_resolve_needed = true;
	if (this._resolve_proc == null)
	    this._fetch();
	return true;
    },

    _fetch: function() {
	let cancellable = null;
	if (this._resolve_proc != null) throw new Error("Attempted multiple fetch");
	let t = this._resolve_taskset.start();
	let taskWorkdir = t.path;

	if (this._autoupdate_self)
	    ProcUtil.runSync(['git', 'pull', '-r'], cancellable)

	let args = ['ostbuild', 'resolve', '--manifest=manifest.json',
		    '--fetch', '--fetch-keep-going'];
	let isFull;
	if (this._full_resolve_needed) {
	    this._full_resolve_needed = false;
	    isFull = true;
	} else if (this._queued_force_resolve.length > 0) {
	    args.push.apply(args, this._queued_force_resolve);
	    isFull = false;
	} else {
	    throw new Error("_fetch() when not needed");
	}
	this._queued_force_resolve = [];
	let context = new GSystem.SubprocessContext({ argv: args });
	context.set_stdout_file_path(t.logfile_path.get_path());
	context.set_stderr_disposition(GSystem.SubprocessStreamDisposition.STDERR_MERGE);
	this._resolve_proc = new GSystem.Subprocess({context: context});
	this._resolve_proc.init(null);
	print(Format.vprintf("Resolve task %s.%s started (%s), pid=%s", [t.major, t.minor,
									 isFull ? "full" : "incremental",
									 this._resolve_proc.get_pid()]));
	this._resolve_proc.wait(null, Lang.bind(this, this._onResolveExited));

	this._updateStatus();

	return false;
    },

    _onResolveExited: function(process, result) {
	this._resolve_proc = null;
	let [success, msg] = ProcUtil.asyncWaitCheckFinish(process, result);
	print(Format.vprintf("resolve exited; success=%s msg=%s", [success, msg]))
	this._resolve_taskset.finish(success);
	this._prev_source_snapshot_path = this._source_snapshot_path;
	this._source_snapshot_path = this._src_db.getLatestPath();
	let changed = (this._prev_source_snapshot_path == null ||
		       !this._prev_source_snapshot_path.equal(this._source_snapshot_path));
        if (changed)
            print(Format.vprintf("New version is %s", [this._source_snapshot_path.get_path()]))
	if (!this._build_needed)
	    this._build_needed = changed;
	if (this._build_needed && this._build_proc == null)
	    this._run_build();

	if (this._full_resolve_needed || this._queued_force_resolve.length > 0) {
	    this._fetch();
	}

	this._updateStatus();
    },
    
    _run_build: function() {
	let cancellable = null;
	if (this._build_proc != null) throw new Error();
	if (!this._build_needed) throw new Error();

	this._build_needed = false;

	let task = this._build_taskset.start();
	let workdir = task.path;
	let transientSnapshotPath = this.workdir.get_child(this.prefix + '-src-snapshot.json');
	GSystem.file_linkcopy(this._source_snapshot_path, transientSnapshotPath, Gio.FileCopyFlags.OVERWRITE, null);
	let args = ['ostbuild', 'build', '--snapshot=' + transientSnapshotPath.get_path()];
	args.push.apply(args, this._queued_force_builds);
	this._queued_force_builds = [];

	let version = this._src_db.parseVersionStr(this._source_snapshot_path.get_basename());
	let meta = {'version': version,
		    'version-path': this._snapshot_dir.get_relative_path(this._source_snapshot_path)};
	let metaPath = workdir.get_child('meta.json');
	JsonUtil.writeJsonFileAtomic(metaPath, meta, cancellable);

	let context = new GSystem.SubprocessContext({ argv: args });
	context.set_stdout_file_path(task.logfile_path.get_path());<commit_msg>autobuilder: Put snapshot copy in taskdir<commit_after>// Copyright (C) 2012,2013 <NAME> <<EMAIL>>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the
// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
// Boston, MA 02111-1307, USA.

const GLib = imports.gi.GLib;
const Gio = imports.gi.Gio;
const Lang = imports.lang;
const Format = imports.format;

const GSystem = imports.gi.GSystem;

const Task = imports.task;
const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const JsonUtil = imports.jsonutil;
const Snapshot = imports.snapshot;
const Config = imports.config;

const loop = GLib.MainLoop.new(null, true);

var AutoBuilderIface = <interface name="org.gnome.OSTreeBuild.AutoBuilder">
<method name="queueBuild">
    <arg type="as" direction="in" />
</method>
<method name="queueResolve">
    <arg type="as" direction="in" />
</method>
<property name="Status" type="s" access="read" />
</interface>;

const AutoBuilder = new Lang.Class({
    Name: 'AutoBuilder',
    
    _init: function() {
	this._resolve_proc = null;
	this._build_proc = null;

	this.config = Config.get();
	this.workdir = Gio.File.new_for_path(this.config.getGlobal('workdir'));
	this.prefix = this.config.getPrefix();
	this._snapshot_dir = this.workdir.get_child('snapshots');
	this._status_path = this.workdir.get_child('autobuilder-' + this.prefix + '.json');

	this._build_needed = true;
	this._full_resolve_needed = true;
	this._queued_force_builds = [];
	this._queued_force_resolve = [];
	this._autoupdate_self = true;
	this._resolve_timeout = 0;
	this._source_snapshot_path = null;
	this._prev_source_snapshot_path = null;
	
	this._impl = Gio.DBusExportedObject.wrapJSObject(AutoBuilderIface, this);
	this._impl.export(Gio.DBus.session, '/org/gnome/OSTreeBuild/AutoBuilder');

	let snapshotdir = this.workdir.get_child('snapshots');
	GSystem.file_ensure_directory(snapshotdir, true, null);
	this._src_db = new JsonDB.JsonDB(snapshotdir, this.prefix + '-src-snapshot');

	let taskdir = new Task.TaskDir(this.workdir.get_child('tasks'));
	this._resolve_taskset = taskdir.get(this.prefix + '-resolve');
	this._build_taskset = taskdir.get(this.prefix + '-build');

	this._source_snapshot_path = this._src_db.getLatestPath();

	this._status_path = this.workdir.get_child('autobuilder-' + this.prefix + '.json');

	this._resolve_timeout = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT,
							 60 * 10, Lang.bind(this, this._fetchAll));
	this._fetchAll();
	if (this._source_snapshot_path != null)
	    this._run_build();

	this._updateStatus();
    },

    _updateStatus: function() {
	let newStatus = "";
	if (this._resolve_proc == null && this._build_proc == null) {
	    newStatus = "idle";
	} else {
	    if (this._resolve_proc != null)
		newStatus += "[resolving] ";
	    if (this._build_proc != null)
		newStatus += "[building] ";
	}
	if (newStatus != this._status) {
	    this._status = newStatus;
	    print(this._status);
	    this._impl.emit_property_changed('Status', new GLib.Variant("s", this._status));
	}

	this._writeStatusFile();
    },

    get Status() {
	return this._status;
    },

    queueBuild: function(components) {
	this._queued_force_builds.push.apply(this._queued_force_builds, components);
	this._build_needed = true;
	print("queued builds: " + this._queued_force_builds);
	if (this._build_proc == null)
	    this._run_build();
    },

    queueResolve: function(components) {
	this._queued_force_resolve.push.apply(this._queued_force_resolve, components);
	print("queued resolves: " + this._queued_force_resolve);
	if (this._resolve_proc == null)
	    this._fetch();
    },
    
    _fetchAll: function() {
	this._full_resolve_needed = true;
	if (this._resolve_proc == null)
	    this._fetch();
	return true;
    },

    _fetch: function() {
	let cancellable = null;
	if (this._resolve_proc != null) throw new Error("Attempted multiple fetch");
	let t = this._resolve_taskset.start();
	let taskWorkdir = t.path;

	if (this._autoupdate_self)
	    ProcUtil.runSync(['git', 'pull', '-r'], cancellable)

	let args = ['ostbuild', 'resolve', '--manifest=manifest.json',
		    '--fetch', '--fetch-keep-going'];
	let isFull;
	if (this._full_resolve_needed) {
	    this._full_resolve_needed = false;
	    isFull = true;
	} else if (this._queued_force_resolve.length > 0) {
	    args.push.apply(args, this._queued_force_resolve);
	    isFull = false;
	} else {
	    throw new Error("_fetch() when not needed");
	}
	this._queued_force_resolve = [];
	let context = new GSystem.SubprocessContext({ argv: args });
	context.set_stdout_file_path(t.logfile_path.get_path());
	context.set_stderr_disposition(GSystem.SubprocessStreamDisposition.STDERR_MERGE);
	this._resolve_proc = new GSystem.Subprocess({context: context});
	this._resolve_proc.init(null);
	print(Format.vprintf("Resolve task %s.%s started (%s), pid=%s", [t.major, t.minor,
									 isFull ? "full" : "incremental",
									 this._resolve_proc.get_pid()]));
	this._resolve_proc.wait(null, Lang.bind(this, this._onResolveExited));

	this._updateStatus();

	return false;
    },

    _onResolveExited: function(process, result) {
	this._resolve_proc = null;
	let [success, msg] = ProcUtil.asyncWaitCheckFinish(process, result);
	print(Format.vprintf("resolve exited; success=%s msg=%s", [success, msg]))
	this._resolve_taskset.finish(success);
	this._prev_source_snapshot_path = this._source_snapshot_path;
	this._source_snapshot_path = this._src_db.getLatestPath();
	let changed = (this._prev_source_snapshot_path == null ||
		       !this._prev_source_snapshot_path.equal(this._source_snapshot_path));
        if (changed)
            print(Format.vprintf("New version is %s", [this._source_snapshot_path.get_path()]))
	if (!this._build_needed)
	    this._build_needed = changed;
	if (this._build_needed && this._build_proc == null)
	    this._run_build();

	if (this._full_resolve_needed || this._queued_force_resolve.length > 0) {
	    this._fetch();
	}

	this._updateStatus();
    },
    
    _run_build: function() {
	let cancellable = null;
	if (this._build_proc != null) throw new Error();
	if (!this._build_needed) throw new Error();

	this._build_needed = false;

	let task = this._build_taskset.start();
	let workdir = task.path;
	let transientSnapshotPath = workdir.get_child(this._source_snapshot_path.get_basename());
	GSystem.file_linkcopy(this._source_snapshot_path, transientSnapshotPath, Gio.FileCopyFlags.OVERWRITE, null);
	let args = ['ostbuild', 'build', '--snapshot=' + transientSnapshotPath.get_path()];
	args.push.apply(args, this._queued_force_builds);
	this._queued_force_builds = [];

	let version = this._src_db.parseVersionStr(this._source_snapshot_path.get_basename());
	let meta = {'version': version,
		    'version-path': this._snapshot_dir.get_relative_path(this._source_snapshot_path)};
	let metaPath = workdir.get_child('meta.json');
	JsonUtil.writeJsonFileAtomic(metaPath, meta, cancellable);

	let context = new GSystem.SubprocessContext({ argv: args });
	context.set_stdout_file_path(task.logfile_path.get_path());
{'fraction_non_alphanumeric': 0.13701418067226892, 'fraction_numerical': 0.003939075630252101, 'mean_word_length': 4.537259178480553, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1675578', 'n_tokens_mistral': 5176, 'n_tokens_neox': 4964, 'n_words': 1065}