<commit_before>#!/usr/bin/env node
import * as fs from "fs";
import * as util from "util";
import * as yargs from "yargs";
import * as path from "path";
import { NodeVM } from "vm2";
import * as glob from "glob";
import * as protos from "./protos";
import * as utils from "./utils";
import * as runners from "./runners";
import * as testcreds from "./testcreds";
import { Executor } from "./executor";

const vm = new NodeVM({
  timeout: 5000,
  wrapper: "none",
  require: {
    context: "sandbox",
    root: "./",
    external: true
  },
  sourceExtensions: ["js", "sql"],
  compiler: (code, file) => {
    if (file.includes(".sql")) {
      return utils.compileSql(code, file);
    } else {
      return code;
    }
  }
});

const argv = yargs
  .option("project-dir", { describe: "The directory of the dataform project to run against", default: "." })
  .command(
    "compile",
    "Compile the dataform project. Produces JSON output describing the entire computation graph.",
    yargs => yargs,
    argv => compile(argv["project-dir"])
  )
  .command(
    "run",
    "Will run the computation graph, with the provided options.",
    yargs =>
      yargs
        .option("dry-run", {
          describe: "Prints the executable computation graph without actually running it",
          type: "boolean",
          default: false,
          alias: "dr"
        })
        .option("full-refresh", {
          describe: "If set, this will rebuild incremental tables from scratch",
          type: "boolean",
          default: false,
          alias: "fr"
        })
        .option("carry-on", {
          describe: "If set, when a task fails it won't stop dependencies from attempting to run.",
          type: "boolean",
          default: false,
          alias: "co"
        })
        .option("retries", {
          describe: "If set, failing tasks will be retried this many times.",
          type: "number",
          default: false,
          alias: "r"
        })
        .option("profile", { describe: "The location of the profile file to run against" })
        .option("nodes", { describe: "A list of computation nodes to run. Defaults to all nodes", type: "array" })
        .option("include-deps", {
          describe: "If set, dependencies for selected nodes will also be run",
          type: "boolean",
          alias: "id"
        }),
    argv => {
      run(
        argv["project-dir"],
        argv["dry-run"],
        argv["full-refresh"],
        argv["profile"],
        argv["nodes"],
        argv["include-deps"],
        argv["carry-on"],
        argv["retries"]
      );
    }
  ).argv;

function run(
  projectDir: string,
  dryRun: boolean,
  fullRefresh: boolean,
  profile: string,
  nodes: string[],
  includeDeps: boolean,
  carryOn: boolean,
  retries: number
) {
  var runConfig: protos.IRunConfig = {
    fullRefresh: fullRefresh,
    includeDependencies: includeDeps,
    nodes: nodes
  };
  var indexScript = genIndex(projectDir, `dft.build(${JSON.stringify(runConfig)})`);
  var executionGraph: protos.IExecutionGraph = vm.run(indexScript, path.resolve(path.join(projectDir, "index.js")));
  if (dryRun) {
    console.log(JSON.stringify(executionGraph, null, 4));
    return;
  }

  var runner = runners.create("bigquery", {
    bigquery: { projectId: "tada-analytics", keyFile: testcreds.bigquery }
  });

  new Executor(runner, executionGraph)<commit_msg>Add init function<commit_after>#!/usr/bin/env node
import * as fs from "fs";
import * as util from "util";
import * as yargs from "yargs";
import * as path from "path";
import { NodeVM } from "vm2";
import * as glob from "glob";
import * as protos from "./protos";
import * as utils from "./utils";
import * as runners from "./runners";
//import * as testcreds from "./testcreds";
import { Executor } from "./executor";

const vm = new NodeVM({
  timeout: 5000,
  wrapper: "none",
  require: {
    context: "sandbox",
    root: "./",
    external: true
  },
  sourceExtensions: ["js", "sql"],
  compiler: (code, file) => {
    if (file.includes(".sql")) {
      return utils.compileSql(code, file);
    } else {
      return code;
    }
  }
});

const argv = yargs
  .option("project-dir", { describe: "The directory of the dataform project to run against", default: "." })
  .command(
    "init",
    "Create a new dataform project in the current, or specified directory.",
    yargs => yargs,
    argv => init(argv["project-dir"])
  )
  .command(
    "compile",
    "Compile the dataform project. Produces JSON output describing the entire computation graph.",
    yargs => yargs,
    argv => compile(argv["project-dir"])
  )
  .command(
    "run",
    "Will run the computation graph, with the provided options.",
    yargs =>
      yargs
        .option("dry-run", {
          describe: "Prints the executable computation graph without actually running it",
          type: "boolean",
          default: false,
          alias: "dr"
        })
        .option("full-refresh", {
          describe: "If set, this will rebuild incremental tables from scratch",
          type: "boolean",
          default: false,
          alias: "fr"
        })
        .option("carry-on", {
          describe: "If set, when a task fails it won't stop dependencies from attempting to run.",
          type: "boolean",
          default: false,
          alias: "co"
        })
        .option("retries", {
          describe: "If set, failing tasks will be retried this many times.",
          type: "number",
          default: false,
          alias: "r"
        })
        .option("profile", { describe: "The location of the profile file to run against" })
        .option("nodes", { describe: "A list of computation nodes to run. Defaults to all nodes", type: "array" })
        .option("include-deps", {
          describe: "If set, dependencies for selected nodes will also be run",
          type: "boolean",
          alias: "id"
        }),
    argv => {
      run(
        argv["project-dir"],
        argv["dry-run"],
        argv["full-refresh"],
        argv["profile"],
        argv["nodes"],
        argv["include-deps"],
        argv["carry-on"],
        argv["retries"]
      );
    }
  ).argv;

function init(projectDir: string) {
  var dataformJsonPath = path.join(projectDir, "dataform.json");
  var packageJsonPath = path.join(projectDir, "dataform.json");
  if (fs.existsSync(dataformJsonPath) || fs.existsSync(packageJsonPath)) {
    throw "Cannot init dataform project, this already appears to be an NPM or Dataform directory.";
  }
  if (!fs.existsSync(projectDir)) {
    fs.mkdirSync(projectDir);
  }
  fs.writeFileSync(
    dataformJsonPath,
    JSON.stringify(
      protos.ProjectConfig.create({
        warehouse: "bigquery",
        defaultSchema: "dataform"
      }),
      null,
      4
    )
  );
  fs.writeFileSync(
    packageJsonPath,
    JSON.stringify(
      {
        name: utils.baseFilename(path.resolve(projectDir)),
        version: "0.0.1",
        description: "New Dataform project.",
        license: "ISC",
        bin: {
          dft: "build/cli.js"
        },
        dependencies: {
          dft: "^0.0.1"
        }
      },
      null,
      4
    )
  );
}

function run(
  projectDir: string,
  dryRun: boolean,
  fullRefresh: boolean,
  profile: string,
  nodes: string[],
  includeDeps: boolean,
  carryOn: boolean,
  retries: number
) {
  var runConfig: protos.IRunConfig = {
    fullRefresh: fullRefresh,
    includeDependencies: includeDeps,
    nodes: nodes
  };
  var indexScript = genIndex(projectDir, `dft.build(${JSON.stringify(runConfig)})`);
  var executionGraph: protos.IExecutionGraph = vm.run(indexScript, path.resolve(path.join(projectDir, "index.js")));
  if (dryRun) {
    console.log(JSON.stringify(executionGraph, null, 4));
    return;
  }

  var runner = runners.create("bigquery", {
    //bigquery: { projectId: "tada-analytics", keyFile: testcreds.bigquery }
  });

  new Executor(runner, executionGraph)
{'fraction_non_alphanumeric': 0.13909444374764832, 'fraction_numerical': 0.002508466072996363, 'mean_word_length': 2.4973684210526317, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1701830', 'n_tokens_mistral': 2406, 'n_tokens_neox': 2369, 'n_words': 702}