<commit_before>import { dataform } from "@dataform/protos";
import * as glob from "glob";
import { util } from "protobufjs";

export function createGenIndexConfig(
  compileConfig: dataform.ICompileConfig,
  returnOverride?: string
): string {
  const includePaths = [];
  glob.sync("includes/*.js", { cwd: compileConfig.projectDir }).forEach(path => {
    if (includePaths.indexOf(path) < 0) {
      includePaths.push(path);
    }
  });

  const definitionPaths = [];
  glob.sync("definitions/**/*.{js,sql,sqlx}", { cwd: compileConfig.projectDir }).forEach(path => {
    if (definitionPaths.indexOf(path) < 0) {
      definitionPaths.push(path);
    }
  });
  // Support projects that don't use the new project structure.
  glob.sync("models/**/*.{js,sql}", { cwd: compileConfig.projectDir }).forEach(path => {
    if (definitionPaths.indexOf(path) < 0) {
      definitionPaths.push(path);
    }
  });
  const encodedConfigBytes = dataform.GenerateIndexConfig.encode({
    compileConfig,
    includePaths,
    definitionPaths,
    returnOverride
  }).finish();
  return util.base64.encode(encodedConfigBytes, 0, encodedConfigBytes.length);
}
<commit_msg>Compile .sqlx files in models/. (#276)<commit_after>import { dataform } from "@dataform/protos";
import * as glob from "glob";
import { util } from "protobufjs";

export function createGenIndexConfig(
  compileConfig: dataform.ICompileConfig,
  returnOverride?: string
): string {
  const includePaths = [];
  glob.sync("includes/*.js", { cwd: compileConfig.projectDir }).forEach(path => {
    if (includePaths.indexOf(path) < 0) {
      includePaths.push(path);
    }
  });

  const definitionPaths = [];
  glob.sync("definitions/**/*.{js,sql,sqlx}", { cwd: compileConfig.projectDir }).forEach(path => {
    if (definitionPaths.indexOf(path) < 0) {
      definitionPaths.push(path);
    }
  });
  // Support projects that don't use the new project structure.
  glob.sync("models/**/*.{js,sql,sqlx}", { cwd: compileConfig.projectDir }).forEach(path => {
    if (definitionPaths.indexOf(path) < 0) {
      definitionPaths.push(path);
    }
  });
  const encodedConfigBytes = dataform.GenerateIndexConfig.encode({
    compileConfig,
    includePaths,
    definitionPaths,
    returnOverride
  }).finish();
  return util.base64.encode(encodedConfigBytes, 0, encodedConfigBytes.length);
}

{'fraction_non_alphanumeric': 0.15256410256410258, 'fraction_numerical': 0.00641025641025641, 'mean_word_length': 4.056155507559395, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6243232', 'n_tokens_mistral': 722, 'n_tokens_neox': 702, 'n_words': 145}