<commit_before>import {ensureDir, walk} from "https://deno.land/std@0.114.0/fs/mod.ts";
import {
  basename,
  dirname,
  join,
  relative,
} from "https://deno.land/std@0.114.0/path/posix.ts";
import {createRequire} from "https://deno.land/std@0.114.0/node/module.ts";

const require = createRequire(import.meta.url);

const ts = require("typescript");

const normalisePath = (path: string) => path.replace(/\\/g, "/");

const denoTestFiles = new Set([
  "test/testbase.ts",
  "test/client.test.ts",
  "test/credentials.test.ts",
]);

run({
  sourceDir: "./src",
  destDir: "./edgedb-deno",
  destEntriesToClean: ["_src", "mod.ts"],
  sourceFilter: path => {
    return !/\/syntax\//.test(path);
  },
  pathRewriteRules: [
    {match: /^src\/index.node.ts$/, replace: "mod.ts"},
    {match: /^src\//, replace: "_src/"},
  ],
  injectImports: [
    {
      imports: ["Buffer", "process"],
      from: "src/globals.deno.ts",
    },
  ],
})
  .then(() =>
    run({
      sourceDir: "./src/syntax",
      destDir: "./edgedb-deno",
      pathRewriteRules: [{match: /^src\//, replace: "_src/"}],
      importRewriteRules: [
        {match: /^edgedb$/, replace: "https://deno.land/x/edgedb/mod.ts"},
        {match: /^\.\.\//, replace: "../"},
        {match: /^@generated\//, replace: "../"},
        {match: /^\.\/setImpl/, replace: "./setImpl.ts"},
        // this file will be implemented after codegen
      ],
      injectImports: [
        {
          imports: ["Buffer"],
          from: "https://deno.land/std@0.114.0/node/buffer.ts",
        },
      ],
    })
  )
  .then(() =>
    run({
      sourceDir: "./test",
      destDir: "./test/deno",
      sourceFilter: path => {
        return denoTestFiles.has(path);
      },
      pathRewriteRules: [{match: /^test\//, replace: ""}],
      importRewriteRules: [
        {
          match: /^\.\.\/src\/index.node$/,
          replace: "../../edgedb-deno/mod.ts",
        },
        {
          match: /^globals.deno.ts$/,
          replace: "../globals.deno.ts",
        },
        {
          match: /^\.\.\/src\/.+/,
          replace: match =>
            `${match.replace(/^\.\.\/src\//, "../../edgedb-deno/_src/")}${
              match.endsWith(".ts") ? "" : ".ts"
            }`,
        },
      ],
      injectImports: [
        {
          imports: ["Buffer", "process"],
          from: "src/globals.deno.ts",
        },
        {
          imports: ["test", "expect", "jest"],
          from: "test/globals.deno.ts",
        },
      ],
    })
  )
  .then(() => {
    return Deno.writeTextFile(
      "./edgedb-deno/generate.ts",
      `
export * from "./_src/reflection/cli.ts";
    `
    );
  });

async function run({
  sourceDir,
  destDir,
  destEntriesToClean,
  pathRewriteRules = [],
  importRewriteRules = [],
  injectImports = [],
  sourceFilter,
}: {
  sourceDir: string;
  destDir: string;
  destEntriesToClean?: string[];
  pathRewriteRules?: {match: RegExp; replace: string}[];
  importRewriteRules?: {
    match: RegExp;
    replace: string | ((match: string) => string);
  }[];
  injectImports?: {imports: string[]; from: string}[];
  sourceFilter?: (path: string) => boolean;
}) {
  console.log(`Denoifying ${sourceDir}...`);
  const destClean = new Set(destEntriesToClean);
  try {
    for await (const entry of Deno.readDir(destDir)) {
      if (!destEntriesToClean || destClean.has(entry.name)) {
        await Deno.remove(join(destDir, entry.name), {recursive: true});
      }
    }
  } catch {}

  const sourceFilePathMap = new Map<string, string>();

  for await (const entry of walk(sourceDir, {includeDirs: false})) {
    const sourcePath = normalisePath(entry.path);
    if (sourceFilter && !sourceFilter(sourcePath)) {
      continue;
    }
    sourceFilePathMap.set(sourcePath, resolveDestPath(sourcePath));
  }

  for (const [sourcePath, destPath] of sourceFilePathMap) {
    compileFileForDeno(sourcePath, destPath);
  }

  async function compileFileForDeno(sourcePath: string, destPath: string) {
    const file = await Deno.readTextFile(sourcePath);
    await ensureDir(dirname(destPath));

    if (destPath.endsWith(".deno.ts")) {
      return await Deno.writeTextFile(destPath, file);
    }

    if (destPath.endsWith(".node.ts")) {
      return;<commit_msg>Fix deno compilation bug<commit_after>import {ensureDir, walk} from "https://deno.land/std@0.114.0/fs/mod.ts";
import {
  basename,
  dirname,
  join,
  relative,
} from "https://deno.land/std@0.114.0/path/posix.ts";
import {createRequire} from "https://deno.land/std@0.114.0/node/module.ts";

const require = createRequire(import.meta.url);

const ts = require("typescript");

const normalisePath = (path: string) => path.replace(/\\/g, "/");
const denoTestFiles = new Set([
  "test/testbase.ts",
  "test/client.test.ts",
  "test/credentials.test.ts",
]);

run({
  sourceDir: "./src",
  destDir: "./edgedb-deno",
  destEntriesToClean: ["_src", "mod.ts"],
  sourceFilter: path => {
    return !/\/syntax\//.test(path);
  },
  pathRewriteRules: [
    {match: /^src\/index.node.ts$/, replace: "mod.ts"},
    {match: /^src\//, replace: "_src/"},
  ],
  injectImports: [
    {
      imports: ["Buffer", "process"],
      from: "src/globals.deno.ts",
    },
  ],
})
  .then(() =>
    run({
      sourceDir: "./src/syntax",
      destDir: "./edgedb-deno",
      destEntriesToClean: ["_src", "mod.ts"],
      pathRewriteRules: [{match: /^src\//, replace: "_src/"}],
      importRewriteRules: [
        {match: /^edgedb$/, replace: "https://deno.land/x/edgedb/mod.ts"},
        {match: /^\.\.\//, replace: "../"},
        {match: /^@generated\//, replace: "../"},
        {match: /^\.\/setImpl/, replace: "./setImpl.ts"},
        // this file will be implemented after codegen
      ],
      injectImports: [
        {
          imports: ["Buffer"],
          from: "https://deno.land/std@0.114.0/node/buffer.ts",
        },
      ],
    })
  )
  .then(async () =>
    run({
      sourceDir: "./test",
      destDir: "./test/deno",
      sourceFilter: path => {
        return denoTestFiles.has(path);
      },
      pathRewriteRules: [{match: /^test\//, replace: ""}],
      importRewriteRules: [
        {
          match: /^\.\.\/src\/index.node$/,
          replace: "../../edgedb-deno/mod.ts",
        },
        {
          match: /^globals.deno.ts$/,
          replace: "../globals.deno.ts",
        },
        {
          match: /^\.\.\/src\/.+/,
          replace: match =>
            `${match.replace(/^\.\.\/src\//, "../../edgedb-deno/_src/")}${
              match.endsWith(".ts") ? "" : ".ts"
            }`,
        },
      ],
      injectImports: [
        {
          imports: ["Buffer", "process"],
          from: "src/globals.deno.ts",
        },
        {
          imports: ["test", "expect", "jest"],
          from: "test/globals.deno.ts",
        },
      ],
    })
  )
  .then(async () =>
    Deno.writeTextFile(
      "./edgedb-deno/generate.ts",
      `
export * from "./_src/reflection/cli.ts";
    `
    )
  );

async function run({
  sourceDir,
  destDir,
  destEntriesToClean,
  pathRewriteRules = [],
  importRewriteRules = [],
  injectImports = [],
  sourceFilter,
}: {
  sourceDir: string;
  destDir: string;
  destEntriesToClean?: string[];
  pathRewriteRules?: {match: RegExp; replace: string}[];
  importRewriteRules?: {
    match: RegExp;
    replace: string | ((match: string) => string);
  }[];
  injectImports?: {imports: string[]; from: string}[];
  sourceFilter?: (path: string) => boolean;
}) {
  console.log(`Denoifying ${sourceDir}...`);
  const destClean = new Set(destEntriesToClean);
  try {
    for await (const entry of Deno.readDir(destDir)) {
      if (!destEntriesToClean || destClean.has(entry.name)) {
        console.log(`Removing ${join(destDir, entry.name)}`);
        await Deno.remove(join(destDir, entry.name), {recursive: true});
      }
    }
  } catch {}

  const sourceFilePathMap = new Map<string, string>();

  for await (const entry of walk(sourceDir, {includeDirs: false})) {
    const sourcePath = normalisePath(entry.path);
    if (sourceFilter && !sourceFilter(sourcePath)) {
      continue;
    }
    sourceFilePathMap.set(sourcePath, resolveDestPath(sourcePath));
  }

  for (const [sourcePath, destPath] of sourceFilePathMap) {
    compileFileForDeno(sourcePath, destPath);
  }

  async function compileFileForDeno(sourcePath: string, destPath: string) {
    console.log(`Compiling ${sourcePath}`);
    const file = await Deno.readTextFile(sourcePath);
    await ensureDir(dirname(destPath));

    if (destPath.endsWith(".deno.ts")) {
      return await Deno.writeTextFile(destPath, file);
    }

    if (destPath.endsWith(".node.ts")) {
      return;
{'fraction_non_alphanumeric': 0.19668164535084687, 'fraction_numerical': 0.004608825901601567, 'mean_word_length': 2.3682576639503297, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1843307', 'n_tokens_mistral': 2998, 'n_tokens_neox': 2950, 'n_words': 527}