<commit_before>import * as path from 'https://deno.land/std@0.56.0/path/mod.ts';
import * as fs from 'https://deno.land/std@0.56.0/fs/mod.ts';
import { green } from 'https://deno.land/std@0.56.0/fmt/colors.ts';

export async function ensureDirAndWriteFileStr(filename: string, content: string) {
  console.log(green('Write'), filename);
  await fs.ensureDir(path.dirname(filename));
  await fs.writeFileStr(filename, content);
}
export async function ensureDirAndCopy(src: string, dest: string, options?: fs.CopyOptions) {
  console.log(green('Copy'), src);
  await fs.ensureDir(path.dirname(dest));
  await fs.copy(src, dest, options);
}
export async function copyPagicFile(pathToPagicRoot: string, dest: string) {
  console.log(green('Copy pagic file'), pathToPagicRoot);
  if (import.meta.url.startsWith('file://')) {
    const src = path.resolve(path.dirname(path.fromFileUrl(import.meta.url)), '../../', pathToPagicRoot);
    await ensureDirAndCopy(src, dest, { overwrite: true });
  } else {
    const res = await fetch(import.meta.url.replace(/\/src\/utils\/copy\.ts$/, `/${pathToPagicRoot}`));
    const content = await res.text();
    await ensureDirAndWriteFileStr(dest, content);
  }
}
<commit_msg>Fix copy<commit_after>import * as path from 'https://deno.land/std@0.56.0/path/mod.ts';
import * as fs from 'https://deno.land/std@0.56.0/fs/mod.ts';
import { green } from 'https://deno.land/std@0.56.0/fmt/colors.ts';

export async function ensureDirAndWriteFileStr(filename: string, content: string) {
  console.log(green('Write'), filename);
  await fs.ensureDir(path.dirname(filename));
  await fs.writeFileStr(filename, content);
}
export async function ensureDirAndCopy(src: string, dest: string, options?: fs.CopyOptions) {
  console.log(green('Copy'), src);
  await fs.ensureDir(path.dirname(dest));
  await fs.copy(src, dest, options);
}
export async function copyPagicFile(pathToPagicRoot: string, dest: string) {
  console.log(green('Copy pagic file'), pathToPagicRoot);
  if (import.meta.url.startsWith('file://')) {
    const src = path.resolve(path.dirname(path.fromFileUrl(import.meta.url)), '../../', pathToPagicRoot);
    await ensureDirAndCopy(src, dest, { overwrite: true });
  } else {
    const res = await fetch(import.meta.url.replace(/\/src\/utils\/copy\.ts$/, `/${pathToPagicRoot}`));
    await fs.ensureDir(path.dirname(dest));
    // https://stackoverflow.com/q/61945050/2777142
    const file = await Deno.open(dest, { create: true, write: true });
    await Deno.writeAll(file, new Uint8Array(await res.arrayBuffer()));
  }
}

{'fraction_non_alphanumeric': 0.1740148263753414, 'fraction_numerical': 0.015606710885680842, 'mean_word_length': 4.672566371681416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3052237', 'n_tokens_mistral': 871, 'n_tokens_neox': 874, 'n_words': 170}