<commit_before>import { expect } from "chai";
import * as parse from "../../src/syntax/parse";
import * as frame from "../../src/frames";

describe.only("Parse", () => {
  describe("ParseToken", () => {
    const content = new frame.FrameString("content");
    const token = new parse.ParseToken(content);
    it("is exported", () => {
      expect(parse.ParseToken).to.be.ok;
    });

    it("is constructed from a Frame", () => {
      expect(token).to.be.ok;
    });

    it("calls argument with content when called", () => {
      const out = new frame.FrameArray([]);
      const result = token.call(out);
      expect(out.asArray().length).to.equal(1);
      expect(out.at(0)).to.equal(content);
    });
  });

  describe("ParsePipe", () => {
    it("is exported", () => {
      expect(parse.ParsePipe).to.be.ok;
    });
  });
});<commit_msg>ParsePipe is constructed from an output Frame<commit_after>import { expect } from "chai";
import * as parse from "../../src/syntax/parse";
import * as frame from "../../src/frames";

describe.only("Parse", () => {
  describe("ParseToken", () => {
    const content = new frame.FrameString("content");
    const token = new parse.ParseToken(content);
    it("is exported", () => {
      expect(parse.ParseToken).to.be.ok;
    });

    it("is constructed from a Frame", () => {
      expect(token).to.be.ok;
    });

    it("calls argument with content when called", () => {
      const out = new frame.FrameArray([]);
      const result = token.call(out);
      expect(out.asArray().length).to.equal(1);
      expect(out.at(0)).to.equal(content);
    });
  });

  describe("ParsePipe", () => {
    const out = new frame.FrameArray([]);
    const pipe = new parse.ParsePipe(out);

    it("is exported", () => {
      expect(parse.ParsePipe).to.be.ok;
    });

    it("is constructed from an output Frame", () => {
      expect(pipe).to.be.ok;
    });
  });
});
{'fraction_non_alphanumeric': 0.20566631689401887, 'fraction_numerical': 0.002098635886673662, 'mean_word_length': 2.8525252525252527, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4791854', 'n_tokens_mistral': 610, 'n_tokens_neox': 599, 'n_words': 125}