<commit_before>(function (global) {
/* global describe, it */
'use strict';
var _inNode = 'process' in global;
var expect, _, JSV, helper, Parser;
if (_inNode) {
  expect = require('expect.js');
  _ = require('lodash');
  JSV = require('JSV').JSV;
  helper = require('./test_helper');
  Parser = require('../src').Parser;
} else {
  expect = global.expect;
  _ = global._;
  JSV = global.JSV;
  helper = global.TestHelper;
  Parser = global.PixivNovelParser.Parser;
}

describe('Parser specifications.', function () {
  /* jshint quotmark: false */
  describe('テキスト', function () {
    it('ただのテキストはちゃんとただのテキストになる', function () {
      var parser = new Parser(),
          novel = 'テキスト\nテキスト',
          expectedAST = [{ type: 'text', val: 'テキスト\nテキスト' }],
          schema = {
            "$schema": "http://json-schema.org/draft-02/hyper-schema#",
            "id": "http://json-schema.org/draft-02/schema#",
            "type": "object",
            "properties": {
              "type": {
                "enum": ["text"]
              },
              "val": {
                "type": "string"
              }
            },
            "required": ["type", "val"]
          };

      parser.parse(novel);
      expect(_.isEqual(parser.tree, expectedAST)).to.be.ok();
      expect(helper.validateJSON(parser.tree[0], schema)).to.be.ok();
    });

    it('改行コードを正規化する', function () {
      var parser = new Parser(),
          novel = '\n\n改行\n改行\r\n改行\n\r\n改行\r\n\n改行\r\n\r\n改行\n',
          expectedAST = [
            { type: 'text', val: '\n\n改行\n改行\n改行\n\n改行\n\n改行\n\n改行\n' },
          ];

      parser.parse(novel);
      expect(_.isEqual(parser.tree, expectedAST)).to.be.ok();
    });
  });

  describe('newpage', function () {
    it('ちゃんとページを分割できる', function () {
      var parser = new Parser(),
          novel = '1ページ目[newpage]2ページ目',
          expectedAST = [
            { type: 'text', val: '1ページ目' },
            { type: 'tag', name: 'newpage' },<commit_msg>Remove trailing comma in array<commit_after>(function (global) {
/* global describe, it */
'use strict';
var _inNode = 'process' in global;
var expect, _, JSV, helper, Parser;
if (_inNode) {
  expect = require('expect.js');
  _ = require('lodash');
  JSV = require('JSV').JSV;
  helper = require('./test_helper');
  Parser = require('../src').Parser;
} else {
  expect = global.expect;
  _ = global._;
  JSV = global.JSV;
  helper = global.TestHelper;
  Parser = global.PixivNovelParser.Parser;
}

describe('Parser specifications.', function () {
  /* jshint quotmark: false */
  describe('テキスト', function () {
    it('ただのテキストはちゃんとただのテキストになる', function () {
      var parser = new Parser(),
          novel = 'テキスト\nテキスト',
          expectedAST = [{ type: 'text', val: 'テキスト\nテキスト' }],
          schema = {
            "$schema": "http://json-schema.org/draft-02/hyper-schema#",
            "id": "http://json-schema.org/draft-02/schema#",
            "type": "object",
            "properties": {
              "type": {
                "enum": ["text"]
              },
              "val": {
                "type": "string"
              }
            },
            "required": ["type", "val"]
          };

      parser.parse(novel);
      expect(_.isEqual(parser.tree, expectedAST)).to.be.ok();
      expect(helper.validateJSON(parser.tree[0], schema)).to.be.ok();
    });

    it('改行コードを正規化する', function () {
      var parser = new Parser(),
          novel = '\n\n改行\n改行\r\n改行\n\r\n改行\r\n\n改行\r\n\r\n改行\n',
          expectedAST = [
            { type: 'text', val: '\n\n改行\n改行\n改行\n\n改行\n\n改行\n\n改行\n' }
          ];

      parser.parse(novel);
      expect(_.isEqual(parser.tree, expectedAST)).to.be.ok();
    });
  });

  describe('newpage', function () {
    it('ちゃんとページを分割できる', function () {
      var parser = new Parser(),
          novel = '1ページ目[newpage]2ページ目',
          expectedAST = [
            { type: 'text', val: '1ページ目' },
            { type: 'tag', name: 'newpage' },
{'fraction_non_alphanumeric': 0.18847508807247107, 'fraction_numerical': 0.004026170105686965, 'mean_word_length': 2.0506523407521104, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6147334', 'n_tokens_mistral': 1468, 'n_tokens_neox': 1426, 'n_words': 236}