<commit_before>const fs = require('fs');
const path = require('path');
const preprocessor = require(path.join('..', 'preprocessor.js'));

describe('Csound preprocessor', () => {
  it('is defined', () => {
    expect(preprocessor).toBeDefined();
  });

  it('lexes empty string', () => {
    preprocessor.setInput('');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('');
    expect(preprocessor.messages.length).toBe(0);
  });

  it('lexes line continuation', () => {
    preprocessor.setInput('\\\n');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('');
    expect(preprocessor.messages.length).toBe(0);
  });

  it('lexes line continuation with trailing semicolon comment', () => {
    preprocessor.setInput('\\ \t;comment\n');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('');
    expect(preprocessor.messages.length).toBe(1);
    expect(preprocessor.messages[0]).toEqual({
      type: 'Warning',
      filePath: undefined,
      text: 'Line continuation is not followed immediately by newline',
      range: [[0, 0], [0, 1]]
    });
  });

  it('lexes block comments', () => {
    preprocessor.setInput('/*\n#include @0 /* @@0 $macro*//**/');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('  ');
    expect(preprocessor.messages.length).toBe(1);
    expect(preprocessor.messages[0]).toEqual({
      type: 'Warning',
      filePath: undefined,
      text: '‘/*’ in block comment',
      range: [[1, 12], [1, 14]]
    });
  });

  it('lexes unterminated block comment', () => {
    preprocessor.setInput('/*');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe(' ');
    expect(preprocessor.messages.length).toBe(1);
    expect(preprocessor.messages[0]).toEqual({
      type: 'Error',
      filePath: undefined,
      text: 'Unterminated block comment',
      range: [[0, 0], [0, 2]]
    });
  });

  it('lexes single-line comments', () => {
    preprocessor.setInput(';#include\n//#define');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('\n\n');
    expect(preprocessor.messages.length).toBe(0);
  });

  it('lexes quoted string', () => {
    const string = '"#include/**/;\\n"';
    preprocessor.setInput(string);
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe(string);
    expect(preprocessor.messages.length).toBe(0);<commit_msg>Add URL of Csound issue<commit_after>const fs = require('fs');
const path = require('path');
const preprocessor = require(path.join('..', 'preprocessor.js'));

describe('Csound preprocessor', () => {
  it('is defined', () => {
    expect(preprocessor).toBeDefined();
  });

  it('lexes empty string', () => {
    preprocessor.setInput('');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('');
    expect(preprocessor.messages.length).toBe(0);
  });

  it('lexes line continuation', () => {
    preprocessor.setInput('\\\n');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('');
    expect(preprocessor.messages.length).toBe(0);
  });

  it('lexes line continuation with trailing semicolon comment', () => {
    preprocessor.setInput('\\ \t;comment\n');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('');
    expect(preprocessor.messages.length).toBe(1);
    expect(preprocessor.messages[0]).toEqual({
      type: 'Warning',
      filePath: undefined,
      text: 'Line continuation is not followed immediately by newline',
      range: [[0, 0], [0, 1]]
    });
  });

  it('lexes block comments', () => {
    preprocessor.setInput('/*\n#include @0 /* @@0 $macro*//**/');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('  ');
    expect(preprocessor.messages.length).toBe(1);
    expect(preprocessor.messages[0]).toEqual({
      type: 'Warning',
      filePath: undefined,
      text: '‘/*’ in block comment',
      range: [[1, 12], [1, 14]]
    });
  });

  // https://github.com/csound/csound/issues/542
  it('lexes unterminated block comment', () => {
    preprocessor.setInput('/*');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe(' ');
    expect(preprocessor.messages.length).toBe(1);
    expect(preprocessor.messages[0]).toEqual({
      type: 'Error',
      filePath: undefined,
      text: 'Unterminated block comment',
      range: [[0, 0], [0, 2]]
    });
  });

  it('lexes single-line comments', () => {
    preprocessor.setInput(';#include\n//#define');
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe('\n\n');
    expect(preprocessor.messages.length).toBe(0);
  });

  it('lexes quoted string', () => {
    const string = '"#include/**/;\\n"';
    preprocessor.setInput(string);
    preprocessor.lex();
    expect(preprocessor.getOutput()).toBe(string);
    expect(preprocessor.messages.length).toBe(0);
{'fraction_non_alphanumeric': 0.203460496143423, 'fraction_numerical': 0.01146549927037732, 'mean_word_length': 3.64922480620155, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3737736', 'n_tokens_mistral': 1520, 'n_tokens_neox': 1481, 'n_words': 250}