<commit_before>
import {legacyUnhtml} from '../../src/restricted/legacy';

describe('legacy conversions', () => {
  it('safe HTML conversion with no options', () => {
    expect(legacyUnhtml('<html><script>alert(0)</script></html>').toString())
        .toEqual('<html><script>alert(0)</script></html>');
    expect(legacyUnhtml('<anything>at>all').toString())
        .toEqual('<anything>at>all');
  });
  it('report-only conversion: inactive HTML', () => {
    const collectedReports: string[] = [];

    expect(legacyUnhtml('<html><b>hi</b></html>', {
             reportingId: 'legacy_conversion_unit_test',
             samplingRate: 1.0,
             heartbeatRate: 1.0,
             sendReport: (_, data) => collectedReports.push(data)
           }).toString())
        .toEqual('<html><b>hi</b></html>');

    expect(collectedReports.map(assertAndClearHostname)).toEqual([
      '{"type":"HEARTBEAT"}', '{"type":"H_ESCAPE"}'
    ])
  });
  it('report-only conversion: plaintext', () => {
    const collectedReports: string[] = [];

    expect(legacyUnhtml('hi', {
             reportingId: 'legacy_conversion_unit_test',
             samplingRate: 1.0,
             heartbeatRate: 1.0,
             sendReport: (_, data) => collectedReports.push(data)
           }).toString())
        .toEqual('hi');

    expect(collectedReports.map(assertAndClearHostname)).toEqual([
      '{"type":"HEARTBEAT"}'
    ])
  });
  it('report-only conversion: reports getting sent doesn\'t crash', () => {
    expect(legacyUnhtml('<html><b>hi</b></html>', {
             reportingId: 'legacy_conversion_unit_test',
             samplingRate: 1.0,
             heartbeatRate: 1.0,
           }).toString())
        .toEqual('<html><b>hi</b></html>');
  });
  it('report-only conversion: low sampling rate', () => {
    const collectedReports: string[] = [];

    expect(legacyUnhtml('<script>alert(0)</script>', {
             reportingId: 'legacy_conversion_unit_test',
             samplingRate: 0.0,
             heartbeatRate: 0.0,
             sendReport: (_, data) => collectedReports.push(data)
           }).toString())
        .toEqual('<script>alert(0)</script>');

    expect(collectedReports.map(assertAndClearHostname)).toEqual([])
  });
});

function assertAndClearHostname(report: string): any {
  const parsed = JSON.parse(report) as any;
  expect(parsed['host']).toBeTruthy();
  delete parsed['host'];
  return JSON.stringify(parsed);
}<commit_msg>Internal change<commit_after>
import {legacyUnhtml} from '../../src/restricted/legacy';

describe('legacy conversions', () => {
  it('safe HTML conversion with no options', () => {
    expect(legacyUnhtml('<html><script>alert(0)</script></html>').toString())
        .toEqual('<html><script>alert(0)</script></html>');
    expect(legacyUnhtml('<anything>at>all').toString())
        .toEqual('<anything>at>all');
  });
  it('report-only conversion: inactive HTML', () => {
    const collectedReports: string[] = [];

    expect(legacyUnhtml('<html><b>hi</b></html>', {
             reportingId: 'legacy_conversion_unit_test',
             samplingRate: 1.0,
             heartbeatRate: 1.0,
             sendReport: (_, data) => collectedReports.push(data)
           }).toString())
        .toEqual('<html><b>hi</b></html>');

    expect(collectedReports.map(assertAndClearHostname)).toEqual([
      '{"type":"HEARTBEAT"}', '{"type":"H_ESCAPE"}'
    ])
  });
  it('report-only conversion: plaintext', () => {
    const collectedReports: string[] = [];

    expect(legacyUnhtml('hi', {
             reportingId: 'legacy_conversion_unit_test',
             samplingRate: 1.0,
             heartbeatRate: 1.0,
             sendReport: (_, data) => collectedReports.push(data)
           }).toString())
        .toEqual('hi');

    expect(collectedReports.map(assertAndClearHostname)).toEqual([
      '{"type":"HEARTBEAT"}'
    ])
  });
  it('report-only conversion: reports getting sent doesn\'t crash', () => {
    expect(legacyUnhtml('<html><b>hi</b></html>', {
             reportingId: 'legacy_conversion_unit_test',
             samplingRate: 1.0,
             heartbeatRate: 1.0,
           }).toString())
        .toEqual('<html><b>hi</b></html>');
  });
  it('report-only conversion: low sampling rate', () => {
    const collectedReports: string[] = [];

    expect(legacyUnhtml('<script>alert(0)</script>', {
             reportingId: 'legacy_conversion_unit_test',
             samplingRate: 0.0,
             heartbeatRate: 0.0,
             sendReport: (_, data) => collectedReports.push(data)
           }).toString())
        .toEqual('<script>alert(0)</script>');

    expect(collectedReports.map(assertAndClearHostname)).toEqual([])
  });
});

function assertAndClearHostname(report: string): any {
  const parsed = JSON.parse(report) as any;
  expect(parsed['host']).toBeTruthy();
  expect(parsed['host']).toBeInstanceOf(String);
  delete parsed['host'];
  return JSON.stringify(parsed);
}
{'fraction_non_alphanumeric': 0.18753781004234724, 'fraction_numerical': 0.008066142367412784, 'mean_word_length': 3.085667215815486, 'pattern_counts': {'":': 6, '<': 63, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '643462', 'n_tokens_mistral': 1525, 'n_tokens_neox': 1504, 'n_words': 222}