<commit_before># Lossless text compression

## Benchmark

The performance gets better with the number of words. 

words | compressed | original | encode  | decode
---|---|---|---|---
100 000 | 226 kB | 654 kB | 381 ms | 51 ms
10 000 | 25 kB | 65 kB | 43 ms | 8.7 ms
1 000 | 4.4 kB | 6.4 kB | 6.7 ms | 1.5 ms
100 | 0.62 kB | 0.73 kB | 0.44 ms | 0.16 ms

## Install

```
npm install lossless-text-compression
```

## How to use

#### Encode

```
const { encode } = require('lossless-text-compression');

console.log(
    encode('1121231234123451234561234567123456781234567891234567890')
);
```

#### Decode

```
const { decode } = require('lossless-text-compression');

console.log(
    decode('11234567890')
);
```

#### With files

```
const { encode, decode } = require('./ltc.js');
const { readFileSync, writeFileSync } = require('fs');
  
const fileName = ...;           // for example: loremipsum.txt
const compressedFile = ...;     // for example: encoded.txt
const restoredFile = ...;       // for example: decoded.txt

// read yourfile file
const content = readFileSync(fileName, 'utf-8');

// encode and save the content
writeFileSync(compressedFile, encode(content), 'ucs2');

// read and decode the encoded content
const decoded = decode(readFileSync(compressedFileName, 'ucs2'));

// save the restored content
writeFileSync(restoredFileName, decoded, 'utf-8');
```

## Author
<commit_msg>minor adjustment readme<commit_after># Lossless text compression

## Benchmark

The performance gets better with the number of words. 

words | compressed | original | encode  | decode
---|---|---|---|---
100 000 | 226 kB | 654 kB | 381 ms | 51 ms
10 000 | 25 kB | 65 kB | 43 ms | 8.7 ms
1 000 | 4.4 kB | 6.4 kB | 6.7 ms | 1.5 ms
100 | 0.62 kB | 0.73 kB | 0.44 ms | 0.16 ms

## Install

```
npm install lossless-text-compression
```

## How to use

#### Encode

```
const { encode } = require('lossless-text-compression');

console.log(
    encode('1121231234123451234561234567123456781234567891234567890')
);
```

#### Decode

```
const { decode } = require('lossless-text-compression');

console.log(
    decode('11234567890')
);
```

#### With files

```
const { encode, decode } = require('lossless-text-compression');
const { readFileSync, writeFileSync } = require('fs');
  
const fileName = ...;           // for example: loremipsum.txt
const compressedFile = ...;     // for example: encoded.txt
const restoredFile = ...;       // for example: decoded.txt

// read yourfile file
const content = readFileSync(fileName, 'utf-8');

// encode and save the content
writeFileSync(compressedFile, encode(content), 'ucs2');

// read and decode the encoded content
const decoded = decode(readFileSync(compressedFileName, 'ucs2'));

// save the restored content
writeFileSync(restoredFileName, decoded, 'utf-8');
```

## Author

{'fraction_non_alphanumeric': 0.16367476240760295, 'fraction_numerical': 0.08940513903555086, 'mean_word_length': 3.4826498422712935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6452530', 'n_tokens_mistral': 1169, 'n_tokens_neox': 964, 'n_words': 272}