<commit_before># lexxy

lexxy is a simple lexer for Node.js.

## Usage

### Installation

```
$ npm install lexxy
```<commit_msg>Added example to README.<commit_after># lexxy

lexxy is a simple lexer for Node.js.

## Usage

### Installation

```
$ npm install lexxy
```

### Example

##### index.js
```javascript
var lexxy = require('lexxy');
var lexer = new lexxy.Lexer();

// patterns for XML-style tags
lexer.type('OPENTAG', /<([^\/]+?)>/);
lexer.type('CLOSETAG', /<\/([^\/]+?)>/);

// pattern for assignments
lexer.type('ASSIGN', /(\w+?)\s+?=\s+(\w+?));

// lex example string
var tokens = lexer.lex('<test>  number \n  =5</test>');

tokens.forEach(function(token, index, arr)
{
	console.log(token.type.name);

	switch(token.type.name)
	{
		case 'OPENTAG':
		case 'CLOSETAG':
			console.log('tag name: ' + token.data[1]);
			break;
		case 'ASSIGN':
			console.log('variable name: ' + token.data[1]);
			console.log('value: ' + token.data[2]);
			break;
	}
});
```
{'fraction_non_alphanumeric': 0.2233712512926577, 'fraction_numerical': 0.004136504653567736, 'mean_word_length': 3.81592039800995, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '152040', 'n_tokens_mistral': 404, 'n_tokens_neox': 368, 'n_words': 82}