<commit_before>{
    "name": "crawler",
    "version": "0.2.5",
    "description": "Crawler is a web spider written with Nodejs. It gives you the full power of jQuery on the server to parse a big number of pages as they are downloaded, asynchronously. Scraping should be simple and fun!",
    "keywords": [
        "dom",
        "javascript",
        "crawling",
        "spider",
        "scraper",
        "scraping",
        "jquery"
    ],
    "maintainers": [
        {
            "name": "<NAME>",
            "email": "<EMAIL>",
            "url": "http://sylvinus.org/"
        }
    ],
    "bugs": {
        "mail": "<EMAIL>",
        "url": "http://github.com/sylvinus/node-crawler/issues"
    },
    "licenses": [
        {
            "type": "MIT",
            "url": "http://github.com/sylvinus/node-crawler/blob/master/LICENSE.txt"
        }
    ],
    "repository":
        {
            "type": "git",
            "url": "https://github.com/sylvinus/node-crawler.git"
        }
    ,
    "dependencies": {
       "request": "2.12.0",
       "jsdom": "0.8.2",
       "generic-pool": "2.0.2",
       "htmlparser": "1.7.6",
       "request": "2.21.0",
       "jsdom": "0.8.2",
       "generic-pool": "2.0.3",
       "underscore": "1.4.4",
       "jschardet": "1.0.2",
       "iconv-lite": "0.2.8"
    },
    "optionalDependencies":{
        "iconv": "2.0.6"
    },
    "devDependencies": {
       "qunit": "0.5.16",
       "express": "2.5.x",
       "memwatch": "0.2.2"
    },
    "scripts": {
        "test": "node test/testrunner.js"
    },
    "engines": [
        "node >=0.6.x"
    ],
    "directories": {
        "lib": "lib"
    },
    "main": "./lib/crawler"
}<commit_msg>remove duplicate declaration<commit_after>{
    "name": "crawler",
    "version": "0.2.5",
    "description": "Crawler is a web spider written with Nodejs. It gives you the full power of jQuery on the server to parse a big number of pages as they are downloaded, asynchronously. Scraping should be simple and fun!",
    "keywords": [
        "dom",
        "javascript",
        "crawling",
        "spider",
        "scraper",
        "scraping",
        "jquery"
    ],
    "maintainers": [
        {
            "name": "<NAME>",
            "email": "<EMAIL>",
            "url": "http://sylvinus.org/"
        }
    ],
    "bugs": {
        "mail": "<EMAIL>",
        "url": "http://github.com/sylvinus/node-crawler/issues"
    },
    "licenses": [
        {
            "type": "MIT",
            "url": "http://github.com/sylvinus/node-crawler/blob/master/LICENSE.txt"
        }
    ],
    "repository":
        {
            "type": "git",
            "url": "https://github.com/sylvinus/node-crawler.git"
        }
    ,
    "dependencies": {
       "htmlparser": "1.7.6",
       "request": "2.21.0",
       "jsdom": "0.8.2",
       "generic-pool": "2.0.3",
       "underscore": "1.4.4",
       "jschardet": "1.0.2",
       "iconv-lite": "0.2.8"
    },
    "optionalDependencies":{
        "iconv": "2.0.6"
    },
    "devDependencies": {
       "qunit": "0.5.16",
       "express": "2.5.x",
       "memwatch": "0.2.2"
    },
    "scripts": {
        "test": "node test/testrunner.js"
    },
    "engines": [
        "node >=0.6.x"
    ],
    "directories": {
        "lib": "lib"
    },
    "main": "./lib/crawler"
}
{'fraction_non_alphanumeric': 0.20018061408789886, 'fraction_numerical': 0.026490066225165563, 'mean_word_length': 1.647808764940239, 'pattern_counts': {'":': 77, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3981660', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1192, 'n_words': 228}