<commit_before>from setuptools import setup, find_packages

version = '0.1'

setup(name='torspider',
      version=version,
      description="Example/prototype of an asynchroneous web-crawler based on Tornado",
      long_description="""\
""",
      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
      keywords='web-crawler web-spider scraper',
      author='<NAME>',
      author_email='<EMAIL>',
      url='http://crawlers.info',
      license='MIT',
      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
      include_package_data=True,
      zip_safe=True,
      install_requires=[
          'tornado',
          'pycurl',
          'bs4',
          'lxml',
          'motor',
          'langdetect',
          'python-dateutil',
      ],
      entry_points={
          'console_scripts': [
               'torspider = torspider.main:run_main',
          ],
        }
)<commit_msg>plugins entry point in setup script<commit_after>from setuptools import setup, find_packages

version = '0.1'

setup(name='torspider',
      version=version,
      description="Example/prototype of an asynchroneous web-crawler based on Tornado",
      long_description="""\
""",
      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
      keywords='web-crawler web-spider scraper',
      author='<NAME>',
      author_email='<EMAIL>',
      url='http://crawlers.info',
      license='MIT',
      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
      include_package_data=True,
      zip_safe=True,
      install_requires=[
          'tornado',
          'pycurl',
          'bs4',
          'lxml',
          'motor',
          'langdetect',
          'python-dateutil',
      ],
      entry_points={
          'console_scripts': [
               'torspider = torspider.main:run_main',
          ],
          # see: http://amir.rachum.com/blog/2017/07/28/python-entry-points/
          'torspider_consumer': [
              'save_to_mongo = torspider_mongo:save_report',
              #'fancy = snek:fancy_snek',
          ],
        }
)
{'fraction_non_alphanumeric': 0.1406613879832324, 'fraction_numerical': 0.007452258965999069, 'mean_word_length': 2.35625, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4162682', 'n_tokens_mistral': 700, 'n_tokens_neox': 673, 'n_words': 106}