<commit_before>import argparse
import glob
import hashlib
import os.path
import socket

from earthreader.web import app

from libearth.crawler import crawl, CrawlError
from libearth.feedlist import FeedList
from libearth.schema import write


def earthreader():
    parser = argparse.ArgumentParser(prog='earthreader')
    subparsers = parser.add_subparsers(dest='command', help='command-help')
    server_parser = subparsers.add_parser('server',
                                          help='Run a server for EarthReader')
    server_parser.add_argument('-H', '--host',
                               default='0.0.0.0',
                               help='Host to listen. default=\'0.0.0.0\'')
    server_parser.add_argument('-p', '--port',
                               type=int,
                               default=5000,
                               help='Port number to listen. default=5000')
    server_parser.add_argument('-d', '--debug',
                               default=False,
                               action='store_true',
                               help='Debug mode. '
                                    'It makes the server possible to '
                                    'automatically restart when files touched.')
    server_parser.add_argument('repository',
                               help='Repository for EarthReader')
    crawl_parser = subparsers.add_parser('crawl',
                                         help='Crawl feeds in the OPML')
    crawl_parser.add_argument('-n', '--threads',
                              type=int,
                              help='Poolsize of the crawler')
    crawl_parser.add_argument('repository',
                              help='Repository which has the OPML')
    args = parser.parse_args()<commit_msg>remove import "feedlist"<commit_after>import argparse
import glob
import hashlib
import os.path
import socket

from earthreader.web import app

from libearth.crawler import crawl, CrawlError
from libearth.schema import write


def earthreader():
    parser = argparse.ArgumentParser(prog='earthreader')
    subparsers = parser.add_subparsers(dest='command', help='command-help')
    server_parser = subparsers.add_parser('server',
                                          help='Run a server for EarthReader')
    server_parser.add_argument('-H', '--host',
                               default='0.0.0.0',
                               help='Host to listen. default=\'0.0.0.0\'')
    server_parser.add_argument('-p', '--port',
                               type=int,
                               default=5000,
                               help='Port number to listen. default=5000')
    server_parser.add_argument('-d', '--debug',
                               default=False,
                               action='store_true',
                               help='Debug mode. '
                                    'It makes the server possible to '
                                    'automatically restart when files touched.')
    server_parser.add_argument('repository',
                               help='Repository for EarthReader')
    crawl_parser = subparsers.add_parser('crawl',
                                         help='Crawl feeds in the OPML')
    crawl_parser.add_argument('-n', '--threads',
                              type=int,
                              help='Poolsize of the crawler')
    crawl_parser.add_argument('repository',
                              help='Repository which has the OPML')
    args = parser.parse_args()
{'fraction_non_alphanumeric': 0.0952247191011236, 'fraction_numerical': 0.008988764044943821, 'mean_word_length': 1.4798050139275767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6243562', 'n_tokens_mistral': 878, 'n_tokens_neox': 843, 'n_words': 197}