<commit_before>from __future__  import print_function
import argparse
import couchdb
import mwclient
import os
import socket
import traceback
import urlparse
import uuid

from datetime import datetime

def fix_server_url(general_siteinfo):
    """
    Get server url from siteinfo's 'general' dict,
    add http if scheme is missing. This will also modify
    given dictionary.

    >>> general_siteinfo = {'server': '//simple.wikipedia.org'}
    >>> fix_server_url(general_siteinfo)
    'http://simple.wikipedia.org'
    >>> general_siteinfo
    {'server': 'http://simple.wikipedia.org'}

    >>> fix_server_url({'server': 'https://en.wikipedia.org'})
    'https://en.wikipedia.org'

    >>> fix_server_url({})
    ''

    """
    server = general_siteinfo.get('server', '')
    if server:
        p = urlparse.urlparse(server)
        if not p.scheme:
            server = urlparse.urlunparse(
                urlparse.ParseResult('http', p.netloc, p.path,
                                     p.params, p.query, p.fragment))
            general_siteinfo['server'] = server
    return server


def update_siteinfo(site, couch_server, db_name):
    try:
        siteinfo_db = couch_server.create('siteinfo')
    except couchdb.PreconditionFailed:
        siteinfo_db = couch_server['siteinfo']

    siteinfo = site.api('query', meta='siteinfo',
                        siprop='general|interwikimap|rightsinfo|statistics'
    )['query']

    fix_server_url(siteinfo['general'])

    siteinfo.pop('userinfo', None)

    siteinfo_doc = siteinfo_db.get(db_name)

    if siteinfo_doc:
        siteinfo_doc.update(siteinfo)
    else:
        siteinfo_doc = siteinfo

    siteinfo_db[db_name] = siteinfo_doc


def parse_args():
    argparser = argparse.ArgumentParser()
    argparser.add_argument('site', nargs='?',
                           help=('MediaWiki site to scrape (host name), '
                                 'e.g. en.m.wikipedia.org'))
    argparser.add_argument('-c', '--couch',
                           help=('CouchDB server URL. '
                                 'Default: %(default)s'),
                           default='http://localhost:5984')
    argparser.add_argument('--db',
                           help=('CouchDB database name. '
                                 'If not specified, the name will be '
                                 'derived from Mediawiki host name.'),
                           default=None)
    argparser.add_argument('--titles', nargs='+',
                           help=('Download article pages with '
                                 'these names (titles). '
                                 'It name starts with @ it is '
                                 'interpreted as name of file containing titles, '
                                 'one per line, utf8 encoded.'))
    argparser.add_argument('--start',
                           help=('Download all article pages '
                                 'beginning with this name'))
    argparser.add_argument('--changes-since',
                           help=('Download all article pages '
                                 'that change since specified time. '
                                 'Timestamp format is yyyymmddhhmmss. '
                                 'See https://www.mediawiki.org/wiki/Timestamp'))
    argparser.add_argument('--timeout',
                           default=30.0,
                           type=float,
                           help=('Network communications timeout. '
                                 'Default: %(default)ss'))
    argparser.add_argument('-S', '--siteinfo-only', action='store_true',
                           help=('Fetch or update siteinfo, then exit'))
    argparser.add_argument('-r', '--resume', nargs='?',
                           default='',
                           metavar='SESSION ID',
                           help=('Resume previous scrape session. '
                                 'This relies on stats saved in '
                                 'mwscrape database.'))
    argparser.add_argument('--sessions-db-name',
                           default='mwscrape',
                           help=('Name of database where '
                                 'session info is stored. '
                                 'Default: %(default)s'))

    return argparser.parse_args()


def main():

    args = parse_args()

    socket.setdefaulttimeout(args.timeout)

    couch_server = couchdb.Server(args.couch)

    sessions_db_name = args.sessions_db_name
    try:
        sessions_db = couch_server.create(sessions_db_name)
    except couchdb.PreconditionFailed:
        sessions_db = couch_server[sessions_db_name]

    if args.resume or args.resume is None:
        session_id = args.resume
        if session_id is None:
            current_doc = sessions_db['$current']
            session_id = current_doc['session_id']
        print('Resuming session %s' % session_id)
        session_doc = sessions_db[session_id]
        site_host = session_doc['site']
        db_name = session_doc['db_name']
        session_doc['resumed_at'] = datetime.utcnow().isoformat()
        start_page_name = session_doc.get('last_page_name')
        if not start_page_name:
            start_page_name = args.start
        sessions_db[session_id] = session_doc
    else:
        site_host = args.site
        db_name = args.db
        start_page_name = args.start
        if not site_host:
            print('Site to scrape is not specified')
            raise SystemExit(1)
        if not db_name:
            db_name = site_host.replace('.', '-')
        session_id = uuid.uuid4().hex
        print('Starting session %s' % session_id)
        sessions_db[session_id] = {
            'created_at': datetime.utcnow().isoformat(),
            'site': site_host,
            'db_name': db_name
        }
        current_doc = sessions_db.get('$current', {})
        current_doc['session_id'] = session_id
        sessions_db['$current'] = current_doc


    site = mwclient.Site(site_host)

    update_siteinfo(site, couch_server, db_name)

    if args.siteinfo_only:
        return

    try:
        db = couch_server.create(db_name)
    except couchdb.PreconditionFailed:
        db = couch_server[db_name]

    def titles_from_args(titles):
        for title in titles:
            if title.startswith('@'):
                with open(os.path.expanduser(title[1:])) as f:
                    for line in f:
                        yield line.strip()
            else:
                yield title

    def titles_from_recent_changes(timestamp):
        changes = site.recentchanges(start=timestamp,
                                     namespace=0,
                                     show='!minor|!redirect')
        for change in changes:
            title = change.get('title')
            if title:
                doc = db.get(title)
                doc_revid = doc.get('parse', {}).get('revid') if doc else None
                revid = change.get('revid')
                if doc_revid == revid:
                    continue
                yield title

    if args.titles:
        pages = (site.Pages[title.decode('utf8')]
                 for title in titles_from_args(args.titles))
    elif args.changes_since:
        print('Getting recent changes (since %s)' % args.changes_since)
        pages = (site.Pages[title]
                 for title in titles_from_recent_changes(args.changes_since))
    else:
        print('Starting at %s' % start_page_name)
        pages = site.allpages(start=start_page_name)<commit_msg>Add option to list all pages in descending order<commit_after>from __future__  import print_function
import argparse
import couchdb
import mwclient
import os
import socket
import traceback
import urlparse
import uuid

from datetime import datetime

def fix_server_url(general_siteinfo):
    """
    Get server url from siteinfo's 'general' dict,
    add http if scheme is missing. This will also modify
    given dictionary.

    >>> general_siteinfo = {'server': '//simple.wikipedia.org'}
    >>> fix_server_url(general_siteinfo)
    'http://simple.wikipedia.org'
    >>> general_siteinfo
    {'server': 'http://simple.wikipedia.org'}

    >>> fix_server_url({'server': 'https://en.wikipedia.org'})
    'https://en.wikipedia.org'

    >>> fix_server_url({})
    ''

    """
    server = general_siteinfo.get('server', '')
    if server:
        p = urlparse.urlparse(server)
        if not p.scheme:
            server = urlparse.urlunparse(
                urlparse.ParseResult('http', p.netloc, p.path,
                                     p.params, p.query, p.fragment))
            general_siteinfo['server'] = server
    return server


def update_siteinfo(site, couch_server, db_name):
    try:
        siteinfo_db = couch_server.create('siteinfo')
    except couchdb.PreconditionFailed:
        siteinfo_db = couch_server['siteinfo']

    siteinfo = site.api('query', meta='siteinfo',
                        siprop='general|interwikimap|rightsinfo|statistics'
    )['query']

    fix_server_url(siteinfo['general'])

    siteinfo.pop('userinfo', None)

    siteinfo_doc = siteinfo_db.get(db_name)

    if siteinfo_doc:
        siteinfo_doc.update(siteinfo)
    else:
        siteinfo_doc = siteinfo

    siteinfo_db[db_name] = siteinfo_doc


def parse_args():
    argparser = argparse.ArgumentParser()
    argparser.add_argument('site', nargs='?',
                           help=('MediaWiki site to scrape (host name), '
                                 'e.g. en.m.wikipedia.org'))
    argparser.add_argument('-c', '--couch',
                           help=('CouchDB server URL. '
                                 'Default: %(default)s'),
                           default='http://localhost:5984')
    argparser.add_argument('--db',
                           help=('CouchDB database name. '
                                 'If not specified, the name will be '
                                 'derived from Mediawiki host name.'),
                           default=None)
    argparser.add_argument('--titles', nargs='+',
                           help=('Download article pages with '
                                 'these names (titles). '
                                 'It name starts with @ it is '
                                 'interpreted as name of file containing titles, '
                                 'one per line, utf8 encoded.'))
    argparser.add_argument('--start',
                           help=('Download all article pages '
                                 'beginning with this name'))
    argparser.add_argument('--changes-since',
                           help=('Download all article pages '
                                 'that change since specified time. '
                                 'Timestamp format is yyyymmddhhmmss. '
                                 'See https://www.mediawiki.org/wiki/Timestamp'))
    argparser.add_argument('--timeout',
                           default=30.0,
                           type=float,
                           help=('Network communications timeout. '
                                 'Default: %(default)ss'))
    argparser.add_argument('-S', '--siteinfo-only', action='store_true',
                           help=('Fetch or update siteinfo, then exit'))
    argparser.add_argument('-r', '--resume', nargs='?',
                           default='',
                           metavar='SESSION ID',
                           help=('Resume previous scrape session. '
                                 'This relies on stats saved in '
                                 'mwscrape database.'))
    argparser.add_argument('--sessions-db-name',
                           default='mwscrape',
                           help=('Name of database where '
                                 'session info is stored. '
                                 'Default: %(default)s'))
    argparser.add_argument('--desc',
                           action='store_true',
                           help=('Request all apges in descending order'))

    return argparser.parse_args()


def main():

    args = parse_args()

    socket.setdefaulttimeout(args.timeout)

    couch_server = couchdb.Server(args.couch)

    sessions_db_name = args.sessions_db_name
    try:
        sessions_db = couch_server.create(sessions_db_name)
    except couchdb.PreconditionFailed:
        sessions_db = couch_server[sessions_db_name]

    if args.resume or args.resume is None:
        session_id = args.resume
        if session_id is None:
            current_doc = sessions_db['$current']
            session_id = current_doc['session_id']
        print('Resuming session %s' % session_id)
        session_doc = sessions_db[session_id]
        site_host = session_doc['site']
        db_name = session_doc['db_name']
        session_doc['resumed_at'] = datetime.utcnow().isoformat()
        start_page_name = session_doc.get('last_page_name')
        if not start_page_name:
            start_page_name = args.start
        sessions_db[session_id] = session_doc
    else:
        site_host = args.site
        db_name = args.db
        start_page_name = args.start
        if not site_host:
            print('Site to scrape is not specified')
            raise SystemExit(1)
        if not db_name:
            db_name = site_host.replace('.', '-')
        session_id = uuid.uuid4().hex
        print('Starting session %s' % session_id)
        sessions_db[session_id] = {
            'created_at': datetime.utcnow().isoformat(),
            'site': site_host,
            'db_name': db_name
        }
        current_doc = sessions_db.get('$current', {})
        current_doc['session_id'] = session_id
        sessions_db['$current'] = current_doc


    site = mwclient.Site(site_host)

    update_siteinfo(site, couch_server, db_name)

    if args.siteinfo_only:
        return

    try:
        db = couch_server.create(db_name)
    except couchdb.PreconditionFailed:
        db = couch_server[db_name]

    def titles_from_args(titles):
        for title in titles:
            if title.startswith('@'):
                with open(os.path.expanduser(title[1:])) as f:
                    for line in f:
                        yield line.strip()
            else:
                yield title

    def titles_from_recent_changes(timestamp):
        changes = site.recentchanges(start=timestamp,
                                     namespace=0,
                                     show='!minor|!redirect')
        for change in changes:
            title = change.get('title')
            if title:
                doc = db.get(title)
                doc_revid = doc.get('parse', {}).get('revid') if doc else None
                revid = change.get('revid')
                if doc_revid == revid:
                    continue
                yield title

    if args.titles:
        pages = (site.Pages[title.decode('utf8')]
                 for title in titles_from_args(args.titles))
    elif args.changes_since:
        print('Getting recent changes (since %s)' % args.changes_since)
        pages = (site.Pages[title]
                 for title in titles_from_recent_changes(args.changes_since))
    else:
        print('Starting at %s' % start_page_name)
        pages = site.allpages(start=start_page_name,
                              dir='descending' if args.desc else 'ascending')
{'fraction_non_alphanumeric': 0.10368260953983562, 'fraction_numerical': 0.001682738981295709, 'mean_word_length': 1.687304347826087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1587342', 'n_tokens_mistral': 4233, 'n_tokens_neox': 4135, 'n_words': 896}