<commit_before>import re
import sys
import gzip
import urllib.request as request
import urllib.parse as parse
import urllib.error
import os.path

import pirate.data
from pirate.print import print

from io import BytesIO


def parse_category(category):
    if str(category) in pirate.data.categories.values():
        return category
    elif category in pirate.data.categories.keys():
        return pirate.data.categories[category]
    else:
        print('Invalid category ignored', color='WARN')
        return '0'


def parse_sort(sort):
    if str(sort) in pirate.data.sorts.values():
        return sort
    elif sort in pirate.data.sorts.keys():
        return pirate.data.sorts[sort]
    else:
        print('Invalid sort ignored', color='WARN')
        return '99'


#todo: redo this with html parser instead of regex
def remote(pages, category, sort, mode, terms, mirror):
    res_l = []
    pages = int(pages)
    if pages < 1:
        raise ValueError('Please provide an integer greater than 0 '
                         'for the number of pages to fetch.')

    # Catch the Ctrl-C exception and exit cleanly
    try:
        sizes = []
        uploaded = []
        identifiers = []
        for page in range(pages):
            if mode == 'browse':
                path = '/browse/'
                if(category == 0):
                    category = 100
                path = '/browse/{}/{}/{}'.format(category, page, sort)
            elif mode == 'recent':
                # This is not a typo. There is no / between 48h and the category.
                path = '/top/48h'
                if(category == 0):
                    path += 'all'<commit_msg>fix bugs with categories and sorts<commit_after>import re
import sys
import gzip
import urllib.request as request
import urllib.parse as parse
import urllib.error
import os.path

import pirate.data
from pirate.print import print

from io import BytesIO


def parse_category(category):
    try:
        category = int(category)
    except ValueError:
        pass
    if category in pirate.data.categories.values():
        return category
    elif category in pirate.data.categories.keys():
        return pirate.data.categories[category]
    else:
        print('Invalid category ignored', color='WARN')
        return '0'


def parse_sort(sort):
    try:
        sort = int(sort)
    except ValueError:
        pass
    if sort in pirate.data.sorts.values():
        return sort
    elif sort in pirate.data.sorts.keys():
        return pirate.data.sorts[sort]
    else:
        print('Invalid sort ignored', color='WARN')
        return '99'


#TODO: redo this with html parser instead of regex
#TODO: warn users when using a sort in a mode that doesn't accept sorts
#TODO: warn users when using search terms in a mode that doesn't accept search terms
#TODO: same with page parameter for top and top48h
#TODO: warn the user if trying to use a minor category with top48h
def remote(pages, category, sort, mode, terms, mirror):
    res_l = []
    pages = int(pages)
    if pages < 1:
        raise ValueError('Please provide an integer greater than 0 '
                         'for the number of pages to fetch.')

    # Catch the Ctrl-C exception and exit cleanly
    try:
        sizes = []
        uploaded = []
        identifiers = []
        for page in range(pages):
            if mode == 'browse':
                path = '/browse/'
                if(category == 0):
                    category = 100
                path = '/browse/{}/{}/{}'.format(category, page, sort)
            elif mode == 'recent':
                # This is not a typo. There is no / between 48h and the category.
                path = '/top/48h'
                # only major categories can be used with this mode
                if(category == 0):
                    path += 'all'
{'fraction_non_alphanumeric': 0.09299296691846835, 'fraction_numerical': 0.008335504037509768, 'mean_word_length': 2.050039714058777, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2530057', 'n_tokens_mistral': 1100, 'n_tokens_neox': 1026, 'n_words': 390}