<commit_before># vim: set ts=4 sw=4 et: -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import re
import sys
import json
import requests
import argparse
import time
import codecs
from bs4 import BeautifulSoup
from six import u

__version__ = '1.0'

# if python 2, disable verify flag in requests.get()
VERIFY = True
if sys.version_info[0] < 3:
    VERIFY = False
    requests.packages.urllib3.disable_warnings()

class PttWebCrawler(object):
    """docstring for PttWebCrawler"""
    def __init__(self, cmdline=None):
        parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description='''
            A crawler for the web version of PTT, the largest online community in Taiwan.
            Input: board name and page indices (or articla ID)
            Output: BOARD_NAME-START_INDEX-END_INDEX.json (or BOARD_NAME-ID.json)
        ''')
        parser.add_argument('-b', metavar='BOARD_NAME', help='Board name', required=True)
        group = parser.add_mutually_exclusive_group(required=True)
        group.add_argument('-i', metavar=('START_INDEX', 'END_INDEX'), type=int, nargs=2, help="Start and end index")
        group.add_argument('-a', metavar='ARTICLE_ID', help="Article ID")
        parser.add_argument('-p', metavar='PushNum',type=int , help="PushNum")
        parser.add_argument('-n',metavar='AuthorName',help = "AuthorName")
        parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)

        if cmdline:
            args = parser.parse_args(cmdline)
        else:
            args = parser.parse_args()
        board = args.b
        PTT_URL = 'https://www.ptt.cc'
        if args.i:
            start = args.i[0]
            if args.i[1] == -1:
                end = self.Stub_getLastPage(board)
                #end = self.getLastPage(board)
            else:
                end = args.i[1]
                #end = args.i[1]
            index = start
            filename = board + '-' + str(start) + '-' + str(end) + '.json'
            AuthorName = args.n
            PushNum = args.p
            #print('AuthorName=',AuthorName)
            #print('pushNum=',PushNum)
            self.store(filename, u'{"articles": [', 'w')
            if(self.InputIsValid(start,end,board,self.Stub_getLastPage(board))):
                for i in range(end-start+1):
                    index = start + i
                    print('Processing index:', str(index))
                    resp = requests.get(
                        url=PTT_URL + '/bbs/' + board + '/index' + str(index) + '.html',
                        cookies={'over18': '1'}, verify=VERIFY
                    )
                    if resp.status_code != 200:
                        print('invalid url:', resp.url)
                        continue
                    soup = BeautifulSoup(resp.text)
                    divs = soup.find_all("div", "r-ent")
                    for div in divs:
                        try:
                            # ex. link would be <a href="/bbs/PublicServan/M.1127742013.A.240.html">Re: [問題] 職等</a>
                            href = div.find('a')['href']
                            link = PTT_URL + href
                            article_id = re.sub('\.html', '', href.split('/')[-1])
                            if((args.p and (self.Stub_getPush(link, article_id, board) > PushNum)) or not args.p  ):
                                if div == divs[-1] and i == end-start:  # last div of last page
                                    self.store(filename, self.parse(link, article_id, board), 'a')
                                else:
                                    self.store(filename, self.parse(link, article_id, board) + ',', 'a')
                        except:
                            pass
                    time.sleep(0.1)
            self.store(filename, u']}', 'a')
        else:  # args.a<commit_msg>change Stub_getLastPage() into functional getLastPage()<commit_after># vim: set ts=4 sw=4 et: -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import print_function

import re
import sys
import json
import requests
import argparse
import time
import codecs
from bs4 import BeautifulSoup
from six import u

__version__ = '1.0'

# if python 2, disable verify flag in requests.get()
VERIFY = True
if sys.version_info[0] < 3:
    VERIFY = False
    requests.packages.urllib3.disable_warnings()

class PttWebCrawler(object):
    """docstring for PttWebCrawler"""
    def __init__(self, cmdline=None):
        parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description='''
            A crawler for the web version of PTT, the largest online community in Taiwan.
            Input: board name and page indices (or articla ID)
            Output: BOARD_NAME-START_INDEX-END_INDEX.json (or BOARD_NAME-ID.json)
        ''')
        parser.add_argument('-b', metavar='BOARD_NAME', help='Board name', required=True)
        group = parser.add_mutually_exclusive_group(required=True)
        group.add_argument('-i', metavar=('START_INDEX', 'END_INDEX'), type=int, nargs=2, help="Start and end index")
        group.add_argument('-a', metavar='ARTICLE_ID', help="Article ID")
        parser.add_argument('-p', metavar='PushNum',type=int , help="PushNum")
        parser.add_argument('-n',metavar='AuthorName',help = "AuthorName")
        parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)

        if cmdline:
            args = parser.parse_args(cmdline)
        else:
            args = parser.parse_args()
        board = args.b
        PTT_URL = 'https://www.ptt.cc'
        if args.i:
            start = args.i[0]
            if args.i[1] == -1:
                end = self.getLastPage(board)
                #end = self.getLastPage(board)
            else:
                end = args.i[1]
                #end = args.i[1]
            index = start
            filename = board + '-' + str(start) + '-' + str(end) + '.json'
            AuthorName = args.n
            PushNum = args.p
            #print('AuthorName=',AuthorName)
            #print('pushNum=',PushNum)
            self.store(filename, u'{"articles": [', 'w')
            if(self.InputIsValid(start,end,board,self.getLastPage(board))):
                for i in range(end-start+1):
                    index = start + i
                    print('Processing index:', str(index))
                    resp = requests.get(
                        url=PTT_URL + '/bbs/' + board + '/index' + str(index) + '.html',
                        cookies={'over18': '1'}, verify=VERIFY
                    )
                    if resp.status_code != 200:
                        print('invalid url:', resp.url)
                        continue
                    soup = BeautifulSoup(resp.text)
                    divs = soup.find_all("div", "r-ent")
                    for div in divs:
                        try:
                            # ex. link would be <a href="/bbs/PublicServan/M.1127742013.A.240.html">Re: [問題] 職等</a>
                            href = div.find('a')['href']
                            link = PTT_URL + href
                            article_id = re.sub('\.html', '', href.split('/')[-1])
                            if((args.p and (self.Stub_getPush(link, article_id, board) > PushNum)) or not args.p  ):
                                if((args.n and self.Stub_isEqualWithName(link, article_id, board,AuthorName)) or not args.n):
                                    if div == divs[-1] and i == end-start:  # last div of last page
                                        self.store(filename, self.parse(link, article_id, board), 'a')
                                    else:
                                        self.store(filename, self.parse(link, article_id, board) + ',', 'a')
                        except:
                            pass
                    time.sleep(0.1)
            self.store(filename, u']}', 'a')
        else:  # args.a
{'fraction_non_alphanumeric': 0.1267865943814687, 'fraction_numerical': 0.009857072449482503, 'mean_word_length': 1.6930988719309887, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3676578', 'n_tokens_mistral': 2408, 'n_tokens_neox': 2325, 'n_words': 533}