<commit_before># -*- encoding: utf-8 -*-

import re
from itertools import zip_longest

import requests
import lxml.html

from utils.config import get_config_section

crawler_config = get_config_section('crawler')

encoding = crawler_config['encoding']  # big5 superset

form_url = crawler_config['form_url']
form_action_url = crawler_config['form_action_url']
syllabus_url = crawler_config['syllabus_url']
attachment_url = crawler_config['attachment_url']
dept_url = crawler_config['dept_url']


def xpath0(element, xpath):
    result = element.xpath(xpath)
    assert len(result) == 1, result
    return result[0]


def extract_text(element, joiner=''):
    return joiner.join(element.itertext()).strip()


def extract_int(element, allow_empty=False):
    text_list = element.xpath('.//text()')
    assert len(text_list) == 1, text_list
    if allow_empty and not text_list[0].strip():
        return None
    return int(text_list[0])


def extract_multirow_text(element):
    return re.sub(
        r'\n{4,}',
        '\n' * 3,
        '\n'.join(
            s.strip() for s in element.itertext()
        )
    )


class EmptyResponse(Exception):
    pass


def with_retry(request_function):
    def function(url, max_retries=32, **kwargs):
        '''
        get a valid response in <max_retries> retries
        change encoding before return
        raises EmptyResponse if not valid
        '''
        for r in range(max_retries):
            response = request_function(url, **kwargs)
            if response.content:
                response.encoding = encoding
                return response
        raise EmptyResponse(url)
    function.__name__ = request_function.__name__
    return function


def get_slfr(text):
    sl, s, fr = text.partition(u'大一新生保留')
    if not sl:
        return None, 0
    sl = int(sl)
    if fr:
        assert fr[-1] == u'人', fr
        fr = int(fr[:-1])
    else:
        fr = 0
    return sl, fr


get = with_retry(requests.get)
post = with_retry(requests.post)


def curriculum_to_trs(html):
    document = lxml.html.fromstring(html)
    course_trs = document.xpath("//tr[contains(@class, 'class3')]")
    assert len(course_trs) % 2 == 0, len(course_trs)
    return course_trs[::2]


def course_from_tr(main_tr):
    '''
    main_tr -> dict: course data

    data info: ([x] means implemented but disabled)
    no                  text            course number
    name_zh             text            Chinese course name<commit_msg>Fix issue #109<commit_after># -*- encoding: utf-8 -*-

import re
from itertools import zip_longest

import requests
import lxml.html

from utils.config import get_config_section

crawler_config = get_config_section('crawler')

encoding = crawler_config['encoding']  # big5 superset

form_url = crawler_config['form_url']
form_action_url = crawler_config['form_action_url']
syllabus_url = crawler_config['syllabus_url']
attachment_url = crawler_config['attachment_url']
dept_url = crawler_config['dept_url']


def xpath0(element, xpath):
    result = element.xpath(xpath)
    assert len(result) == 1, result
    return result[0]


def extract_text(element, joiner=''):
    return joiner.join(element.itertext()).strip()


def extract_int(element, allow_empty=False):
    text_list = element.xpath('.//text()')
    assert len(text_list) == 1, text_list
    if allow_empty and not text_list[0].strip():
        return None
    return int(text_list[0])


def extract_multirow_text(element):
    return re.sub(
        r'\n{4,}',
        '\n' * 3,
        '\n'.join(
            s.strip() for s in element.itertext()
        )
    )


class EmptyResponse(Exception):
    pass


def with_retry(request_function):
    def function(url, max_retries=32, **kwargs):
        '''
        get a valid response in <max_retries> retries
        change encoding before return
        raises EmptyResponse if not valid
        '''
        for r in range(max_retries):
            response = request_function(url, **kwargs)
            if response.content:
                response.encoding = encoding
                return response
        raise EmptyResponse(url)
    function.__name__ = request_function.__name__
    return function


def get_slfr(text):
    sl, s, fr = text.partition(u'新生保留')
    if not sl:
        return None, 0
    sl = int(sl)
    if fr:
        assert fr[-1] == u'人', fr
        fr = int(fr[:-1])
    else:
        fr = 0
    return sl, fr


get = with_retry(requests.get)
post = with_retry(requests.post)


def curriculum_to_trs(html):
    document = lxml.html.fromstring(html)
    course_trs = document.xpath("//tr[contains(@class, 'class3')]")
    assert len(course_trs) % 2 == 0, len(course_trs)
    return course_trs[::2]


def course_from_tr(main_tr):
    '''
    main_tr -> dict: course data

    data info: ([x] means implemented but disabled)
    no                  text            course number
    name_zh             text            Chinese course name
{'fraction_non_alphanumeric': 0.1112906483538679, 'fraction_numerical': 0.008685114118359927, 'mean_word_length': 2.7715156130997713, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3170575', 'n_tokens_mistral': 1616, 'n_tokens_neox': 1575, 'n_words': 385}