<commit_before>require 'net/http'
require 'nokogiri'

class PeoplesoftParser

    REDIRECT_LIMIT = 10

    NOT_FOUND_TEXT = 'No student record found for this Campus ID'
    FAIL_TEXT = "It's now at <a href=\"https://srvslspsw001.uct.ac.za/psc/public/EMPLOYEE/HRMS/c/?cmd=logout\""

    def initialize
        @cookies = ''
    end

    def retrieve(student_number)
        Net::HTTP.start('srvslspsw001.uct.ac.za', use_ssl: true) do |http|

            #response = follow_get(http, URI('https://srvslspsw001.uct.ac.za/psp/public/EMPLOYEE/HRMS/c/UCT_PUBLIC_MENU.UCT_SS_ADV_PUBLIC.GBL?FolderPath=PORTAL_ROOT_OBJECT.UCT_SS_ADV_PUBLIC_GBL_1&IsFolder=false&IgnoreParamTempl=FolderPath%252cIsFolder'))

            response = follow_get(http, URI('https://srvslspsw001.uct.ac.za/psc/public/EMPLOYEE/HRMS/c/UCT_PUBLIC_MENU.UCT_SS_ADV_PUBLIC.GBL?FolderPath=PORTAL_ROOT_OBJECT.UCT_SS_ADV_PUBLIC_GBL_1&IsFolder=false&IgnoreParamTempl=FolderPath%252cIsFolder&PortalActualURL=https%3a%2f%2fsrvslspsw001.uct.ac.za%2fpsc%2fpublic%2fEMPLOYEE%2fHRMS%2fc%2fUCT_PUBLIC_MENU.UCT_SS_ADV_PUBLIC.GBL&PortalContentURL=https%3a%2f%2fsrvslspsw001.uct.ac.za%2fpsc%2fpublic%2fEMPLOYEE%2fHRMS%2fc%2fUCT_PUBLIC_MENU.UCT_SS_ADV_PUBLIC.GBL&PortalContentProvider=HRMS&PortalCRefLabel=Public%20Access&PortalRegistryName=EMPLOYEE&PortalServletURI=https%3a%2f%2fsrvslspsw001.uct.ac.za%2fpsp%2fpublic%2f&PortalURI=https%3a%2f%2fsrvslspsw001.uct.ac.za%2fpsc%2fpublic%2f&PortalHostNode=HRMS&NoCrumbs=yes&PortalKeyStruct=yes'))

            doc = Nokogiri::HTML(response.body)

            icsid = doc.css('#ICSID').attr('value')

            response = http.request_post(URI('https://srvslspsw001.uct.ac.za/psc/public/EMPLOYEE/HRMS/c/UCT_PUBLIC_MENU.UCT_SS_ADV_PUBLIC.GBL'), construct_post_data(student_number, icsid), {'Cookie' => @cookies})

            if response.body.include? FAIL_TEXT
                raise 'Failed. Please retry.'
            elsif response.body.include? NOT_FOUND_TEXT
                return nil
            else
                open('dump3.txt', "w") { |io| io.write(response.body) }

                doc = Nokogiri::HTML(response.body)
                rows = doc.xpath("//table[@id='TERM_CLASSES$scroll$0']/tr/td/table/tr[position()>1]")

                return rows.map { |r| construct_course_model(r.css('span'))  }
            end
        end
    end

    private

        def follow_get(http, uri, limit = REDIRECT_LIMIT)
            raise ArgumentError('too many HTTP redirects') if limit == 0

            response = http.request_get(uri, {'Cookie' => @cookies})

            if response.to_hash.include? 'set-cookie'
                @cookies = response.get_fields('set-cookie').map { |c| c.split('; ')[0] }.join('; ')
            end

            case response
            when Net::HTTPSuccess then
                response
            when Net::HTTPRedirection then
                follow_get(http, URI(response['location']), limit - 1)
            else<commit_msg>don't have to use uri when you can use absolute paths<commit_after>require 'net/http'
require 'nokogiri'

class PeoplesoftParser

    REDIRECT_LIMIT = 10

    NOT_FOUND_TEXT = 'No student record found for this Campus ID'
    FAIL_TEXT = "It's now at <a href=\"https://srvslspsw001.uct.ac.za/psc/public/EMPLOYEE/HRMS/c/?cmd=logout\""

    PAGE_PATH = '/psc/public/EMPLOYEE/HRMS/c/UCT_PUBLIC_MENU.UCT_SS_ADV_PUBLIC.GBL'

    FORM_PATH_ARGS = '?FolderPath=PORTAL_ROOT_OBJECT.UCT_SS_ADV_PUBLIC_GBL_1&IsFolder=false&IgnoreParamTempl=FolderPath%252cIsFolder&PortalActualURL=https%3a%2f%2fsrvslspsw001.uct.ac.za%2fpsc%2fpublic%2fEMPLOYEE%2fHRMS%2fc%2fUCT_PUBLIC_MENU.UCT_SS_ADV_PUBLIC.GBL&PortalContentURL=https%3a%2f%2fsrvslspsw001.uct.ac.za%2fpsc%2fpublic%2fEMPLOYEE%2fHRMS%2fc%2fUCT_PUBLIC_MENU.UCT_SS_ADV_PUBLIC.GBL&PortalContentProvider=HRMS&PortalCRefLabel=Public%20Access&PortalRegistryName=EMPLOYEE&PortalServletURI=https%3a%2f%2fsrvslspsw001.uct.ac.za%2fpsp%2fpublic%2f&PortalURI=https%3a%2f%2fsrvslspsw001.uct.ac.za%2fpsc%2fpublic%2f&PortalHostNode=HRMS&NoCrumbs=yes&PortalKeyStruct=yes'

    def initialize
        @cookies = ''
    end

    def retrieve(student_number)
        Net::HTTP.start('srvslspsw001.uct.ac.za', use_ssl: true) do |http|

            response = follow_get(http, PAGE_PATH + FORM_PATH_ARGS)

            doc = Nokogiri::HTML(response.body)

            icsid = doc.css('#ICSID').attr('value')

            response = http.request_post(PAGE_PATH, construct_post_data(student_number, icsid), {'Cookie' => @cookies})

            if response.body.include? FAIL_TEXT
                raise 'Failed. Please retry.'
            elsif response.body.include? NOT_FOUND_TEXT
                return nil
            else
                doc = Nokogiri::HTML(response.body)
                rows = doc.xpath("//table[@id='TERM_CLASSES$scroll$0']/tr/td/table/tr[position()>1]")

                return rows.map { |r| construct_course_model(r.css('span'))  }
            end
        end
    end

    private

        def follow_get(http, uri, limit = REDIRECT_LIMIT)
            raise ArgumentError('too many HTTP redirects') if limit == 0

            response = http.request_get(uri, {'Cookie' => @cookies})

            if response.to_hash.include? 'set-cookie'
                @cookies = response.get_fields('set-cookie').map { |c| c.split('; ')[0] }.join('; ')
            end

            case response
            when Net::HTTPSuccess then
                response
            when Net::HTTPRedirection then
                follow_get(http, URI(response['location']), limit - 1)
            else
{'fraction_non_alphanumeric': 0.12280387235568305, 'fraction_numerical': 0.024381498745069917, 'mean_word_length': 3.2265151515151516, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1325642', 'n_tokens_mistral': 2190, 'n_tokens_neox': 2041, 'n_words': 247}