<commit_before>"use strict";

import { workspace } from 'vscode';
import { ArrayUtility } from './common/arrayUtility';
import { HttpRequest } from './models/httpRequest';
import { IRequestParser } from './models/IRequestParser';
import { RequestParserUtil } from './requestParserUtil';
import { HttpClient } from './httpClient';
import { MimeUtility } from './mimeUtility';
import { EOL } from 'os';
import * as fs from 'fs';
import * as path from 'path';
import { Stream } from 'stream';

const CombinedStream = require('combined-stream');
const encodeurl = require('encodeurl');

export class HttpRequestParser implements IRequestParser {
    private static readonly defaultMethod = 'GET';
    private static readonly uploadFromFileSyntax = /^<\s+([\S]*)\s*$/;

    public parseHttpRequest(requestRawText: string, requestAbsoluteFilePath: string, parseFileContentAsStream: boolean): HttpRequest {
        // parse follows http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html
        // split the request raw text into lines
        let lines: string[] = requestRawText.split(EOL);

        // skip leading empty lines
        lines = ArrayUtility.skipWhile(lines, value => value.trim() === '');

        // skip trailing empty lines
        lines = ArrayUtility.skipWhile(lines.reverse(), value => value.trim() === '').reverse();

        if (lines.length === 0) {
            return null;
        }

        // parse request line
        let requestLine = HttpRequestParser.parseRequestLine(lines[0]);

        // get headers range
        let headers: { [key: string]: string };
        let body: string | Stream;
        let bodyLines: string[] = [];
        let headerStartLine = ArrayUtility.firstIndexOf(lines, value => value.trim() !== '', 1);
        if (headerStartLine !== -1) {
            if (headerStartLine === 1) {
                // parse request headers
                let firstEmptyLine = ArrayUtility.firstIndexOf(lines, value => value.trim() === '', headerStartLine);
                let headerEndLine = firstEmptyLine === -1 ? lines.length : firstEmptyLine;
                let headerLines = lines.slice(headerStartLine, headerEndLine);
                let index = 0;
                let queryString = '';
                for (; index < headerLines.length; ) {
                    let headerLine = (headerLines[index]).trim();
                    if (headerLine[0] in { '?': '', '&': '' } && headerLine.split('=').length === 2) {
                        queryString += headerLine;
                        index++;
                        continue;
                    }
                    break;
                }

                if (queryString !== '') {
                    requestLine.url += queryString;
                }
                headers = RequestParserUtil.parseRequestHeaders(headerLines.slice(index));

                // get body range
                let bodyStartLine = ArrayUtility.firstIndexOf(lines, value => value.trim() !== '', headerEndLine);
                if (bodyStartLine !== -1) {
                    let contentTypeHeader = HttpRequestParser.getContentTypeHeader(headers);
                    firstEmptyLine = ArrayUtility.firstIndexOf(lines, value => value.trim() === '', bodyStartLine);
                    let bodyEndLine = MimeUtility.isMultiPartFormData(contentTypeHeader) || firstEmptyLine === -1 ? lines.length : firstEmptyLine;
                    bodyLines = lines.slice(bodyStartLine, bodyEndLine);
                }
            } else {
                // parse body, since no headers provided
                let firstEmptyLine = ArrayUtility.firstIndexOf(lines, value => value.trim() === '', headerStartLine);
                let bodyEndLine = firstEmptyLine === -1 ? lines.length : firstEmptyLine;
                bodyLines = lines.slice(headerStartLine, bodyEndLine);
            }
        }

        // if Host header provided and url is relative path, change to absolute url
        if (HttpClient.getHeaderValue(headers, 'Host') && requestLine.url[0] === '/') {
            requestLine.url = `http://${HttpClient.getHeaderValue(headers, 'Host')}${requestLine.url}`;
        }

        // parse body
        let contentTypeHeader = HttpRequestParser.getContentTypeHeader(headers);<commit_msg>Set scheme according to host header port number<commit_after>"use strict";

import { workspace } from 'vscode';
import { ArrayUtility } from './common/arrayUtility';
import { HttpRequest } from './models/httpRequest';
import { IRequestParser } from './models/IRequestParser';
import { RequestParserUtil } from './requestParserUtil';
import { HttpClient } from './httpClient';
import { MimeUtility } from './mimeUtility';
import { EOL } from 'os';
import * as fs from 'fs';
import * as path from 'path';
import { Stream } from 'stream';

const CombinedStream = require('combined-stream');
const encodeurl = require('encodeurl');

export class HttpRequestParser implements IRequestParser {
    private static readonly defaultMethod = 'GET';
    private static readonly uploadFromFileSyntax = /^<\s+([\S]*)\s*$/;

    public parseHttpRequest(requestRawText: string, requestAbsoluteFilePath: string, parseFileContentAsStream: boolean): HttpRequest {
        // parse follows http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html
        // split the request raw text into lines
        let lines: string[] = requestRawText.split(EOL);

        // skip leading empty lines
        lines = ArrayUtility.skipWhile(lines, value => value.trim() === '');

        // skip trailing empty lines
        lines = ArrayUtility.skipWhile(lines.reverse(), value => value.trim() === '').reverse();

        if (lines.length === 0) {
            return null;
        }

        // parse request line
        let requestLine = HttpRequestParser.parseRequestLine(lines[0]);

        // get headers range
        let headers: { [key: string]: string };
        let body: string | Stream;
        let bodyLines: string[] = [];
        let headerStartLine = ArrayUtility.firstIndexOf(lines, value => value.trim() !== '', 1);
        if (headerStartLine !== -1) {
            if (headerStartLine === 1) {
                // parse request headers
                let firstEmptyLine = ArrayUtility.firstIndexOf(lines, value => value.trim() === '', headerStartLine);
                let headerEndLine = firstEmptyLine === -1 ? lines.length : firstEmptyLine;
                let headerLines = lines.slice(headerStartLine, headerEndLine);
                let index = 0;
                let queryString = '';
                for (; index < headerLines.length; ) {
                    let headerLine = (headerLines[index]).trim();
                    if (headerLine[0] in { '?': '', '&': '' } && headerLine.split('=').length === 2) {
                        queryString += headerLine;
                        index++;
                        continue;
                    }
                    break;
                }

                if (queryString !== '') {
                    requestLine.url += queryString;
                }
                headers = RequestParserUtil.parseRequestHeaders(headerLines.slice(index));

                // get body range
                let bodyStartLine = ArrayUtility.firstIndexOf(lines, value => value.trim() !== '', headerEndLine);
                if (bodyStartLine !== -1) {
                    let contentTypeHeader = HttpRequestParser.getContentTypeHeader(headers);
                    firstEmptyLine = ArrayUtility.firstIndexOf(lines, value => value.trim() === '', bodyStartLine);
                    let bodyEndLine = MimeUtility.isMultiPartFormData(contentTypeHeader) || firstEmptyLine === -1 ? lines.length : firstEmptyLine;
                    bodyLines = lines.slice(bodyStartLine, bodyEndLine);
                }
            } else {
                // parse body, since no headers provided
                let firstEmptyLine = ArrayUtility.firstIndexOf(lines, value => value.trim() === '', headerStartLine);
                let bodyEndLine = firstEmptyLine === -1 ? lines.length : firstEmptyLine;
                bodyLines = lines.slice(headerStartLine, bodyEndLine);
            }
        }

        // if Host header provided and url is relative path, change to absolute url
        if (HttpClient.getHeaderValue(headers, 'Host') && requestLine.url[0] === '/') {
            let host = HttpClient.getHeaderValue(headers, 'Host');
            let [, port] = host.split(':');
            let scheme = port === '443' ? 'https' : 'http';
            requestLine.url = `${scheme}://${host}${requestLine.url}`;
        }

        // parse body
        let contentTypeHeader = HttpRequestParser.getContentTypeHeader(headers);
{'fraction_non_alphanumeric': 0.12482774460266421, 'fraction_numerical': 0.005627009646302251, 'mean_word_length': 2.2018382352941175, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '206095', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2147, 'n_words': 556}