<commit_before>const pathPosix = require("path-posix");
const joinURL = require("url-join");
const { merge } = require("../merge.js");
const { handleResponseCode, processGlobFilter, processResponsePayload } = require("../response.js");
const { normaliseHREF, normalisePath } = require("../url.js");
const { getSingleValue, getValueForKey, parseXML, propsToStat } = require("./dav.js");
const { encodePath, prepareRequestOptions, request } = require("../request.js");

function getDirectoryContents(remotePath, options) {
    // Join the URL and path for the request
    const requestOptions = {
        url: joinURL(options.remoteURL, encodePath(remotePath), "/"),
        method: "PROPFIND",
        headers: {
            Accept: "text/plain",
            Depth: options.deep ? "infinity" : 1
        },
        responseType: "text"
    };
    let response = null;
    prepareRequestOptions(requestOptions, options);
    return request(requestOptions)
        .then(handleResponseCode)
        .then(res => {
            response = res;
            return res.data;
        })
        .then(parseXML)
        .then(result => getDirectoryFiles(result, options.remotePath, remotePath, options.details))
        .then(files => processResponsePayload(response, files, options.details))
        .then(files => {
            if (options.glob && options.glob.pattern) {
                return processGlobFilter(files, options.glob);
            }
            return files;
        });
}

function getDirectoryFiles(result, serverBasePath, requestPath, isDetailed = false) {
    const remoteTargetPath = pathPosix.join(serverBasePath, requestPath, "/");
    const serverBase = pathPosix.join(serverBasePath, "/");
    // Extract the response items (directory contents)
    const multiStatus = getValueForKey("multistatus", result);
    const responseItems = getValueForKey("response", multiStatus);
    return (
        responseItems
            // Filter out the item pointing to the current directory (not needed)
            .filter(item => {
                let href = getSingleValue(getValueForKey("href", item));
                href = pathPosix.join(normalisePath(normaliseHREF(href)), "/");
                return href !== serverBase && href !== remoteTargetPath;
            })
            // Map all items to a consistent output structure (results)
            .map(item => {
                // HREF is the file path (in full)
                let href = getSingleValue(getValueForKey("href", item));
                href = normaliseHREF(href);<commit_msg>simplified based on Perry's recommendation<commit_after>const pathPosix = require("path-posix");
const joinURL = require("url-join");
const { merge } = require("../merge.js");
const { handleResponseCode, processGlobFilter, processResponsePayload } = require("../response.js");
const { normaliseHREF, normalisePath } = require("../url.js");
const { getSingleValue, getValueForKey, parseXML, propsToStat } = require("./dav.js");
const { encodePath, prepareRequestOptions, request } = require("../request.js");

function getDirectoryContents(remotePath, options) {
    // Join the URL and path for the request
    const requestOptions = {
        url: joinURL(options.remoteURL, encodePath(remotePath), "/"),
        method: "PROPFIND",
        headers: {
            Accept: "text/plain",
            Depth: options.deep ? "infinity" : 1
        },
        responseType: "text"
    };
    let response = null;
    prepareRequestOptions(requestOptions, options);
    return request(requestOptions)
        .then(handleResponseCode)
        .then(res => {
            response = res;
            return res.data;
        })
        .then(parseXML)
        .then(result => getDirectoryFiles(result, options.remotePath, remotePath, options.details))
        .then(files => processResponsePayload(response, files, options.details))
        .then(files => options.glob && options.glob.pattern ? processGlobFilter(files, options.glob) : files)
        ;
}

function getDirectoryFiles(result, serverBasePath, requestPath, isDetailed = false) {
    const remoteTargetPath = pathPosix.join(serverBasePath, requestPath, "/");
    const serverBase = pathPosix.join(serverBasePath, "/");
    // Extract the response items (directory contents)
    const multiStatus = getValueForKey("multistatus", result);
    const responseItems = getValueForKey("response", multiStatus);
    return (
        responseItems
            // Filter out the item pointing to the current directory (not needed)
            .filter(item => {
                let href = getSingleValue(getValueForKey("href", item));
                href = pathPosix.join(normalisePath(normaliseHREF(href)), "/");
                return href !== serverBase && href !== remoteTargetPath;
            })
            // Map all items to a consistent output structure (results)
            .map(item => {
                // HREF is the file path (in full)
                let href = getSingleValue(getValueForKey("href", item));
                href = normaliseHREF(href);
{'fraction_non_alphanumeric': 0.12206665352001578, 'fraction_numerical': 0.00039439952672056796, 'mean_word_length': 2.737656595431098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '806270', 'n_tokens_mistral': 1330, 'n_tokens_neox': 1302, 'n_words': 326}