<commit_before>/**
 * This example script expects a JSON blob generated by react-docgen as input,
 * e.g. react-docgen components/* | buildDocs.sh
 */
const through = require('through2');
const File = require('vinyl');
const docgen = require('react-docgen');

const paths = require('../paths.json');

/**
 * The quick-and-dirty react-docgen plugin
 * @return {Stream} The stream
 */
const docGen = function docGen(options = {}) {
    // implement the default options
    //
    const opts = Object.assign({
        name: 'react-docgen.json',
    }, options);

    // declare a variable to hold the list of files
    //
    const fileList = [];

    return through.obj(

        // eslint-disable-next-line prefer-arrow-callback
        function write(file, encoding, callback) {
            // is the file null? if so, just carry on
            //
            if (file.isNull()) {
                return callback(null, file);
            }

            // add the file to the list of files
            //
            fileList.push(file);

            // carry on...
            //
            return callback();
        },

        function end(callback) {
            // declare a variable to hold the results
            //
            const result = {};

            // iterate over the list of files
            //
            fileList.forEach((file) => {
                // get the file contents
                //
                const content = file.contents.toString();

                // declare a variable to hold the results
                //
                let doc = null;

                // attempt to parse the content
                //
                try {
                    doc = docgen.parse(content);
                } catch (err) {
                    // log the error?
                }

                // did we get some parsed content? if so, add it to the results
                if (doc) {
                    result[file.relative] = doc;
                }
            });

            // push the new file containing the JSON-encoded results
            //
            this.push(new File({
                contents: new Buffer(JSON.stringify(result, null, '  ')),
                base:     '/tmp',
                path:     `/tmp/${opts.name}`,
            }));

            // carry on...
            //
            callback();
        }

    );
};

/**
 * Copyright (c) 2015, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

const stringOfLength = (string, length) => {
    let newString = '';
    for (let i = 0; i < length; i++) { // eslint-disable-line id-length
        newString += string;
    }
    return newString;
};

const generateTitle = (name) => {
    const title = `\`${name}\` (component)`;
    return `${title}\n${stringOfLength('=', title.length)}\n`;
};

const generateDescription = (description) => `${description}\n`;

const generatePropType = (type) => {
    let values;
    if (Array.isArray(type.value)) {
        values = `(${type.value.map((typeValue) => typeValue.name || typeValue.value).join('|')})`;
    } else {
        values = type.value;
    }

    return `type: \`${type.name}${values || ''}\`\n`;
};

const generatePropDefaultValue = (value) => `defaultValue: \`${value.value}\`\n`;

const generateProp = (propName, prop) => {
    const descriptor = `${propName}${prop.required ? ' (required)' : ''}`;

    const description = prop.description ? `${prop.description}\n\n` : '';

    const type = prop.type ? generatePropType(prop.type) : '';

    const def = prop.defaultValue ? generatePropDefaultValue(prop.defaultValue) : '';

    return `### ${descriptor}\n\n${description}${type}${def}\n`;
};

const generateProps = (props) => {
    const description = Object.keys(props)
        .sort()
        .map((propName) => generateProp(propName, props[propName]))
        .join('\n');
    return `Props\n-----\n\n${description}`;
};

const generateMarkdown = function generateMarkdown(name, reactAPI) {

    const markdownString =
        `${generateTitle(name)}\n`
        + `${generateDescription(reactAPI.description)}\n`
        + `${generateProps(reactAPI.props)}`;

    return markdownString;
};


const toMarkdown = function toMarkdown(options = {}) {
    // implement the default options
    //
    const opts = Object.assign({
        name: 'react-docgen.md',
    }, options);

    return through.obj(function write(file, encoding, callback) {
        if (file.isNull()) {
            return callback(null, file);
        }

        const reactAPI = JSON.parse(file.contents.toString());

        let markdown = '';

        Object.keys(reactAPI).forEach((filename) => {
            markdown += generateMarkdown(filename, reactAPI[filename]);
        });

        // push the new file containing the JSON-encoded results
        //
        this.push(new File({
            contents: new Buffer(markdown),
            base:     '/tmp',
            path:     `/tmp/${opts.name}`,
        }));

        // carry on...
        //
        return callback();

    });
};


module.exports = (gulp) =>
    gulp.src(paths.source)
        .pipe(docGen())
        .pipe(toMarkdown({ name: 'REACT.md' }))
        .pipe(gulp.dest('./'));
<commit_msg>moved docgen plugin file to separate file<commit_after>/**
 * This example script expects a JSON blob generated by react-docgen as input,
 * e.g. react-docgen components/* | buildDocs.sh
 */
const File = require('vinyl');
const through = require('through2');

const docgen = require('../plugins/docgen');
const paths = require('../paths.json');

/**
 * Copyright (c) 2015, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

const stringOfLength = (string, length) => {
    let newString = '';
    for (let i = 0; i < length; i++) { // eslint-disable-line id-length
        newString += string;
    }
    return newString;
};

const generateTitle = (name) => {
    const title = `\`${name}\` (component)`;
    return `${title}\n${stringOfLength('=', title.length)}\n`;
};

const generateDescription = (description) => `${description}\n`;

const generatePropType = (type) => {
    let values;
    if (Array.isArray(type.value)) {
        values = `(${type.value.map((typeValue) => typeValue.name || typeValue.value).join('|')})`;
    } else {
        values = type.value;
    }

    return `type: \`${type.name}${values || ''}\`\n`;
};

const generatePropDefaultValue = (value) => `defaultValue: \`${value.value}\`\n`;

const generateProp = (propName, prop) => {
    const descriptor = `${propName}${prop.required ? ' (required)' : ''}`;

    const description = prop.description ? `${prop.description}\n\n` : '';

    const type = prop.type ? generatePropType(prop.type) : '';

    const def = prop.defaultValue ? generatePropDefaultValue(prop.defaultValue) : '';

    return `### ${descriptor}\n\n${description}${type}${def}\n`;
};

const generateProps = (props) => {
    const description = Object.keys(props)
        .sort()
        .map((propName) => generateProp(propName, props[propName]))
        .join('\n');
    return `Props\n-----\n\n${description}`;
};

const generateMarkdown = function generateMarkdown(name, reactAPI) {

    const markdownString =
        `${generateTitle(name)}\n`
        + `${generateDescription(reactAPI.description)}\n`
        + `${generateProps(reactAPI.props)}`;

    return markdownString;
};


const toMarkdown = function toMarkdown(options = {}) {
    // implement the default options
    //
    const opts = Object.assign({
        name: 'react-docgen.md',
    }, options);

    return through.obj(function write(file, encoding, callback) {
        if (file.isNull()) {
            return callback(null, file);
        }

        const reactAPI = JSON.parse(file.contents.toString());

        let markdown = '';

        Object.keys(reactAPI).forEach((filename) => {
            markdown += generateMarkdown(filename, reactAPI[filename]);
        });

        // push the new file containing the JSON-encoded results
        //
        this.push(new File({
            contents: new Buffer(markdown),
            base:     '/tmp',
            path:     `/tmp/${opts.name}`,
        }));

        // carry on...
        //
        return callback();

    });
};


module.exports = (gulp) =>
    gulp.src(paths.source)
        .pipe(docgen())
        .pipe(toMarkdown({ name: 'REACT.md' }))
        .pipe(gulp.dest('./'));

{'fraction_non_alphanumeric': 0.15426435609921849, 'fraction_numerical': 0.0013591573224600749, 'mean_word_length': 2.1015103617843343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6770083', 'n_tokens_mistral': 2536, 'n_tokens_neox': 2479, 'n_words': 642}