<commit_before>import koaRouter from 'koa-router';
import fs from 'fs';
import path from 'path'
import readline from 'readline';
import markdownIt from 'markdown-it';

const router = koaRouter({prefix: '/api/posts'});

function readPostFile(postFilePath) {
    return function(done) {
        var postData = { content: "" };
        var headerCounter = 0;

        var lineReader = readline.createInterface({
            input: fs.createReadStream(postFilePath)
        });

        lineReader.on('line', function(line){
            if(line === "---") {
                headerCounter++;
            } else if(headerCounter == 1) {
                let headerComponents = line.split(':', 2);
                postData[headerComponents[0]] = headerComponents[1].trim();
            } else if(headerCounter == 2 && line.trim() !== "") {
                postData.content += (line + "<br/>");
            }
        });

        lineReader.on('close', function(){
            done(null, postData);
        });
    }
}

router.get('/', function*(next) {
    const postsBaseDir = path.resolve(__dirname, '../../../posts');
    const posts = fs.readdirSync(postsBaseDir);
    var postsData = [];
    let md = markdownIt({html: true, typographer: true});

    for (let i = 0, len = posts.length; i < len; i++) {
        let postFilename = posts[i];
        const postFilePath = path.resolve(postsBaseDir, postFilename);
        let postData = yield readPostFile(postFilePath);

        postData.renderedContent = md.render(postData.content);
        delete postData.content;<commit_msg>Update small issue typos.<commit_after>import koaRouter from 'koa-router';
import fs from 'fs';
import path from 'path'
import readline from 'readline';
import markdownIt from 'markdown-it';

const router = koaRouter({prefix: '/api/posts'});

function readPostFile(postFilePath) {
    return function(done) {
        var postData = { content: "" };
        var headerCounter = 0;

        var lineReader = readline.createInterface({
            input: fs.createReadStream(postFilePath)
        });

        lineReader.on('line', function(line){
            if(line === "---") {
                headerCounter++;
            } else if(headerCounter == 1) {
                let headerComponents = line.split(':', 2);
                postData[headerComponents[0]] = headerComponents[1].trim();
            } else if(headerCounter == 2) {
                postData.content += (line + "<br/>");
            }
        });

        lineReader.on('close', function(){
            done(null, postData);
        });
    }
}

router.get('/', function*(next) {
    const postsBaseDir = path.resolve(__dirname, '../../../posts');
    const posts = fs.readdirSync(postsBaseDir);
    const POSTS_PAGE_SIZE = parseInt(process.env.POSTS_PAGE_SIZE);

    let offset = this.request.query.offset ? this.request.query.offset : 1;
    if(offset > posts.length) {
        offset = posts.length - POSTS_PAGE_SIZE
    }

    console.log("limit1", offset + POSTS_PAGE_SIZE);

    let limit = (offset + POSTS_PAGE_SIZE - 1) > posts.length ? posts.length : offset + POSTS_PAGE_SIZE - 1;

    console.log("offset", offset);
    console.log("limit", limit);

    var postsData = [];
    let md = markdownIt({html: true});

    for (let i = offset - 1; i < limit; i++) {
        let postFilename = posts[i];
        const postFilePath = path.resolve(postsBaseDir, postFilename);

        let postData = yield readPostFile(postFilePath);

        postData.renderedContent = md.render(postData.content);
        delete postData.content;
{'fraction_non_alphanumeric': 0.1434965034965035, 'fraction_numerical': 0.005034965034965035, 'mean_word_length': 2.4819863680623175, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2191787', 'n_tokens_mistral': 1047, 'n_tokens_neox': 1002, 'n_words': 236}