<commit_before>import { Router, Request, Response, NextFunction } from "express";
import { AbstractController } from "./AbstractController";
import { IRepositoryService } from "../app/services/RepositoryService";
import { RepositoryDocument } from "../app/entities/documents/RepositoryDocument";
import { IRepositoryEntity, RepositoryEntity } from "../app/entities/RepositoryEntity";
import { IReviewService } from "../app/services/ReviewService";
import { EntityUtil } from "../app/util/EntityUtil";
import * as json2csv from "json2csv";

/**
 * Repository controller interface.
 * @author <NAME> <<EMAIL>>
 */
export interface IRepositoryController {
    get(req: Request, res: Response): Promise<void>;
    getPage(req: Request, res: Response): Promise<void>;
    getByNamePage(req: Request, res: Response): Promise<void>;
    getByReviewsPage(req: Request, res: Response): Promise<void>;
    getByPullRequestsPage(req: Request, res: Response): Promise<void>;
    getList(req: Request, res: Response): Promise<void>;
    getStatsMeans(req: Request, res: Response): Promise<void>;
    getCSV(req: Request, res: Response): Promise<void>;
}

/**
 * Repository controller.
 * Defines Repository requests handling.
 * @implements IRepositoryController.
 */
export class RepositoryController extends AbstractController implements IRepositoryController {

    public async get(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        const owner: string = req.params.owner;
        const repository: string = req.params.repository;
        try {
            const entity: IRepositoryEntity = await service.getRepository(owner, repository);
            const json: Object = EntityUtil.toJSON(entity);
            res.json(json);
        } catch (error) {
            res.status(500).json({ message: "Oops, something went wrong." });
        }
    }

    public async getPage(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        await this.getOrderedPage(req, res, service, handler);

        async function handler(page: number, direction: number): Promise<IRepositoryEntity[]> {
            return service.getRepositoriesPage(page, direction);
        }
    }

    public async getByNamePage(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        await this.getOrderedPage(req, res, service, handler);

        async function handler(page: number, direction: number): Promise<IRepositoryEntity[]> {
            return service.getRepositoriesByNamePage(page, direction);
        }
    }

    public async getByReviewsPage(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        await this.getOrderedPage(req, res, service, handler);

        async function handler(page: number, direction: number): Promise<IRepositoryEntity[]> {
            return service.getRepositoriesByReviewsPage(page, direction);
        }
    }

    public async getByPullRequestsPage(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        await this.getOrderedPage(req, res, service, handler);

        async function handler(page: number, direction: number): Promise<IRepositoryEntity[]> {
            return service.getRepositoriesByPullRequestsPage(page, direction);
        }
    }

    public async getList(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        try {
            const entities: IRepositoryEntity[] = await service.getRepositoriesList();
            const json: any[] = EntityUtil.toJSONArray(entities);
            let list: string[] = json.map((jsonobj) => { return jsonobj.full_name });
            res.json(list);
        } catch (error) {
            res.status(500).json({ message: "Oops, something went wrong." });
        }
    }

    public async getStatsMeans(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        try {
            const means: any = await service.getRepositoriesStatsMeans();
            res.json(means);
        } catch (error) {
            console.log(error);
            res.status(500).json({ message: "Oops, something went wrong." });
        }
    }

    public async getCSV(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        const reviewService: IReviewService = this._services.review;
        const owner: string = req.params.owner;
        const repository: string = req.params.repository;
        try {
            const filter: any = { "repository.owner": owner, "repository.name": repository };
            const lastPage: number = await reviewService.numPages(filter);
            const fields: string[] = ["review_id", "repository_id", "repository_owner", "repository_name",
                "language", "repository_creation_date", "repository_update_date", "pull_request_id",
                "pull_request_title", "pull_request_body", "review_status", "review_body", "reviewer_login"];
            res.setHeader('Content-disposition', `attachment; filename=reviews_${owner}_${repository}.csv`);
            res.set('Content-Type', 'text/csv');
            for (let i = 1; i <= lastPage; i++) {
                const csvPage: any[] = await service.getRepositoryCSVPage(owner, repository, i);
                const csv = json2csv({ data: csvPage, fields });
                if (i < lastPage) res.write(csv);
                else res.end(csv);
            }
        } catch (error) {
            console.log(error);
            res.status(500).json({ message: "Oops, something went wrong." });
        }

    }

}<commit_msg>csv file streaming to client #69.<commit_after>import { Router, Request, Response, NextFunction } from "express";
import { AbstractController } from "./AbstractController";
import { IRepositoryService } from "../app/services/RepositoryService";
import { RepositoryDocument } from "../app/entities/documents/RepositoryDocument";
import { IRepositoryEntity, RepositoryEntity } from "../app/entities/RepositoryEntity";
import { IReviewService } from "../app/services/ReviewService";
import { EntityUtil } from "../app/util/EntityUtil";
import * as json2csv from "json2csv";

/**
 * Repository controller interface.
 * @author M<NAME> <<EMAIL>>
 */
export interface IRepositoryController {
    get(req: Request, res: Response): Promise<void>;
    getPage(req: Request, res: Response): Promise<void>;
    getByNamePage(req: Request, res: Response): Promise<void>;
    getByReviewsPage(req: Request, res: Response): Promise<void>;
    getByPullRequestsPage(req: Request, res: Response): Promise<void>;
    getList(req: Request, res: Response): Promise<void>;
    getStatsMeans(req: Request, res: Response): Promise<void>;
    getCSV(req: Request, res: Response): Promise<void>;
}

/**
 * Repository controller.
 * Defines Repository requests handling.
 * @implements IRepositoryController.
 */
export class RepositoryController extends AbstractController implements IRepositoryController {

    public async get(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        const owner: string = req.params.owner;
        const repository: string = req.params.repository;
        try {
            const entity: IRepositoryEntity = await service.getRepository(owner, repository);
            const json: Object = EntityUtil.toJSON(entity);
            res.json(json);
        } catch (error) {
            res.status(500).json({ message: "Oops, something went wrong." });
        }
    }

    public async getPage(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        await this.getOrderedPage(req, res, service, handler);

        async function handler(page: number, direction: number): Promise<IRepositoryEntity[]> {
            return service.getRepositoriesPage(page, direction);
        }
    }

    public async getByNamePage(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        await this.getOrderedPage(req, res, service, handler);

        async function handler(page: number, direction: number): Promise<IRepositoryEntity[]> {
            return service.getRepositoriesByNamePage(page, direction);
        }
    }

    public async getByReviewsPage(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        await this.getOrderedPage(req, res, service, handler);

        async function handler(page: number, direction: number): Promise<IRepositoryEntity[]> {
            return service.getRepositoriesByReviewsPage(page, direction);
        }
    }

    public async getByPullRequestsPage(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        await this.getOrderedPage(req, res, service, handler);

        async function handler(page: number, direction: number): Promise<IRepositoryEntity[]> {
            return service.getRepositoriesByPullRequestsPage(page, direction);
        }
    }

    public async getList(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        try {
            const entities: IRepositoryEntity[] = await service.getRepositoriesList();
            const json: any[] = EntityUtil.toJSONArray(entities);
            let list: string[] = json.map((jsonobj) => { return jsonobj.full_name });
            res.json(list);
        } catch (error) {
            res.status(500).json({ message: "Oops, something went wrong." });
        }
    }

    public async getStatsMeans(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        try {
            const means: any = await service.getRepositoriesStatsMeans();
            res.json(means);
        } catch (error) {
            console.log(error);
            res.status(500).json({ message: "Oops, something went wrong." });
        }
    }

    public async getCSV(req: Request, res: Response): Promise<void> {
        const service: IRepositoryService = this._services.repo;
        const reviewService: IReviewService = this._services.review;
        const owner: string = req.params.owner;
        const repository: string = req.params.repository;
        try {
            const filter: any = { "repository.owner": owner, "repository.name": repository };
            const lastPage: number = await reviewService.numPages(filter);
            const fields: string[][] = [["review_id", "repository_id", "repository_owner", "repository_name",
                "language", "repository_creation_date", "repository_update_date", "pull_request_id",
                "pull_request_title", "pull_request_body", "review_state", "review_body", "reviewer_login"]];
            res.setHeader('Content-disposition', `attachment; filename=reviews_${owner}_${repository}.csv`);
            res.set('Content-Type', 'text/csv');
            const title: string = json2csv({ data: fields, hasCSVColumnTitle: false });
            res.write(`${title}\n`);
            for (let i = 1; i <= lastPage; i++) {
                const csvPage: any[] = await service.getRepositoryCSVPage(owner, repository, i);
                const csv: string = json2csv({ data: csvPage, hasCSVColumnTitle: false });
                if (i < lastPage) res.write(`${csv}\n`);
                else res.end(csv);
            }
        } catch (error) {
            console.log(error);
            res.status(500).json({ message: "Oops, something went wrong." });
        }

    }

}
{'fraction_non_alphanumeric': 0.12070410729253982, 'fraction_numerical': 0.002933780385582565, 'mean_word_length': 3.092967409948542, 'pattern_counts': {'":': 4, '<': 53, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4997936', 'n_tokens_mistral': 3189, 'n_tokens_neox': 3142, 'n_words': 814}