<commit_before>import {LoaderStrategy} from '../load/loader-strategy.interface';
import {FilterStrategy} from '../filter/filter-strategy.interface';
import {AllEntriesStrategy} from '../filter/all-entries.strategy';
import {Observable} from 'rxjs/Observable';
import {SitemapEntry} from '../model/sitemap-entry.model';
import {UrlReplacer} from './url-replacer';
import {RegressionViolation} from './regression-violation';
import * as request from 'request';
import {Request, RequestResponse} from 'request';
import winston = require('winston');

export class SitemapRegressionTest {

    private loader: LoaderStrategy;
    private filter: FilterStrategy = new AllEntriesStrategy();
    private urlReplacer: UrlReplacer = new UrlReplacer();

    constructor(loader: LoaderStrategy) {
        this.loader = loader;
    }

    public withFilter(filter: FilterStrategy): this {
        this.filter = filter;
        return this;
    }

    public withReplacement(replace: string, by: string): this {
        this.urlReplacer.withReplacement(replace, by);
        return this;
    }

    public regressionTest(): Observable<RegressionViolation> {
        let entries: Observable<SitemapEntry[]> = this.loader.load();
        entries = this.filter.filter(entries)
            .do((all: SitemapEntry[]) => winston.info(`About to check ${all.length} URLs`));

        return entries.flatMap(entries => entries)
            .map(entry => new SitemapEntry(this.urlReplacer.replace(entry.url)))
            .flatMap((entry: SitemapEntry): any => {
                return new Observable(observer => {
                    winston.debug(`About to check ${entry.url}`);
                    const req: Request = request(entry.url, {
                        timeout: 1500
                    }, (error: any, response: RequestResponse, body: any) => {<commit_msg>Feature: limit concurrent http requests to 5 with customized http agent<commit_after>import {LoaderStrategy} from '../load/loader-strategy.interface';
import {FilterStrategy} from '../filter/filter-strategy.interface';
import {AllEntriesStrategy} from '../filter/all-entries.strategy';
import {Observable} from 'rxjs/Observable';
import {SitemapEntry} from '../model/sitemap-entry.model';
import {UrlReplacer} from './url-replacer';
import {RegressionViolation} from './regression-violation';
import * as request from 'request';
import * as http from 'http';
import {Request, RequestResponse} from 'request';
import winston = require('winston');

export class SitemapRegressionTest {

    private loader: LoaderStrategy;
    private filter: FilterStrategy = new AllEntriesStrategy();
    private urlReplacer: UrlReplacer = new UrlReplacer();

    constructor(loader: LoaderStrategy) {
        this.loader = loader;
    }

    public withFilter(filter: FilterStrategy): this {
        this.filter = filter;
        return this;
    }

    public withReplacement(replace: string, by: string): this {
        this.urlReplacer.withReplacement(replace, by);
        return this;
    }

    public regressionTest(): Observable<RegressionViolation> {
        let entries: Observable<SitemapEntry[]> = this.loader.load();
        entries = this.filter.filter(entries)
            .do((all: SitemapEntry[]) => winston.info(`About to check ${all.length} URLs`));

        var agent = new http.Agent({maxSockets: 5}); // 5 concurrent connections per origin

        return entries.flatMap(entries => entries)
            .map(entry => new SitemapEntry(this.urlReplacer.replace(entry.url)))
            .flatMap((entry: SitemapEntry): any => {
                return new Observable(observer => {
                    winston.debug(`About to check ${entry.url}`);
                    const req: Request = request(entry.url, {
                        timeout: 1500,
                        agent: agent
                    }, (error: any, response: RequestResponse, body: any) => {
{'fraction_non_alphanumeric': 0.12292358803986711, 'fraction_numerical': 0.002811142346026067, 'mean_word_length': 3.064382139148494, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2222579', 'n_tokens_mistral': 1085, 'n_tokens_neox': 1044, 'n_words': 273}