<commit_before><link rel="import" href="../connector.html">

<script>

    /**
     * 
     */
    class ReadManga extends Connector {

        /**
         *
         */
        constructor() {
            super();
            // Public members for usage in UI (mandatory)
            super.id         = 'readmanga';
            super.label      = 'ReadManga';
            this.tags        = [ 'manga', 'russian' ];
            super.isLocked   = false;
            // Private members for internal usage only (convenience)
            this.url         = 'http://readmanga.me';
            // Private members for internal use that can be configured by the user through settings menu (set to undefined or false to hide from settings menu!)
            this.config = {
                throttle: {
                    label: 'Throttle Requests [ms]',
                    description: 'Enter the timespan in [ms] to delay consecuitive HTTP requests.\nThe website may reject to many consecuitive requests.\nSlightly increase the value when getting 429 errors during manga list update.',
                    input: Input.numeric,
                    min: 0,
                    max: 5000,
                    value: 1500
                }
            };

            this.preferSubtitleAsMangaTitle = true;
        }

        /**
         *
         */
        _getMangaListFromPages( mangaPageLinks, index ) {
            if( index === undefined ) {
                index = 0;
            }
            return this.wait( this.config.throttle.value )
            .then ( () => this.fetchDOM( mangaPageLinks[ index ], 'div.tile div.desc', 5 ) )
            .then( data => {
                // TODO: use english titles instead of russian titles?
                let mangaList = data.map( element => {
                    let a = element.querySelector( 'h3 a' );
                    let h4 = element.querySelector( 'h4' );
                    return {
                        id: this.getRelativeLink( a ),
                        title: ( this.preferSubtitleAsMangaTitle && h4 ? h4.title : a.title )
                    };
                } );
                if( index < mangaPageLinks.length - 1 ) {
                    return this._getMangaListFromPages( mangaPageLinks, index + 1 )
                    .then( mangas => mangas.concat( mangaList ) );
                } else {
                    return Promise.resolve( mangaList );
                }
            } );
        }

        /**
         *
         */
        _getMangaList( callback ) {
            this.fetchDOM( this.url + '/list', 'span.pagination a:nth-last-child(2)' )
            .then( data => {
                let pageCount = parseInt( data[0].text.trim() );
                let pageLinks = [...( new Array( pageCount ) ).keys()].map( page => this.url + '/list?offset=' + ( page * 70 ) );
                return this._getMangaListFromPages( pageLinks );
            } )
            .then( data => {
                callback( null, data );
            } )
            .catch( error => {
                console.error( error, this );
                callback( error, undefined );
            } );
        }

        /**
         *
         */
        _getChapterList( manga, callback ) {
            this.fetchDOM( this.url + manga.id, 'div.chapters-link table tr td a' )
            .then( data => {
                let chapterList = data.map( element => {
                    return {
                        id: this.getRelativeLink( element ),
                        title: element.text.replace( manga.title, '' ).trim(),
                        language: 'ru'
                    };
                } );
                callback( null, chapterList );
            } )
            .catch( error => {
                console.error( error, manga );
                callback( error, undefined );
            } );
        }

        /**
         *
         */
        _getPageList( manga, chapter, callback ) {
            fetch( this.url + chapter.id + '?mtr=1', this.requestOptions )
            .then( response => {
                if( response.status !== 200 ) {
                    throw new Error( `Failed to receive page list (status: ${response.status}) - ${response.statusText}` );
                }
                return response.text();
            } )
            .then( data => {
                let pages = data.match( /init\s*\(\s*(\[\[.*?\]\])/ )[1];
                let pageList = JSON.parse( pages.replace( /'/g, '"' ) ).map( p => p[1] + p[2] );
                callback( null, pageList );<commit_msg>[readmanga] improved chapter name extraction<commit_after><link rel="import" href="../connector.html">

<script>

    /**
     * 
     */
    class ReadManga extends Connector {

        /**
         *
         */
        constructor() {
            super();
            super.id         = 'readmanga';
            super.label      = 'ReadManga';
            this.tags        = [ 'manga', 'russian' ];
            this.url         = 'http://readmanga.me';
            // Private members for internal use that can be configured by the user through settings menu (set to undefined or false to hide from settings menu!)
            this.config = {
                throttle: {
                    label: 'Throttle Requests [ms]',
                    description: 'Enter the timespan in [ms] to delay consecuitive HTTP requests.\nThe website may reject to many consecuitive requests.\nSlightly increase the value when getting 429 errors during manga list update.',
                    input: Input.numeric,
                    min: 0,
                    max: 5000,
                    value: 1500
                }
            };

            this.preferSubtitleAsMangaTitle = true;
        }

        /**
         *
         */
        _getMangaListFromPages( mangaPageLinks, index ) {
            if( index === undefined ) {
                index = 0;
            }
            return this.wait( this.config.throttle.value )
            .then ( () => this.fetchDOM( mangaPageLinks[ index ], 'div.tile div.desc', 5 ) )
            .then( data => {
                // TODO: use english titles instead of russian titles?
                let mangaList = data.map( element => {
                    let a = element.querySelector( 'h3 a' );
                    let h4 = element.querySelector( 'h4' );
                    return {
                        id: this.getRelativeLink( a ),
                        title: ( this.preferSubtitleAsMangaTitle && h4 ? h4.title : a.title )
                    };
                } );
                if( index < mangaPageLinks.length - 1 ) {
                    return this._getMangaListFromPages( mangaPageLinks, index + 1 )
                    .then( mangas => mangas.concat( mangaList ) );
                } else {
                    return Promise.resolve( mangaList );
                }
            } );
        }

        /**
         *
         */
        _getMangaList( callback ) {
            this.fetchDOM( this.url + '/list', 'span.pagination a:nth-last-child(2)' )
            .then( data => {
                let pageCount = parseInt( data[0].text.trim() );
                let pageLinks = [...( new Array( pageCount ) ).keys()].map( page => this.url + '/list?offset=' + ( page * 70 ) );
                return this._getMangaListFromPages( pageLinks );
            } )
            .then( data => {
                callback( null, data );
            } )
            .catch( error => {
                console.error( error, this );
                callback( error, undefined );
            } );
        }

        /**
         *
         */
        _getChapterList( manga, callback ) {
            this.fetchDOM( this.url + manga.id, 'div#mangaBox' )
            .then( data => {
                let content = data[0];
                let mangaTitle = content.querySelector( 'h1.names span.name' ).innerText.trim();
                let chapterList = [...content.querySelectorAll( 'div.chapters-link table tr td a' ) ];
                chapterList = chapterList.map( element => {
                    return {
                        // TODO: fix chapter title (e.g. battle through heaven)
                        id: this.getRelativeLink( element ),
                        title: element.text.replace( /\s{1,}/g, ' ' ).replace( mangaTitle, '' ).trim(),
                        language: 'ru'
                    };
                } );
                callback( null, chapterList );
            } )
            .catch( error => {
                console.error( error, manga );
                callback( error, undefined );
            } );
        }

        /**
         *
         */
        _getPageList( manga, chapter, callback ) {
            fetch( this.url + chapter.id + '?mtr=1', this.requestOptions )
            .then( response => {
                if( response.status !== 200 ) {
                    throw new Error( `Failed to receive page list (status: ${response.status}) - ${response.statusText}` );
                }
                return response.text();
            } )
            .then( data => {
                let pages = data.match( /init\s*\(\s*(\[\[.*?\]\])/ )[1];
                let pageList = JSON.parse( pages.replace( /'/g, '"' ) ).map( p => p[1] + p[2] );
                callback( null, pageList );
{'fraction_non_alphanumeric': 0.12208929715872677, 'fraction_numerical': 0.007156590472121342, 'mean_word_length': 1.187105816398038, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6983360', 'n_tokens_mistral': 2525, 'n_tokens_neox': 2334, 'n_words': 602}