<commit_before>import _ from 'lodash';
import http from 'https';
import url from 'url';
// import sizeOf from 'image-size';
import ApiDataInstance from './ApiDataInstance';
import { ENTITY } from './CONSTANT';

/*
function getImageSize(imageUrl) {
    return new Promise((resolve, reject) => {
        imageUrl = url.parse(imageUrl);
        https.get(options, (response) => {
            let chuncks = [];
            response.on('data', (chunk) => {
                chunks.push(chunk);
            })
            .on('end', () => {
                let buffer = Buffer.concat(chunks);
                // resolve(sizeOf(buffer));
                resolve({});
            })
            .on('error', (err) => {
                reject(err);
            });
        });
    });
    }
    */

function getResizedImageUrls(imageUrl) {
    let desktop = '';
    let mobile = '';
    let tablet = '';
    // TODO check url pattern
    if (imageUrl && typeof imageUrl === 'string') {
        const matched = imageUrl.match(/(\.[^.]+)?$/);
        let ext = '';
        let url = '';
        if (matched !== null) {
            url = imageUrl.substr(0, matched.index);
            ext = matched[0];
        } else {
            url = imageUrl;
        }
        desktop = url + '-desktop' + ext;
        mobile = url + '-mobile' + ext;
        tablet = url + '-tablet' + ext;
    }
    return {
        desktop,
        mobile,
        tablet
    };
}

const processor = {
    convertBlock(entityMap, block) {
        let alignment = 'center';
        let content;
        let entityRange = block.entityRanges[0];
        const entity = entityMap[entityRange.key];
        const type = entity && entity.type;
        switch (type) {
            case ENTITY.embeddedCode.type:
                alignment = entity.data && entity.data.alignment || alignment;
                content = [ {
                    caption: entity.data.caption,
                    embeddedCode: entity.data.embeddedCode
                }];
                break;
            case ENTITY.slideshow.type:
            case ENTITY.imageDiff.type:
                alignment = entity.data && entity.data.alignment || alignment;
                content = this.convertImagesBlock(entity);
                break;
            case ENTITY.image.type:
                alignment = entity.data && entity.data.alignment || alignment;
                content = [this.convertImageBlock(entity)];
                break;
            default:
                return;
        }
        return new ApiDataInstance({alignment, type, content})
    },

    convertImageBlock(entity) {
        let image = entity.data || {};
        let rtn = {};
        let urls = getResizedImageUrls(image.url);<commit_msg>insert infobox atomic block<commit_after>import _ from 'lodash';
import http from 'https';
import url from 'url';
// import sizeOf from 'image-size';
import ApiDataInstance from './ApiDataInstance';
import { ENTITY } from './CONSTANT';

/*
function getImageSize(imageUrl) {
    return new Promise((resolve, reject) => {
        imageUrl = url.parse(imageUrl);
        https.get(options, (response) => {
            let chuncks = [];
            response.on('data', (chunk) => {
                chunks.push(chunk);
            })
            .on('end', () => {
                let buffer = Buffer.concat(chunks);
                // resolve(sizeOf(buffer));
                resolve({});
            })
            .on('error', (err) => {
                reject(err);
            });
        });
    });
    }
    */

function getResizedImageUrls(imageUrl) {
    let desktop = '';
    let mobile = '';
    let tablet = '';
    // TODO check url pattern
    if (imageUrl && typeof imageUrl === 'string') {
        const matched = imageUrl.match(/(\.[^.]+)?$/);
        let ext = '';
        let url = '';
        if (matched !== null) {
            url = imageUrl.substr(0, matched.index);
            ext = matched[0];
        } else {
            url = imageUrl;
        }
        desktop = url + '-desktop' + ext;
        mobile = url + '-mobile' + ext;
        tablet = url + '-tablet' + ext;
    }
    return {
        desktop,
        mobile,
        tablet
    };
}

const processor = {
    convertBlock(entityMap, block) {
        let alignment = 'center';
        let content;
        let entityRange = block.entityRanges[0];
        const entity = entityMap[entityRange.key];
        const type = entity && entity.type;
        switch (type) {
            case ENTITY.infobox.type:
                alignment = entity.data && entity.data.alignment || alignment;
                content = [ {
                    title: entity.data.title,
                    body: entity.data.body
                }];
                break;
            case ENTITY.embeddedCode.type:
                alignment = entity.data && entity.data.alignment || alignment;
                content = [ {
                    caption: entity.data.caption,
                    embeddedCode: entity.data.embeddedCode
                }];
                break;
            case ENTITY.slideshow.type:
            case ENTITY.imageDiff.type:
                alignment = entity.data && entity.data.alignment || alignment;
                content = this.convertImagesBlock(entity);
                break;
            case ENTITY.image.type:
                alignment = entity.data && entity.data.alignment || alignment;
                content = [this.convertImageBlock(entity)];
                break;
            default:
                return;
        }
        return new ApiDataInstance({alignment, type, content})
    },

    convertImageBlock(entity) {
        let image = entity.data || {};
        let rtn = {};
        let urls = getResizedImageUrls(image.url);
{'fraction_non_alphanumeric': 0.12817412333736397, 'fraction_numerical': 0.0010364484366902746, 'mean_word_length': 1.5261780104712042, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1446686', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1511, 'n_words': 348}