<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import os

import grequests
from django.conf import settings
from django.core.management.base import BaseCommand

from digest.models import Item


def check_exist_link(data, item):
    for info in data.get('links'):
        if info['link'] == item.link:
            return True
    else:
        return False


def create_dataset(queryset_items, name):
    out_filepath = os.path.join(settings.DATASET_FOLDER, name)

    if os.path.exists(out_filepath):
        with open(out_filepath, 'r') as fio:
            try:
                data = json.load(fio)
            except ValueError as e:
                print(e)
                data = {'links': []}
    else:
        data = {'links': []}

    items = {x.link: x for x in queryset_items if not check_exist_link(data, x)}

    rs = (grequests.get(u) for u in items.keys())
    resps = grequests.map(rs)

    for res in resps:
        data['links'].append(items[res.url].get_data4cls(status=True, text=items[res.url].get_text(res.text)))

    if not os.path.exists(os.path.dirname(out_filepath)):
        os.makedirs(os.path.dirname(out_filepath))

    with open(out_filepath, 'w') as fio:
        json.dump(data, fio)


class Command(BaseCommand):
    help = u'Create dataset'
<commit_msg>ADD: добавил комманду create_dataset_ids для создания датасета на основе id'шников<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import os

import grequests
from django.conf import settings
from django.core.management.base import BaseCommand

from digest.models import Item


def check_exist_link(data, item):
    for info in data.get('links'):
        if info['link'] == item.link:
            return True
    else:
        return False


def create_dataset(queryset_items, name):
    if not queryset_items:
        return
    out_filepath = os.path.join(settings.DATASET_FOLDER, name)

    if os.path.exists(out_filepath):
        with open(out_filepath, 'r') as fio:
            try:
                data = json.load(fio)
            except ValueError as e:
                print(e)
                data = {'links': []}
    else:
        data = {'links': []}

    items = {x.link: x for x in queryset_items if not check_exist_link(data, x)}

    rs = (grequests.get(u) for u in items.keys())
    resps = grequests.map(rs)

    for res in resps:
        data['links'].append(items[res.url].get_data4cls(status=True, text=items[res.url].get_text(res.text)))

    if not os.path.exists(os.path.dirname(out_filepath)):
        os.makedirs(os.path.dirname(out_filepath))

    with open(out_filepath, 'w') as fio:
        json.dump(data, fio)


class Command(BaseCommand):
    help = u'Create dataset'

{'fraction_non_alphanumeric': 0.1163963963963964, 'fraction_numerical': 0.0014414414414414415, 'mean_word_length': 2.7311827956989245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4222436', 'n_tokens_mistral': 918, 'n_tokens_neox': 902, 'n_words': 227}