<commit_before>import os
import requests
import zipfile
import json

from collections import defaultdict
from cStringIO import StringIO
from datetime import datetime
from slyd.gitstorage.repoman import Repoman, retry_operation

from slybot.validation.schema import get_schema_validator


DASH_API_URL = None
DEFAULT_DASH_ITEM = '''{
  "default": {
    "fields": {
      "images": {"type": "image", "required": true, "vary": false},
      "price": {"type": "price", "required": true, "vary": false},
      "name": {"type": "text", "required": true, "vary": false},
      "description": {"type": "safe html", "required": false, "vary": false}
    }
  }
}'''
REQUIRED_FILES = {'extractors.json', 'items.json', 'project.json'}


def set_dash_url(dash_url):
    global DASH_API_URL
    DASH_API_URL = dash_url


class DeployError(Exception):

    def __init__(self, message='', data={}):
        self.message = message
        self.data = data


def import_project(name, apikey, repo):
    """Download a project from Dash and create a GIT repo for it."""

    def validump_resource(jsonres, restype):
        get_schema_validator(restype).validate(jsonres)
        return json.dumps(jsonres)

    def split_templates(spider, spider_filename, files):
        templates = spider['templates']
        spider['templates'] = []
        spider['template_names'] = []
        for template in templates:
            template['name'] = template['page_id']
            spider['template_names'].append(template['name'])
            template_fname = os.path.join(
                spider_filename.rpartition('.')[0],
                str(template['name']) + '.json')
            files[template_fname] = validump_resource(template, 'template')

    archive = zipfile.ZipFile(StringIO(_download_project(name, apikey)))
    files = {}
    for filename in archive.namelist():
        contents = archive.read(filename)
        if filename == 'items.json':
            resource = 'items'
        elif filename == 'extractors.json':
            resource = 'extractors'
        elif filename.startswith('spiders'):
            resource = 'spider'
        else:
            resource = None
        if resource in ['items', 'spider', 'extractors']:
            as_json = json.loads(contents)
            if resource == 'items':
                as_json = _fix_items(as_json)
            elif resource == 'spider':
                split_templates(as_json, filename, files)
            contents = validump_resource(as_json, resource)
        files[filename] = contents
    if 'extractors.json' not in files:
        files['extractors.json'] = '{}'
    if ('items.json' not in files or not files['items.json'] or
            files['items.json'] == '{}'):
        files['items.json'] = DEFAULT_DASH_ITEM
    repo.save_files(files, 'master', 'Publishing initial import.')
    # XXX: Tell dash that project has been opened in Portia
    deploy_project(name, apikey, changed_files=[])<commit_msg>Remove default item when loading from dash<commit_after>import os
import requests
import zipfile
import json

from collections import defaultdict
from cStringIO import StringIO
from datetime import datetime
from slyd.gitstorage.repoman import Repoman, retry_operation

from slybot.validation.schema import get_schema_validator


DASH_API_URL = None
REQUIRED_FILES = {'extractors.json', 'items.json', 'project.json'}


def set_dash_url(dash_url):
    global DASH_API_URL
    DASH_API_URL = dash_url


class DeployError(Exception):

    def __init__(self, message='', data={}):
        self.message = message
        self.data = data


def import_project(name, apikey, repo):
    """Download a project from Dash and create a GIT repo for it."""

    def validump_resource(jsonres, restype):
        get_schema_validator(restype).validate(jsonres)
        return json.dumps(jsonres)

    def split_templates(spider, spider_filename, files):
        templates = spider['templates']
        spider['templates'] = []
        spider['template_names'] = []
        for template in templates:
            template['name'] = template['page_id']
            spider['template_names'].append(template['name'])
            template_fname = os.path.join(
                spider_filename.rpartition('.')[0],
                str(template['name']) + '.json')
            files[template_fname] = validump_resource(template, 'template')

    archive = zipfile.ZipFile(StringIO(_download_project(name, apikey)))
    files = {}
    for filename in archive.namelist():
        contents = archive.read(filename)
        if filename == 'items.json':
            resource = 'items'
        elif filename == 'extractors.json':
            resource = 'extractors'
        elif filename.startswith('spiders'):
            resource = 'spider'
        else:
            resource = None
        if resource in ['items', 'spider', 'extractors']:
            as_json = json.loads(contents)
            if resource == 'items':
                as_json = _fix_items(as_json)
            elif resource == 'spider':
                split_templates(as_json, filename, files)
            contents = validump_resource(as_json, resource)
        files[filename] = contents
    if 'extractors.json' not in files:
        files['extractors.json'] = '{}'
    if ('items.json' not in files or not files['items.json'] or
            files['items.json'] == '{}'):
        files['items.json'] = '{}'
    repo.save_files(files, 'master', 'Publishing initial import.')
    # XXX: Tell dash that project has been opened in Portia
    deploy_project(name, apikey, changed_files=[])
{'fraction_non_alphanumeric': 0.12316505549588257, 'fraction_numerical': 0.00035803795202291446, 'mean_word_length': 2.8906685236768803, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7433517', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1571, 'n_words': 402}