<commit_before>"""
scrapple.utils.form
~~~~~~~~~~~~~~~~~~~

Functions related to form handling.
"""

import os
import json
import itertools


def form_to_json(form):
	config = dict()
	if form['project_name'] == "":
		raise Exception('Project name cannot be empty.')
	if form['selector_type'] not in ["css", "xpath"]:
		raise Exception('Selector type has to css or xpath')
	config['project_name'] = form['project_name']
	config['selector_type'] = form['selector_type']
	config['scraping'] = dict()
	if form['url'] == "":
		raise Exception('URL cannot be empty')
	config['scraping']['url'] = form['url']
	config['scraping']['data'] = list()
	for i in itertools.count(start=1):
		try:
			data = {
				'field': form['field_' + str(i)],
				'selector': form['selector_' + str(i)],
				'attr': form['attribute_' + str(i)]
			}
			config['scraping']['data'].append(data)
		except KeyError:
			break
	# TODO : Crawler 'next' parameter handling
	with open(os.path.join(os.getcwd(), form['project_name'] + '.json'), 'w') as f:
		json.dump(config, f)
	return
<commit_msg>Include default in form handling util function<commit_after>"""
scrapple.utils.form
~~~~~~~~~~~~~~~~~~~

Functions related to form handling.
"""

import os
import json
import itertools


def form_to_json(form):
	config = dict()
	if form['project_name'] == "":
		raise Exception('Project name cannot be empty.')
	if form['selector_type'] not in ["css", "xpath"]:
		raise Exception('Selector type has to css or xpath')
	config['project_name'] = form['project_name']
	config['selector_type'] = form['selector_type']
	config['scraping'] = dict()
	if form['url'] == "":
		raise Exception('URL cannot be empty')
	config['scraping']['url'] = form['url']
	config['scraping']['data'] = list()
	for i in itertools.count(start=1):
		try:
			data = {
				'field': form['field_' + str(i)],
				'selector': form['selector_' + str(i)],
				'attr': form['attribute_' + str(i)],
				'default': form['default_' + str(i)]
			}
			config['scraping']['data'].append(data)
		except KeyError:
			break
	# TODO : Crawler 'next' parameter handling
	with open(os.path.join(os.getcwd(), form['project_name'] + '.json'), 'w') as f:
		json.dump(config, f)
	return

{'fraction_non_alphanumeric': 0.19817767653758542, 'fraction_numerical': 0.0009111617312072893, 'mean_word_length': 5.016438356164383, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4974252', 'n_tokens_mistral': 794, 'n_tokens_neox': 744, 'n_words': 193}