<commit_before># -*- coding: utf-8 -*-

from __future__ import print_function
import json
import os

from scrapple.selectors.{{ config.selector_type }} import {{ config.selector_type|title }}Selector


def task_{{ config.project_name }}():
	"""
	Script generated using `Scrapple <http://scrappleapp.github.io/scrapple>`_
	"""
	results = dict()
	results['project'] = "{{ config.project_name }}"
	results['data'] = list()
	try:
		r0 = dict()
		page0 = {{ config.selector_type|title }}Selector("{{ config['scraping']['url'] }}")
		{%- for attr in config['scraping']['data'] %}
		{%- if not attr.field == "" %}
		r0["{{ attr.field }}"] = page0.extract_content("{{ attr.selector }}", "{{ attr.attr }}", "{{ attr.default }}")
		{%- endif %}
		{%- endfor %}
		{%- if config['scraping']['next'] %}
		{%- for next in config['scraping']['next'] recursive %}
		
		for page{{ loop.depth }} in page{{ loop.depth - 1 }}.extract_links("{{ next['follow_link'] }}"):
			r{{ loop.depth }} = r{{ loop.depth - 1 }}.copy()
			{%- set outer_loop = loop %}
			{%- for attr in next['scraping']['data'] %}
			{%- if not attr.field == "" %}
			r{{ outer_loop.depth }}["{{ attr.field }}"] = page{{ outer_loop.depth }}.extract_content("{{ attr.selector }}", "{{ attr.attr }}", "{{ attr.default }}")
			{%- endif %}
			{%- endfor %}
			{%- if next['scraping']['next'] %}
			{{ loop(next['scraping']['next'])|indent(3 + loop.depth , true) }}
			{%- else %}
			results['data'].append(r{{ loop.depth }})
			{%- endif %}
			{%- endfor %}
		{%- else %}
		results.append(r)
		{%- endif %}
	except KeyboardInterrupt:
		pass
	except Exception as e:
		print(e)
	finally:
		with open(os.path.join(os.getcwd(), '{{ config.output_file }}.json'), 'w') as f:
			json.dump(results, f)
	

if __name__ == '__main__':
	task_{{ config.project_name }}()<commit_msg>Fix mistake in generate template<commit_after># -*- coding: utf-8 -*-

from __future__ import print_function
import json
import os

from scrapple.selectors.{{ config.selector_type }} import {{ config.selector_type|title }}Selector


def task_{{ config.project_name }}():
	"""
	Script generated using `Scrapple <http://scrappleapp.github.io/scrapple>`_
	"""
	results = dict()
	results['project'] = "{{ config.project_name }}"
	results['data'] = list()
	try:
		r0 = dict()
		page0 = {{ config.selector_type|title }}Selector("{{ config['scraping']['url'] }}")
		{%- for attr in config['scraping']['data'] %}
		{%- if not attr.field == "" %}
		r0["{{ attr.field }}"] = page0.extract_content("{{ attr.selector }}", "{{ attr.attr }}", "{{ attr.default }}")
		{%- endif %}
		{%- endfor %}
		{%- if config['scraping']['next'] %}
		{%- for next in config['scraping']['next'] recursive %}
		
		for page{{ loop.depth }} in page{{ loop.depth - 1 }}.extract_links("{{ next['follow_link'] }}"):
			r{{ loop.depth }} = r{{ loop.depth - 1 }}.copy()
			{%- set outer_loop = loop %}
			{%- for attr in next['scraping']['data'] %}
			{%- if not attr.field == "" %}
			r{{ outer_loop.depth }}["{{ attr.field }}"] = page{{ outer_loop.depth }}.extract_content("{{ attr.selector }}", "{{ attr.attr }}", "{{ attr.default }}")
			{%- endif %}
			{%- endfor %}
			{%- if next['scraping']['next'] %}
			{{ loop(next['scraping']['next'])|indent(3 + loop.depth , true) }}
			{%- else %}
			results['data'].append(r{{ loop.depth }})
			{%- endif %}
			{%- endfor %}
		{%- else %}
		results['data'].append(r0)
		{%- endif %}
	except KeyboardInterrupt:
		pass
	except Exception as e:
		print(e)
	finally:
		with open(os.path.join(os.getcwd(), '{{ config.output_file }}.json'), 'w') as f:
			json.dump(results, f)
	

if __name__ == '__main__':
	task_{{ config.project_name }}()
{'fraction_non_alphanumeric': 0.23825136612021858, 'fraction_numerical': 0.004644808743169399, 'mean_word_length': 3.823451910408432, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6069328', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1271, 'n_words': 268}