<commit_before>#!/usr/bin/env python
"""Scrapy admin script is used to create new scrapy projects and similar
tasks"""

import os
import shutil
from optparse import OptionParser

import scrapy

usage = """
scrapy-admin.py [options] [command]
 
Available commands:
     
    startproject <project_name>
      Starts a new project with name 'project_name'
"""

def main():
    parser = OptionParser(usage=usage)
    opts, args = parser.parse_args()
    
    if not args:
        parser.print_help()

    cmd = args[0]
    if cmd == "startproject":
        if len(args) >= 2:
            project_name = args[1]
            project_tplpath = os.path.join(scrapy.__path__[0], "conf", "project_template")
            shutil.copytree(project_tplpath, project_name)
        else:
            print "scrapy-admin.py: missing project name"
    else:
        print "scrapy-admin.py: unknown command: %s" % cmd

if __name__ == '__main__':
    main()
<commit_msg>substitute scrapy_settings template variables on startproject<commit_after>#!/usr/bin/env python
"""Scrapy admin script is used to create new scrapy projects and similar
tasks"""
from __future__ import with_statement

import os
import shutil
import string
from optparse import OptionParser

import scrapy

usage = """
scrapy-admin.py [options] [command]
 
Available commands:
     
    startproject <project_name>
      Starts a new project with name 'project_name'
"""

TEMPLATES = (
        'scrapy_settings.py',
        )

def render_templatefile(path, **kwargs):
    with open(path, 'rb') as file:
        raw = file.read()

    content = string.Template(raw).substitute(**kwargs)

    with open(path, 'wb') as file:
        file.write(content)


def main():
    parser = OptionParser(usage=usage)
    opts, args = parser.parse_args()
    
    if not args:
        parser.print_help()
        return

    cmd = args[0]
    if cmd == "startproject":
        if len(args) >= 2:
            project_name = args[1]
            project_tplpath = os.path.join(scrapy.__path__[0], "conf", "project_template")
            shutil.copytree(project_tplpath, project_name)
            for path in TEMPLATES:
                render_templatefile(os.path.join(project_name, path), project_name=project_name)
        else:
            print "scrapy-admin.py: missing project name"
    else:
        print "scrapy-admin.py: unknown command: %s" % cmd

if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.10671611042439226, 'fraction_numerical': 0.003296250515039143, 'mean_word_length': 2.8357030015797786, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1872069', 'n_tokens_mistral': 763, 'n_tokens_neox': 739, 'n_words': 207}