<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""ppcheckout

Usage:
  ppcheckout [options] <projectid> [<projectname>]
  ppcheckout -h | --help
  ppcheckout --version

Translates pgdp.org formatted text files into ppgen syntax.

Examples:
  ppcheckout projectID509579cf64be8
  ppcheckout projectID509579cf64be8 proj-name

Options:
  -q, --quiet           Print less text.
  -v, --verbose         Print more text.
  -h, --help            Show help.
  --version             Show version.
"""

from docopt import docopt
from robobrowser import RoboBrowser
import urllib
import logging
import re
import shutil
import os
import zipfile


VERSION="0.1.0" # MAJOR.MINOR.PATCH | http://semver.org

def main():
    args = docopt(__doc__, version="ppcheckout v{}".format(VERSION))

    # Configure logging
    logLevel = logging.INFO #default
    if args['--verbose']:
        logLevel = logging.DEBUG
    elif args['--quiet']:
        logLevel = logging.ERROR

    logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel)
    logging.debug(args)

    # Process command line arguments
    projectId = args['<projectid>']
    projectName = args['<projectname>']
    logging.info("Checking out project {} to {}".format(projectId,projectName))

    # Parse project page for title and author
    logging.info("Parsing project page for {}".format(projectId))
    projectURL = "http://www.pgdp.net/c/project.php?id={}".format(projectId)
    browser = RoboBrowser()
    browser.open(projectURL)
    projectInfo = browser.find('table',id='project_info_table')
    title = projectInfo.find('b',text="Title").parent.next_sibling.string
    author = projectInfo.find('b',text="Author").parent.next_sibling.string
    print(title)
    print(author)

    if not projectName:
        projectName = generateProjectName(title)

    print(projectId)
    print(projectName)

    # Download project files
    logging.info("Downloading project files for {} by {}".format(title,author))
    goodWordsURL = "http://www.pgdp.net/projects/{}/good_words.txt".format(projectId)
    badWordsURL = "http://www.pgdp.net/projects/{}/bad_words.txt".format(projectId)
    imagesURL = "http://www.pgdp.net/c/tools/download_images.php?projectid={}&dummy={}images.zip".format(projectId, projectId)
    textURL = "http://www.pgdp.net/projects/{}/{}.zip".format(projectId, projectId)
    downloadFile(goodWordsURL,"good_words.txt")
    downloadFile(badWordsURL,"bad_words.txt")
    downloadFile(imagesURL,"images.zip")
    downloadFile(textURL,"text.zip")

    # Init project skeleton
    logging.info("Building project base structure")
    srcDir = os.path.abspath("_NEW_PROJECT_TEMPLATE")
    dstDir = os.path.abspath(projectName)
    shutil.copytree(srcDir, dstDir)

    # Copy/unzip project files
    logging.info("Adding project files")
    zipText = zipfile.ZipFile("text.zip","r")
    zipImages = zipfile.ZipFile("images.zip","r")

    print (zipText.names)

    return


def downloadFile( src, dest ):
    logging.info("Saving {} to {}".format(src,dest))
    try:
        urllib.request.urlretrieve(src,dest)
    except urllib.error.HTTPError as e:<commit_msg>Add copying of project files<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""ppcheckout

Usage:
  ppcheckout [options] <projectid> [<projectname>]
  ppcheckout -h | --help
  ppcheckout --version

Translates pgdp.org formatted text files into ppgen syntax.

Examples:
  ppcheckout projectID509579cf64be8
  ppcheckout projectID509579cf64be8 proj-name

Options:
  -q, --quiet           Print less text.
  -v, --verbose         Print more text.
  -h, --help            Show help.
  --version             Show version.
"""

from docopt import docopt
from robobrowser import RoboBrowser
import urllib
import logging
import re
import shutil
import os
import zipfile
import glob


VERSION="0.1.0" # MAJOR.MINOR.PATCH | http://semver.org

def main():
    args = docopt(__doc__, version="ppcheckout v{}".format(VERSION))

    # Configure logging
    logLevel = logging.INFO #default
    if args['--verbose']:
        logLevel = logging.DEBUG
    elif args['--quiet']:
        logLevel = logging.ERROR

    logging.basicConfig(format='%(levelname)s: %(message)s', level=logLevel)
    logging.debug(args)

    # Process command line arguments
    projectId = args['<projectid>']
    projectName = args['<projectname>']
    logging.info("Checking out project {} to {}".format(projectId,projectName))

    # Parse project page for title and author
    logging.info("Parsing project page for {}".format(projectId))
    projectURL = "http://www.pgdp.net/c/project.php?id={}".format(projectId)
    browser = RoboBrowser()
    browser.open(projectURL)
    projectInfo = browser.find('table',id='project_info_table')
    title = projectInfo.find('b',text="Title").parent.next_sibling.string
    author = projectInfo.find('b',text="Author").parent.next_sibling.string
    print(title)
    print(author)

    if not projectName:
        projectName = generateProjectName(title)

    print(projectId)
    print(projectName)

    # Download project files
    logging.info("Downloading project files for {} by {}".format(title,author))
    imagesURL = "http://www.pgdp.net/c/tools/download_images.php?projectid={}&dummy={}images.zip".format(projectId, projectId)
    textURL = "http://www.pgdp.net/projects/{}/{}.zip".format(projectId, projectId)
    downloadFile(imagesURL,"images.zip")
    downloadFile(textURL,"text.zip")

    # Init project skeleton
    logging.info("Building project base structure")
    srcDir = os.path.abspath("_NEW_PROJECT_TEMPLATE")
    dstDir = os.path.abspath(projectName)
    shutil.copytree(srcDir, dstDir)

    # Copy/unzip project files
    logging.info("Adding project files")
    zipText = zipfile.ZipFile("text.zip","r")
    zipImages = zipfile.ZipFile("images.zip","r")
    zipText.extractall(path=os.path.abspath("{}/originals/".format(projectName)))
    zipImages.extractall(path=os.path.abspath("{}/pngs/".format(projectName)))
    moveFiles(glob.glob(os.path.abspath("{}/pngs/*.jpg".format(projectName))), os.path.abspath("{}/originals/illustrations/".format(projectName)))
    shutil.copy(os.path.abspath("{}/originals/{}".format(projectName,"{}.txt".format(projectId))),os.path.abspath("{}/{}-src.txt".format(projectName,projectName)))

    return


def moveFiles( files, dest ):
    for f in files:
        print(f)
        shutil.move(f,dest)


def downloadFile( src, dest ):
    logging.info("Saving {} to {}".format(src,dest))
    try:
        urllib.request.urlretrieve(src,dest)
    except urllib.error.HTTPError as e:
{'fraction_non_alphanumeric': 0.14305156509904732, 'fraction_numerical': 0.00695599576591562, 'mean_word_length': 3.7412186379928314, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6221737', 'n_tokens_mistral': 2181, 'n_tokens_neox': 2117, 'n_words': 412}