<commit_before>"""Script to import staged TWiki export (see twiki-extract.pl) into
a git repository via git fast-import.

./git-import.py  > fast-import.data
git fast-import < fast-import.data
"""

#!/usr/bin/python
import os, sys, time, re
from subprocess import check_call, Popen, PIPE

sys.path.extend(('lib', '../lib'))
from booki import xhtml_utils


#XXX need to import -- and correctly link -- images.

STAGING_DIR = '/home/douglas/fm-data/import-tests/staging'

DEST_DIR = '/home/douglas/fm-data/import-tests/booki-data.git'
SEPARATOR = "\n------8<-----------------\n"

WRITE = sys.stdout.write

branches = set(['master'])

def to_git(branch, chapter, text, author, email, date, version, comments,
           write=WRITE, fetch_images=False, lang='en'):
    if fetch_images:
        c = xhtml_utils.ImportedChapter(lang, branch, chapter, text, author, email, date)
        text, images = c.localise_links()
        for image in images:
            imgbin = xhtml_utils._read_local_url(image)
            img_comments = 'Image associated with %s\n%s' % (chapter, comments)
            to_git(branch, image, imgbin, author, email, date, version, img_comments,
                   write=WRITE, fetch_images=False)

    if comments:
        comments = '\n' + comments
    write("commit refs/heads/%s\n" % branch)
    write("committer %s <%s> %s\n" % (author, email, date))
    _data_blob("TWiki import: %s r%s%s" % (chapter, version, comments), write)
    if branch not in branches:
        write('merge refs/heads/master\n')
        branches.add(branch)

    write('M 644 inline %s\n' % (chapter))
    _data_blob(text, write)<commit_msg>try to keep git-import up to date with xhtml_utils changes<commit_after>"""Script to import staged TWiki export (see twiki-extract.pl) into
a git repository via git fast-import.

./git-import.py  > fast-import.data
git fast-import < fast-import.data
"""

#!/usr/bin/python
import os, sys, time, re
from subprocess import check_call, Popen, PIPE

sys.path.extend(('lib', '../lib'))
from booki import xhtml_utils


#XXX need to import -- and correctly link -- images.

STAGING_DIR = '/home/douglas/fm-data/import-tests/staging'

DEST_DIR = '/home/douglas/fm-data/import-tests/booki-data.git'
SEPARATOR = "\n------8<-----------------\n"

WRITE = sys.stdout.write

branches = set(['master'])

def to_git(branch, chapter, text, author, email, date, version, comments,
           write=WRITE, fetch_images=False, lang='en'):
    if fetch_images:
        c = xhtml_utils.ImportedChapter(lang, branch, chapter, text, author, email, date)
        images = c.localise_links()
        text = c.text
        for image in images:
            imgbin = c.image_cache.read_local_url(image)
            img_comments = 'Image associated with %s\n%s' % (chapter, comments)
            to_git(branch, image, imgbin, author, email, date, version, img_comments,
                   write=WRITE, fetch_images=False)

    if comments:
        comments = '\n' + comments
    write("commit refs/heads/%s\n" % branch)
    write("committer %s <%s> %s\n" % (author, email, date))
    _data_blob("TWiki import: %s r%s%s" % (chapter, version, comments), write)
    if branch not in branches:
        write('merge refs/heads/master\n')
        branches.add(branch)

    write('M 644 inline %s\n' % (chapter))
    _data_blob(text, write)
{'fraction_non_alphanumeric': 0.14802631578947367, 'fraction_numerical': 0.0023923444976076554, 'mean_word_length': 3.0252707581227436, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7419554', 'n_tokens_mistral': 1117, 'n_tokens_neox': 1087, 'n_words': 293}