<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © <NAME>. See NOTICE for more information.
#
import sys
sys.path.append('.')
sys.path.append('./apps')
sys.path.append('./lib')

from django.core.management import setup_environ
from redakcja import settings

setup_environ(settings)

from catalogue.models import Book, Chunk
import re

fixed = {}

datepd = r"(<dc:date.pd[^>]+>)([0-9]+)(</dc:date.pd>)"
def fix(book, author, dry_run=True):
    if len(book) == 0:
        print "%s does not contain chunks" % book.slug
        return
    fc = book[0]
    txt = fc.materialize()

    if dry_run:
        m = re.search(datepd, txt)
        if m:
            print("%s: %s->%d" % (book.slug, m.groups()[1], int(m.groups()[1])+70))
        else:
            print("%s: date.pd not found??" % (book.slug,))
    else:
        dates = {}
        def up_date(match):
            tagopen, date, tagclose = match.groups()
            olddate=date
            date = str(int(date)+70)
            dates['date'] = date
            dates['olddate'] = olddate
            return tagopen+date+tagclose
        new_txt = re.sub(datepd, up_date, txt)
        print "%s: %s->%s" % (book.slug, dates['olddate'], dates['date'])
        fc.commit(new_txt, author=author, description=u"Automatyczne poprawienie daty przejścia do domeny publicznej z %s na %s" % (dates['olddate'], dates['date']))

import sys
import getopt
from django.contrib.auth.models import User
opts, oth_ = getopt.getopt(sys.argv[1:],[],[ "seriously"])<commit_msg>71 years instead of 70. detect dated which do not need changing<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © <NAME>. See NOTICE for more information.
#
import sys
sys.path.append('.')
sys.path.append('./apps')
sys.path.append('./lib')

from django.core.management import setup_environ
from redakcja import settings

setup_environ(settings)

from catalogue.models import Book, Chunk
import re

fixed = {}

datepd = r"(<dc:date.pd[^>]+>)([0-9]+)(</dc:date.pd>)"
def fix(book, author, dry_run=True):
    if len(book) == 0:
        print "%s does not contain chunks" % book.slug
        return
    fc = book[0]
    txt = fc.materialize()

    if dry_run:
        m = re.search(datepd, txt)
        if m:
            print("%s: %s->%d" % (book.slug, m.groups()[1], int(m.groups()[1])+71))
        else:
            print("%s: date.pd not found??" % (book.slug,))
    else:
        dates = {}
        def up_date(match):
            tagopen, date, tagclose = match.groups()
            olddate=date
            date = str(int(date)+71)
            dates['date'] = date
            dates['olddate'] = olddate
            dates['overflow'] = False
            if int(date) > 2012:
               dates['overflow'] = True
               return tagopen+date+tagclose

        new_txt = re.sub(datepd, up_date, txt)
        if dates:
            print "%s: %s->%s" % (book.slug, dates['olddate'], dates['date'])
            if dates['overflow']:
                print "oops, new date would overfow to the future, i'm not changing"
                return	
           # fc.commit(new_txt, author=author, description=u"Automatyczne poprawienie daty przejścia do domeny publicznej z %s na %s" % (dates['olddate'], dates['date']))
        else:
            print "skipping %s" % book.slug
import sys
import getopt
from django.contrib.auth.models import User
opts, oth_ = getopt.getopt(sys.argv[1:],[],[ "seriously"])
{'fraction_non_alphanumeric': 0.15433462175593596, 'fraction_numerical': 0.009387078961899503, 'mean_word_length': 2.6157684630738522, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5855158', 'n_tokens_mistral': 1208, 'n_tokens_neox': 1194, 'n_words': 319}