<commit_before>#!/usr/bin/python

'''
Extract articles from Die Zeit epub, store them as plain text, one article per file.
Optionally sort the articles by resort by prefixing them with a running number.
'''

import xml.etree.ElementTree as ET
import re
import argparse
import os.path
import os
import subprocess
import logging
import shutil
import glob
from difflib import SequenceMatcher as SM



# Use this list to set the default order of all resorts.
# they only need to fuzzy match, see
RESORTS_ORDER = [
    'Politik',
    'Wissen',
    'Doktor',
    'Seite 1',
    'Recht und Unrecht',
    'Wirtschaft',
    'Dossier',
    'Geschichte',
    'Chancen',
    'Glauben und Zweifeln',
    'geld-spezial',
    'zeitmagazin',
    'ZEIT magazin',
    'Krimi Spezial',
    'Chancen Spezial',
    'Reisen',
    'reisen spezial',
    'Kultursommer',
    'Feuilleton',
    'Musik-Spezial',
    'Literatur-Spezial',
    'leserbriefe',
    'Zeit zum Entdecken',
    'Krimi Spezial',
    'ZEIT der Leser',
    'Kinder und Jugendbuch',
    'Olympia',
    'Hamburg',
    'Schweiz',
    'Oesterreich',
    'ZEIT im Osten',
    'Leo',
    'Kinderzeit',
    'Fussball',
    ]

RESORT_ORDER_MATCH_TRESHOLD = 0.75

# Use this list to define some resorts to skip in any case.
RESORTS_BLACKLIST = [
    'Zeit zum Entdecken',
    'Golfen',
    'Hamburg',
    'Schweiz',
    'Oesterreich',
    'ZEIT im Osten',
    'Kinderzeit',
    'Fussball',
    'ZEIT magazin',
    'Feuilleton',
    'Leo',
        ]

RESORT_BLACKLIST_MATCH_TRESHOLD = 0.8


class Article():
    '''Class to parse/process one article.'''

    def __init__(self, article_id=0):
        '''Initialize fields to None, set the article's ID.'''
        self._id = article_id
        self._resort = None
        self._title = None
        self._subtitle = None
        self._author = None
        # list of strings, each one paragraph
        self._content = None<commit_msg>zeit_extractor: fix RESORT_BLACKLIST<commit_after>#!/usr/bin/python

'''
Extract articles from Die Zeit epub, store them as plain text, one article per file.
Optionally sort the articles by resort by prefixing them with a running number.
'''

import xml.etree.ElementTree as ET
import re
import argparse
import os.path
import os
import subprocess
import logging
import shutil
import glob
from difflib import SequenceMatcher as SM



# Use this list to set the default order of all resorts.
# they only need to fuzzy match, see
RESORTS_ORDER = [
    'Politik',
    'Wissen',
    'Doktor',
    'Seite 1',
    'Recht und Unrecht',
    'Wirtschaft',
    'Dossier',
    'Geschichte',
    'Chancen',
    'Glauben und Zweifeln',
    'geld-spezial',
    'zeitmagazin',
    'ZEIT magazin',
    'Krimi Spezial',
    'Chancen Spezial',
    'Reisen',
    'reisen spezial',
    'Kultursommer',
    'Feuilleton',
    'Musik-Spezial',
    'Literatur-Spezial',
    'leserbriefe',
    'Zeit zum Entdecken',
    'Krimi Spezial',
    'ZEIT der Leser',
    'Kinder und Jugendbuch',
    'Olympia',
    'Hamburg',
    'Schweiz',
    'Oesterreich',
    'ZEIT im Osten',
    'Leo',
    'Kinderzeit',
    'Fussball',
    ]

RESORT_ORDER_MATCH_TRESHOLD = 0.75

# Use this list to define some resorts to skip in any case.
RESORTS_BLACKLIST = [
    'Zeit zum Entdecken',
    'Golfen',
    'Hamburg',
    'Schweiz',
    'Oesterreich',
    'ZEIT im Osten',
    'Kinderzeit',
    'Fussball',
    'ZEIT magazin',
    'Feuilleton',
    'Leo - ZEIT fuer Kinder',
        ]

RESORT_BLACKLIST_MATCH_TRESHOLD = 0.8


class Article():
    '''Class to parse/process one article.'''

    def __init__(self, article_id=0):
        '''Initialize fields to None, set the article's ID.'''
        self._id = article_id
        self._resort = None
        self._title = None
        self._subtitle = None
        self._author = None
        # list of strings, each one paragraph
        self._content = None
{'fraction_non_alphanumeric': 0.11076764554353426, 'fraction_numerical': 0.0036063884595569293, 'mean_word_length': 2.75531914893617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '65401', 'n_tokens_mistral': 1386, 'n_tokens_neox': 1364, 'n_words': 394}