<commit_before>#!/usr/bin/env python3

from osmfilter.filters import BaseFilter, VersionIncrementor
from osmfilter.filtering import run
import logging
import sys


class WebsiteFilter(BaseFilter):
    def website(self, e):
        modified = False
        for tag in ('website', 'contact:website'):
            if tag in e.tags.keys():
                modified = True
                e.tags[tag] = "Modified"
        if modified:
            e.version += 1
            return e
        else:
            return True

    def node(self, e):
        return self.website(e)

    def way(self, e):
        return self.website(e)

    def relation(self, e):
        return self.website(e)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    input_file_path = sys.argv[1]
    output_file_path = sys.argv[2]

    input_file = open(input_file_path, "rb")
    output_file = open(output_file_path, "wb")

    filters = (WebsiteFilter(), VersionIncrementor())

    try:
        run(input_file, output_file, filters, threads=20)
    finally:
        input_file.close()
        output_file.close()
        logging.debug("Closed files")
<commit_msg>Add phone tag correction to example.py and adapt to library change<commit_after>#!/usr/bin/env python3

import logging
import sys
import re

from osmfilter.filters import BaseFilter, VersionIncrementor
from osmfilter.filtering import run, FilteringModifier
from osmfilter.writing import OSMWriter


class WebsiteFilter(BaseFilter):
    def __init__(self):
        self.protocol_re = re.compile("^https?://")
    def clean_url(self, url):
        if not self.protocol_re.search(url):
            url = "http://" + url
        return url

    def act(self, e):
        state = FilteringModifier.discard
        for tag in ('website', 'contact:website'):
            if tag in e.tags.keys():
                state = FilteringModifier.modified
                e.tags[tag] = self.clean_url(e.tags[tag])
        return (e, state)

    def node(self, e, flags):
        return self.act(e)

    def way(self, e, flags):
        return self.act(e)

    def relation(self, e, flags):
        return self.act(e)


class PhoneFilter(BaseFilter):
    def __init__(self, writer):
        self.writer = writer

        self.parsable_re = re.compile(".*([1-9]).*?(\d).*?(\d).*?(\d).*?(\d).*?(\d).*?(\d).*?(\d).*?(\d)$")
        self.france_re = re.compile("^\+33 ?[1-9]( ?[0-9]){8}$")
    def test_phone(self, number):
        if self.france_re.match(number):
            return True
        match = self.parsable_re.search(number)
        if match:
            return "+33 {0} {1}{2} {3}{4} {5}{6} {7}{8}".format(*match.groups())
        else:
            return False

    def act(self, e):
        state = FilteringModifier.discard
        for tag in ('phone', 'contact:phone', 'fax', 'contact:fax'):
            if tag in e.tags.keys():
                new_phone = self.test_phone(e.tags[tag])
                if new_phone is True:
                    continue
                if new_phone is False:
                    self.writer.write(e)
                else:
                    state = FilteringModifier.modified
                    e.tags[tag] = new_phone

        return (e, state)

    def node(self, e, flags):
        return self.act(e)

    def way(self, e, flags):
        return self.act(e)

    def relation(self, e, flags):
        return self.act(e)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    input_file_path = sys.argv[1]
    output_file_path = sys.argv[2]

    input_file = open(input_file_path, "rb")
    output_file = open(output_file_path, "wb")

    bad_phone_file = open("bad_phone.osm", "wb")
    writer = OSMWriter(bad_phone_file)
    writer.initialize_document()

    #filters = (PhoneFilter(writer), VersionIncrementor())
    filters = (WebsiteFilter(), VersionIncrementor())

    try:
        run(input_file, output_file, filters, threads=20)
    finally:
        writer.finalize_document()
        input_file.close()
        output_file.close()
        logging.debug("Closed files")

{'fraction_non_alphanumeric': 0.12872496336101613, 'fraction_numerical': 0.007572056668295066, 'mean_word_length': 2.305084745762712, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7000756', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1261, 'n_words': 299}