<commit_before># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
 #the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Publish notifications about mails to pony mail.

Enable this by adding the following to your mailman.cfg file::

    [archiver.ponymail]
    # The class implementing the IArchiver interface.
    class: mailman3_ponymail_plugin.Archiver
    enable: yes


"""

# Change this index name to whatever you picked!!
indexname = "ponymail_alpha"
if __name__ != '__main__':
    from zope.interface import implements
    from mailman.interfaces.archiver import IArchiver
else:
    import sys

from elasticsearch import Elasticsearch
import hashlib
import email.utils
import datetime, time
import json
from collections import namedtuple
import re


def pm_charsets(msg):
    charsets = set({})
    for c in msg.get_charsets():<commit_msg>add a little note about using archiver as standalone<commit_after># Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
 #the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" Publish notifications about mails to pony mail.

Enable this by adding the following to your mailman.cfg file::

    [archiver.ponymail]
    # The class implementing the IArchiver interface.
    class: mailman3_ponymail_plugin.Archiver
    enable: yes

OR, to use the STDIN version (non-MM3 mailing list managers),
sub someone to the list(s) and add this to their .forward file:
"|/usr/bin/python /path/to/plugin.py"

"""

# Change this index name to whatever you picked!!
indexname = "ponymail_alpha"
if __name__ != '__main__':
    from zope.interface import implements
    from mailman.interfaces.archiver import IArchiver
else:
    import sys

from elasticsearch import Elasticsearch
import hashlib
import email.utils
import datetime, time
import json
from collections import namedtuple
import re


def pm_charsets(msg):
    charsets = set({})
    for c in msg.get_charsets():
{'fraction_non_alphanumeric': 0.062289045719545875, 'fraction_numerical': 0.0033752684872660325, 'mean_word_length': 4.309446254071661, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2169003', 'n_tokens_mistral': 853, 'n_tokens_neox': 842, 'n_words': 415}