<commit_before>#!/usr/bin/env python

import argparse
import errno
import glob
import os
import re
import shutil
import sys
import zlib
zlib.Z_DEFAULT_COMPRESSION = zlib.Z_BEST_COMPRESSION
import zipfile

import distutils.ccompiler
import distutils.sysconfig

import pkg_resources

import wx

import openslides
import openslides_gui

COMMON_EXCLUDE = [
    r".pyc$",
    r".pyo$",
    r".po$",
    r".egg-info",
    r"\blocale/(?!de/|en/|fr/)[^/]+/"
]

LIBEXCLUDE = [
    r"^site-packages/",
    r"^test/",
    r"^curses/",
    r"^idlelib/",
    r"^lib2to3/",
    r"^lib-tk/",
    r"^msilib/",
]


SITE_PACKAGES = {
    # openslides dependencies
    "bleach": {
        "copy": ["bleach"],
    },
    "django": {
        "copy": ["django"],
        "exclude": [
            r"^django/contrib/flatpages/",
            r"^django/contrib/gis/",
            r"^django/contrib/redirects/",
            r"^django/contrib/sitemaps/",
            r"^django/contrib/syndication/",
        ]
    },
    "djangorestframework": {
        "copy": ["rest_framework"],
    },
    "html5lib": {
        "copy": ["html5lib"],
    },
    "jsonfield": {
        "copy": ["jsonfield"],
    },
    "pypdf2": {
        "copy": ["PyPDF2"],
    },
    "roman": {
        "copy": ["roman.py"],
    },
    "setuptools": {
        "copy": [
            "pkg_resources",
            "setuptools",
            "easy_install.py",
        ],
    },
    "six": {
        "copy": ["six.py"],
    },
    "wxpython-phoenix": {
        "copy": ["wx"],
        "exclude": [
            r"^wx/lib/",
            r"^wx/py/",
            r"^wx/tools/",
            r"wx/adv.pi",
            r"wx/core.pi",
            r"wx/dataview.pi",
            r"wx/dataview.py",
            r"wx/_dataview.pyd",
            r"wx/freetype6.dll",
            r"wx/glcanvas.pi",
            r"wx/glcanvas.py",
            r"wx/_glcanvas.pyd",
            r"wx/grid.pi",
            r"wx/grid.py",
            r"wx/_grid.pyd",
            r"wx/html.pi",
            r"wx/html.py",
            r"wx/_html.pyd",
            r"wx/html2.pi",
            r"wx/html2.py",
            r"wx/_html2.pyd",
            r"wx/libcairo-2.dll",
            r"wx/libcairo-gobject-2.dll",
            r"wx/libcairo-script-interpreter-2.dll",
            r"wx/libexpat-1.dll",
            r"wx/libfontconfig-1.dll",
            r"wx/libpng14-14.dll",
            r"wx/stc.pi",
            r"wx/stc.py",
            r"wx/_stc.pyd",
            r"wx/richtext.pi",
            r"wx/richtext.py",
            r"wx/_richtext.pyd",
            r"wx/webkit.pi",
            r"wx/webkit.py",
            r"wx/wxmsw30u_aui_vc100.dll",
            r"wx/wxmsw30u_gl_vc100.dll",
            r"wx/wxmsw30u_html_vc100.dll",
            r"wx/wxmsw30u_media_vc100.dll",
            r"wx/wxmsw30u_propgrid_vc100.dll",
            r"wx/wxmsw30u_qa_vc100.dll",
            r"wx/wxmsw30u_ribbon_vc100.dll",
            r"wx/wxmsw30u_richtext_vc100.dll",
            r"wx/wxmsw30u_stc_vc100.dll",
            r"wx/wxmsw30u_webview_vc100.dll",
            r"wx/wxmsw30u_xml_vc100.dll",
            r"wx/wxmsw30u_xrc_vc100.dll",
            r"wx/xrc.pi",
            r"wx/xrc.py",
            r"wx/_xrc.pyd",
            r"wx/xml.pi",
            r"wx/xml.py",
            r"wx/_xml.pyd",
            r"wx/zlib1.dll",
        ],
    },
    # channels and all dependencies
    "channels": {
        "copy": ["channels"],
    },
    "daphne": {
        "copy": ["daphne"],
    },
    "asgiref": {
        "copy": ["asgiref"],
    },
    "autobahn": {
        "copy": ["autobahn"],
    },
    "txaio": {
        "copy": ["txaio"],
    },
    "twisted": {
        "copy": [
            "twisted",
            "win32",
            "pywin32.pth",
        ],
    },
    "packaging": {
        "copy": ["packaging"],
    },
    "pyparsing": {
        "copy": ["pyparsing.py"],
    },
    "appdirs": {
        "copy": ["appdirs.py"],
    },
    "incremental": {
        "copy": ["incremental"],
    },
    "constantly": {
        "copy": ["constantly"],
    },
    "zope.interface": {
        "copy": ["zope"],
    },
    # openslides core
    "openslides": {
        "copy": ["openslides"],
    },
    # openslides-gui
    "openslides_gui": {
        "copy": ["openslides_gui"],
    },
    "psutil": {
        "copy": ["psutil"],
    },
}

PY_DLLS = [
    "unicodedata.pyd",
    "sqlite3.dll",
    "_sqlite3.pyd",
    "_socket.pyd",
    "select.pyd",
    "_ctypes.pyd",
    "_ssl.pyd",
    "_multiprocessing.pyd",
    "pyexpat.pyd",
]


OPENSLIDES_RC_TMPL = """
#include <winresrc.h><commit_msg>Added requirements for redis and postgres support.<commit_after>#!/usr/bin/env python

import argparse
import errno
import glob
import os
import re
import shutil
import sys
import zlib
zlib.Z_DEFAULT_COMPRESSION = zlib.Z_BEST_COMPRESSION
import zipfile

import distutils.ccompiler
import distutils.sysconfig

import pkg_resources

import wx

import openslides
import openslides_gui

COMMON_EXCLUDE = [
    r".pyc$",
    r".pyo$",
    r".po$",
    r".egg-info",
    r"\blocale/(?!de/|en/|fr/)[^/]+/"
]

LIBEXCLUDE = [
    r"^site-packages/",
    r"^test/",
    r"^curses/",
    r"^idlelib/",
    r"^lib2to3/",
    r"^lib-tk/",
    r"^msilib/",
]


SITE_PACKAGES = {
    # openslides dependencies
    "bleach": {
        "copy": ["bleach"],
    },
    "django": {
        "copy": ["django"],
        "exclude": [
            r"^django/contrib/flatpages/",
            r"^django/contrib/gis/",
            r"^django/contrib/redirects/",
            r"^django/contrib/sitemaps/",
            r"^django/contrib/syndication/",
        ]
    },
    "djangorestframework": {
        "copy": ["rest_framework"],
    },
    "html5lib": {
        "copy": ["html5lib"],
    },
    "jsonfield": {
        "copy": ["jsonfield"],
    },
    "pypdf2": {
        "copy": ["PyPDF2"],
    },
    "roman": {
        "copy": ["roman.py"],
    },
    "setuptools": {
        "copy": [
            "pkg_resources",
            "setuptools",
            "easy_install.py",
        ],
    },
    "six": {
        "copy": ["six.py"],
    },
    "wxpython-phoenix": {
        "copy": ["wx"],
        "exclude": [
            r"^wx/lib/",
            r"^wx/py/",
            r"^wx/tools/",
            r"wx/adv.pi",
            r"wx/core.pi",
            r"wx/dataview.pi",
            r"wx/dataview.py",
            r"wx/_dataview.pyd",
            r"wx/freetype6.dll",
            r"wx/glcanvas.pi",
            r"wx/glcanvas.py",
            r"wx/_glcanvas.pyd",
            r"wx/grid.pi",
            r"wx/grid.py",
            r"wx/_grid.pyd",
            r"wx/html.pi",
            r"wx/html.py",
            r"wx/_html.pyd",
            r"wx/html2.pi",
            r"wx/html2.py",
            r"wx/_html2.pyd",
            r"wx/libcairo-2.dll",
            r"wx/libcairo-gobject-2.dll",
            r"wx/libcairo-script-interpreter-2.dll",
            r"wx/libexpat-1.dll",
            r"wx/libfontconfig-1.dll",
            r"wx/libpng14-14.dll",
            r"wx/stc.pi",
            r"wx/stc.py",
            r"wx/_stc.pyd",
            r"wx/richtext.pi",
            r"wx/richtext.py",
            r"wx/_richtext.pyd",
            r"wx/webkit.pi",
            r"wx/webkit.py",
            r"wx/wxmsw30u_aui_vc100.dll",
            r"wx/wxmsw30u_gl_vc100.dll",
            r"wx/wxmsw30u_html_vc100.dll",
            r"wx/wxmsw30u_media_vc100.dll",
            r"wx/wxmsw30u_propgrid_vc100.dll",
            r"wx/wxmsw30u_qa_vc100.dll",
            r"wx/wxmsw30u_ribbon_vc100.dll",
            r"wx/wxmsw30u_richtext_vc100.dll",
            r"wx/wxmsw30u_stc_vc100.dll",
            r"wx/wxmsw30u_webview_vc100.dll",
            r"wx/wxmsw30u_xml_vc100.dll",
            r"wx/wxmsw30u_xrc_vc100.dll",
            r"wx/xrc.pi",
            r"wx/xrc.py",
            r"wx/_xrc.pyd",
            r"wx/xml.pi",
            r"wx/xml.py",
            r"wx/_xml.pyd",
            r"wx/zlib1.dll",
        ],
    },
    # channels and all dependencies
    "channels": {
        "copy": ["channels"],
    },
    "daphne": {
        "copy": ["daphne"],
    },
    "asgiref": {
        "copy": ["asgiref"],
    },
    "autobahn": {
        "copy": ["autobahn"],
    },
    "txaio": {
        "copy": ["txaio"],
    },
    "twisted": {
        "copy": [
            "twisted",
            "win32",
            "pywin32.pth",
        ],
    },
    "packaging": {
        "copy": ["packaging"],
    },
    "pyparsing": {
        "copy": ["pyparsing.py"],
    },
    "appdirs": {
        "copy": ["appdirs.py"],
    },
    "incremental": {
        "copy": ["incremental"],
    },
    "constantly": {
        "copy": ["constantly"],
    },
    "zope.interface": {
        "copy": ["zope"],
    },
    # redis and postgres requirements
    "asgi-redis": {
        "copy": ["asgi_redis"],
    },
    "redis": {
        "copy": ["redis"],
    },
    "msgpack-python": {
        "copy": ["msgpack"],
    },
    "django-redis": {
        "copy": ["django_redis"],
    },
    "django-redis-sessions": {
        "copy": ["redis_sessions"],
    },
    "psycopg2": {
        "copy": ["psycopg2"],
    },
    # openslides core
    "openslides": {
        "copy": ["openslides"],
    },
    # openslides-gui
    "openslides_gui": {
        "copy": ["openslides_gui"],
    },
    "psutil": {
        "copy": ["psutil"],
    },
}

PY_DLLS = [
    "unicodedata.pyd",
    "sqlite3.dll",
    "_sqlite3.pyd",
    "_socket.pyd",
    "select.pyd",
    "_ctypes.pyd",
    "_ssl.pyd",
    "_multiprocessing.pyd",
    "pyexpat.pyd",
]


OPENSLIDES_RC_TMPL = """
#include <winresrc.h>
{'fraction_non_alphanumeric': 0.1862327129042321, 'fraction_numerical': 0.018092960382655714, 'mean_word_length': 1.5579787234042553, 'pattern_counts': {'":': 116, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2435369', 'n_tokens_mistral': 3855, 'n_tokens_neox': 3698, 'n_words': 449}