<commit_before>{
    "type" : "remote",
    "remote" : {
        "urlGetDocuments"       : "https://community-dev.jboss.org/data-api/content.jspa?spaceId={space}&type=forum&updatedAfter={updatedAfter}&maxSize=50",
        "getDocsResFieldDocuments"  : "items",
        "urlGetSpaces"					: "https://community-dev.jboss.org/data-api/spaces.jspa",
        "getSpacesResField"			: "spaces",
        "timeout"               : "20s",
        "indexUpdatePeriod"     : "10m",
        "indexFullUpdatePeriod" : "10d",
        "maxIndexingThreads"    : 2
    },
    "index" : {
        "index"             : "data_jbossorg_sbs_forum",
        "type"              : "jbossorg_sbs_forum",
        "remote_field_document_id" : "id",
	      "remote_field_updated"     : "published",
        "field_space_key"   : "space_key",
        "field_document_id" : "dcp_id",
        "fields" : {
            "dcp_content_id"        : {"remote_field" : "id_remote"},
            "dcp_type"              : {"remote_field" : "dcp_type"},
            "dcp_updated"           : {"remote_field" : "dcp_updated"},
            "dcp_content_provider"  : {"remote_field" : "dcp_content_provider"},
            "dcp_content_type"      : {"remote_field" : "dcp_content_type"},
            "dcp_activity_dates"    : {"remote_field" : "dcp_activity_dates"},
            "dcp_last_activity_date": {"remote_field" : "dcp_last_activity_date"},
            "dcp_project"           : {"remote_field" : "dcp_project"},
            "dcp_project_name"      : {"remote_field" : "dcp_project_name"},
            "dcp_url_view"					: {"remote_field" : "url"},
            "dcp_contributors"      : {"remote_field" : "dcp_contributors"},
            "dcp_title"             : {"remote_field" : "title"},
            "dcp_created"           : {"remote_field" : "published"},
            "dcp_description"       : {"remote_field" : "dcp_description"},
            "dcp_content"           : {"remote_field" : "content"},
            "dcp_content_content-type" : {"remote_field" : "dcp_content_content-type"},
            "tags"                  : {"remote_field" : "tags"},
            "dcp_tags"              : {"remote_field" : "tags"},
            "authors"               : {"remote_field" : "authors", "value_filter" : "user"}
        },
        "value_filters" : {
            "user" : {
                "email"           : "email_address",
                "full_name"       : "display_name",
                "dcp_contributor" : "dcp_contributor" 
            }
        },
        "comment_mode"   : "embedded",
        "remote_field_comments" : "comments",
        "field_comments" : "dcp_comments",
        "comment_fields" : {
            "comment_body"    : {"remote_field" : "content"},
            "comment_author"  : {"remote_field" : "author", "value_filter" : "user"},
            "comment_created" : {"remote_field" : "published"},
            "comment_updated" : {"remote_field" : "published"}
        },
        "preprocessors" : [
            {
                "name"     : "Remote id copy",
                "class"    : "org.jboss.elasticsearch.tools.content.AddMultipleValuesPreprocessor",
                "settings" : {
                    "id_remote"             : "{id}"
                 } 
            },{
                "name"     : "DCP values filler",
                "class"    : "org.jboss.elasticsearch.tools.content.AddMultipleValuesPreprocessor",
                "settings" : {
                    "id"                    : "jbossorg_sbs_forum-{id}",
                    "dcp_type"              : "forumthread",
                    "dcp_content_provider"  : "jbossorg",
                    "dcp_content_type"      : "jbossorg_sbs_forum",
                    "dcp_content_content-type" : "text/html"
                 } 
            },{ 
                "name"     : "DCP Updated filler",
                "class"    : "org.jboss.elasticsearch.tools.content.AddCurrentTimestampPreprocessor",
                "settings" : { "field" : "dcp_updated" }
            },{ 
                "name"     : "comment body HTML stripper",
                "class"    : "org.jboss.elasticsearch.tools.content.StripHtmlPreprocessor",
                "settings" : {
                    "source_field" : "content",
                    "target_field" : "content",
                    "source_bases" : ["comments"]
                } 
            },{ 
                "name"     : "DCP description HTML stripper",
                "class"    : "org.jboss.elasticsearch.tools.content.StripHtmlPreprocessor",
                "settings" : {
                    "source_field" : "content",
                    "target_field" : "dcp_description"
                } 
            },{ 
                "name"     : "DCP description trimmer",
                "class"    : "org.jboss.elasticsearch.tools.content.TrimStringValuePreprocessor",
                "settings" : {
                    "source_field" : "dcp_description",<commit_msg>sbs forum river configuration improved<commit_after>{
    "type" : "remote",
    "remote" : {
        "urlGetDocuments"       : "https://community-dev.jboss.org/data-api/content.jspa?spaceId={space}&type=forum&updatedAfter={updatedAfter}&maxSize=50",
        "getDocsResFieldDocuments"  : "items",
        "urlGetSpaces"					: "https://community-dev.jboss.org/data-api/spaces.jspa",
        "getSpacesResField"			: "spaces",
        "timeout"               : "20s",
        "indexUpdatePeriod"     : "10m",
        "indexFullUpdatePeriod" : "10d",
        "maxIndexingThreads"    : 2
    },
    "index" : {
        "index"             : "data_jbossorg_sbs_forum",
        "type"              : "jbossorg_sbs_forum",
        "remote_field_document_id" : "id",
	      "remote_field_updated"     : "published",
        "field_space_key"   : "space_key",
        "field_document_id" : "dcp_id",
        "fields" : {
            "dcp_content_id"        : {"remote_field" : "id_remote"},
            "dcp_type"              : {"remote_field" : "dcp_type"},
            "dcp_updated"           : {"remote_field" : "dcp_updated"},
            "dcp_content_provider"  : {"remote_field" : "dcp_content_provider"},
            "dcp_content_type"      : {"remote_field" : "dcp_content_type"},
            "dcp_activity_dates"    : {"remote_field" : "dcp_activity_dates"},
            "dcp_last_activity_date": {"remote_field" : "dcp_last_activity_date"},
            "dcp_project"           : {"remote_field" : "dcp_project"},
            "dcp_project_name"      : {"remote_field" : "dcp_project_name"},
            "dcp_url_view"					: {"remote_field" : "url"},
            "dcp_contributors"      : {"remote_field" : "dcp_contributors"},
            "dcp_title"             : {"remote_field" : "title"},
            "dcp_created"           : {"remote_field" : "published"},
            "dcp_description"       : {"remote_field" : "dcp_description"},
            "dcp_content"           : {"remote_field" : "content"},
            "dcp_content_content-type" : {"remote_field" : "dcp_content_content-type"},
            "tags"                  : {"remote_field" : "tags"},
            "dcp_tags"              : {"remote_field" : "tags"},
            "authors"               : {"remote_field" : "authors", "value_filter" : "user"}
        },
        "value_filters" : {
            "user" : {
                "email"           : "email_address",
                "full_name"       : "display_name",
                "dcp_contributor" : "dcp_contributor" 
            }
        },
        "comment_mode"   : "embedded",
        "remote_field_comments" : "comments",
        "field_comments" : "dcp_comments",
        "comment_fields" : {
            "comment_body"    : {"remote_field" : "content"},
            "comment_author"  : {"remote_field" : "author", "value_filter" : "user"},
            "comment_created" : {"remote_field" : "published"}
        },
        "preprocessors" : [
            {
                "name"     : "Remote id copy",
                "class"    : "org.jboss.elasticsearch.tools.content.AddMultipleValuesPreprocessor",
                "settings" : {
                    "id_remote"             : "{id}"
                 } 
            },{
                "name"     : "DCP values filler",
                "class"    : "org.jboss.elasticsearch.tools.content.AddMultipleValuesPreprocessor",
                "settings" : {
                    "id"                    : "jbossorg_sbs_forum-{id}",
                    "dcp_type"              : "forumthread",
                    "dcp_content_provider"  : "jbossorg",
                    "dcp_content_type"      : "jbossorg_sbs_forum",
                    "dcp_content_content-type" : "text/html"
                 } 
            },{ 
                "name"     : "DCP Updated filler",
                "class"    : "org.jboss.elasticsearch.tools.content.AddCurrentTimestampPreprocessor",
                "settings" : { "field" : "dcp_updated" }
            },{
                "name"     : "published timestamp convertor",
                "class"    : "org.jboss.elasticsearch.river.remote.content.LongToTimestampValuePreprocessor",
                "settings" : {
                    "source_field" : "published",
                    "target_field" : "published"
                } 
            },{
                "name"     : "updated timestamp convertor",
                "class"    : "org.jboss.elasticsearch.river.remote.content.LongToTimestampValuePreprocessor",
                "settings" : {
                    "source_field" : "updated",
                    "target_field" : "updated"
                } 
            },{
                "name"     : "comments timestamp convertor",
                "class"    : "org.jboss.elasticsearch.river.remote.content.LongToTimestampValuePreprocessor",
                "settings" : {
                    "source_field" : "published",
                    "target_field" : "published",
                    "source_bases" : ["comments"]
                } 
            },{ 
                "name"     : "comment body HTML stripper",
                "class"    : "org.jboss.elasticsearch.tools.content.StripHtmlPreprocessor",
                "settings" : {
                    "source_field" : "content",
                    "target_field" : "content",
                    "source_bases" : ["comments"]
                } 
            },{ 
                "name"     : "DCP description HTML stripper",
                "class"    : "org.jboss.elasticsearch.tools.content.StripHtmlPreprocessor",
                "settings" : {
                    "source_field" : "content",
                    "target_field" : "dcp_description"
                } 
            },{ 
                "name"     : "DCP description trimmer",
                "class"    : "org.jboss.elasticsearch.tools.content.TrimStringValuePreprocessor",
                "settings" : {
                    "source_field" : "dcp_description",
{'fraction_non_alphanumeric': 0.13523844193665818, 'fraction_numerical': 0.0016381507098653077, 'mean_word_length': 1.5402219140083218, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3738474', 'n_tokens_mistral': 3313, 'n_tokens_neox': 3103, 'n_words': 421}