<commit_before>﻿{
  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "logicappname": {
      "defaultValue": "LogicApp",
      "type": "string"
    },
    "resourcegroup": {
      "defaultValue": "stresourcegroup",
      "type": "string"
    },
    "subscription": {
      "defaultValue": "idrequired",
      "type": "string"
    },
    "imagecachelogicapp": {
      "defaultValue": "ImageCacheLogicApp",
      "type": "string"
    },
    "sitename": {
      "defaultValue": "sitename",
      "type": "string"
    },
    "searchquery": {
      "defaultValue": "searchQuery",
      "type": "string"
    }
  },
    "variables": {},
    "resources": [
      {
        "type": "Microsoft.Logic/workflows",
        "name": "[parameters('logicappname')]",
        "apiVersion": "2016-06-01",
        "location": "[resourceGroup().location]",
        "properties": {
          "state": "Enabled",
          "definition": {
            "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#",
            "contentVersion": "1.0.0.0",
            "parameters": {
              "$connections": {
                "defaultValue": {},
                "type": "Object"
              }
            },
            "triggers": {
              "On_new_news_article": {
                "inputs": {
                  "host": {
                    "connection": {
                      "name": "@parameters('$connections')['bingsearch']['connectionId']"
                    }
                  },
                  "method": "get",
                  "path": "/trigger/news/search",
                  "queries": {
                    "count": "20",
                    "mkt": "en-US",
                    "offset": "0",
                    "q": "[parameters('searchquery')]",
                    "safeSearch": "Moderate"
                  }
                },
                "recurrence": {
                  "frequency": "Minute",
                  "interval": 5
                },
                "splitOn": "@triggerBody()",
                "type": "ApiConnection"
              }
            },
            "actions": {
			            "AbstractExtractor": {
                "inputs": {
                    "body": {
                        "query": "@{trigger()['inputs']['queries']['q']}",
                        "text": "@{body('HTMLStripperWebHook')['Scrubbed']}"
                    },
                    "function": {
                        "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/AbstractExtractor')]"
                    }
                },
                "runAfter": {
                    "HTMLStripperWebHook": [
                        "Succeeded"
                    ]
                },
                "type": "Function"
            },
              "ArticleCleanerWebHook": {
                "inputs": {
                  "body": {
                    "text": "@body('ExtractArticle')['bodyInHtml']"
                  },
                  "function": {
                    "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/ASCIICleanerWebhook')]"
                  }
                },
                "runAfter": {
                  "ExtractArticle": [
                    "Succeeded"
                  ]
                },
                "type": "Function"
              },
              "Detect_Sentiment": {
                "inputs": {
                  "body": {
                    "text": "@{substring(body('HTMLStripperWebHook')['NoTags'], 0, min(length(body('HTMLStripperWebHook')['NoTags']),5000))}"
                  },
                  "host": {

                    "connection": {
                      "name": "@parameters('$connections')['cognitiveservicestextanalytics']['connectionId']"
                    }
                  },
                  "method": "post",
                  "path": "/sentiment"
                },
                "runAfter": {
                    "AbstractExtractor": [
                        "Succeeded"
                    ]
                },
                "type": "ApiConnection"
              },
              "ExtractArticle": {
                "inputs": {
                  "body": {
                    "html": "@{body('HTTP')}",
                    "url": "@Json(body('UrlCleaner'))?['newUri']"
                  },
                  "function": {
                    "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/ArticleExtractor')]"
                  }
                },
                "runAfter": {
                  "HTTP": [
                    "Succeeded"
                  ]
                },
                "type": "Function"
              },
              "UrlCleaner": {
                "inputs": {
                  "body": {
                    "url": "@{triggerBody()?['url']}"
                  },
                  "function": {
                    "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/UrlCleaner')]"
                  }
                },
                "runAfter": {},
                "type": "Function"
              },
              "HTMLStripperWebHook": {
                "inputs": {
                  "body": {
                    "html": "@{body('ArticleCleanerWebHook')['text']}"
                  },
                  "function": {
                    "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/HTMLCleanerWebhook')]"
                  }
                },
                "runAfter": {
                  "ArticleCleanerWebHook": [
                    "Succeeded"
                  ]
                },
                "type": "Function"
              },
              "HTTP": {
                "inputs": {
                  "method": "GET",
                  "uri": "@Json(body('UrlCleaner'))?['newUri']"
                },
                "runAfter": {
                  "UrlCleaner": [
                    "Succeeded"
                  ]
                },
                "type": "Http"
              },
              "ImageCacherApp": {
                "inputs": {
                  "body": {
                    "Request": {
                      "DocumentID": "@Json(body('UrlCleaner'))?['triggerId']",
                      "imgUrl": "@triggerBody()?['image']?['thumbnail']?['contentUrl']"
                    }
                  },
                  "host": {
                    "triggerName": "manual",
                    "workflow": {
                      "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Logic/workflows/', parameters('imagecachelogicapp'))]"
                    }
                  }
                },
                "runAfter": {
                  "UrlDomainWebHook": [
                    "Succeeded"
                  ]
                },
                "type": "Workflow"
              },
              "InsertDocument": {
                "runAfter": {
                  "Detect_Sentiment": [
                    "Succeeded"
                  ],
                  "TimeUtilsWebHook": [
                    "Succeeded"
                  ],
                  "TimeUtilsWebHook_2": [
                    "Succeeded"
                  ],
                  "ImageCacherApp": [
                    "Succeeded"
                  ],
                  "Key_Phrases": [
                    "Succeeded"
                  ]
                },
                "type": "ApiConnection",
                "inputs": {
                  "body": {
                    "category": "@{triggerBody()?['category']}",
                    "cleanedText": "@{body('HTMLStripperWebHook')['NoTags']}",
                    "cleanedTextLength": "@{body('HTMLStripperWebHook')['NoTagsLength']}",
                    "abstract": "@{body('AbstractExtractor')?['Snippet']}",
                    "docid": "@Json(body('UrlCleaner'))?['triggerId']",
                    "imageHeight": "@triggerBody()?['image']?['thumbnail']?['height']",
                    "imageUrl": "@{body('ImageCacherApp')}",
                    "imageWidth": "@triggerBody()?['image']?['thumbnail']?['width']",<commit_msg>Updates to logic app for news<commit_after>﻿{
  "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "logicappname": {
      "defaultValue": "LogicApp",
      "type": "string"
    },
    "resourcegroup": {
      "defaultValue": "stresourcegroup",
      "type": "string"
    },
    "subscription": {
      "defaultValue": "idrequired",
      "type": "string"
    },
    "sitename": {
      "defaultValue": "sitename",
      "type": "string"
    },
    "searchquery": {
      "defaultValue": "searchQuery",
      "type": "string"
    }
  },
    "variables": {},
    "resources": [
      {
        "type": "Microsoft.Logic/workflows",
        "name": "[parameters('logicappname')]",
        "apiVersion": "2016-06-01",
        "location": "[resourceGroup().location]",
        "properties": {
          "state": "Enabled",
          "definition": {
            "$schema": "https://schema.management.azure.com/providers/Microsoft.Logic/schemas/2016-06-01/workflowdefinition.json#",
            "contentVersion": "1.0.0.0",
            "parameters": {
              "$connections": {
                "defaultValue": {},
                "type": "Object"
              }
            },
            "triggers": {
              "On_new_news_article": {
                "inputs": {
                  "host": {
                    "connection": {
                      "name": "@parameters('$connections')['bingsearch']['connectionId']"
                    }
                  },
                  "method": "get",
                  "path": "/trigger/news/search",
                  "queries": {
                    "count": "20",
                    "mkt": "en-US",
                    "offset": "0",
                    "q": "[parameters('searchquery')]",
                    "safeSearch": "Moderate"
                  }
                },
                "recurrence": {
                  "frequency": "Minute",
                  "interval": 5
                },
                "splitOn": "@triggerBody()",
                "type": "ApiConnection"
              }
            },
            "actions": {
			            "AbstractExtractor": {
                "inputs": {
                    "body": {
                        "query": "@{trigger()['inputs']['queries']['q']}",
                        "text": "@{body('HTMLStripperWebHook')['Scrubbed']}"
                    },
                    "function": {
                        "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/AbstractExtractor')]"
                    }
                },
                "runAfter": {
                    "HTMLStripperWebHook": [
                        "Succeeded"
                    ]
                },
                "type": "Function"
            },
              "ArticleCleanerWebHook": {
                "inputs": {
                  "body": {
                    "text": "@body('ExtractArticle')['bodyInHtml']"
                  },
                  "function": {
                    "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/ASCIICleanerWebhook')]"
                  }
                },
                "runAfter": {
                  "ExtractArticle": [
                    "Succeeded"
                  ]
                },
                "type": "Function"
              },
              "Detect_Sentiment": {
                "inputs": {
                  "body": {
                    "text": "@{substring(body('HTMLStripperWebHook')['NoTags'], 0, min(length(body('HTMLStripperWebHook')['NoTags']),5000))}"
                  },
                  "host": {

                    "connection": {
                      "name": "@parameters('$connections')['cognitiveservicestextanalytics']['connectionId']"
                    }
                  },
                  "method": "post",
                  "path": "/sentiment"
                },
                "runAfter": {
                    "AbstractExtractor": [
                        "Succeeded"
                    ]
                },
                "type": "ApiConnection"
              },
              "ExtractArticle": {
                "inputs": {
                  "body": {
                    "html": "@{body('HTTP')}",
                    "url": "@Json(body('UrlCleaner'))?['newUri']"
                  },
                  "function": {
                    "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/ArticleExtractor')]"
                  }
                },
                "runAfter": {
                  "HTTP": [
                    "Succeeded"
                  ]
                },
                "type": "Function"
              },
              "UrlCleaner": {
                "inputs": {
                  "body": {
                    "url": "@{triggerBody()?['url']}"
                  },
                  "function": {
                    "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/UrlCleaner')]"
                  }
                },
                "runAfter": {},
                "type": "Function"
              },
              "HTMLStripperWebHook": {
                "inputs": {
                  "body": {
                    "html": "@{body('ArticleCleanerWebHook')['text']}"
                  },
                  "function": {
                    "id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/HTMLCleanerWebhook')]"
                  }
                },
                "runAfter": {
                  "ArticleCleanerWebHook": [
                    "Succeeded"
                  ]
                },
                "type": "Function"
              },
              "HTTP": {
                "inputs": {
                  "method": "GET",
                  "uri": "@Json(body('UrlCleaner'))?['newUri']"
                },
                "runAfter": {
                  "UrlCleaner": [
                    "Succeeded"
                  ]
                },
                "type": "Http"
              },
              "MainImageExtractor": {
                "inputs": {
                  "body": {
                    "Request": {
                      "DocumentID": "@Json(body('UrlCleaner'))?['triggerId']",
                      "imgUrl": "@triggerBody()?['image']?['thumbnail']?['contentUrl']"
                    }
                  },
					"function": {
					"id": "[concat('/subscriptions/', parameters('subscription'), '/resourceGroups/',parameters('resourcegroup'),'/providers/Microsoft.Web/sites/', parameters('sitename'), '/Functions/MainImageExtractor')]"
                  }
                },
                "runAfter": {
                  "HTTP": [
                    "Succeeded"
                  ]
                },
                "type": "Function"
              },
              "InsertDocument": {
                "runAfter": {
                  "Detect_Sentiment": [
                    "Succeeded"
                  ],
                  "TimeUtilsWebHook": [
                    "Succeeded"
                  ],
                  "TimeUtilsWebHook_2": [
                    "Succeeded"
                  ],
                  "MainImageExtractor": [
                    "Succeeded"
                  ],
				  "UrlDomainWebHook": [
				  "Succeeded"
				  ],
                  "Key_Phrases": [
                    "Succeeded"
                  ]
                },
                "type": "ApiConnection",
                "inputs": {
                  "body": {
                    "category": "@{triggerBody()?['category']}",
                    "cleanedText": "@{body('HTMLStripperWebHook')['NoTags']}",
                    "cleanedTextLength": "@{body('HTMLStripperWebHook')['NoTagsLength']}",
                    "abstract": "@{body('AbstractExtractor')?['Snippet']}",
                    "docid": "@Json(body('UrlCleaner'))?['triggerId']",
                    "imageHeight": "@triggerBody()?['image']?['thumbnail']?['height']",
                    "imageUrl": "@Json(body('MainImageExtractor'))?['ImageUrl']",
                    "imageWidth": "@triggerBody()?['image']?['thumbnail']?['width']",
{'fraction_non_alphanumeric': 0.16335291459557164, 'fraction_numerical': 0.004744690465431541, 'mean_word_length': 1.1473620375985445, 'pattern_counts': {'":': 300, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '382031', 'n_tokens_mistral': 4694, 'n_tokens_neox': 4223, 'n_words': 525}