<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f561a08-a7db-46a8-8efa-fb3637a7e580",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "import pyspark\n",
    "from pyspark.sql.functions import sum\n",
    "from pyspark.sql.functions import col, udf\n",
    "from pyspark.sql.types import StringType\n",
    "from pyspark.sql.types import IntegerType\n",
    "from wordcloud import WordCloud, STOPWORDS\n",
    "import matplotlib.pyplot as plt\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb2c9256-8978-43a1-a10f-be8dbe103280",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "#Creating spark session.\n",
    "spark = SparkSession.builder.appName('twitter_data').getOrCreate()"
   ]<commit_msg>Added Apache header<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f561a08-a7db-46a8-8efa-fb3637a7e580",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "import pyspark\n",
    "from pyspark.sql.functions import sum\n",
    "from pyspark.sql.functions import col, udf\n",
    "from pyspark.sql.types import StringType\n",
    "from pyspark.sql.types import IntegerType\n",
    "from wordcloud import WordCloud, STOPWORDS\n",
    "import matplotlib.pyplot as plt\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb2c9256-8978-43a1-a10f-be8dbe103280",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "#Copyright 2022 Google LLC.\n",
    "#SPDX-License-Identifier: Apache-2.0\n",
    "#Creating spark session.\n",
    "spark = SparkSession.builder.appName('twitter_data').getOrCreate()"
   ]
{'fraction_non_alphanumeric': 0.17667493796526054, 'fraction_numerical': 0.042679900744416875, 'mean_word_length': 3.0159362549800797, 'pattern_counts': {'":': 34, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '929905', 'n_tokens_mistral': 832, 'n_tokens_neox': 782, 'n_words': 138}