<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import from regular python\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sb\n",
    "\n",
    "cvr_Path = \"/home/svanhmic/workspace/DABAI/ReadData\" # TODB alter path!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports from Pyspark\n",
    "\n",
    "from pyspark.sql import functions as F\n",
    "from pyspark.ml.feature import VectorAssembler, StandardScaler\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.clustering import KMeans\n",
    "\n",
    "sc.addPyFile(cvr_Path+\"/ConvertAllToVecToMl.py\")\n",
    "import ConvertAllToVecToMl # included homebrewed transformation for converting input vector to double vector. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_Pdf = sb.load_dataset(\"flights\") # naive data set from seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create spark data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],<commit_msg>seneste ændringer<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import from regular python\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sb\n",
    "\n",
    "cvr_Path = \"/home/svanhmic/workspace/DABAI/ReadData\" # TODB alter path!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#imports from Pyspark\n",
    "\n",
    "from pyspark.sql import functions as F\n",
    "from pyspark.ml.feature import VectorAssembler, StandardScaler\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.clustering import KMeans\n",
    "\n",
    "sc.addPyFile(cvr_Path+\"/ConvertAllToVecToMl.py\")\n",
    "import ConvertAllToVecToMl # included homebrewed transformation for converting input vector to double vector. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>passengers</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1949</td>\n",
       "      <td>January</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1949</td>\n",
       "      <td>February</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1949</td>\n",
       "      <td>March</td>\n",
       "      <td>132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1949</td>\n",
       "      <td>April</td>\n",
       "      <td>129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1949</td>\n",
       "      <td>May</td>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1949</td>\n",
       "      <td>June</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1949</td>\n",
       "      <td>July</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1949</td>\n",
       "      <td>August</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1949</td>\n",
       "      <td>September</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1949</td>\n",
       "      <td>October</td>\n",
       "      <td>119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1949</td>\n",
       "      <td>November</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1949</td>\n",
       "      <td>December</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1950</td>\n",
       "      <td>January</td>\n",
       "      <td>115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1950</td>\n",
       "      <td>February</td>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1950</td>\n",
       "      <td>March</td>\n",
       "      <td>141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1950</td>\n",
       "      <td>April</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1950</td>\n",
       "      <td>May</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1950</td>\n",
       "      <td>June</td>\n",
       "      <td>149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1950</td>\n",
       "      <td>July</td>\n",
       "      <td>170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1950</td>\n",
       "      <td>August</td>\n",
       "      <td>170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1950</td>\n",
       "      <td>September</td>\n",
       "      <td>158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1950</td>\n",
       "      <td>October</td>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1950</td>\n",
       "      <td>November</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1950</td>\n",
       "      <td>December</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1951</td>\n",
       "      <td>January</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1951</td>\n",
       "      <td>February</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1951</td>\n",
       "      <td>March</td>\n",
       "      <td>178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1951</td>\n",
       "      <td>April</td>\n",
       "      <td>163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1951</td>\n",
       "      <td>May</td>\n",
       "      <td>172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1951</td>\n",
       "      <td>June</td>\n",
       "      <td>178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>1958</td>\n",
       "      <td>July</td>\n",
       "      <td>491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>1958</td>\n",
       "      <td>August</td>\n",
       "      <td>505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>1958</td>\n",
       "      <td>September</td>\n",
       "      <td>404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>1958</td>\n",
       "      <td>October</td>\n",
       "      <td>359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>1958</td>\n",
       "      <td>November</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>1958</td>\n",
       "      <td>December</td>\n",
       "      <td>337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>1959</td>\n",
       "      <td>January</td>\n",
       "      <td>360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>1959</td>\n",
       "      <td>February</td>\n",
       "      <td>342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>1959</td>\n",
       "      <td>March</td>\n",
       "      <td>406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>1959</td>\n",
       "      <td>April</td>\n",
       "      <td>396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>1959</td>\n",
       "      <td>May</td>\n",
       "      <td>420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>1959</td>\n",
       "      <td>June</td>\n",
       "      <td>472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>1959</td>\n",
       "      <td>July</td>\n",
       "      <td>548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>1959</td>\n",
       "      <td>August</td>\n",
       "      <td>559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>1959</td>\n",
       "      <td>September</td>\n",
       "      <td>463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>1959</td>\n",
       "      <td>October</td>\n",
       "      <td>407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>1959</td>\n",
       "      <td>November</td>\n",
       "      <td>362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>1959</td>\n",
       "      <td>December</td>\n",
       "      <td>405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>1960</td>\n",
       "      <td>January</td>\n",
       "      <td>417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>1960</td>\n",
       "      <td>February</td>\n",
       "      <td>391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>1960</td>\n",
       "      <td>March</td>\n",
       "      <td>419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>1960</td>\n",
       "      <td>April</td>\n",
       "      <td>461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>1960</td>\n",
       "      <td>May</td>\n",
       "      <td>472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>1960</td>\n",
       "      <td>June</td>\n",
       "      <td>535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>1960</td>\n",
       "      <td>July</td>\n",
       "      <td>622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>1960</td>\n",
       "      <td>August</td>\n",
       "      <td>606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>1960</td>\n",
       "      <td>September</td>\n",
       "      <td>508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>1960</td>\n",
       "      <td>October</td>\n",
       "      <td>461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>1960</td>\n",
       "      <td>November</td>\n",
       "      <td>390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>1960</td>\n",
       "      <td>December</td>\n",
       "      <td>432</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>144 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     year      month  passengers\n",
       "0    1949    January         112\n",
       "1    1949   February         118\n",
       "2    1949      March         132\n",
       "3    1949      April         129\n",
       "4    1949        May         121\n",
       "5    1949       June         135\n",
       "6    1949       July         148\n",
       "7    1949     August         148\n",
       "8    1949  September         136\n",
       "9    1949    October         119\n",
       "10   1949   November         104\n",
       "11   1949   December         118\n",
       "12   1950    January         115\n",
       "13   1950   February         126\n",
       "14   1950      March         141\n",
       "15   1950      April         135\n",
       "16   1950        May         125\n",
       "17   1950       June         149\n",
       "18   1950       July         170\n",
       "19   1950     August         170\n",
       "20   1950  September         158\n",
       "21   1950    October         133\n",
       "22   1950   November         114\n",
       "23   1950   December         140\n",
       "24   1951    January         145\n",
       "25   1951   February         150\n",
       "26   1951      March         178\n",
       "27   1951      April         163\n",
       "28   1951        May         172\n",
       "29   1951       June         178\n",
       "..    ...        ...         ...\n",
       "114  1958       July         491\n",
       "115  1958     August         505\n",
       "116  1958  September         404\n",
       "117  1958    October         359\n",
       "118  1958   November         310\n",
       "119  1958   December         337\n",
       "120  1959    January         360\n",
       "121  1959   February         342\n",
       "122  1959      March         406\n",
       "123  1959      April         396\n",
       "124  1959        May         420\n",
       "125  1959       June         472\n",
       "126  1959       July         548\n",
       "127  1959     August         559\n",
       "128  1959  September         463\n",
       "129  1959    October         407\n",
       "130  1959   November         362\n",
       "131  1959   December         405\n",
       "132  1960    January         417\n",
       "133  1960   February         391\n",
       "134  1960      March         419\n",
       "135  1960      April         461\n",
       "136  1960        May         472\n",
       "137  1960       June         535\n",
       "138  1960       July         622\n",
       "139  1960     August         606\n",
       "140  1960  September         508\n",
       "141  1960    October         461\n",
       "142  1960   November         390\n",
       "143  1960   December         432\n",
       "\n",
       "[144 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_Pdf = sb.load_dataset(\"flights\") # naive data set from seaborn\n",
    "data_Pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#create spark data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
{'fraction_non_alphanumeric': 0.23604030823947836, 'fraction_numerical': 0.06799051570835803, 'mean_word_length': 1.043483527131783, 'pattern_counts': {'":': 63, '<': 633, '<?xml version=': 0, '>': 633, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3363312', 'n_tokens_mistral': 7309, 'n_tokens_neox': 6448, 'n_words': 885}