<commit_before>// Databricks notebook source exported at Thu, 31 Mar 2016 03:05:11 UTC
// MAGIC %md
// MAGIC 
// MAGIC # [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)
// MAGIC 
// MAGIC 
// MAGIC ### prepared by [<NAME>](https://nz.linkedin.com/in/raazesh-sainudiin-45955845) and [<NAME>](https://www.linkedin.com/in/sivanand)
// MAGIC 
// MAGIC *supported by* [![](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/databricks_logoTM_200px.png)](https://databricks.com/)
// MAGIC and 
// MAGIC [![](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/AWS_logoTM_200px.png)](https://www.awseducate.com/microsite/CommunitiesEngageHome)

// COMMAND ----------

// MAGIC %md
// MAGIC # Tweet Collector - capture live tweets
// MAGIC 
// MAGIC ### First let's take the twitter stream and write to DBFS as json files

// COMMAND ----------

import org.apache.spark._
import org.apache.spark.storage._
import org.apache.spark.streaming._<commit_msg>remareked on non-robustness of this and the pointer to TweetGenericCollector notebook<commit_after>// Databricks notebook source exported at Thu, 19 May 2016 22:46:37 UTC
// MAGIC %md
// MAGIC 
// MAGIC # [Scalable Data Science](http://www.math.canterbury.ac.nz/~r.sainudiin/courses/ScalableDataScience/)
// MAGIC 
// MAGIC 
// MAGIC ### prepared by [<NAME>](https://nz.linkedin.com/in/raazesh-sainudiin-45955845) and [<NAME>](https://www.linkedin.com/in/sivanand)
// MAGIC 
// MAGIC *supported by* [![](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/databricks_logoTM_200px.png)](https://databricks.com/)
// MAGIC and 
// MAGIC [![](https://raw.githubusercontent.com/raazesh-sainudiin/scalable-data-science/master/images/AWS_logoTM_200px.png)](https://www.awseducate.com/microsite/CommunitiesEngageHome)

// COMMAND ----------

// MAGIC %md
// MAGIC # Tweet Collector - capture live tweets
// MAGIC 
// MAGIC ### First let's take the twitter stream and write to DBFS as json files
// MAGIC 
// MAGIC #### See the notebook 022_TweetGenericCollector (this notebook is not robust and it is only for demo)

// COMMAND ----------

import org.apache.spark._
import org.apache.spark.storage._
import org.apache.spark.streaming._
{'fraction_non_alphanumeric': 0.16588084012001714, 'fraction_numerical': 0.02357479639948564, 'mean_word_length': 3.773006134969325, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 12, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5006507', 'n_tokens_mistral': 915, 'n_tokens_neox': 785, 'n_words': 160}