<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> This notebook convert a mailing list (or a set of mailing lists) into a network of interaction</b>\n",
    "\n",
    "What it does:\n",
    "-it creates a network of interaction between senders and receivers of emails, on one or more mailing lists\n",
    "-it generates a .gexf file that can be imported in Gephi for visualization and analysis\n",
    "\n",
    "Parameters to set options:\n",
    "-it can look in one or more mailing lists, according to how many urls are set in the ‘urls’ variable; networks are aggregated across mailing lists\n",
    "-it can filter the network by date; set the variable 'date_from' and 'date_to' with a date frame consistent with the data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from bigbang.archive import Archive\n",
    "import bigbang.parse as parse\n",
    "import bigbang.graph as graph\n",
    "import bigbang.mailman as mailman\n",
    "import bigbang.process as process\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from pprint import pprint as pp\n",
    "import pytz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Opening 19 archive files\n",
      "Opening 5 archive files\n",
      "Opening 19 archive files\n"
     ]
    }
   ],
   "source": [
    "#Insert the list of urls (one or more) from which to gather the data\n",
    "#e.g. urls = [urls = [\"http://mm.icann.org/pipermail/cc-humanrights/\", \n",
    "                    # \"http://mm.icann.org/pipermail/wp4/\", \n",
    "                    # \"http://mm.icann.org/pipermail/ge/\"]\n",
    "\n",
    "        \n",
    "urls = [\"http://mm.icann.org/pipermail/cc-humanrights/\", \n",
    "                \"http://mm.icann.org/pipermail/wp4/\", \n",
    "                \"http://mm.icann.org/pipermail/ge/\"]\n",
    "\n",
    "\n",
    "archives= [mailman.open_list_archives(url) for url in urls]\n",
    "archives_merged = pd.concat(archives)\n",
    "archives_data = Archive(archives_merged).data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set a valid date frame for building the network. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2014-12-02 01:06:24\n",
      "2016-06-30 09:48:43\n"
     ]
    }
   ],
   "source": [
    "#The oldest date and more recent date for the whole mailing lists are displayed, so you WON't set an invalid time frame \n",
    "print archives_data['Date'].min()\n",
    "print archives_data['Date'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#set the date frame\n",
    "date_from = pd.datetime(2000,11,1,tzinfo=pytz.utc)\n",
    "date_to = pd.datetime(2111,12,1,tzinfo=pytz.utc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter data according to date frame and export to .gexf file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def filter_by_date(df,d_from,d_to):\n",
    "    return df[(df['Date'] > d_from) & (df['Date'] < d_to)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#create filtered network\n",
    "archives_data_filtered = filter_by_date(archives_data, date_from, date_to)\n",
    "network = graph.messages_to_interaction_graph(archives_data_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#export the network in a format that you can open in Gephi. \n",
    "\n",
    "#insert a valid path and file name (e.g. path = 'c:/bigbang/network.gexf')\n",
    "path = 'c:/users/davide/bigbang/network_for_gephi.gexf'\n",
    "\n",
    "nx.write_gexf(network, path)\n",
    "    \n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2<commit_msg>fixed ml import issue<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b> This notebook convert a mailing list (or a set of mailing lists) into a network of interaction</b>\n",
    "\n",
    "What it does:\n",
    "-it creates a network of interaction between senders and receivers of emails, on one or more mailing lists\n",
    "-it generates a .gexf file that can be imported in Gephi for visualization and analysis\n",
    "\n",
    "Parameters to set options:\n",
    "-it can look in one or more mailing lists, according to how many urls are set in the ‘urls’ variable; networks are aggregated across mailing lists\n",
    "-it can filter the network by date; set the variable 'date_from' and 'date_to' with a date frame consistent with the data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from bigbang.archive import Archive\n",
    "from bigbang.archive import load as load_archive\n",
    "import bigbang.parse as parse\n",
    "import bigbang.graph as graph\n",
    "import bigbang.mailman as mailman\n",
    "import bigbang.process as process\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from pprint import pprint as pp\n",
    "import pytz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Insert the list of urls (one or more) from which to gather the data\n",
    "#e.g. urls = [urls = [\"http://mm.icann.org/pipermail/cc-humanrights/\", \n",
    "                    # \"http://mm.icann.org/pipermail/wp4/\", \n",
    "                    # \"http://mm.icann.org/pipermail/ge/\"]\n",
    "\n",
    "        \n",
    "urls = [\"http://mm.icann.org/pipermail/cc-humanrights/\", \n",
    "                \"http://mm.icann.org/pipermail/wp4/\", \n",
    "                \"http://mm.icann.org/pipermail/wp1/\"]\n",
    "\n",
    "\n",
    "try:\n",
    "    arch_paths =[]\n",
    "    for url in urls:\n",
    "        arch_paths.append('../archives/'+url[:-1].replace('://','_/')+'.csv')\n",
    "    archives = [load_archive(arch_path).data for arch_path in arch_paths]\n",
    "except:\n",
    "    arch_paths =[]\n",
    "    for url in urls:\n",
    "        arch_paths.append('../archives/'+url[:-1].replace('//','/')+'.csv')\n",
    "    archives = [load_archive(arch_path).data for arch_path in arch_paths]\n",
    "archives_merged = pd.concat(archives)\n",
    "archives_data = Archive(archives_merged).data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set a valid date frame for building the network. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2014-12-02 01:06:24\n",
      "2016-06-27 14:39:29\n"
     ]
    }
   ],
   "source": [
    "#The oldest date and more recent date for the whole mailing lists are displayed, so you WON't set an invalid time frame \n",
    "print archives_data['Date'].min()\n",
    "print archives_data['Date'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#set the date frame\n",
    "date_from = pd.datetime(2000,11,1,tzinfo=pytz.utc)\n",
    "date_to = pd.datetime(2111,12,1,tzinfo=pytz.utc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filter data according to date frame and export to .gexf file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def filter_by_date(df,d_from,d_to):\n",
    "    return df[(df['Date'] > d_from) & (df['Date'] < d_to)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#create filtered network\n",
    "archives_data_filtered = filter_by_date(archives_data, date_from, date_to)\n",
    "network = graph.messages_to_interaction_graph(archives_data_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#export the network in a format that you can open in Gephi. \n",
    "\n",
    "#insert a valid path and file name (e.g. path = 'c:/bigbang/network.gexf')\n",
    "path = 'c:/users/davide/bigbang/network_for_gephi.gexf'\n",
    "\n",
    "nx.write_gexf(network, path)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
{'fraction_non_alphanumeric': 0.18629737609329447, 'fraction_numerical': 0.012244897959183673, 'mean_word_length': 2.492025788937903, 'pattern_counts': {'":': 155, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2970357', 'n_tokens_mistral': 3913, 'n_tokens_neox': 3752, 'n_words': 870}