<commit_before>// where jdk is installed
export JAVA_HOME="/usr/local/jdk-6u22-64"

// At least current dir and mysql connector jar should be in CLASSPATH
// Mysql connector jar can be downloaded from http://dev.mysql.com/downloads/connector/j/.
// Download version 5.0.8 or higher. Copy the mysql jar to the lib directory.
export CLASSPATH=.:<path to mysql-connector-java-5.0.8-bin.jar>

export PATH=$JAVA_HOME/bin:$PATH

// compile
ant clean dist

// create tables in mysql
create database linkdb;

use linkdb;

CREATE TABLE `linktable` (
  `id1` bigint(20) unsigned NOT NULL DEFAULT '0',
  `id1_type` int(10) unsigned NOT NULL DEFAULT '0',
  `id2` bigint(20) unsigned NOT NULL DEFAULT '0',
  `id2_type` int(10) unsigned NOT NULL DEFAULT '0',
  `link_type` bigint(20) unsigned NOT NULL DEFAULT '0',
  `visibility` tinyint(3) NOT NULL DEFAULT '0',
  `data` varchar(255) NOT NULL DEFAULT '',
  `time` bigint(20) unsigned NOT NULL DEFAULT '0',
  `version` int(11) unsigned NOT NULL DEFAULT '0',
  PRIMARY KEY (`id1`,`id2`,`link_type`),
  KEY `id2_vis` (`id2`,`visibility`),
  KEY `id1_type` (`id1`,`link_type`,`visibility`,`time`,`version`,`data`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

CREATE TABLE `counttable` (
  `id` bigint(20) unsigned NOT NULL DEFAULT '0',
  `id_type` int(10) unsigned NOT NULL DEFAULT '0',
  `link_type` bigint(20) unsigned NOT NULL DEFAULT '0',
  `count` int(10) unsigned NOT NULL DEFAULT '0',
  `time` bigint(20) unsigned NOT NULL DEFAULT '0',
  `version` bigint(20) unsigned NOT NULL DEFAULT '0',
  PRIMARY KEY (`id`,`link_type`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

// Examine LinkConfigMysql.properties and change values as needed. At a minimum u need to set
// username, password of account to use for accessing mysql and set dbid to database u want to use.

// Set mysql buffer pool size to 1G and reboot

// OPTIONAL OPTIMIATIONS TO MAKE THE LOAD FAST

// drop secondary indexes
mysql> alter table linktable drop key `id2_vis`, drop key `id1_type`;

set global innodb_flush_log_at_trx_commit = 2;

set global sync_binlog = 0;

// LOAD DATA . takes about 11000 seconds.
java com.facebook.LinkBench.LinkBenchDriver com/facebook/LinkBench/LinkConfigMysql.properties 1

// You will see last line of output like this<commit_msg>Update the documentation to pick up the mysql library from the lib directory<commit_after>// where jdk is installed
export JAVA_HOME="/usr/local/jdk-6u22-64"

// At least current dir and mysql connector jar should be in CLASSPATH
// Mysql connector jar can be downloaded from http://dev.mysql.com/downloads/connector/j/.
// Download version 5.0.8 or higher. Copy the mysql jar to the lib directory.
export CLASSPATH=.:lib/mysql-connector-java-5.1.16-bin.jar:dist/FacebookLinkBench-20120608.jar

export PATH=$JAVA_HOME/bin:$PATH

// compile
ant clean dist

// create tables in mysql
create database linkdb;

use linkdb;

CREATE TABLE `linktable` (
  `id1` bigint(20) unsigned NOT NULL DEFAULT '0',
  `id1_type` int(10) unsigned NOT NULL DEFAULT '0',
  `id2` bigint(20) unsigned NOT NULL DEFAULT '0',
  `id2_type` int(10) unsigned NOT NULL DEFAULT '0',
  `link_type` bigint(20) unsigned NOT NULL DEFAULT '0',
  `visibility` tinyint(3) NOT NULL DEFAULT '0',
  `data` varchar(255) NOT NULL DEFAULT '',
  `time` bigint(20) unsigned NOT NULL DEFAULT '0',
  `version` int(11) unsigned NOT NULL DEFAULT '0',
  PRIMARY KEY (`id1`,`id2`,`link_type`),
  KEY `id2_vis` (`id2`,`visibility`),
  KEY `id1_type` (`id1`,`link_type`,`visibility`,`time`,`version`,`data`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

CREATE TABLE `counttable` (
  `id` bigint(20) unsigned NOT NULL DEFAULT '0',
  `id_type` int(10) unsigned NOT NULL DEFAULT '0',
  `link_type` bigint(20) unsigned NOT NULL DEFAULT '0',
  `count` int(10) unsigned NOT NULL DEFAULT '0',
  `time` bigint(20) unsigned NOT NULL DEFAULT '0',
  `version` bigint(20) unsigned NOT NULL DEFAULT '0',
  PRIMARY KEY (`id`,`link_type`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;

// Examine LinkConfigMysql.properties and change values as needed. At a minimum u need to set
// username, password of account to use for accessing mysql and set dbid to database u want to use.

// Set mysql buffer pool size to 1G and reboot

// OPTIONAL OPTIMIATIONS TO MAKE THE LOAD FAST

// drop secondary indexes
mysql> alter table linktable drop key `id2_vis`, drop key `id1_type`;

set global innodb_flush_log_at_trx_commit = 2;

set global sync_binlog = 0;

// LOAD DATA . takes about 11000 seconds.
java com.facebook.LinkBench.LinkBenchDriver config/LinkConfigMysql.properties 1

// You will see last line of output like this
{'fraction_non_alphanumeric': 0.11413043478260869, 'fraction_numerical': 0.035869565217391305, 'mean_word_length': 4.523409363745499, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1322317', 'n_tokens_mistral': 1623, 'n_tokens_neox': 1493, 'n_words': 555}