<commit_before># snippet crawler dev notes

## mongodb

download: http://www.mongodb.org/downloads

### install manually
```bash
# download on azure
screen -S down
curl -O https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1410-clang-3.0.2.tgz

# upload to dev vm
scp root@aint:~/tmp/mongodb-linux-x86_64-ubuntu1410-clang-3.0.2.tgz .
scp mongodb-linux-x86_64-ubuntu1410-clang-3.0.2.tgz root@ls:~/

# install
tar -zxvf mongodb-linux-x86_64-ubuntu1410-clang-3.0.2.tgz
mkdir -p /opt/mongodb
cp -R -n mongodb-linux-x86_64-ubuntu1410-clang-3.0.2/* /opt/mongodb
echo 'export PATH=/opt/mongodb/bin:$PATH' >> ~/.bashrc
apt-get install libc++-dev
```

### run manually
```bash
screen -S mdb
mkdir -p /data/mongodb
mongod --dbpath /data/mongodb
```

### enable authentication
login via `mongo` to create the administrator and close localhost exception:
```javascript
use admin
db.createUser(
  {
    user: "root",
    pwd: "<PASSWORD>",
    roles: [ { role: "userAdminAnyDatabase", db: "admin" } ]
  }
)
```

restart the `mongod` with auth required:
```bash
mongod --dbpath /data/mongodb --bind_ip=0.0.0.0 --auth
```

login as `root` with auth:
```bash
mongo --port 27017 -u root -p YOUR_PASSWORD --authenticationDatabase admin
```

### create user
```javascript
use snippetcrawl
db.createUser(
  {
    user: "YOUR_USERNAME",
    pwd: "<PASSWORD>",
    roles: [ { role: "dbOwner", db: "snippetcrawl" } ]
  }
)
```


## python

### setup devenv for python3

```bash
sudo apt-get install python3-pip python3-dev python3-setuptools
```

### install via pip
```bash
sudo pip3 install pymongo
sudo pip3 install requests
sudo pip3 install beautifulsoup4
sudo pip3 install flask
```

### fix `ImportError: cannot import name 'IncompleteRead'` 
```bash
sudo easy_install3 -U pip
```
<commit_msg>dev notes - python lib setup<commit_after># snippet crawler dev notes

## mongodb

download: http://www.mongodb.org/downloads

### install manually
```bash
# download on azure
screen -S down
curl -O https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1410-clang-3.0.2.tgz

# upload to dev vm
scp root@aint:~/tmp/mongodb-linux-x86_64-ubuntu1410-clang-3.0.2.tgz .
scp mongodb-linux-x86_64-ubuntu1410-clang-3.0.2.tgz root@ls:~/

# install
tar -zxvf mongodb-linux-x86_64-ubuntu1410-clang-3.0.2.tgz
mkdir -p /opt/mongodb
cp -R -n mongodb-linux-x86_64-ubuntu1410-clang-3.0.2/* /opt/mongodb
echo 'export PATH=/opt/mongodb/bin:$PATH' >> ~/.bashrc
apt-get install libc++-dev
```

### run manually
```bash
screen -S mdb
mkdir -p /data/mongodb
mongod --dbpath /data/mongodb
```

### enable authentication
login via `mongo` to create the administrator and close localhost exception:
```javascript
use admin
db.createUser(
  {
    user: "root",
    pwd: "<PASSWORD>",
    roles: [ { role: "userAdminAnyDatabase", db: "admin" } ]
  }
)
```

restart the `mongod` with auth required:
```bash
mongod --dbpath /data/mongodb --bind_ip=0.0.0.0 --auth
```

login as `root` with auth:
```bash
mongo --port 27017 -u root -p YOUR_PASSWORD --authenticationDatabase admin
```

### create user
```javascript
use snippetcrawl
db.createUser(
  {
    user: "YOUR_USERNAME",
    pwd: "<PASSWORD>",
    roles: [ { role: "dbOwner", db: "snippetcrawl" } ]
  }
)
```


## python

### setup devenv for python3

```bash
sudo apt-get install python3-pip python3-dev python3-setuptools
```

### install via pip
```bash
sudo pip3 install pymongo
sudo pip3 install requests
sudo pip3 install beautifulsoup4
sudo pip3 install flask
```

### install by requirements.txt 
```bash
pip3 install -r requirements.txt
```

### fix `ImportError: cannot import name 'IncompleteRead'` 
```bash
sudo easy_install3 -U pip
```

### upload to devbox
```bash
scp -r *.* root@stcaraa:~/snipcrawl/
```
{'fraction_non_alphanumeric': 0.15658458244111348, 'fraction_numerical': 0.039882226980728054, 'mean_word_length': 3.930079155672823, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4563430', 'n_tokens_mistral': 1581, 'n_tokens_neox': 1373, 'n_words': 363}