<commit_before>=======================
SAIO - Swift All In One
=======================

-----------------------------------
Instructions for seting up a dev VM
-----------------------------------

This documents setting up a virtual machine for doing Swift development. The
virtual machine will emulate running a four node Swift cluster. It assumes
you're using *VMware Fusion 3* on *Mac OS X Snow Leopard*, but should give a
good idea what to do on other environments.

* Get the *Ubuntu 10.04 LTS (Lucid Lynx)* server image from: 
  http://cdimage.ubuntu.com/releases/10.04/release/ubuntu-10.04-dvd-amd64.iso
* Create guest virtual machine:

  #. `Continue without disc`
  #. `Use operating system installation disc image file`, pick the .iso
     from above.
  #. Select `Linux` and `Ubuntu 64-bit`.
  #. Fill in the *Linux Easy Install* details.
  #. `Customize Settings`, name the image whatever you want 
     (`SAIO` for instance.)
  #. When the `Settings` window comes up, select `Hard Disk`, create an
     extra disk (the defaults are fine).
  #. Start the virtual machine up and wait for the easy install to
     finish.

* As root on guest (you'll have to log in as you, then `sudo su -`):

  #. `apt-get install python-software-properties`
  #. `add-apt-repository ppa:swift-core/ppa`
  #. `apt-get update`
  #. `apt-get install curl gcc bzr memcached python-configobj
     python-coverage python-dev python-nose python-setuptools python-simplejson
     python-xattr sqlite3 xfsprogs python-webob python-eventlet
     python-greenlet python-pastedeploy`
  #. Install anything else you want, like screen, ssh, vim, etc.
  #. `fdisk /dev/sdb` (set up a single partition)
  #. `mkfs.xfs -i size=1024 /dev/sdb1`
  #. `mkdir /mnt/sdb1`
  #. Edit `/etc/fstab` and add
     `/dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0`
  #. `mount /mnt/sdb1`
  #. `mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 /mnt/sdb1/test`
  #. `chown <your-user-name>:<your-group-name> /mnt/sdb1/*`
  #. `mkdir /srv`
  #. `for x in {1..4}; do ln -s /mnt/sdb1/$x /srv/$x; done`
  #. `mkdir -p /etc/swift/object-server /etc/swift/container-server /etc/swift/account-server /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 /var/run/swift`
  #. `chown -R <your-user-name>:<your-group-name> /etc/swift /srv/[1-4]/ /var/run/swift` -- **Make sure to include the trailing slash after /srv/[1-4]/**
  #. Add to `/etc/rc.local` (before the `exit 0`)::

        mkdir /var/run/swift
        chown <your-user-name>:<your-user-name> /var/run/swift

  #. Create /etc/rsyncd.conf::

        uid = <Your user name>
        gid = <Your group name>
        log file = /var/log/rsyncd.log
        pid file = /var/run/rsyncd.pid


        [account6012]
        max connections = 25
        path = /srv/1/node/
        read only = false
        lock file = /var/lock/account6012.lock

        [account6022]
        max connections = 25
        path = /srv/2/node/
        read only = false
        lock file = /var/lock/account6022.lock

        [account6032]
        max connections = 25
        path = /srv/3/node/
        read only = false
        lock file = /var/lock/account6032.lock

        [account6042]
        max connections = 25
        path = /srv/4/node/
        read only = false
        lock file = /var/lock/account6042.lock


        [container6011]
        max connections = 25
        path = /srv/1/node/
        read only = false
        lock file = /var/lock/container6011.lock

        [container6021]
        max connections = 25
        path = /srv/2/node/
        read only = false
        lock file = /var/lock/container6021.lock

        [container6031]
        max connections = 25
        path = /srv/3/node/
        read only = false
        lock file = /var/lock/container6031.lock

        [container6041]
        max connections = 25
        path = /srv/4/node/
        read only = false
        lock file = /var/lock/container6041.lock


        [object6010]
        max connections = 25
        path = /srv/1/node/
        read only = false
        lock file = /var/lock/object6010.lock

        [object6020]
        max connections = 25
        path = /srv/2/node/
        read only = false
        lock file = /var/lock/object6020.lock

        [object6030]
        max connections = 25
        path = /srv/3/node/
        read only = false
        lock file = /var/lock/object6030.lock

        [object6040]
        max connections = 25
        path = /srv/4/node/
        read only = false
        lock file = /var/lock/object6040.lock

  #. Edit the following line in /etc/default/rsync::

        RSYNC_ENABLE=true

  #. `service rsync restart`

* As you on guest:

  #. `mkdir ~/bin`
  #. Create `~/.bazaar/bazaar.conf`::

        [DEFAULT]
                email = Your Name <your-email-address>
  #. If you are using launchpad to get the code or make changes, run
     `bzr launchpad-login <launchpad_id>`
  #. Create the swift repo with `bzr init-repo swift`
  #. Check out your bzr branch of swift, for example:
     `cd ~/swift; bzr branch lp:swift trunk`
  #. `cd ~/swift/trunk; sudo python setup.py develop`
  #. Edit `~/.bashrc` and add to the end::

        export PATH_TO_TEST_XFS=/mnt/sdb1/test
        export SWIFT_TEST_CONFIG_FILE=/etc/swift/func_test.conf
        export PATH=${PATH}:~/bin

  #. `. ~/.bashrc`
  #. Create `/etc/swift/auth-server.conf`::

        [DEFAULT]
        user = <your-user-name>

        [pipeline:main]
        pipeline = auth-server

        [app:auth-server]
        use = egg:swift#auth
        default_cluster_url = http://127.0.0.1:8080/v1

  #. Create `/etc/swift/proxy-server.conf`::

        [DEFAULT]
        bind_port = 8080
        user = <your-user-name>

        [pipeline:main]
        pipeline = healthcheck cache auth proxy-server
        
        [app:proxy-server]
        use = egg:swift#proxy

        [filter:auth]
        use = egg:swift#auth

        [filter:healthcheck]
        use = egg:swift#healthcheck

        [filter:cache]
        use = egg:swift#cache

  #. Create `/etc/swift/account-server/1.conf`::

        [DEFAULT]
        devices = /srv/1/node
        mount_check = false
        bind_port = 6012
        user = <your-user-name>

        [pipeline:main]
        pipeline = account-server

        [app:account-server]
        use = egg:swift#account

        [account-replicator]
        vm_test_mode = yes

        [account-auditor]

        [account-reaper]

  #. Create `/etc/swift/account-server/2.conf`::
<commit_msg>Fixed memcache middleware config for proxy<commit_after>=======================
SAIO - Swift All In One
=======================

-----------------------------------
Instructions for seting up a dev VM
-----------------------------------

This documents setting up a virtual machine for doing Swift development. The
virtual machine will emulate running a four node Swift cluster. It assumes
you're using *VMware Fusion 3* on *Mac OS X Snow Leopard*, but should give a
good idea what to do on other environments.

* Get the *Ubuntu 10.04 LTS (Lucid Lynx)* server image from: 
  http://cdimage.ubuntu.com/releases/10.04/release/ubuntu-10.04-dvd-amd64.iso
* Create guest virtual machine:

  #. `Continue without disc`
  #. `Use operating system installation disc image file`, pick the .iso
     from above.
  #. Select `Linux` and `Ubuntu 64-bit`.
  #. Fill in the *Linux Easy Install* details.
  #. `Customize Settings`, name the image whatever you want 
     (`SAIO` for instance.)
  #. When the `Settings` window comes up, select `Hard Disk`, create an
     extra disk (the defaults are fine).
  #. Start the virtual machine up and wait for the easy install to
     finish.

* As root on guest (you'll have to log in as you, then `sudo su -`):

  #. `apt-get install python-software-properties`
  #. `add-apt-repository ppa:swift-core/ppa`
  #. `apt-get update`
  #. `apt-get install curl gcc bzr memcached python-configobj
     python-coverage python-dev python-nose python-setuptools python-simplejson
     python-xattr sqlite3 xfsprogs python-webob python-eventlet
     python-greenlet python-pastedeploy`
  #. Install anything else you want, like screen, ssh, vim, etc.
  #. `fdisk /dev/sdb` (set up a single partition)
  #. `mkfs.xfs -i size=1024 /dev/sdb1`
  #. `mkdir /mnt/sdb1`
  #. Edit `/etc/fstab` and add
     `/dev/sdb1 /mnt/sdb1 xfs noatime,nodiratime,nobarrier,logbufs=8 0 0`
  #. `mount /mnt/sdb1`
  #. `mkdir /mnt/sdb1/1 /mnt/sdb1/2 /mnt/sdb1/3 /mnt/sdb1/4 /mnt/sdb1/test`
  #. `chown <your-user-name>:<your-group-name> /mnt/sdb1/*`
  #. `mkdir /srv`
  #. `for x in {1..4}; do ln -s /mnt/sdb1/$x /srv/$x; done`
  #. `mkdir -p /etc/swift/object-server /etc/swift/container-server /etc/swift/account-server /srv/1/node/sdb1 /srv/2/node/sdb2 /srv/3/node/sdb3 /srv/4/node/sdb4 /var/run/swift`
  #. `chown -R <your-user-name>:<your-group-name> /etc/swift /srv/[1-4]/ /var/run/swift` -- **Make sure to include the trailing slash after /srv/[1-4]/**
  #. Add to `/etc/rc.local` (before the `exit 0`)::

        mkdir /var/run/swift
        chown <your-user-name>:<your-user-name> /var/run/swift

  #. Create /etc/rsyncd.conf::

        uid = <Your user name>
        gid = <Your group name>
        log file = /var/log/rsyncd.log
        pid file = /var/run/rsyncd.pid


        [account6012]
        max connections = 25
        path = /srv/1/node/
        read only = false
        lock file = /var/lock/account6012.lock

        [account6022]
        max connections = 25
        path = /srv/2/node/
        read only = false
        lock file = /var/lock/account6022.lock

        [account6032]
        max connections = 25
        path = /srv/3/node/
        read only = false
        lock file = /var/lock/account6032.lock

        [account6042]
        max connections = 25
        path = /srv/4/node/
        read only = false
        lock file = /var/lock/account6042.lock


        [container6011]
        max connections = 25
        path = /srv/1/node/
        read only = false
        lock file = /var/lock/container6011.lock

        [container6021]
        max connections = 25
        path = /srv/2/node/
        read only = false
        lock file = /var/lock/container6021.lock

        [container6031]
        max connections = 25
        path = /srv/3/node/
        read only = false
        lock file = /var/lock/container6031.lock

        [container6041]
        max connections = 25
        path = /srv/4/node/
        read only = false
        lock file = /var/lock/container6041.lock


        [object6010]
        max connections = 25
        path = /srv/1/node/
        read only = false
        lock file = /var/lock/object6010.lock

        [object6020]
        max connections = 25
        path = /srv/2/node/
        read only = false
        lock file = /var/lock/object6020.lock

        [object6030]
        max connections = 25
        path = /srv/3/node/
        read only = false
        lock file = /var/lock/object6030.lock

        [object6040]
        max connections = 25
        path = /srv/4/node/
        read only = false
        lock file = /var/lock/object6040.lock

  #. Edit the following line in /etc/default/rsync::

        RSYNC_ENABLE=true

  #. `service rsync restart`

* As you on guest:

  #. `mkdir ~/bin`
  #. Create `~/.bazaar/bazaar.conf`::

        [DEFAULT]
                email = <NAME> <your-email-address>
  #. If you are using launchpad to get the code or make changes, run
     `bzr launchpad-login <launchpad_id>`
  #. Create the swift repo with `bzr init-repo swift`
  #. Check out your bzr branch of swift, for example:
     `cd ~/swift; bzr branch lp:swift trunk`
  #. `cd ~/swift/trunk; sudo python setup.py develop`
  #. Edit `~/.bashrc` and add to the end::

        export PATH_TO_TEST_XFS=/mnt/sdb1/test
        export SWIFT_TEST_CONFIG_FILE=/etc/swift/func_test.conf
        export PATH=${PATH}:~/bin

  #. `. ~/.bashrc`
  #. Create `/etc/swift/auth-server.conf`::

        [DEFAULT]
        user = <your-user-name>

        [pipeline:main]
        pipeline = auth-server

        [app:auth-server]
        use = egg:swift#auth
        default_cluster_url = http://127.0.0.1:8080/v1

  #. Create `/etc/swift/proxy-server.conf`::

        [DEFAULT]
        bind_port = 8080
        user = <your-user-name>

        [pipeline:main]
        pipeline = healthcheck cache auth proxy-server
        
        [app:proxy-server]
        use = egg:swift#proxy

        [filter:auth]
        use = egg:swift#auth

        [filter:healthcheck]
        use = egg:swift#healthcheck

        [filter:cache]
        use = egg:swift#memcache

  #. Create `/etc/swift/account-server/1.conf`::

        [DEFAULT]
        devices = /srv/1/node
        mount_check = false
        bind_port = 6012
        user = <your-user-name>

        [pipeline:main]
        pipeline = account-server

        [app:account-server]
        use = egg:swift#account

        [account-replicator]
        vm_test_mode = yes

        [account-auditor]

        [account-reaper]

  #. Create `/etc/swift/account-server/2.conf`::

{'fraction_non_alphanumeric': 0.13719071183859916, 'fraction_numerical': 0.032127902550437765, 'mean_word_length': 1.9943013448826077, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6062802', 'n_tokens_mistral': 4776, 'n_tokens_neox': 4249, 'n_words': 1164}