CLI
    README.md
    config
        custom-environment-variables.js
        development.js
    package.json
    src
        controllers
            cli.ts
        helpers
            commands.ts
            constants.ts
            functions.ts
            questions.ts
            types.ts
        index.ts
    tsconfig.json
CommentSubmission
    index.html
    main.js
    package.json
Filecoin
    Server
        README.md
        config
            custom-environment-variables.js
            development.js
        package.json
        src
            controllers
                orchestrator.ts
                powergate.ts
            helpers
                constants.ts
                functions.ts
                types.ts
            index.ts
            server
                cors.ts
                express.ts
                router.ts
                routes.ts
                server.ts
        tsconfig.json
    powergate
        CONTRIBUTING.md
        README.md
        api
            client
                admin
                    admin.go
                    data.go
                    indices.go
                    options.go
                    records.go
                    storageinfo.go
                    storagejobs.go
                    users.go
                    wallet.go
                admin_test.go
                client.go
                client_test.go
                data.go
                deals.go
                storageconfig.go
                storageinfo.go
                storagejobs.go
                utils_test.go
                wallet.go
                wallet_test.go
            gen
                powergate
                    admin
                        v1
                            admin.pb.go
                            admin_grpc.pb.go
                    user
                        v1
                            user.pb.go
                            user_grpc.pb.go
            server
                admin
                    hs.go
                    indices.go
                    jobs.go
                    records.go
                    service.go
                    storageinfo.go
                    users.go
                    wallet.go
                server.go
                user
                    data.go
                    deals.go
                    service.go
                    storageconfig.go
                    storageinfo.go
                    storagejobs.go
                    util.go
                    wallet.go
                util
                    util.go
        buildinfo
            buildinfo.go
        chainstore
            chainstore.go
            chainstore_test.go
        chainsync
            chainsync.go
            chainsync_test.go
        cli-docs
            pow
                pow.md
                pow_admin.md
                pow_admin_data.md
                pow_admin_data_gcstaged.md
                pow_admin_data_pinnedcids.md
                pow_admin_storage-info.md
                pow_admin_storage-info_get.md
                pow_admin_storage-info_list.md
                pow_admin_storage-jobs.md
                pow_admin_storage-jobs_list.md
                pow_admin_storage-jobs_summary.md
                pow_admin_users.md
                pow_admin_users_create.md
                pow_admin_users_list.md
                pow_admin_users_regenerate.md
                pow_admin_wallet.md
                pow_admin_wallet_addrs.md
                pow_admin_wallet_new.md
                pow_admin_wallet_send.md
                pow_config.md
                pow_config_apply.md
                pow_config_default.md
                pow_config_remove.md
                pow_config_set-default.md
                pow_data.md
                pow_data_get.md
                pow_data_info.md
                pow_data_log.md
                pow_data_replace.md
                pow_data_stage.md
                pow_data_summary.md
                pow_deals.md
                pow_deals_retrievals.md
                pow_deals_storage.md
                pow_id.md
                pow_offline.md
                pow_offline_car.md
                pow_offline_commp.md
                pow_offline_prepare.md
                pow_storage-info.md
                pow_storage-info_get.md
                pow_storage-info_list.md
                pow_storage-jobs.md
                pow_storage-jobs_cancel-executing.md
                pow_storage-jobs_cancel-queued.md
                pow_storage-jobs_cancel.md
                pow_storage-jobs_get.md
                pow_storage-jobs_list.md
                pow_storage-jobs_storage-config.md
                pow_storage-jobs_summary.md
                pow_storage-jobs_watch.md
                pow_version.md
                pow_wallet.md
                pow_wallet_addrs.md
                pow_wallet_balance.md
                pow_wallet_new-addr.md
                pow_wallet_send.md
                pow_wallet_sign.md
                pow_wallet_verify.md
        cmd
            migrtest
                README.md
                main.go
            pow
                cmd
                    admin
                        admin.go
                        data
                            data.go
                            gcstaged
                                gcstaged.go
                            pinnedcids
                                pinnedcids.go
                        storageinfo
                            get
                                get.go
                            list
                                list.go
                            storageinfo.go
                        storagejobs
                            list
                                list.go
                            storagejobs.go
                            summary
                                summary.go
                        users
                            create
                                create.go
                            list
                                list.go
                            regenerate
                                regenerate.go
                            users.go
                        wallet
                            addrs
                                addrs.go
                            new
                                new.go
                            send
                                send.go
                            wallet.go
                    config
                        apply
                            apply.go
                        config.go
                        getdefault
                            getdefault.go
                        remove
                            remove.go
                        setdefault
                            setdefault.go
                    data
                        data.go
                        get
                            get.go
                        info
                            info.go
                        log
                            log.go
                        replace
                            replace.go
                        stage
                            stage.go
                        summary
                            summary.go
                            tui.go
                    deals
                        deals.go
                        retrievals
                            retrievals.go
                        storage
                            storage.go
                    id
                        id.go
                    pow.go
                    prepare
                        prepare.go
                        prepare_test.go
                    storageinfo
                        get
                            get.go
                        list
                            list.go
                        storageinfo.go
                    storagejobs
                        cancel
                            cancel.go
                        cancelexecuting
                            cancelexecuting.go
                        cancelqueued
                            cancelqueued.go
                        get
                            get.go
                        list
                            list.go
                        storageconfig
                            storageconfig.go
                        storagejobs.go
                        summary
                            summary.go
                        watch
                            watch.go
                    version
                        version.go
                    wallet
                        addrs
                            addrs.go
                        balance
                            balance.go
                        newaddr
                            newaddr.go
                        send
                            send.go
                        sign
                            sign.go
                        verify
                            verify.go
                        wallet.go
                common
                    common.go
                main.go
            powbench
                README.md
                main.go
                runner
                    runner.go
                    runner_test.go
            powcfg
                main.go
                ratelim
                    ratelim.go
                transform.go
                transform_test.go
                transformations.go
            powd
                main.go
        dataprep
            dataprep.go
        deals
            module
                deals.go
                deals_test.go
                dealwatcher
                    dealwatcher.go
                    metrics.go
                metrics.go
                module.go
                records.go
                retrieve.go
                store
                    metrics.go
                    store.go
                    store_test.go
                util.go
            options.go
            types.go
        docker
            dashboards
                System_Monitoring.json
            grafana
                provisioning
                    dashboards
                        TextileFC.json
                        dashboard.yml
                    datasources
                        datasource.yml
            prometheus
                prometheus.yml
        docs
            manual_installation.md
        fchost
            config.go
            fchost.go
            fchost_test.go
        ffs
            Design.md
            api
                api.go
                api_actions.go
                api_deals.go
                api_jobs.go
                api_logs.go
                api_retrieval.go
                api_storageinfo.go
                api_wallet.go
                istore.go
                option.go
                types.go
            auth
                auth.go
            coreipfs
                coreipfs.go
                coreipfs_test.go
                internal
                    pinstore
                        pinstore.go
            filcold
                filcold.go
                metrics.go
            integrationtest
                config
                    config_test.go
                filters
                    filters_test.go
                general
                    general_test.go
                importing
                    importing_test.go
                integrationtest.go
                logger
                    logger_test.go
                manager
                    manager.go
                renew
                    renew_test.go
                repair
                    repair_test.go
                repfactor
                    repfactor_test.go
                replace
                    replace_test.go
                retrieval
                    retrieval_test.go
                scheduler
                    scheduler_test.go
                unfreeze
                    unfreeze_test.go
                wallet
                    wallet_test.go
            interfaces.go
            joblogger
                joblogger.go
            manager
                manager.go
                manager_test.go
            minerselector
                fixed
                    fixed.go
                reptop
                    reptop.go
                sr2
                    sr2.go
                    sr2_test.go
            scheduler
                internal
                    astore
                        astore.go
                    cistore
                        cistore.go
                    ristore
                        cistore.go
                    rjstore
                        rjstore.go
                        rjstore_test.go
                    sjstore
                        metrics.go
                        sjstore.go
                        sjstore_test.go
                    trackstore
                        trackstore.go
                        trackstore_test.go
                scheduler.go
                scheduler_import.go
                scheduler_retrieval.go
                scheduler_storage.go
            types.go
        filchain
            filchain.go
        gateway
            assets.go
            gateway.go
            public
                css
                    all.min.css
                    style.css
                img
                    hex.svg
        index
            ask
                internal
                    store
                        store.go
                runner
                    metrics.go
                    runner.go
                    runner_test.go
                types.go
            faults
                module
                    faults.go
                    faults_test.go
                types.go
            miner
                lotusidx
                    meta.go
                    metrics.go
                    miner.go
                    miner_test.go
                    onchain.go
                    store
                        store.go
                types.go
        iplocation
            iplocation.go
            maxmind
                maxmind.go
                maxmind_test.go
        lotus
            client.go
            client_test.go
            metrics.go
        migration
            migration.go
            migration1.go
            migration1_joblogger.go
            migration1_joblogger_test.go
            migration1_pinstore.go
            migration1_pinstore_test.go
            migration1_starteddeals.go
            migration1_starteddeals_test.go
            migration1_storageinfo.go
            migration1_storageinfo_test.go
            migration1_trackstore.go
            migration1_trackstore_test.go
            migration2.go
            migration2_storageinfo_test.go
            migration3.go
            migration3_test.go
            migration4.go
            migration4_test.go
            migration5.go
            migration_test.go
        reputation
            internal
                source
                    source.go
                    store.go
            reputation.go
        scripts
            gen-js-protos.sh
            gen-py-protos.sh
        signaler
            signaler.go
        tests
            auth.go
            docker.go
            flaky.go
            ldevnet.go
            mocks
                mocks.go
            txmapds.go
        txndstransform
            txndstransform.go
        util
            util.go
            util_test.go
        wallet
            lotuswallet
                metrics.go
                wallet.go
            type.go
LICENSE.txt
NEAR
    Contracts
        README.md
        assembly
            identity-provider
                as_types.d.ts
                asconfig.json
                main.ts
                model.ts
                tsconfig.json
            network
                as_types.d.ts
                asconfig.json
                main.ts
                model.ts
                neardev
                    dev-account.env
                tsconfig.json
            publisher
                as_types.d.ts
                asconfig.json
                main.ts
                model.ts
                tsconfig.json
            repository
                as_types.d.ts
                asconfig.json
                main.ts
                model.ts
                neardev
                    dev-account.env
                tsconfig.json
        package.json
    Server
        README.md
        config
            custom-environment-variables.js
            development.js
        package.json
        src
            controllers
                near.ts
                orchestrator.ts
            helpers
                constants.ts
                functions.ts
                types.ts
            index.ts
            server
                cors.ts
                express.ts
                router.ts
                routes.ts
                server.ts
        tsconfig.json
README.md
Website
    cloud.html
    covid.html
    home.html
article.json
comment.json

# Server

## Table of Contents

-   [Build](#build)
-   [Deploy](#deploy)
-   [Configuration](#configuration)
-   [Endpoints](#endpoints)
-   [License](#license)

## Build

To build this server, install npm packages:
```bash
$ npm install
```

Then start this script:
```bash
$ npm run build
```

## Deploy

You should first run a local network of Filecoin. This can be done by executing, from the __powergate__ folder:
```bash
$ make localnet
```

Then, deploy the server:
```bash
$ npm install && npm run deploy
```

## Endpoints
Two main endpoints are available.

The first one is:
```bash
$ GET http://<host>:<port>/data/:cid
```
(where :cid is the unique identifier of the content hosted on Filecoin)

The second one is:
```bash
$ POST http://<host>:<port>/data
```
(where the submitted data as part of the HTTP request is the content to be hosted on Filecoin)

## Configuration

A configuration file is available in __config/development.js__. Here, you can specify the endpoint used to connect to Powergate, itself connected to your Filecoin network running locally.

It is also possible to specify the endpoint of the server.

## License

[MIT](LICENSE)
# Server

## Table of Contents

-   [Build](#build)
-   [Deploy](#deploy)
-   [Configuration](#configuration)
-   [Endpoints](#endpoints)
-   [License](#license)

## Build

To build this server, install npm packages:
```bash
$ npm install
```

Then start this script:
```bash
$ npm run build
```

## Deploy

For the first run, you should configure the tool appropriately and set the __deployment__ variable to *true* (see [Configuration](#configuration)).

Deployment of the server is done as follow:
```bash
$ npm install && npm run deploy
```

## Endpoints
Available endpoints are described below.

```bash
$ GET http://<host>:<port>/articles/:identifier
```
(where :identifier is the unique identifier of the article hosted on Filecoin)

```bash
$ POST http://<host>:<port>/articles
```
(where the submitted data are valid article metada, as per the model of the *repository* smart contract (see __Contracts__ folder))

```bash
$ GET http://<host>:<port>/articles/:article/version/:version/comments/:comment
```
(where :article is the unique identifier of the article hosted on Filecoin, :version the version of the article for which the comment should be added, and :comment the unique identifier of the comment hosted on Filecoin)

```bash
$ POST http://<host>:<port>/articles/:article/version/:version/comments
```
(with the same remarks as for the previous call, and the submitted data being valid comment metada, as per the model of the *repository* smart contract (see __Contracts__ folder))

## Configuration

A configuration file is available in __config/development.js__. Here, you can specify information about the NEAR blockchain and about the server.

For NEAR (*near* object of the configuration file):
- *network* is the network to use (by default, testnet)
- *credentials* is the folder where the keypairs of the keystore are hosted
- *deployment* should be *true* only during the initialization (i.e. when smart contracts should be deployed), otherwise the variable should be *false*
- endpoints are the different endpoints to use for the connection, as per NEAR documentation
- *accounts* is the list of accounts to use for the demonstration (each account having at least an *account* property referencing the address of the account, and a *private_key* related to this account)
- *contracts* is the set of contracts to deploy and use (*base* refers to where they are hosted, and each contract defines its own *file* (related to the base), its *name*, the *account* owning it, and the *methods* that it exposes)

## License

[MIT](LICENSE)
# migrtest

`migrtest` is a tool for testing migrations on real data. This tools allows to run migrations on databases in a safe way since they're run in a copy of the target database. This tool is useful while developing new migrations, or to Powergate operations which might want to test a new release which has migrations on real data to see if something bad would happen in the real update.

## Usage

Run `migrtest -h` to know about the available flags.

### Target a remote `go-datastore` and run locally
```bash
migrtool --origin-remote "$MONGO_URI;$MONGO_DBNAME"
```
This command will:
- Create a folder `run_<TIMESTAMP>` which will have the run assets.
- Make a copy of the remote `go-datastore` to `run_<TIMESTAMP>/badger-migrated`.
- Run all detected/needed migrations in the copied datastore.
- Print to stdout Powergate output while doing the migrations.
- `run_<TIMESTAMP>/badger-migrated` will be the migrated `go-datastore` that might be useful for inspection.

###  Target a remote `go-datastore` and run remotely
```bash
migrtool --origin-remote "$MONGO_URI;$MONGO_DBNAME" --run-remote
```
This command will:
- Create a folder `run_<TIMESTAMP>` which will have the run assets.
- Make a copy of the remote `go-datastore` to the same MongoDB cluster, in a fixed `tmp_powergate_migrtest` database, with a collection name `run_<TIMESTAMP>`.
- Run all detected/needed migrations in the copied datastore.
- Print to stdout Powergate output while doing the migrations.
- The remote collection is kept that might be useful for inspection.

This setup is useful to double-check that running using `go-ds-mongo` client works correctly. This may be slower since it's targeting a remote datastore, but as close as simulating the real migration as possible.

Recall that you should delete the remote databases if not needed anymore. They're all under the same database `tmp_pow_migrtest`, so you can simply delete that database for cleaning all runs.

### Target a remote `go-datastore` and skip the migration
This setup might be useful to just have a local copy of the remote `go-datastore`.
```bash
migrtool --origin-remote "$MONGO_URI;$MONGO_DBNAME" --skip-migrations
```
This command will:
- Create a folder `run_<TIMESTAMP>` which will have the run assets.
- Make a copy of the remote `go-datastore` to the same MongoDB cluster, in a fixed `tmp_powergate_migrtest` database, with a collection name `run_<TIMESTAMP>`.
# Contracts

## Table of Contents
-   [Description](#description)
-   [Build](#build)
-   [Deploy](#deploy)
-   [License](#license)

## Description

The internal and public functions of each smart contract are defined in the __main.ts__ file.
The __model.ts__ file is used to specify the model of the objects being manipulated by the smart contracts.
The __out__ folder is where the compiled file is saved.

## Build

To build each contract manually, you should first install AssemblyScript tool, e.g., globally:
```bash
$ npm install -g asbuild
```

Then, for each smart contract, run:
```bash
asb main.ts
```

## Deploy

The smart contracts are deployed setting the *deployment* configuration property to *true* and deploying the server.
To manually deploy contracts using NEAR CLI (and assuming CLI is installed, as per NEAR official documentation), you can run:
```bash
near dev-deploy
```

Or for production deployment:
```bash
near deploy
```

## License

[MIT](LICENSE)
# CLI

## Table of Contents
-   [Description](#description)
-   [Deploy](#deploy)
-   [License](#license)

## Description

The tool is used to submit articles and comments to the servers deployed as gateways to Filecoin platform and NEAR blockchain. Users interact with the tool using the Terminal.

## Deploy

To build and run the tool, execute:
```bash
$ npm install && npm run deploy
```

## License

[MIT](LICENSE)
# News For Trust

NewsForTrust aims to provide a platform for certifying and incentivize experts' contributions and comments on publishing platforms valuing high-quality dialogue.

Publishers joining the network are able to build expertise-defined collections of articles and allow for contributors to participate new information to them.
Whether these are from parties externally validated as experts in the field, or simply from readers voicing their takes with or without source, contributions and comments are keyed in with their writer' persona ID, and rewarded depending on a publisher-defined scheme.

Contracts are built on the NEAR Platform and allow:
- ChainedTrust to integrate new actors, Publishers and ID providers
- ID Providers to validate the real identity of an actor, linking their key to data representing their expertise field and level
- ID Providers to validate the real identity of an actor, linking their key to data representing their expertise field and level
- Collections to manage their sets of articles and corresponding comments, and value and reward contributions

## Project organisation and FILECOIN, NEAR, CLI tools codebases

_Please see the following repositories, and their respective README.md_

For Filecoin:
- the Server developed is hosted here: https://github.com/Ornythoma/NewsForTrust/tree/master/Filecoin/Server
- specifically, the controllers (main logic) are there: https://github.com/Ornythoma/NewsForTrust/tree/master/Filecoin/Server/src/controllers 

For NEAR:
- the smart contracts are here: https://github.com/Ornythoma/NewsForTrust/tree/master/NEAR/Contracts 
- the Server managing NEAR contracts, it is hosted here: https://github.com/Ornythoma/NewsForTrust/tree/master/NEAR/Server 
- specifically, the controllers (main logic) are there: https://github.com/Ornythoma/NewsForTrust/tree/master/NEAR/Server/src/controllers 
- the appropriate configuration is here: https://github.com/Ornythoma/NewsForTrust/blob/master/NEAR/Server/config/development.js 

For the CLI tool to manage the whole logic (submission of articles and comments to Filecoin (content) and NEAR (metadata)), we use:
- this repository : https://github.com/Ornythoma/NewsForTrust/tree/master/CLI .
- specifically, the logic related to the CLI is hosted in this file: https://github.com/Ornythoma/NewsForTrust/blob/master/CLI/src/controllers/cli.ts

## DEVELOPMENT 
- TypeScript
- Express.JS for REST API
- configuration (cf. configuration files + environment variables defined for that), 
- clear modularization (main entry point, controllers, ExpressJS server (including router and related routes), helpers, etc.)

## Why NEAR?

- Multichain support, which enables a whole ecosystem where identity providers and publishers may interact while being on different chains
- Design scalable-friendly and enforcement of low fees for transactions and contract calls, which is really important in this context, as articles, comments and responses are expected to be published regularly and in a consequent way
- Support for account names, which facilitates the identification of the actors of the ChainedTrust ecosystem easily, abstracting intrinsic complexity of traditional blockchain where users have to use private/public keys and long addresses
- Support of AssemblyScript in addition to Rust, which eases the integration within web applications where articles, comments and responses are expected to be submitted and verified

  
## Authors

Arbër Salihi & Emmanuel Kellner
# Powergate bench

Powergate `powbench` is an utility tool that allows to run benchmark scenarios against a properly configured Powergate server.

To build and install `powbench`, from the root of the Powergate repo, run:

```bash
> make install-powbench
> powbench -h
Usage of powbench:
      --maxParallel int    Max parallel file storage (default 1)
      --minerAddr string   Miner address to force Powergate to select for making deals (default "t01000")
      --pgAddr string      Powergate server multiaddress (default "/ip4/127.0.0.1/tcp/5002")
      --randSeed int       Random seed used to generate random samples data (default 42)
      --sampleSize int     Size of randomly generated files in bytes (default 1024)
      --totalSamples int   Total samples to run (default 3)
```

The targeted Powergate server should have enabled the auto-funding of newly created FFS instances wallet addresses.
This means:
- The Lotus node owned by Powergate must have a wallet address; the _master address_. 
- _master address_ should have funds.
- This address will be used to automatically send funds to the new FFS instance that will run the benchmark.
- Powergate should be started with two flags:
  - `--lotusmasteraddr`: with the above mentioned address.
  - `--walletinitialfund`: an reasonable amount of _attoFIL_ that will be transferred from the master address to the created FFS instance. It should be enough to fund _all_ deals that will be ran in the scenario.
  - _Note: env variables can be used instead of command line flags, i.e: `POWD_LOTUSMASTERADDR` and `POWD_WALLETINITIALFUND` respectively._
# Powergate

[![Made by Textile](https://img.shields.io/badge/made%20by-Textile-informational.svg?style=popout-square)](https://textile.io)
[![Chat on Slack](https://img.shields.io/badge/slack-slack.textile.io-informational.svg?style=popout-square)](https://slack.textile.io)
[![GitHub license](https://img.shields.io/github/license/textileio/filecoin.svg?style=popout-square)](./LICENSE)
[![Go Report Card](https://goreportcard.com/badge/github.com/textileio/powergate?style=flat-square)](https://goreportcard.com/report/github.com/textileio/powergate?style=flat-square)
[![GitHub action](https://github.com/textileio/powergate/workflows/Tests/badge.svg?style=popout-square)](https://github.com/textileio/powergate/actions)

Powergate is a multitiered file storage API built on Filecoin and IPFS, and an index builder for Filecoin data. It's designed to be modular and extensible.

Join us on our [public Slack channel](https://slack.textile.io/) for news, discussions, and status updates. [Check out our blog](https://medium.com/textileio) for the latest posts and announcements.

*Warning* This project is still **pre-release** and is not ready for production usage.

## Table of Contents

-   [Prerequisites](#prerequisites)
-   [Design](#design)
-   [Installation](#installation)
-   [Localnet mode](#localnet-mode)
-   [Production setup](#production-setup)
-   [Tests](#tests)
-   [Benchmark](#benchmark)
-   [Contributing](#contributing)
-   [Changelog](#changelog)
-   [License](#license)

## Prerequisites

To build from source, you need to have Go 1.14 or newer installed.

## Design

Powergate is composed of different modules which can be used independently, and compose together to provide other higher-level modules.

Here's a high-level overview of the main components, and how Powergate interacts with IPFS and a Filecoin client:
![Powergate Design](https://user-images.githubusercontent.com/6136245/86490337-6dbd5200-bd3d-11ea-9895-3689dd1c8a8f.png)

Note in the diagram that the Lotus and Filecoin client node _doesn't need_ to be in the same host where Powergate is running. They can, but isn't necessary.

### 📢 Deals module
The Deals module provides a lower layer of abstraction to a Filecoin client node. It provides simple APIs to store, watch, and retrieve data in the Filecoin network. Currently, it interacts with the Lotus client but we have plans to support other Filecoin clients.

### 👷 Indices and Reputation scoring
Powergate builds three indexes related to on-chain and off-chain data.

The _Miners index_ provides processed data regarding registered miners (on-chain and off-chain), such as: total miner power, relative power, online status, geolocation, and more!

The _Ask index_ provides a fast-retrieval up to date snapshot of miner's asking prices for data storage.

The _Faults index_ provides history data about miners faults while proving their storage on-chain. 

Built on top of the previous indexes, a _Reputation_ module constructs a weighted-scoring system that allows to sort miners considering multiple on-chain and off-chain data, such as: compared price to the median of the market, low storage-fault history, power on network, and external sources (soon!).

###  ⚡ FFS
This module provides a multitiered file storage API built on Filecoin and IPFS. Storing data on IPFS and Filecoin is as easy as expressing your desired configuration for storing a Cid.

Want to know more about this Powergate module? Check out the [FFS design document](https://github.com/textileio/powergate/blob/master/ffs/Design.md) and our presentation and demo at the _IPFS Pinning Summit_:

[![Video](https://img.youtube.com/vi/aiOTSkz_6aY/0.jpg)](https://youtu.be/aiOTSkz_6aY)

### 💫 API + CLI

Powergate exposes an API built from the various modules through gRPC endpoints. 
You can explore our [`.proto` files](https://github.com/textileio/powergate/tree/master/proto/powergate) to generate your clients, or take advange of a ready-to-use Powergate Go and [JS client](https://github.com/textileio/js-powergate-client). 🙌

We have a CLI that supports most of Powergate features.

To build and install the CLI, run:
```bash
$ make install-pow
```
The binary will be placed automatically in `$GOPATH/bin`. You may have to set the Path variables using the below commands
```bash
$ export PATH=$PATH:$(go env GOPATH)/bin
$ export GOPATH=$(go env GOPATH)
```
You can then run `pow` in your terminal.

You can read the [generated CLI docs](https://github.com/textileio/powergate/blob/master/cli-docs/pow/pow.md) in this repo, or run `pow` with the `--help` flag to see the available commands:

```
$ pow --help
A client for storage and retreival of powergate data

Usage:
  pow [flags]
  pow [command]

Available Commands:
  admin        Provides admin commands
  config       Provides commands to interact with cid storage configs
  data         Provides commands to interact with general data APIs
  deals        Provides commands to view Filecoin deal information
  help         Help about any command
  id           Returns the user id
  storage-jobs Provides commands to query for storage jobs in various states
  version      Display version information for pow and the connected server
  wallet       Provides commands about filecoin wallets

Flags:
  -h, --help                   help for pow
      --serverAddress string   address of the powergate service api (default "127.0.0.1:5002")
  -t, --token string           user auth token
  -v, --version                display version information for pow and the connected server

Use "pow [command] --help" for more information about a command.
```

## Installation

Powergate installation involves running external dependencies, and wiring them correctly with Powergate.

### External dependencies
Powergate needs external dependencies in order to provide full functionality, in particular a synced Filecoin client and a IPFS node.

#### Filecoin client
Currently, we support the Lotus Filecoin client but we plan to support other clients.

All described modules of Powergate need to comunicate with Lotus to build indices data, and provide storing and retrieving features in FFS. To install Lotus refer to its [official](https://lotu.sh/) documentation, taking special attention to [its dependencies](https://docs.lotu.sh/en+install-lotus-ubuntu). 

Fully syncing a Lotus node can take time, so be sure to check you're fully synced doing `./lotus sync status`.

We also automatically generate a public Docker image targeting the `master` branch of Lotus. This image is a pristine version of Lotus, with a sidecar reverse proxy to provide external access to the containerized API. For more information, refer to [textileio/lotus-build](https://github.com/textileio/lotus-build) and its [Dockerhub repository](https://hub.docker.com/repository/docker/textile/lotus).

In short, a fully-synced Lotus node should be available with its API (`127.0.0.1:1234`, by default) port accessible to Powergate.

### IPFS node
A running IPFS node is needed if you plan to use the FFS module.

If that's the case, you can refer [here](https://docs.ipfs.io/guides/guides/install/) for installation instructions, or its [Dockerhub repository](https://hub.docker.com/r/ipfs/go-ipfs) if you want to run a contanerized version. Currently we're supporting v0.5.1. The API endpoint should be accessible to Powergate (port 5001, by default). 

Since FFS _HotStorage_ is pinning Cids in the IPFS node, Powergate should be the only party controlling the pinset of the node. Other systems can share the same IPFS node if can  **guarantee** not unpinning Cids pinned by Powergate FFS instances. 

### Geolite database
Powergate needs an offline geo-location database to resolve miners country using their IP address. The same folder in which `powd` is executing, should have the Geolite2 database file `GeoLite2-City.mmdb` or you can pass the `--maxminddbfolder` flag to `powd` to specify the path of the folder containing `GeoLite2-City.mmdb`.
You can copy this file from the GitHub repo at `iplocation/maxmind/GeoLite2-City.mmdb`. If you run Powergate using Docker, this database is bundeled in the image so isn't necessary to have extra considerations.

### Server
To build and install the Powergate server, run:
```bash
make install-powd
```
You can run the `-h` flag to see the configurable flags:
```bash
$ powd -h 
Usage of powd:
      --askindexmaxparallel string       Max parallel query ask to execute while updating index (default "3")
      --askindexqueryasktimeout string   Timeout in seconds for a query ask (default "15")
      --askindexrefreshinterval string   Refresh interval measured in minutes (default "60")
      --askindexrefreshonstart           If true it will refresh the index on start
      --autocreatemasteraddr             Automatically creates & funds a master address if none is provided.
      --dealwatchpollduration string     Poll interval in seconds used by Deals Module watch to detect state changes (default "900")
      --debug                            Enable debug log level in all loggers.
      --devnet                           Indicate that will be running on an ephemeral devnet. --repopath will be autocleaned on exit.
      --disableindices                   Disable all indices updates, useful to help Lotus syncing process
      --disablenoncompliantapis          Disable APIs that may not easily comply with US law
      --ffsadmintoken string             FFS admin token for authorized APIs. If empty, the APIs will be open to the public.
      --ffsdealfinalitytimeout string    Deadline in minutes in which a deal must prove liveness changing status before considered abandoned (default "4320")
      --ffsminerselector string          Miner selector to be used by FFS: 'sr2', 'reputation' (default "sr2")
      --ffsminerselectorparams string    Miner selector configuration parameter, depends on --ffsminerselector (default "https://raw.githubusercontent.com/filecoin-project/slingshot/master/miners.json")
      --ffsminimumpiecesize string       Minimum piece size in bytes allowed to be stored in Filecoin (default "67108864")
      --ffsschedmaxparallel string       Maximum amount of Jobs executed in parallel (default "1000")
      --ffsusemasteraddr                 Use the master address as the initial address for all new FFS instances instead of creating a new unique addess for each new FFS instance.
      --gatewaybasepath string           Gateway base path. (default "/")
      --gatewayhostaddr string           Gateway host listening address. (default "0.0.0.0:7000")
      --grpchostaddr string              gRPC host listening address. (default "/ip4/0.0.0.0/tcp/5002")
      --grpcwebproxyaddr string          gRPC webproxy listening address. (default "0.0.0.0:6002")
      --ipfsapiaddr string               IPFS API endpoint multiaddress. (Optional, only needed if FFS is used) (default "/ip4/127.0.0.1/tcp/5001")
      --lotushost string                 Lotus client API endpoint multiaddress. (default "/ip4/127.0.0.1/tcp/1234")
      --lotusmasteraddr string           Existing wallet address in Lotus to be used as source of funding for new FFS instances. (Optional)
      --lotustoken string                Lotus API authorization token. This flag or --lotustoken file are mandatory.
      --lotustokenfile string            Path of a file that contains the Lotus API authorization token.
      --maxminddbfolder string           Path of the folder containing GeoLite2-City.mmdb (default ".")
      --mongodb string                   Mongo database name. (if --mongouri is used, is mandatory
      --mongouri string                  Mongo URI to connect to MongoDB database. (Optional: if empty, will use Badger)
      --repopath string                  Path of the repository where Powergate state will be saved. (default "~/.powergate")
      --walletinitialfund int            FFS initial funding transaction amount in attoFIL received by --lotusmasteraddr. (if set) (default 250000000000000000)
```

## Localnet mode

Having a fully synced Lotus node can take a considerable amount of time and effort to mantain. We have built [lotus-devnet](https://github.com/textileio/lotus-devnet) which runs a local network with a _sectorbuilder_ mock. This provides a fast way to spinup a local network where the sealing process if mocked, but the rest of the node logic is the same as production The _localnet_ supports both 2Kib and 512Kib sectors, and the speed of block production is configurable. Refer to [lotus-devnet](https://github.com/textileio/lotus-devnet) readme for more information.

If you're interested in running Powergate and experiment with the CLI, the fastest way is to replace the Lotus client dependency with a running localnet, which runs a local Lotus client connected to a network with local miners. 

A simple docker-compose setup is available that will run Powergate connected to a Lotus local network with 512Mib sectors and allows to use the gRPC API or CLI without any extra config flags! 🎊  Note: you will first need to [install Docker compose](https://docs.docker.com/compose/install/) in order to get started.
```bash
cd docker
make localnet
```
This will build Powergate `powd`, a Lotus local network with `BIGSECTORS=true` by default, an IPFS node and wire them correctly to be ready to use.

**Note**: Running `BIGSECTORS=false make localnet` will create the Lotus devent using 2Kib sectors. This may be more appropriate for certain development or testing scenarios. 

Here is a full example of using the local network:
Terminal 1:
```bash
cd docker
make localnet
```

Wait for seeing logs about the height of the chain increase in a regular cadence.

Terminal 2:
```bash
make build
❯ head -c 700 </dev/urandom > myfile
❯ pow admin user create
{
  "user":  {
    "id":  "c06382e0-2021-4234-be53-6e07a8d40065",
    "token":  "883f57b1-4e66-47f8-b291-7cf8b10f6370"
  }
}
❯ pow data stage -t 883f57b1-4e66-47f8-b291-7cf8b10f6370 myfile
{
  "cid":  "QmQJxVtp61Y7UrdjUKuWvse3TxGHaPDyA7RobrBhFwqcBM"
}
❯ pow config apply -w -t 883f57b1-4e66-47f8-b291-7cf8b10f6370 QmYaAK8SSsKJsJdtahCbUe7MZzQdkPBybFCcQJJ3dKZpfm
{
  "jobId":  "b4110048-5367-4ae5-8508-709bf7969748"
}
                 JOB ID                |       STATUS       | MINER  |  PRICE   |    DEAL STATUS     
---------------------------------------+--------------------+--------+----------+--------------------
  b4110048-5367-4ae5-8508-709bf7969748 | JOB_STATUS_SUCCESS |        |          |                    
                                       |                    | f01000 | 62500000 | StorageDealActive
❯ pow data get -t 883f57b1-4e66-47f8-b291-7cf8b10f6370 QmYaAK8SSsKJsJdtahCbUe7MZzQdkPBybFCcQJJ3dKZpfm myfile2
> Success! Data written to myfile2
```

In this example we created a random 700 bytes file for the test, but since the localnet supports 512Mib sectors you can store store bigger files.

## Production setup

A production setup is also provided in the `docker` folder. It launches `powd` connected to `lotus` and `ipfs`, plus a set of monitoring components:
- _Prometheus_, which is the backend for metrics processing.
- _Grafana_, for metrics dashboard.
- _cAdvisor_, for container metrics.
- _Lotus_, node running on the current mainnet.
- _IPFS_, node running to back Powergate FFS.
- _Powergate_, wired with all of above components.

Depending on which network you want to connect to, you have to run different commands:
- `make up`, to connect to `mainnet`.

Remember that you should wait for _Lotus_ to be fully-synced which might take a long time; you can check your current node sync status running `lotus sync status` inside the Lotus container. We also provide automatically generated Dockerhub images of Powergate server, see [textile/powergate](https://hub.docker.com/r/textile/powergate).

If you're interested in a more detailed explanation about Powergate installation, please refer to the [installation docs](docs/manual_installation.md).

## Tests
We have a big set of tests for covering most important Powergate features.

For integration tests, we leverage our `textileio/lotus-devnet` configured with 2Kib sectors to provide fast iteration and CI runs.

If you want to run tests locally:
```bash
make test
```
It will auto-download any necessary dependencies and run all tests.

## Benchmark
There's a dedicated binary to run benchmarks against a Powergate server. For more information see the [specific README](cmd/powbench/README.md). 

Soon we'll add benchmark results against real miners in mainnet, so stay tuned. ⌛ 

## Contributing

This project is a work in progress. As such, there's a few things you can do right now to help out:

-   **Ask questions**! We'll try to help. Be sure to drop a note (on the above issue) if there is anything you'd like to work on and we'll update the issue to let others know. Also [get in touch](https://slack.textile.io) on Slack.
-   **Open issues**, [file issues](https://github.com/textileio/powergate/issues), submit pull requests!
-   **Perform code reviews**. More eyes will help a) speed the project along b) ensure quality and c) reduce possible future bugs.
-   **Take a look at the code**. Contributions here that would be most helpful are **top-level comments** about how it should look based on your understanding. Again, the more eyes the better.
-   **Add tests**. There can never be enough tests.

Before you get started, be sure to read our [contributors guide](./CONTRIBUTING.md) and our [contributor covenant code of conduct](./CODE_OF_CONDUCT.md).

## Changelog

[Changelog is published to Releases.](https://github.com/textileio/powergate/releases)

## License

[MIT](LICENSE)
