.github
    ISSUE_TEMPLATE
        bug_report.md
    weekly-digest.yml
    workflows
        mac_binary.yml
.gitpod.yml
ATTRIBUTIONS.md
CODE_OF_CONDUCT.md
CONTRIBUTING.md
Cargo.toml
METRICS.md
README.md
SECURITY.md
chain
    chain
        Cargo.toml
        src
            chain.rs
            doomslug.rs
            error.rs
            lib.rs
            lightclient.rs
            metrics.rs
            store.rs
            store_validator.rs
            store_validator
                validate.rs
            test_utils.rs
            types.rs
            validate.rs
        tests
            challenges.rs
            doomslug.rs
            gc.rs
            simple_chain.rs
            sync_chain.rs
    chunks
        Cargo.toml
        src
            chunk_cache.rs
            lib.rs
            test_utils.rs
            types.rs
    client
        Cargo.toml
        src
            client.rs
            client_actor.rs
            info.rs
            lib.rs
            metrics.rs
            sync.rs
            test_utils.rs
            types.rs
            view_client.rs
        tests
            bug_repros.rs
            catching_up.rs
            challenges.rs
            chunks_management.rs
            consensus.rs
            cross_shard_tx.rs
            process_blocks.rs
            query_client.rs
    epoch_manager
        Cargo.toml
        src
            lib.rs
            proposals.rs
            reward_calculator.rs
            test_utils.rs
            types.rs
        tests
            random_epochs.rs
    indexer
        Cargo.toml
        README.md
        src
            lib.rs
            streamer
                mod.rs
                streamer.rs
    jsonrpc
        Cargo.toml
        build_errors_schema.sh
        client
            Cargo.toml
            src
                lib.rs
                message.rs
        fuzz
            Cargo.toml
            fuzz_targets
                fuzz_target_1.rs
        res
            rpc_errors_schema.json
        src
            lib.rs
            metrics.rs
        test-utils
            Cargo.toml
            src
                lib.rs
        tests
            http_query.rs
            rpc_query.rs
            rpc_transactions.rs
            test_utils
                mod.rs
    network
        Cargo.toml
        benches
            graph.rs
        src
            cache.rs
            codec.rs
            lib.rs
            metrics.rs
            peer.rs
            peer_manager.rs
            peer_store.rs
            rate_counter.rs
            recorder.rs
            routing.rs
            test_utils.rs
            types.rs
            utils.rs
        tests
            ban_peers.rs
            cache.rs
            cache_edges.rs
            churn_attack.rs
            full_network.rs
            infinite_loop.rs
            peer_handshake.rs
            routing.rs
            runner
                mod.rs
            stress_network.rs
    pool
        Cargo.toml
        src
            lib.rs
            types.rs
    rosetta-rpc
        CHANGELOG.md
        Cargo.toml
        README.md
        src
            adapters.rs
            config.rs
            consts.rs
            errors.rs
            lib.rs
            models.rs
            utils.rs
    telemetry
        Cargo.toml
        src
            lib.rs
1
|
codecov.yml
core
    chain-configs
        Cargo.toml
        src
            client_config.rs
            genesis_config.rs
            lib.rs
    crypto
        Cargo.toml
        src
            hash.rs
            key_conversion.rs
            key_file.rs
            lib.rs
            randomness.rs
            signature.rs
            signer.rs
            test_utils.rs
            traits.rs
            util.rs
            vrf.rs
    metrics
        Cargo.toml
        src
            lib.rs
    primitives
        Cargo.toml
        benches
            serialization.rs
        src
            account.rs
            block.rs
            block_header.rs
            challenge.rs
            contract.rs
            epoch_manager.rs
            errors.rs
            hash.rs
            lib.rs
            logging.rs
            merkle.rs
            network.rs
            receipt.rs
            rpc.rs
            serialize.rs
            sharding.rs
            state_record.rs
            syncing.rs
            telemetry.rs
            test_utils.rs
            transaction.rs
            trie_key.rs
            types.rs
            utils.rs
            validator_signer.rs
            version.rs
            views.rs
    runtime-configs
        Cargo.toml
        src
            lib.rs
    store
        Cargo.toml
        benches
            trie_bench.rs
        src
            db.rs
            db
                migration_utils.rs
                refcount.rs
                v6_to_v7.rs
            lib.rs
            migrations.rs
            migrations
                v6_to_v7.rs
                v8_to_v9.rs
            test_utils.rs
            trie
                insert_delete.rs
                iterator.rs
                mod.rs
                nibble_slice.rs
                shard_tries.rs
                state_parts.rs
                trie_storage.rs
                trie_tests.rs
                update.rs
deny.toml
docs
    logo.svg
genesis-tools
    README.md
    genesis-csv-to-json
        Cargo.toml
        src
            csv_parser.rs
            csv_to_json_configs.rs
            main.rs
            serde_with.rs
    genesis-populate
        Cargo.toml
        src
            lib.rs
            main.rs
    keypair-generator
        Cargo.toml
        src
            main.rs
neard
    Cargo.toml
    res
        genesis_config.json
        mainnet_genesis.json
    src
        config.rs
        genesis_validate.rs
        lib.rs
        main.rs
        runtime.rs
        shard_tracker.rs
    tests
        economics.rs
        rpc_nodes.rs
        run_nodes.rs
        stake_nodes.rs
        sync_nodes.rs
        sync_state_nodes.rs
        track_shards.rs
nightly
    mocknet.txt
    nayduck.py
    nightly.txt
    nightly_small.txt
    tests_for_nayduck.txt
pytest
    __init__.py
    empty-contract-rs
        Cargo.toml
        build.sh
        src
            lib.rs
    lib
        __init__.py
        account.py
        branches.py
        cluster.py
        data.py
        lightclient.py
        messages
            __init__.py
            block.py
            crypto.py
            network.py
            shard.py
            tx.py
        metrics.py
        mocknet.py
        network.py
        peer.py
        populate.py
        proxy.py
        proxy_instances.py
        retrying.py
        serializer.py
        transaction.py
        utils.py
    remote.json
    remote.md
    requirements.txt
    tests
        __init__.py
        adversarial
            fork_sync.py
            gc_rollback.py
            malicious_chain.py
            start_from_genesis.py
            wrong_sync_info.py
        contracts
            deploy_call_smart_contract.py
            gibberish.py
        delete_remote_nodes.py
        mocknet
            bounce.py
            load_testing.py
            load_testing_helper.py
            outage.py
            sanity.py
        sanity
            backward_compatible.py
            block_production.py
            block_sync.py
            block_sync_archival.py
            concurrent_function_calls.py
            controlled_edge_nonce.py
            db_migration.py
            enable_inflation.py
            epoch_switches.py
            garbage_collection.py
            garbage_collection1.py
            gc_after_sync.py
            gc_after_sync1.py
            gc_sync_after_sync.py
            large_messages.py
            lightclnt.py
            network_drop_package.py
            one_val.py
            proxy_example.py
            proxy_restart.py
            proxy_simple.py
            repro_2916.py
            restaked.py
            restart.py
            rpc_finality.py
            rpc_light_client_execution_outcome_proof.py
            rpc_query.py
            rpc_state_changes.py
            rpc_tx_forwarding.py
            rpc_tx_submission.py
            skip_epoch.py
            staking1.py
            staking2.py
            staking_repro1.py
            staking_repro2.py
            state_migration.py
            state_sync.py
            state_sync1.py
            state_sync2.py
            state_sync3.py
            state_sync4.py
            state_sync5.py
            state_sync_late.py
            state_sync_massive.py
            state_sync_routed.py
            switch_node_key.py
            sync_ban.py
            transactions.py
            upgradable.py
            validator_switch.py
            validator_switch_key.py
        spec
            network
                future_handshake.py
                handshake.py
                peers_request.py
        stress
            hundred_nodes
                100_node_block_production.py
                README.md
                block_chunks.py
                collect_logs.py
                create_gcloud_image.py
                create_prometheus_targets.py
                node_rotation.py
                start_100_nodes.py
                watch_fork.py
            network_stress.py
            saturate_routing_table.py
            stress.py
runtime
    CHANGELOG.md
    near-runtime-fees
        Cargo.toml
        README.md
        src
            lib.rs
    near-runtime-utils
        Cargo.toml
        README.md
        src
            lib.rs
    near-vm-errors
        Cargo.toml
        README.md
        src
            lib.rs
    near-vm-logic
        Cargo.toml
        README.md
        src
            config.rs
            context.rs
            dependencies.rs
            gas_counter.rs
            lib.rs
            logic.rs
            mocks
                mock_external.rs
                mock_memory.rs
                mod.rs
            serde_with.rs
            types.rs
            utils.rs
        tests
            fixtures.rs
            helpers.rs
            test_context.rs
            test_iterators.rs
            test_miscs.rs
            test_promises.rs
            test_registers.rs
            test_storage_read_write.rs
            test_storage_usage.rs
            test_view_method.rs
            vm_logic_builder.rs
    near-vm-runner-standalone
        Cargo.toml
        README.md
        example-contract
            Cargo.toml
            build.js
            run.js
            src
                lib.rs
        src
            main.rs
    near-vm-runner
        Cargo.toml
        FAQ.md
        README.md
        benches
            bench.rs
        src
            cache.rs
            errors.rs
            imports.rs
            lib.rs
            memory.rs
            prepare.rs
            runner.rs
            wasmer_runner.rs
            wasmtime_runner.rs
        tests
            test-contract-rs
                Cargo.toml
                build.sh
                src
                    lib.rs
            test-contract-ts
                README.md
                assembly
                    index.ts
                    tsconfig.json
                package-lock.json
                package.json
            test_error_cases.rs
            test_invalid_contracts.rs
            test_rs_contract.rs
            test_ts_contract.rs
            test_utils
                mod.rs
    publish.sh
    runtime-params-estimator
        Cargo.toml
        README.md
        compiler.sh
        emu-cost
            README.md
            build.sh
            ci.sh
            compare_costs.py
            counter_plugin
                counter.c
                test.c
            run.sh
            safety_mutiplier.py
        estimate.sh
        setup.sh
        small-test-contract
            Cargo.toml
            build.sh
            src
                lib.rs
        src
            cases.rs
            ext_costs_generator.rs
            lib.rs
            main.rs
            runtime_fees_generator.rs
            stats.rs
            testbed.rs
            testbed_runners.rs
            vm_estimator.rs
        test-contract
            Cargo.toml
            build.sh
            src
                lib.rs
    runtime-standalone
        Cargo.toml
        contracts
            cross-contract-high-level
                Cargo.toml
                build.sh
                src
                    lib.rs
            status-message
                Cargo.toml
                README.md
                build.sh
                src
                    lib.rs
        src
            lib.rs
    runtime
        Cargo.toml
        src
            actions.rs
            adapter.rs
            balance_checker.rs
            cache.rs
            config.rs
            ext.rs
            lib.rs
            metrics.rs
            state_viewer.rs
            verifier.rs
        tests
            runtime_group_tools
                mod.rs
                random_config.rs
            test_async_calls.rs
            test_evil_contracts.rs
            test_regression.rs
            tiny-contract-rs
                Cargo.toml
                build.sh
                src
                    lib.rs
Host functions
Actions
|:--------------:|
rustfmt.toml
scripts
    binary-release.sh
    build_wasm.sh
    check_nightly.py
    create_service.py
    docker-release.sh
    flaky_test_check.py
    install_precommit.sh
    migrations
        10-gas-price-fix.py
        11-runtime-cost-adjustment.py
        12-fix-inflation.py
        13-block-merkle-root.py
        14-update-wasmer.py
        15-col-trie-changes.py
        16-expose-validator-method.py
        17-lightclient.py
        18-exclude-debug-info-from-outcome-proof.py
        19-col-chunks-height.py
        20-refund-allowance.py
        21-minimum-attached-gas.py
        22-protocol-upgrade.py
        23-delete_action_last.py
        24-max-gas-price.py
        25-minimum-stake-divisor.py
        26-introduce-pessimistic-gas.py
        27-add-amount-burnt-to-execution-outcome.py
        28-add-executor-id-to-execution-outcome.py
        29-add-compile-time-fees.py
        5-preserve-height.py
        6-state-stake.py
        7-account-registrar.py
        8-fraction.py
        9-state-record-data.py
    nodelib.py
    parallel_coverage.py
    parallel_run_tests.py
    run_clippy.sh
    run_docker.sh
    setup_hooks.sh
    start_localnet.py
    start_staging_testnet.py
    start_stakewars.py
    start_testnet.py
    start_unittest.py
    state
        mega-migrate.py
        split-genesis.py
        update_res.py
    stop.py
    tag_release.sh
    test_nearlib.sh
    testlib.py
    waitonserver.sh
test-utils
    loadtester
        Cargo.toml
        README.md
        src
            main.rs
            remote_node.rs
            sampler.rs
            stats.rs
            transactions_executor.rs
            transactions_generator.rs
    logger
        Cargo.toml
        src
            lib.rs
    state-viewer
        Cargo.toml
        src
            main.rs
            state_dump.rs
    store-validator
        Cargo.toml
        src
            main.rs
    testlib
        Cargo.toml
        src
            actix_utils.rs
            fees_utils.rs
            lib.rs
            node
                mod.rs
                process_node.rs
                runtime_node.rs
                thread_node.rs
            run_nodes.rs
            runtime_utils.rs
            standard_test_cases.rs
            test_helpers.rs
            user
                mod.rs
                rpc_user.rs
                runtime_user.rs
tests
    test_cases_runtime.rs
    test_cases_testnet_rpc.rs
    test_catchup.rs
    test_errors.rs
    test_overflows.rs
    test_rejoin.rs
    test_simple.rs
    test_tps_regression.rs
tools
    delay_detector
        Cargo.toml
        src
            lib.rs
    indexer
        example
            Cargo.toml
            README.md
            src
                configs.rs
                main.rs
    restaked
        Cargo.toml
        src
            main.rs
    rpctypegen
        core
            Cargo.toml
            src
                lib.rs
        macro
            Cargo.toml
            src
                lib.rs
utils
    actix
        Cargo.toml
        src
            lib.rs

# NEAR Indexer

NEAR Indexer is a micro-framework, which provides you with a stream of blocks that are recorded on NEAR network. It is useful to handle real-time "events" on the chain.

## Rationale

As scaling dApps enter NEAR’s mainnet, an issue may arise: how do they quickly and efficiently access state from our deployed smart contracts, and cut out the cruft? Contracts may grow to have complex data structures and querying the network RPC may not be the optimal way to access state data. The NEAR Indexer Framework allows for streams to be captured and indexed in a customized manner. The typical use-case is for this data to make its way to a relational database. Seeing as this is custom per project, there is engineering work involved in using this framework.

NEAR Indexer is already in use for several new projects, namely, we index all the events for NEAR Blockchain Explorer, and we also dig into Access Keys and index all of them for NEAR Wallet passphrase recovery and multi-factor authentication. With NEAR Indexer you can do high-level aggregation as well as low-level introspection of all the events inside the blockchain.

We are going to build more Indexers in the future, and will also consider building Indexer integrations with streaming solutions like Kafka, RabbitMQ, ZeroMQ, and NoSQL databases. Feel free to [join our discussions](https://github.com/nearprotocol/nearcore/issues/2996).

See the [example](https://github.com/nearprotocol/nearcore/tree/master/tools/indexer/example) for further technical details.

## How to set up and test NEAR Indexer

Before you proceed, make sure you have the following software installed:
* [rustup](https://rustup.rs/) or Rust version that is mentioned in `rust-toolchain` file in the root of nearcore project.

### localnet

Clone [nearcore](https://github.com/nearprotocol/nearcore)

To run the NEAR Indexer connected to a network we need to have configs and keys prepopulated. To generate configs for localnet do the following

```bash
$ git clone git@github.com:nearprotocol/nearcore.git
$ cd nearcore/tools/indexer/example
$ cargo run --release -- --home-dir ~/.near/localnet init
```

The above commands should initialize necessary configs and keys to run localnet in `~/.near/localnet`.

```bash
$ cargo run --release -- --home-dir ~/.near/localnet/ run
```

After the node is started, you should see logs of every block produced in your localnet. Get back to the code to implement any custom handling of the data flowing into the indexer.

Use [near-shell](https://github.com/near/near-shell) to submit transactions. For example, to create a new user you run the following command:

```
$ env NEAR_ENV=local near --keyPath ~/.near/localnet/validator_key.json create_account new-account.test.near --masterAccount test.near
```

### testnet / betanet

To run the NEAR Indexer connected to testnet or betanet we need to have configs and keys prepopulated, you can get them with the NEAR Indexer Example like above with a little change. Follow the instructions below to run non-validating node (leaving account ID empty).

```bash
$ cargo run --release --home-dir ~/.near/testnet init --chain-id testnet --download
```

The above code will download the official genesis config and generate necessary configs. You can replace `testnet` in the command above to different network ID `betanet`.

**NB!** According to changes in `nearcore` config generation we don't fill all the necessary fields in the config file. While this issue is open https://github.com/nearprotocol/nearcore/issues/3156 you need to download config you want and replace the generated one manually.
 - [testnet config.json](https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/testnet/config.json)
 - [betanet config.json](https://s3-us-west-1.amazonaws.com/build.nearprotocol.com/nearcore-deploy/betanet/config.json)
 
Replace `config.json` in your `--home-dir` (e.g. `~/.near/testnet/config.json`) with downloaded one.

Configs for the specified network are in the `--home-dir` provided folder. We need to ensure that NEAR Indexer follows all the necessary shards, so `"tracked_shards"` parameters in `~/.near/testnet/config.json` needs to be configured properly. For example, with a single shared network, you just add the shard #0 to the list:

```
...
"tracked_shards": [0],
...
```

Hint: See the Tweaks section below to learn more about further configuration options.

After that we can run NEAR Indexer.

```bash
$ cargo run --release -- --home-dir ~/.near/testnet run
```

After the network is synced, you should see logs of every block produced in Testnet. Get back to the code to implement any custom handling of the data flowing into the indexer.

## Tweaks

By default, nearcore is configured to do as little work as possible while still operating on an up-to-date state. Indexers may have different requirements, so there is no solution that would work for everyone, and thus we are going to provide you with the set of knobs you can tune for your requirements.

As already has been mentioned in this README, the most common tweak you need to apply is listing all the shards you want to index data from; to do that, you should ensure that `"tracked_shards"` in the `config.json` lists all the shard IDs, e.g. for the current betanet and testnet, which have a single shard:

```
...
"tracked_shards": [0],
...
```

You can choose Indexer Framework sync mode by setting what to stream:
 - `LatestSynced` - Real-time syncing, always taking the latest finalized block to stream
 - `FromInterruption` - Starts syncing from the block NEAR Indexer was interrupted last time 
 - `BlockHeight(u64)` - Specific block height to start syncing from
 
 Refer to `main()` function in [Indexer Example](https://github.com/nearprotocol/nearcore/blob/master/tools/indexer/example/src/main.rs)
 

Another tweak changes the default "fast" sync process to a "full" sync process. When the node gets online and observes that its state is missing or outdated, it will do state sync, and that can be done in two strategies:

1. ("fast" / default) sync enough information (only block headers) to ensure that the chain is valid; that means that the node won't have transactions, receipts, and execution outcomes, only the proofs, so Indexer will skip these blocks
2. (very slow / full sync) sync all the blocks, chunks, transactions, receipts, and execution outcomes starting from the genesis.

To force full sync (don't forget to track shards [see the previous tweak]), make the following change to your `config.json`:

```
...
"consensus": {
  ...
  "block_fetch_horizon": 18446744073709551615,
  ...
},
...
```

Indexer Framework also exposes access to the internal APIs (see `Indexer::client_actors` method), so you can fetch data about any block, transaction, etc, yet by default, nearcore is configured to remove old data (garbage collection [GC]), so querying the data that was observed a few epochs before may return an error saying that the data is not found. If you only need blocks streaming, you don't need this tweak, but if you need access to the historical data right from your Indexer, consider updating `"archive"` setting in `config.json` to `true`:

```
...
"archive": true,
...
```

## Who is using NEAR Indexer?

*This list is not exclusive, feel free to submit your project by sending a pull request.*

* [Indexer for NEAR Wallet](https://github.com/near/near-indexer-for-wallet)
* [Indexer for NEAR Explorer](https://github.com/near/near-indexer-for-explorer)
# Exact gas price estimator

## Theory of operations

 Operation execution cost (aka gas cost) is computed basing on the number of userland x86 instructions required to perform the
particular operation in current NEAR runtime implementation. To compute this cost, we use instrumented QEMU binary
translating engine to execute required operations in the userland Linux simulator.
Thus, to measure the execution cost we have to compile NEAR runtime benchmark for Linux, execute the benchmark under
instrumented QEMU running in Docker, and count how many x86 instructions are executed between start and end of execution.

 Instrumentation of QEMU is implemented in the following manner. We install instrumentation callback which conditionally increments
the instruction counter on every instruction during translation by QEMU's JIT, TCG. We activate counting when specific Linux syscall
(currently, 0 aka sys_read) is executed with the certain arguments (file descriptor argument == 0xcafebabe or 0xcafebabf).
On start event we clear instruction counter, on stop event we stop counting and return counted instructions into the buffer provided
to read syscall. As result, NEAR benchmark will know the exact instruction counter passed between two moments and this value
is the pure function of Docker image used, Rust compiler version and the NEAR implementation and is fully reproducible.

## Usage

We build and run the cost estimator in the Docker container to make sure config is fully reproducible.
Please make sure that Docker is given at least 4G of RAM, as running under emulator is rather resouce consuming.
First fetch appropriate base image, with `docker pull rust`.
Then create a Docker image with `build.sh`, it will create a Docker image with additional build deps.

Set `HOST_DIR` environment variable to local folder where relevant sources are present.
It will be mounted under `/host` in the Docker container.

Start container and build estimator with:

    host> ./run.sh
    docker> cd /host/nearcore/runtime/runtime-params-estimator
    docker> cargo run --release --package neard --bin neard -- --home /tmp/data init --chain-id= --test-seed=alice.near --account-id=test.near --fast
    docker> cargo run --release --package genesis-populate --bin genesis-populate -- --additional-accounts-num=200000 --home /tmp/data
    docker> cargo build --release --package runtime-params-estimator

Now start the estimator under QEMU with the counter plugin enabled (note, that Rust compiler produces SSE4, so specify recent CPU):

    docker> ./emu-cost/counter_plugin/qemu-x86_64 -cpu Westmere-v1 -plugin file=./emu-cost/counter_plugin/libcounter.so ../../target/release/runtime-params-estimator --home /tmp/data --accounts-num 20000 --iters 1 --warmup-iters 1

Note that it may take some time, as we execute instrumented code under the binary translator.

## Optional: re-building QEMU and the instruction counter plugin

We ship prebuilt QEMU and TCG instruction counter plugin, so in many cases one doesn't have to build it.
However, in case you still want to build it - use the following steps.

Important: we build QEMU and the TCG plugin inside the container, so execute following commands inside Docker.
Set environment variable HOST_DIR (on the host) to location where both QEMU and nearcore source code is checked
out, it will be mounted as `/host` inside the Docker container.
Start container with:

    ./run.sh

To build QEMU use:

    cd /host/qemu
    ./configure --disable-system --enable-user --enable-plugins  --prefix=/host/qemu-linux --target-list=x86_64-linux-user
    make && make install

Then build and test the QEMU's JIT plugin:

    cd /host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin
    make QEMU_DIR=/host/qemu
    make test

To execute commands in already running container first find its id with:

    > docker ps

    CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                  NAMES
    e9dcb52cc91b        ubuntu-emu          "/bin/bash"   2 hours ago         Up 2 hours          0.0.0.0:5000->22/tcp   reverent_carson

and the use container ID for `docker exec` command, like:

    docker exec -it e9dcb52cc91b /host/qemu-linux/bin/qemu-x86_64 -d plugin -plugin file=/host/qemu-linux/plugins/libcounter.so /host/nearcore/runtime/runtime-params-estimator/emu-cost/counter_plugin/test_binary

## Useful scripts

The script `../estimate.sh` will handle all the setup mentioned above. It uses `wasmer` by default, but you can pass `wasmtime` and `lightbeam` as well.

To just test the compile time costs use `../compile.sh`, which also takes an optional vm argument.  Its final output is (fee per byte, base fee)\
# Genesis Tools

* `genesis-populate` -- tool for creating genesis state dump populated with large number of accounts;
* TODO `genesis-rebase`-- tool for rebasing the entire chain to a new genesis;
* TODO `genesis-mainnet` -- tool for creating the main genesis used at the MainNet launch;

## `genesis-populate`

Performance of our node varies drastically depending on the size of the trie it operates with.
As the baseline, we agreed to take the trie roughly equivalent to the trie of Ethereum (as of October 2019) in
its complexity. We achieve it by populating trie with 80M accounts and uploading a smart contract to each of them.
We also make sure the trie does not compress the subtrees due to similar account names. We then use this trie for
benchmarking, loadtesting, and estimating system parameters.

To start node with 20k accounts first create configs:
```bash
cargo run --package neard --bin neard -- init --chain-id= --test-seed=alice.near --account-id=test.near --fast
```

Then create state dump with home many accounts you want:
```bash
cargo run --package genesis-populate --bin genesis-populate -- --additional-accounts-num=20000
```

This will produce `state_dump` and `genesis_roots` files in home directory of the node, you can also
use `--home` on all commands here to specify an absolute path to a home directory to use.

Then start a single local node:
```bash
cargo run --package neard --bin neard -- run --boot-nodes=
```
# near-vm-runner

An engine that run smart contracts compiled to Wasm by exposing
them `near-vm-logic` through the host functions. Currently is using Wasmer and singlepass compiler.

Can be used for benchmarks of smart contracts.
# near-runtime-fees

Fees applied to near runtime encapsulated in a separate crate. Might merge it later.
# Runtime Parameters Estimator

Use this tool to measure the running time of elementary runtime operations that have associated fees.

1. Create genesis config and generate state dump
    ```bash
    cargo run --release --package neard --bin neard -- --home /tmp/data init --chain-id= --test-seed=alice.near --account-id=test.near --fast
    cargo run --release --package genesis-populate --bin genesis-populate -- --additional-accounts-num=200000 --home /tmp/data
    ```

2. Run the estimator
    ```bash
    cargo run --release --package runtime-params-estimator --bin runtime-params-estimator -- --home /tmp/data --accounts-num 20000 --iters 1 --warmup-iters 1 --metric time
    ```

    With the given parameters above estimator will run relatively fast. We will be using different parameters to do the actual parameter estimation. Also note that the defualt metric is `icount`, instruction count, but requires using qemu to emulate the processor. So this example provides a way to get a  quick way to test out the estimator based on time, but the instructions in [`emu-cost/README.md`](./emu-cost/README.md) should be followed to get the real data.

Note, if you use the plotting functionality you would need to install [gnuplot](http://gnuplot.info/) to see the graphs.
# NEAR Runtime utilities

This crate contains utility functions for NEAR runtime.

The crate doesn't include dependencies on `near-primitives` and `near-crypto` to keep the dependencies limited.
Historically, `near-primitives` contained a bunch of dependencies that prevented this crate from being compiled to Wasm.
# Status Message

Records the status messages of the accounts that call this contract.

## Testing
To test run:
```bash
cargo test --package status-message -- --nocapture
```
A smart contract written in AssemblyScript that can be used to make
sure near runtime is compatible with AssemblyScript smart contracts.

# Pre-requisites

Switch to the smart contract directory and point npm to AssemblyScript:
```bash
npm install --save-dev AssemblyScript/assemblyscript
```

Then install dependencies with
```bash
npm install
```

# Building

Build smart contract with:
```bash
npm run asbuild:untouched
```

And copy the smart contract into `res` directory:
```bash
cp build/untouched.wasm ../res/test_contract_ts.wasm
```

Then run the Rust integration test with:
```bash
cargo test --package near-vm-runner --test test_ts_contract "" -- --nocapture
```
# Load testing tool

This tool can be used to test a local or remote set of nodes. It submits transactions at a given rate and monitors
the output TPS by periodically requesting the most recent block from the leading nodes.

## Example of a local testnet

The following is an example of starting a local testnet suitable for load testing and load testing it.

Create 10 validator accounts and totally 400 accounts (these are default arguments):
```bash
target/debug/loadtester create_genesis
```

Start a local testnet from home dir of `~/.near/near.x`

Launch the load tester:
```bash
target/debug/loadtester run --addrs 127.0.0.1:3030 127.0.0.1:3031 127.0.0.1:3032 127.0.0.1:3033
```

## Example of a launch and load test a remote testnet
Delete `sudo rm -rf ~/.near/near.*/data` if there is any. Create 10 validator accounts and totally 400 accounts (these are default arguments):
```bash
target/debug/loadtester create_genesis
```

Launch a testnet in several remote nodes.

Running loadtester with 100 tps, using 10 of accounts created in the first step.
```
target/debug/loadtester run --tps 100 --accounts 10 --addrs <list-of-node-socket-addrs>
```

## More usages

More parameters, like tps, number of accounts to create for loadtest network config, etc. is customizable. See them by
`loadtester run --help` and `loadtester create_genesis --help`.

## Further work
- [x] run loadtester with different kind of txns.
- [x] run loadtester from gcloud to a all gcloud network
- [1/2] run loadtester from local to hybrid of local/gcloud network.
- [1/2] exposing peer node information from nearcore, so you can specify only one address to post transaction to all public facing nodes.
NEAR Indexer Simple Logger Example
==================================

This is an example project featuring [NEAR Indexer Framework](https://github.com/nearprotocol/nearcore/tree/master/chain/indexer). This Indexer prints out all the blocks, chunks, transactions, receipts, execution outcomes, and state changes block by block immediately once it gets finalized in the network.

Refer to the NEAR Indexer Framework README to learn how to run this example.
# Steps to run hundred node tests

## Prerequisites

- Have a [gcloud cli installed](https://cloud.google.com/sdk/install), [being logged in](https://cloud.google.com/sdk/gcloud/reference/auth/login), default project set.
(If you can `gcloud compute instances list` and your account can create gcloud instances then it's good)
- python3, virtualenv, pip3
- Locally compile these packages (used for create config, keys and genesis)

```bash
cargo build -p neard --release
cargo build -p genesis-csv-to-json --release
cargo build -p keypair-generator --release
```

## Steps

1. Install python dependencies:

    ```bash
    sudo apt install python3-dev
    cd pytest
    virtualenv venv -p `which python3` # First time only
    . venv/bin/activate
    pip install -r requirements.txt
    ```

    Note: You need python3.6 or greater.

2. Create a gcloud (vm disk) image that has compiled near binary

    ```bash
    # This will build from current branch, image name as near-<branch>-YYYYMMDD-<username>, cargo build -p near --release
    python tests/stress/hundred_nodes/create_gcloud_image.py
    # If you want different branch, image name or additional flags passed to cargo
    python tests/stress/hundred_nodes/create_gcloud_image image_name branch 'additional flags'
    ```

3. Start hundred nodes

    ```bash
    # will use the near-<branch>-YYYYMMDD-<username> image, instance name will be pytest-node-<username>-0 to 99
    python tests/stress/hundred_nodes/start_100_nodes.py
    # If you have a different image name, or want different instance name
    ... start_100_nodes.py image_name instance_name_prefix
    ```

    Nodes are running after this step

4. Access every node

    ```bash
    gcloud compute ssh pytest-node-<i>
    tmux a
    ```

## Clean up

- Logs are stored in each instance in `/tmp/python-rc.log`. You can collect them all by `tests/stress/hundred_nodes/collect_logs.py`.
- Delete all instances quickly with `tests/delete_remote_nodes.py [prefix]`

## Some notes

If you have volatile or slow ssh access to gcloud instances, these scripts can fail at any step. I recommend create an instance on digitalocean, mosh to digitalocean instance (mosh is reliable), running all pytest script there (access gcloud from digital ocean is fast). In the reliable office network or mosh-digitalocean over an unreliable network, scripts never failed.
# Rosetta API Extension for nearcore

Rosetta is a public API spec defined to be a common denominator for blockchain projects.

Rosetta RPC is built into nearcore and it happily co-exist with JSON RPC.

- [Rosetta Homepage](https://www.rosetta-api.org/docs/welcome.html)
- [Rosetta API Specification](https://github.com/coinbase/rosetta-specifications)
- [Rosetta Tooling](https://github.com/coinbase/rosetta-cli)

You can view Rosetta API specification in [OpenAPI (Swagger) UI](https://petstore.swagger.io/)
passing the link to Rosetta OpenAPI specification:

```
https://raw.githubusercontent.com/coinbase/rosetta-specifications/master/api.json
```

Also, Rosetta implementation in nearcore exposes auto-generated OpenAPI
specification that has some extra comments regarding to the particular
implementation, and you can always access it from the running node:

```
http://localhost:3040/api/spec
```

## Supported Features

Our current goal is to have a minimal yet feature-complete implementation of
Rosetta RPC serving
[the main use-case Rosetta was designed for](https://community.rosetta-api.org/t/what-is-rosetta-main-use-case/92/2),
that is exposing balance-changing operations in a consistent way enabling
reconciliation through tracking individual blocks and transactions.

The Rosetta APIs are organized into two distinct categories, the Data API and
the Construction API. Simply put, the Data API is for retrieving data from a
blockchain network and the Construction API is for constructing and submitting
transactions to a blockchain network.

| Feature                      | Status                                                                                                                              |
| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
| Data API                     | Feature-complete with some quirks                                                                                                   |
| - `/network/list`            | Done                                                                                                                                |
| - `/network/status`          | Done                                                                                                                                |
| - `/network/options`         | Done                                                                                                                                |
| - `/block`                   | Feature-complete (exposes only balance-changing operations)                                                                         |
| - `/block/transaction`       | Feature-complete (exposes only balance-changing operations and the implementation is suboptimal from the performance point of view) |
| - `/account/balance`         | Done (properly exposes liquid, liquid for storage, and locked [staked] balances through sub-accounts)                               |
| - `/mempool`                 | Not implemented as mempool does not hold transactions for any meaningful time                                                       |
| - `/mempool/transaction`     | Not implemented (see above)                                                                                                         |
| Construction API             | Partially implemented                                                                                                               |
| - `/construction/derive`     | Done (not applicable to NEAR)                                                                                                       |
| - `/construction/preprocess` | Not implemented                                                                                                                     |
| - `/construction/metadata`   | Not implemented                                                                                                                     |
| - `/construction/payloads`   | Not implemented                                                                                                                     |
| - `/construction/combine`    | Not implemented                                                                                                                     |
| - `/construction/parse`      | Not implemented                                                                                                                     |
| - `/construction/hash`       | Done                                                                                                                                |
| - `/construction/submit`     | Done                                                                                                                                |

To verify the API compliance use:

```
rosetta-cli check:data --configuration-file=./rosetta.cfg
rosetta-cli check:construction --configuration-file=./rosetta.cfg
```

## How to Compile

Follow [the standard nearcore procedures to run a node compiled from the source code](https://docs.near.org/docs/contribution/nearcore)
enabling `rosetta_rpc` feature:

```bash
cargo build --release --package neard --bin neard --features rosetta_rpc
```

## How to Configure

You need `neard` binary to proceed; if you compiled it from the source code (see
above), you can find it in `./target/release/neard`.

### Initial Configuration

#### mainnet

```bash
neard --home ~/.near/mainnet init --chain-id mainnet --download-genesis
```

#### testnet

```bash
neard --home ~/.near/testnet init --chain-id testnet --download-genesis
```

NOTE: The genesis of testnet is around 1GB, so it will take a while to download it.

#### localnet (for local development)

```bash
neard --home ~/.near/localnet init
```

### Tuning

You are free to configure your node the way you feel necessary through the
config file: `~/.near/<chain-id>/config.json`. Here are some useful
configuration options for Rosetta API.

#### Enable Rosetta Server

By default, Rosetta API is disabled even if `neard` is compiled with the
feature enabled. Thus, we need to add the following section to the top-level
of the `config.json` (next to the `"rpc"` section):

```json
  ...
  "rosetta_rpc": {
    "addr": "0.0.0.0:3040",
    "cors_allowed_origins": [
      "*"
    ]
  },
  ...
```

#### Keep Track of Everything

By default, nearcore is configured to do as little work as possible while still
operating on an up-to-date state. Indexers may have different requirements, so
there is no solution that would work for everyone, and thus we are going to
provide you with the set of knobs you can tune for your requirements.

As already has been mentioned in this README, the most common tweak you need to
apply is listing all the shards you want to index data from; to do that, you
should ensure that `"tracked_shards"` lists all the shard IDs, e.g. for the
current betanet and testnet, which have a single shard:

```json
  ...
  "tracked_shards": [0],
  ...
```

By default, nearcore is configured to automatically clean old data (performs
garbage collection [GC]), so querying the data that was observed a few epochs
before may return an error saying that the data is missing. If you only need
recent blocks, you don't need this tweak, but if you need access to the
historical data, consider updating `"archive"` setting in `config.json` to
`true`:

```json
  ...
  "archive": true,
  ...
```

## How to Run

Once you have configured the node, just execute `neard` with the relevant home dir:

```bash
neard --home ~/.near/mainnet run
```

To confirm that everything is fine, you should be able to query the Rosetta API:

```bash
curl http://127.0.0.1:3040/network/list --header 'Content-Type: application/json' --data '{"metadata": {}}'
```

Expect to see the following response:

```json
{ "network_identifiers": [{ "blockchain": "nearprotocol", "network": "mainnet" }] }
```

The `network` value should reflect the chain id you specified during
configuration (`mainnet`, `testnet`, `betanet`, or a random string like
`test-chain-ztmbv` for localnet development).
# near-vm-logic

This crate implements the specification of the interface that Near blockchain exposes to the smart contracts.
It is not dependent on the specific way the smart contract code is executed, e.g. through Wasmer or whatnot, and
therefore can be used for unit tests in smart contracts.

Note, this logic assumes the little endian byte ordering of the memory used by the smart contract.

# Run tests

`cargo test --features mocks`
# near-vm-errors

Error that can occur inside Near Runtime encapsulated in a separate crate. Might merge it later.
# near-vm-runner-standalone

A command line wrapper around `near-vm-runner.`
All error messages that can be raised during the contract execution are raised by `near-vm-runner`
and the all effects of computing the execution result of a smart contract are encapsulated inside `near-vm-runner`.

One can use `near-vm-runner-standalone` to test the smart contracts, e.g. with integration tests
to make sure it has expected behavior once deployed to the blockchain.

To use run like this:

   cargo run -- --wasm-file ./status_message.wasm --method-name set_status --input '{"message": "12345"}'
   сargo run -- --wasm-file ./status_message.wasm --method-name get_status --input '{"account_id": "bob"}' --state '{"U1RBVEU=":"AQAAAAMAAABib2IFAAAAMTIzNDU="}'

I.e. persistent state could be passed across runs via `--state` parameter.
<br />
<br />

<p align="center">
<img src="docs/logo.svg" width="240">
</p>

<br />
<br />

## Reference implementation of NEAR Protocol

[![Build Status][ci-badge-master]][ci-url] 
![Stable Status][stable-release]
![Prerelease Status][prerelease]
[![codecov][codecov-badge]][codecov-url]
[![Discord chat][discord-badge]][discord-url]
[![Telegram Group][telegram-badge]][telegram-url]

[stable-release]: https://img.shields.io/github/v/release/nearprotocol/nearcore?label=stable
[prerelease]: https://img.shields.io/github/v/release/nearprotocol/nearcore?include_prereleases&label=prerelease
[ci-badge-master]: https://badge.buildkite.com/a81147cb62c585cc434459eedd1d25e521453120ead9ee6c64.svg?branch=master
[ci-url]: https://buildkite.com/nearprotocol/nearcore
[codecov-badge]: https://codecov.io/gh/nearprotocol/nearcore/branch/master/graph/badge.svg
[codecov-url]: https://codecov.io/gh/nearprotocol/nearcore
[discord-badge]: https://img.shields.io/discord/490367152054992913.svg
[discord-url]: https://near.chat
[telegram-badge]: https://cdn.jsdelivr.net/gh/Patrolavia/telegram-badge@8fe3382b3fd3a1c533ba270e608035a27e430c2e/chat.svg
[telegram-url]: https://t.me/cryptonear

## About NEAR

NEAR's purpose is to enable community-driven innovation to benefit people around the world.

To achieve this purpose, *NEAR* provides a developer platform where developers and entrepreneurs can create apps that put users back in control of their data and assets, which is the foundation of ["Open Web" movement][open-web-url].

One of the components of *NEAR* is NEAR Protocol, an infrastructure for server-less applications and smart contracts powered by blockchain.
NEAR Protocol is built to deliver usability and scalability of modern PaaS like Firebase at fraction of prices that blockchains like Ethereum charge.

*NEAR* overall provides wide range of tools for developers to easily build applications:
 - [JS Client library][js-api] to connect to NEAR Protocol from your applications.
 - [Rust][rust-sdk] and [AssemblyScript][as-sdk] SDKs to write smart contracts and stateful server-less functions.
 - [Numerous examples][examples-url] with links to hack on them right inside your browser.
 - [Lots of documentation][docs-url], with [Tutorials][tutorials-url] and [API docs][api-docs-url].

[open-web-url]: https://techcrunch.com/2016/04/10/1301496/ 
[js-api]: https://github.com/near/near-api-js 
[rust-sdk]: https://github.com/near/near-sdk-rs
[as-sdk]: https://github.com/near/near-sdk-as
[examples-url]: https://near.dev
[docs-url]: http://docs.nearprotocol.com
[tutorials-url]: https://docs.nearprotocol.com/docs/roles/developer/tutorials/introduction
[api-docs-url]: https://docs.nearprotocol.com/docs/roles/developer/examples/nearlib/introduction

## Join the Network

The easiest way to join the network, is by using `nearup` command, which you can install:

```bash
curl --proto '=https' --tlsv1.2 -sSfL https://up.near.dev | python3
```

You can join all the active networks:
* TestNet: `nearup testnet`
* BetaNet: `nearup betanet`

Check `nearup` repository for [more details](https://github.com/near/nearup) how to run with or without docker.

To learn how to become validator, checkout [documentation](https://docs.nearprotocol.com/docs/validator/staking-overview).

## Development Status

This project is currently under heavy development toward MainNet launch.

We are using [ZenHub](https://zenhub.com) to manage the development process. You can either login on their website or [install ZenHub extension](https://www.zenhub.com/extension) to see additional information right on Github.

For the high-level roadmap, checkout [Chain&Middleware's roadmap](https://app.zenhub.com/workspaces/chainmiddleware-5cea2bcf78297c385cf0ec81/roadmap). 

## Contributing

The workflow and details of setup to contribute are described in [CONTRIBUTING.md](CONTRIBUTING.md), and security policy is described in [SECURITY.md](SECURITY.md).
To propose new protocol change or standard use [Specification & Standards repository](https://github.com/nearprotocol/NEPs).
