text:
#==============================================================================#
# CognitoIdentity.jl
#
# This file is generated from:
# https://github.com/aws/aws-sdk-js/blob/master/apis/cognito-identity-2014-06-30.normal.json
#==============================================================================#

__precompile__()

module CognitoIdentity

using AWSCore

"""
    using AWSSDK.CognitoIdentity.create_identity_pool
    create_identity_pool([::AWSConfig], arguments::Dict)
    create_identity_pool([::AWSConfig]; IdentityPoolName=, AllowUnauthenticatedIdentities=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "CreateIdentityPool", arguments::Dict)
    cognito_identity([::AWSConfig], "CreateIdentityPool", IdentityPoolName=, AllowUnauthenticatedIdentities=, <keyword arguments>)

# CreateIdentityPool Operation

Creates a new identity pool. The identity pool is a store of user identity information that is specific to your AWS account. The limit on identity pools is 60 per account. The keys for `SupportedLoginProviders` are as follows:

*   Facebook: `graph.facebook.com`

*   Google: `accounts.google.com`

*   Amazon: `www.amazon.com`

*   Twitter: `api.twitter.com`

*   Digits: `www.digits.com`

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolName = ::String` -- *Required*
A string that you provide.

## `AllowUnauthenticatedIdentities = ::Bool` -- *Required*
TRUE if the identity pool supports unauthenticated logins.

## `SupportedLoginProviders = ::Dict{String,String}`
Optional key:value pairs mapping provider names to provider app IDs.

## `DeveloperProviderName = ::String`
The "domain" by which Cognito will refer to your users. This name acts as a placeholder that allows your backend and the Cognito service to communicate about the developer provider. For the `DeveloperProviderName`, you can use letters as well as period (`.`), underscore (`_`), and dash (`-`).

Once you have set a developer provider name, you cannot change it. Please take care in setting this parameter.

## `OpenIdConnectProviderARNs = [::String, ...]`
A list of OpendID Connect provider ARNs.

## `CognitoIdentityProviders = [[ ... ], ...]`
An array of Amazon Cognito Identity user pools and their client IDs.
```
 CognitoIdentityProviders = [[
        "ProviderName" =>  ::String,
        "ClientId" =>  ::String,
        "ServerSideTokenCheck" =>  ::Bool
    ], ...]
```

## `SamlProviderARNs = [::String, ...]`
An array of Amazon Resource Names (ARNs) of the SAML provider for your identity pool.

# Returns

`IdentityPool`

# Exceptions

`InvalidParameterException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException`, `InternalErrorException` or `LimitExceededException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/CreateIdentityPool)
"""
@inline create_identity_pool(aws::AWSConfig=default_aws_config(); args...) = create_identity_pool(aws, args)

@inline create_identity_pool(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "CreateIdentityPool", args)

@inline create_identity_pool(args) = create_identity_pool(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.delete_identities
    delete_identities([::AWSConfig], arguments::Dict)
    delete_identities([::AWSConfig]; IdentityIdsToDelete=)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "DeleteIdentities", arguments::Dict)
    cognito_identity([::AWSConfig], "DeleteIdentities", IdentityIdsToDelete=)

# DeleteIdentities Operation

Deletes identities from an identity pool. You can specify a list of 1-60 identities that you want to delete.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityIdsToDelete = [::String, ...]` -- *Required*
A list of 1-60 identities that you want to delete.

# Returns

`DeleteIdentitiesResponse`

# Exceptions

`InvalidParameterException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/DeleteIdentities)
"""
@inline delete_identities(aws::AWSConfig=default_aws_config(); args...) = delete_identities(aws, args)

@inline delete_identities(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "DeleteIdentities", args)

@inline delete_identities(args) = delete_identities(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.delete_identity_pool
    delete_identity_pool([::AWSConfig], arguments::Dict)
    delete_identity_pool([::AWSConfig]; IdentityPoolId=)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "DeleteIdentityPool", arguments::Dict)
    cognito_identity([::AWSConfig], "DeleteIdentityPool", IdentityPoolId=)

# DeleteIdentityPool Operation

Deletes a user pool. Once a pool is deleted, users will not be able to authenticate with the pool.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/DeleteIdentityPool)
"""
@inline delete_identity_pool(aws::AWSConfig=default_aws_config(); args...) = delete_identity_pool(aws, args)

@inline delete_identity_pool(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "DeleteIdentityPool", args)

@inline delete_identity_pool(args) = delete_identity_pool(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.describe_identity
    describe_identity([::AWSConfig], arguments::Dict)
    describe_identity([::AWSConfig]; IdentityId=)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "DescribeIdentity", arguments::Dict)
    cognito_identity([::AWSConfig], "DescribeIdentity", IdentityId=)

# DescribeIdentity Operation

Returns metadata related to the given identity, including when the identity was created and any associated linked logins.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityId = ::String` -- *Required*
A unique identifier in the format REGION:GUID.

# Returns

`IdentityDescription`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/DescribeIdentity)
"""
@inline describe_identity(aws::AWSConfig=default_aws_config(); args...) = describe_identity(aws, args)

@inline describe_identity(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "DescribeIdentity", args)

@inline describe_identity(args) = describe_identity(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.describe_identity_pool
    describe_identity_pool([::AWSConfig], arguments::Dict)
    describe_identity_pool([::AWSConfig]; IdentityPoolId=)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "DescribeIdentityPool", arguments::Dict)
    cognito_identity([::AWSConfig], "DescribeIdentityPool", IdentityPoolId=)

# DescribeIdentityPool Operation

Gets details about a particular identity pool, including the pool name, ID description, creation date, and current number of users.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

# Returns

`IdentityPool`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/DescribeIdentityPool)
"""
@inline describe_identity_pool(aws::AWSConfig=default_aws_config(); args...) = describe_identity_pool(aws, args)

@inline describe_identity_pool(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "DescribeIdentityPool", args)

@inline describe_identity_pool(args) = describe_identity_pool(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.get_credentials_for_identity
    get_credentials_for_identity([::AWSConfig], arguments::Dict)
    get_credentials_for_identity([::AWSConfig]; IdentityId=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "GetCredentialsForIdentity", arguments::Dict)
    cognito_identity([::AWSConfig], "GetCredentialsForIdentity", IdentityId=, <keyword arguments>)

# GetCredentialsForIdentity Operation

Returns credentials for the provided identity ID. Any provided logins will be validated against supported login providers. If the token is for cognito-identity.amazonaws.com, it will be passed through to AWS Security Token Service with the appropriate role for the token.

This is a public API. You do not need any credentials to call this API.

# Arguments

## `IdentityId = ::String` -- *Required*
A unique identifier in the format REGION:GUID.

## `Logins = ::Dict{String,String}`
A set of optional name-value pairs that map provider names to provider tokens.

## `CustomRoleArn = ::String`
The Amazon Resource Name (ARN) of the role to be assumed when multiple roles were received in the token from the identity provider. For example, a SAML-based identity provider. This parameter is optional for identity providers that do not support role customization.

# Returns

`GetCredentialsForIdentityResponse`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException`, `InvalidIdentityPoolConfigurationException`, `InternalErrorException` or `ExternalServiceException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/GetCredentialsForIdentity)
"""
@inline get_credentials_for_identity(aws::AWSConfig=default_aws_config(); args...) = get_credentials_for_identity(aws, args)

@inline get_credentials_for_identity(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "GetCredentialsForIdentity", args)

@inline get_credentials_for_identity(args) = get_credentials_for_identity(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.get_id
    get_id([::AWSConfig], arguments::Dict)
    get_id([::AWSConfig]; IdentityPoolId=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "GetId", arguments::Dict)
    cognito_identity([::AWSConfig], "GetId", IdentityPoolId=, <keyword arguments>)

# GetId Operation

Generates (or retrieves) a Cognito ID. Supplying multiple logins will create an implicit linked account.

This is a public API. You do not need any credentials to call this API.

# Arguments

## `AccountId = ::String`
A standard AWS account ID (9+ digits).

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

## `Logins = ::Dict{String,String}`
A set of optional name-value pairs that map provider names to provider tokens. The available provider names for `Logins` are as follows:

*   Facebook: `graph.facebook.com`

*   Amazon Cognito Identity Provider: `cognito-idp.us-east-1.amazonaws.com/us-east-1_123456789`

*   Google: `accounts.google.com`

*   Amazon: `www.amazon.com`

*   Twitter: `api.twitter.com`

*   Digits: `www.digits.com`

# Returns

`GetIdResponse`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException`, `InternalErrorException`, `LimitExceededException` or `ExternalServiceException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/GetId)
"""
@inline get_id(aws::AWSConfig=default_aws_config(); args...) = get_id(aws, args)

@inline get_id(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "GetId", args)

@inline get_id(args) = get_id(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.get_identity_pool_roles
    get_identity_pool_roles([::AWSConfig], arguments::Dict)
    get_identity_pool_roles([::AWSConfig]; IdentityPoolId=)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "GetIdentityPoolRoles", arguments::Dict)
    cognito_identity([::AWSConfig], "GetIdentityPoolRoles", IdentityPoolId=)

# GetIdentityPoolRoles Operation

Gets the roles for an identity pool.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

# Returns

`GetIdentityPoolRolesResponse`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/GetIdentityPoolRoles)
"""
@inline get_identity_pool_roles(aws::AWSConfig=default_aws_config(); args...) = get_identity_pool_roles(aws, args)

@inline get_identity_pool_roles(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "GetIdentityPoolRoles", args)

@inline get_identity_pool_roles(args) = get_identity_pool_roles(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.get_open_id_token
    get_open_id_token([::AWSConfig], arguments::Dict)
    get_open_id_token([::AWSConfig]; IdentityId=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "GetOpenIdToken", arguments::Dict)
    cognito_identity([::AWSConfig], "GetOpenIdToken", IdentityId=, <keyword arguments>)

# GetOpenIdToken Operation

Gets an OpenID token, using a known Cognito ID. This known Cognito ID is returned by [GetId](@ref). You can optionally add additional logins for the identity. Supplying multiple logins creates an implicit link.

The OpenId token is valid for 15 minutes.

This is a public API. You do not need any credentials to call this API.

# Arguments

## `IdentityId = ::String` -- *Required*
A unique identifier in the format REGION:GUID.

## `Logins = ::Dict{String,String}`
A set of optional name-value pairs that map provider names to provider tokens. When using graph.facebook.com and www.amazon.com, supply the access_token returned from the provider's authflow. For accounts.google.com, an Amazon Cognito Identity Provider, or any other OpenId Connect provider, always include the `id_token`.

# Returns

`GetOpenIdTokenResponse`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException`, `InternalErrorException` or `ExternalServiceException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/GetOpenIdToken)
"""
@inline get_open_id_token(aws::AWSConfig=default_aws_config(); args...) = get_open_id_token(aws, args)

@inline get_open_id_token(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "GetOpenIdToken", args)

@inline get_open_id_token(args) = get_open_id_token(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.get_open_id_token_for_developer_identity
    get_open_id_token_for_developer_identity([::AWSConfig], arguments::Dict)
    get_open_id_token_for_developer_identity([::AWSConfig]; IdentityPoolId=, Logins=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "GetOpenIdTokenForDeveloperIdentity", arguments::Dict)
    cognito_identity([::AWSConfig], "GetOpenIdTokenForDeveloperIdentity", IdentityPoolId=, Logins=, <keyword arguments>)

# GetOpenIdTokenForDeveloperIdentity Operation

Registers (or retrieves) a Cognito `IdentityId` and an OpenID Connect token for a user authenticated by your backend authentication process. Supplying multiple logins will create an implicit linked account. You can only specify one developer provider as part of the `Logins` map, which is linked to the identity pool. The developer provider is the "domain" by which Cognito will refer to your users.

You can use `GetOpenIdTokenForDeveloperIdentity` to create a new identity and to link new logins (that is, user credentials issued by a public provider or developer provider) to an existing identity. When you want to create a new identity, the `IdentityId` should be null. When you want to associate a new login with an existing authenticated/unauthenticated identity, you can do so by providing the existing `IdentityId`. This API will create the identity in the specified `IdentityPoolId`.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

## `IdentityId = ::String`
A unique identifier in the format REGION:GUID.

## `Logins = ::Dict{String,String}` -- *Required*
A set of optional name-value pairs that map provider names to provider tokens. Each name-value pair represents a user from a public provider or developer provider. If the user is from a developer provider, the name-value pair will follow the syntax `"developer_provider_name": "developer_user_identifier"`. The developer provider is the "domain" by which Cognito will refer to your users; you provided this domain while creating/updating the identity pool. The developer user identifier is an identifier from your backend that uniquely identifies a user. When you create an identity pool, you can specify the supported logins.

## `TokenDuration = ::Int`
The expiration time of the token, in seconds. You can specify a custom expiration time for the token so that you can cache it. If you don't provide an expiration time, the token is valid for 15 minutes. You can exchange the token with Amazon STS for temporary AWS credentials, which are valid for a maximum of one hour. The maximum token duration you can set is 24 hours. You should take care in setting the expiration time for a token, as there are significant security implications: an attacker could use a leaked token to access your AWS resources for the token's duration.

# Returns

`GetOpenIdTokenForDeveloperIdentityResponse`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException`, `InternalErrorException` or `DeveloperUserAlreadyRegisteredException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/GetOpenIdTokenForDeveloperIdentity)
"""
@inline get_open_id_token_for_developer_identity(aws::AWSConfig=default_aws_config(); args...) = get_open_id_token_for_developer_identity(aws, args)

@inline get_open_id_token_for_developer_identity(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "GetOpenIdTokenForDeveloperIdentity", args)

@inline get_open_id_token_for_developer_identity(args) = get_open_id_token_for_developer_identity(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.list_identities
    list_identities([::AWSConfig], arguments::Dict)
    list_identities([::AWSConfig]; IdentityPoolId=, MaxResults=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "ListIdentities", arguments::Dict)
    cognito_identity([::AWSConfig], "ListIdentities", IdentityPoolId=, MaxResults=, <keyword arguments>)

# ListIdentities Operation

Lists the identities in a pool.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

## `MaxResults = ::Int` -- *Required*
The maximum number of identities to return.

## `NextToken = ::String`
A pagination token.

## `HideDisabled = ::Bool`
An optional boolean parameter that allows you to hide disabled identities. If omitted, the ListIdentities API will include disabled identities in the response.

# Returns

`ListIdentitiesResponse`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/ListIdentities)
"""
@inline list_identities(aws::AWSConfig=default_aws_config(); args...) = list_identities(aws, args)

@inline list_identities(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "ListIdentities", args)

@inline list_identities(args) = list_identities(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.list_identity_pools
    list_identity_pools([::AWSConfig], arguments::Dict)
    list_identity_pools([::AWSConfig]; MaxResults=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "ListIdentityPools", arguments::Dict)
    cognito_identity([::AWSConfig], "ListIdentityPools", MaxResults=, <keyword arguments>)

# ListIdentityPools Operation

Lists all of the Cognito identity pools registered for your account.

You must use AWS Developer credentials to call this API.

# Arguments

## `MaxResults = ::Int` -- *Required*
The maximum number of identities to return.

## `NextToken = ::String`
A pagination token.

# Returns

`ListIdentityPoolsResponse`

# Exceptions

`InvalidParameterException`, `NotAuthorizedException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/ListIdentityPools)
"""
@inline list_identity_pools(aws::AWSConfig=default_aws_config(); args...) = list_identity_pools(aws, args)

@inline list_identity_pools(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "ListIdentityPools", args)

@inline list_identity_pools(args) = list_identity_pools(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.lookup_developer_identity
    lookup_developer_identity([::AWSConfig], arguments::Dict)
    lookup_developer_identity([::AWSConfig]; IdentityPoolId=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "LookupDeveloperIdentity", arguments::Dict)
    cognito_identity([::AWSConfig], "LookupDeveloperIdentity", IdentityPoolId=, <keyword arguments>)

# LookupDeveloperIdentity Operation

Retrieves the `IdentityID` associated with a `DeveloperUserIdentifier` or the list of `DeveloperUserIdentifier`s associated with an `IdentityId` for an existing identity. Either `IdentityID` or `DeveloperUserIdentifier` must not be null. If you supply only one of these values, the other value will be searched in the database and returned as a part of the response. If you supply both, `DeveloperUserIdentifier` will be matched against `IdentityID`. If the values are verified against the database, the response returns both values and is the same as the request. Otherwise a `ResourceConflictException` is thrown.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

## `IdentityId = ::String`
A unique identifier in the format REGION:GUID.

## `DeveloperUserIdentifier = ::String`
A unique ID used by your backend authentication process to identify a user. Typically, a developer identity provider would issue many developer user identifiers, in keeping with the number of users.

## `MaxResults = ::Int`
The maximum number of identities to return.

## `NextToken = ::String`
A pagination token. The first call you make will have `NextToken` set to null. After that the service will return `NextToken` values as needed. For example, let's say you make a request with `MaxResults` set to 10, and there are 20 matches in the database. The service will return a pagination token as a part of the response. This token can be used to call the API again and get results starting from the 11th match.

# Returns

`LookupDeveloperIdentityResponse`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/LookupDeveloperIdentity)
"""
@inline lookup_developer_identity(aws::AWSConfig=default_aws_config(); args...) = lookup_developer_identity(aws, args)

@inline lookup_developer_identity(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "LookupDeveloperIdentity", args)

@inline lookup_developer_identity(args) = lookup_developer_identity(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.merge_developer_identities
    merge_developer_identities([::AWSConfig], arguments::Dict)
    merge_developer_identities([::AWSConfig]; SourceUserIdentifier=, DestinationUserIdentifier=, DeveloperProviderName=, IdentityPoolId=)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "MergeDeveloperIdentities", arguments::Dict)
    cognito_identity([::AWSConfig], "MergeDeveloperIdentities", SourceUserIdentifier=, DestinationUserIdentifier=, DeveloperProviderName=, IdentityPoolId=)

# MergeDeveloperIdentities Operation

Merges two users having different `IdentityId`s, existing in the same identity pool, and identified by the same developer provider. You can use this action to request that discrete users be merged and identified as a single user in the Cognito environment. Cognito associates the given source user (`SourceUserIdentifier`) with the `IdentityId` of the `DestinationUserIdentifier`. Only developer-authenticated users can be merged. If the users to be merged are associated with the same public provider, but as two different users, an exception will be thrown.

You must use AWS Developer credentials to call this API.

# Arguments

## `SourceUserIdentifier = ::String` -- *Required*
User identifier for the source user. The value should be a `DeveloperUserIdentifier`.

## `DestinationUserIdentifier = ::String` -- *Required*
User identifier for the destination user. The value should be a `DeveloperUserIdentifier`.

## `DeveloperProviderName = ::String` -- *Required*
The "domain" by which Cognito will refer to your users. This is a (pseudo) domain name that you provide while creating an identity pool. This name acts as a placeholder that allows your backend and the Cognito service to communicate about the developer provider. For the `DeveloperProviderName`, you can use letters as well as period (.), underscore (_), and dash (-).

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

# Returns

`MergeDeveloperIdentitiesResponse`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/MergeDeveloperIdentities)
"""
@inline merge_developer_identities(aws::AWSConfig=default_aws_config(); args...) = merge_developer_identities(aws, args)

@inline merge_developer_identities(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "MergeDeveloperIdentities", args)

@inline merge_developer_identities(args) = merge_developer_identities(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.set_identity_pool_roles
    set_identity_pool_roles([::AWSConfig], arguments::Dict)
    set_identity_pool_roles([::AWSConfig]; IdentityPoolId=, Roles=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "SetIdentityPoolRoles", arguments::Dict)
    cognito_identity([::AWSConfig], "SetIdentityPoolRoles", IdentityPoolId=, Roles=, <keyword arguments>)

# SetIdentityPoolRoles Operation

Sets the roles for an identity pool. These roles are used when making calls to [GetCredentialsForIdentity](@ref) action.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

## `Roles = ::Dict{String,String}` -- *Required*
The map of roles associated with this pool. For a given role, the key will be either "authenticated" or "unauthenticated" and the value will be the Role ARN.

## `RoleMappings = ::Dict{String,String}`
How users for a specific identity provider are to mapped to roles. This is a string to [RoleMapping](@ref) object map. The string identifies the identity provider, for example, "graph.facebook.com" or "cognito-idp-east-1.amazonaws.com/us-east-1_abcdefghi:app_client_id".

Up to 25 rules can be specified per identity provider.

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException`, `InternalErrorException` or `ConcurrentModificationException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/SetIdentityPoolRoles)
"""
@inline set_identity_pool_roles(aws::AWSConfig=default_aws_config(); args...) = set_identity_pool_roles(aws, args)

@inline set_identity_pool_roles(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "SetIdentityPoolRoles", args)

@inline set_identity_pool_roles(args) = set_identity_pool_roles(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.unlink_developer_identity
    unlink_developer_identity([::AWSConfig], arguments::Dict)
    unlink_developer_identity([::AWSConfig]; IdentityId=, IdentityPoolId=, DeveloperProviderName=, DeveloperUserIdentifier=)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "UnlinkDeveloperIdentity", arguments::Dict)
    cognito_identity([::AWSConfig], "UnlinkDeveloperIdentity", IdentityId=, IdentityPoolId=, DeveloperProviderName=, DeveloperUserIdentifier=)

# UnlinkDeveloperIdentity Operation

Unlinks a `DeveloperUserIdentifier` from an existing identity. Unlinked developer users will be considered new identities next time they are seen. If, for a given Cognito identity, you remove all federated identities as well as the developer user identifier, the Cognito identity becomes inaccessible.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityId = ::String` -- *Required*
A unique identifier in the format REGION:GUID.

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

## `DeveloperProviderName = ::String` -- *Required*
The "domain" by which Cognito will refer to your users.

## `DeveloperUserIdentifier = ::String` -- *Required*
A unique ID used by your backend authentication process to identify a user.

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException` or `InternalErrorException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/UnlinkDeveloperIdentity)
"""
@inline unlink_developer_identity(aws::AWSConfig=default_aws_config(); args...) = unlink_developer_identity(aws, args)

@inline unlink_developer_identity(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "UnlinkDeveloperIdentity", args)

@inline unlink_developer_identity(args) = unlink_developer_identity(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.unlink_identity
    unlink_identity([::AWSConfig], arguments::Dict)
    unlink_identity([::AWSConfig]; IdentityId=, Logins=, LoginsToRemove=)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "UnlinkIdentity", arguments::Dict)
    cognito_identity([::AWSConfig], "UnlinkIdentity", IdentityId=, Logins=, LoginsToRemove=)

# UnlinkIdentity Operation

Unlinks a federated identity from an existing account. Unlinked logins will be considered new identities next time they are seen. Removing the last linked login will make this identity inaccessible.

This is a public API. You do not need any credentials to call this API.

# Arguments

## `IdentityId = ::String` -- *Required*
A unique identifier in the format REGION:GUID.

## `Logins = ::Dict{String,String}` -- *Required*
A set of optional name-value pairs that map provider names to provider tokens.

## `LoginsToRemove = [::String, ...]` -- *Required*
Provider names to unlink from this identity.

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException`, `InternalErrorException` or `ExternalServiceException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/UnlinkIdentity)
"""
@inline unlink_identity(aws::AWSConfig=default_aws_config(); args...) = unlink_identity(aws, args)

@inline unlink_identity(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "UnlinkIdentity", args)

@inline unlink_identity(args) = unlink_identity(default_aws_config(), args)

"""
    using AWSSDK.CognitoIdentity.update_identity_pool
    update_identity_pool([::AWSConfig], arguments::Dict)
    update_identity_pool([::AWSConfig]; IdentityPoolId=, IdentityPoolName=, AllowUnauthenticatedIdentities=, <keyword arguments>)

    using AWSCore.Services.cognito_identity
    cognito_identity([::AWSConfig], "UpdateIdentityPool", arguments::Dict)
    cognito_identity([::AWSConfig], "UpdateIdentityPool", IdentityPoolId=, IdentityPoolName=, AllowUnauthenticatedIdentities=, <keyword arguments>)

# UpdateIdentityPool Operation

Updates a user pool.

You must use AWS Developer credentials to call this API.

# Arguments

## `IdentityPoolId = ::String` -- *Required*
An identity pool ID in the format REGION:GUID.

## `IdentityPoolName = ::String` -- *Required*
A string that you provide.

## `AllowUnauthenticatedIdentities = ::Bool` -- *Required*
TRUE if the identity pool supports unauthenticated logins.

## `SupportedLoginProviders = ::Dict{String,String}`
Optional key:value pairs mapping provider names to provider app IDs.

## `DeveloperProviderName = ::String`
The "domain" by which Cognito will refer to your users.

## `OpenIdConnectProviderARNs = [::String, ...]`
A list of OpendID Connect provider ARNs.

## `CognitoIdentityProviders = [[ ... ], ...]`
A list representing an Amazon Cognito Identity User Pool and its client ID.
```
 CognitoIdentityProviders = [[
        "ProviderName" =>  ::String,
        "ClientId" =>  ::String,
        "ServerSideTokenCheck" =>  ::Bool
    ], ...]
```

## `SamlProviderARNs = [::String, ...]`
An array of Amazon Resource Names (ARNs) of the SAML provider for your identity pool.

# Returns

`IdentityPool`

# Exceptions

`InvalidParameterException`, `ResourceNotFoundException`, `NotAuthorizedException`, `ResourceConflictException`, `TooManyRequestsException`, `InternalErrorException`, `ConcurrentModificationException` or `LimitExceededException`.

See also: [AWS API Documentation](https://docs.aws.amazon.com/goto/WebAPI/cognito-identity-2014-06-30/UpdateIdentityPool)
"""
@inline update_identity_pool(aws::AWSConfig=default_aws_config(); args...) = update_identity_pool(aws, args)

@inline update_identity_pool(aws::AWSConfig, args) = AWSCore.Services.cognito_identity(aws, "UpdateIdentityPool", args)

@inline update_identity_pool(args) = update_identity_pool(default_aws_config(), args)

end # module CognitoIdentity

#==============================================================================#
# End of file
#==============================================================================#


text:
# tests for multinomial logit models

include("biogeme_swissmetro.jl")
include("biogeme_swissmetro_distributed.jl")
include("convergence_failure.jl")

ext: jl


text:
module CARPData

import Base: show

export Data, Vertex, Edge, load, isRequired

const data_path = joinpath(pkgdir(CARPData), "data")

include("Vertex.jl")
include("Edge.jl")
include("Data.jl")
include("Loader.jl")
include("Util.jl")

end


text:
#  we assume
# * outidx already has grouping columns
# * any fun except byrow will get input from input data set, and byrow gets input from the output data set.
# * except Tuple every other column selector will assume funcions are univariate (byrow is exception)

# col => fun => dst, the job is to create col => fun => :dst
function normalize_combine!(outidx::Index, idx,
    @nospecialize(sel::Pair{<:ColumnIndex,
                            <:Pair{<:Union{Function},
                                <:Union{Symbol, AbstractString}}})
                                )
    src, (fun, dst) = sel
    _check_ind_and_add!(outidx, Symbol(dst))
    return _names(idx)[idx[src]] => fun => Symbol(dst)
end

# Tuple => fun => dst, the job is to create Tuple => fun => :dst
function normalize_combine!(outidx::Index, idx,
    @nospecialize(sel::Pair{<:NTuple{N, ColumnIndex},
                            <:Pair{<:Union{Function},
                                <:Union{Symbol, AbstractString}}})
                                ) where N
    src, (fun, dst) = sel
    N < 2 && throw(ArgumentError("For multivariate functions (Tuple of column names), the number of input columns must be greater than 1"))
    _check_ind_and_add!(outidx, Symbol(dst))
    return ntuple(i -> _names(idx)[idx[src[i]]], N) => fun => Symbol(dst)
end

# col => fun, the job is to create col => fun => :colname
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:ColumnIndex,
                                <:Union{Function}}))

    src, fun = sel
    nname = Symbol(funname(fun), "_", _names(idx)[idx[src]])
    _check_ind_and_add!(outidx, nname)
    return _names(idx)[idx[src]] => fun => nname
end

# Tuple => fun  normalise as Tuple => fun => :genname
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:NTuple{N, ColumnIndex},
                                <:Union{Function}})) where N

    src, fun = sel
    N < 2 && throw(ArgumentError("For multivariate functions (Tuple of column names), the number of input columns must be greater than 1"))
    col1, col2 = idx[src[1]], idx[src[2]]
    var1, var2 = _names(idx)[col1], _names(idx)[col2]
    if N > 2
        nname = Symbol(funname(fun), "_", var1, "_", var2, "_etc")
    else
        nname = Symbol(funname(fun), "_", var1, "_", var2)
    end

    _check_ind_and_add!(outidx, nname)
    return ntuple(i -> _names(idx)[idx[src[i]]], N) => fun => nname
end

# handling vector of tuples
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:Vector{<:NTuple{N, ColumnIndex}},
                                <:Union{Function}})) where N
    normalize_combine!(outidx, idx, sel.first .=> sel.second)
end
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:Vector{<:NTuple{N, ColumnIndex}},
                                <:Vector{<:Function}})) where N
    normalize_combine!(outidx, idx, Ref(sel.first) .=> sel.second)
end

# col => byrow
#TODO if we define byrow(fun) as a type rather than an Expr, we should modify this part
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:ColumnIndex,
                                <:Vector{Expr}}))
    if sel.second[1].head == :BYROW
        colsidx = outidx[sel.first]
        dsc_sym = Symbol(funname(sel.second[1].args[1]), "_", _names(outidx)[outidx[sel.first]])
        _check_ind_and_add!(outidx, dsc_sym )
        return _names(outidx)[outidx[colsidx]] => sel.second[1] => dsc_sym
    end
    throw(ArgumentError("only byrow is accepted when using expressions"))
end
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:ColumnIndex,
                                <:Expr}))
    if sel.second.head == :BYROW
        colsidx = outidx[sel.first]

        dsc_sym = Symbol(funname(sel.second.args[1]), "_", _names(outidx)[outidx[sel.first]])
        _check_ind_and_add!(outidx, dsc_sym )
        return _names(outidx)[outidx[colsidx]] => sel.second => dsc_sym
    end
    throw(ArgumentError("only byrow is accepted when using expressions"))
end

# col => byrow => dst
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:ColumnIndex,
                                <:Pair{<:Vector{Expr},
                                    <:Union{Symbol, AbstractString}}}))
    if sel.second.first[1].head == :BYROW
        colsidx = outidx[sel.first]
        _check_ind_and_add!(outidx, Symbol(sel.second.second))
        return _names(outidx)[outidx[colsidx]] => sel.second.first[1] => Symbol(sel.second.second)
    end
    throw(ArgumentError("only byrow is accepted when using expressions"))
end
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:ColumnIndex,
                                <:Pair{<:Expr,
                                    <:Union{Symbol, AbstractString}}}))
    if sel.second.first.head == :BYROW
        colsidx = outidx[sel.first]
        _check_ind_and_add!(outidx, Symbol(sel.second.second))
        return _names(outidx)[outidx[colsidx]] => sel.second.first => Symbol(sel.second.second)
    end
    throw(ArgumentError("only byrow is accepted when using expressions"))
end

# cols => byrow
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:MultiColumnIndex,
                                <:Vector{Expr}}))
    if sel.second[1] isa Expr
        if sel.second[1].head == :BYROW
            colsidx = outidx[sel.first]
        end
        _check_ind_and_add!(outidx, Symbol("row_", funname(sel.second[1].args[1])))
        return _names(outidx)[outidx[colsidx]] => sel.second[1] => Symbol("row_", funname(sel.second[1].args[1]))
    end
    throw(ArgumentError("only byrow is accepted when using expressions"))
end
# cols => fun/byrow normalise as cols .=> fun or cols => byrow
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:MultiColumnIndex,
                                <:Union{Function,Expr}}))
    if sel.second isa Expr
        if sel.second.head == :BYROW
            colsidx = outidx[sel.first]
        end
    # TODO needs a better name for destination
        _check_ind_and_add!(outidx, Symbol("row_", funname(sel.second.args[1])))
        return _names(outidx)[outidx[colsidx]] => sel.second => Symbol("row_", funname(sel.second.args[1]))
    end
    normalize_combine!(outidx, idx, idx[sel.first] .=> sel.second)
end

# cols => funs normalize cols .=> Ref(funs)
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:MultiColumnIndex,
                                <:Vector{<:Function}}))
    colsidx = idx[sel.first]
    normalize_combine!(outidx, idx, colsidx .=> Ref(sel.second))
end

# col => funs normalise as col .=> funs
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:ColumnIndex,
                                <:Vector{<:Function}}))
    colsidx = idx[sel.first]
    normalize_combine!(outidx, idx, colsidx .=> sel.second)
end

# special case cols => byrow(...) => :name
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:MultiColumnIndex,
                                <:Pair{<:Vector{Expr},
                                    <:Union{Symbol, AbstractString}}}))
    if sel.second.first[1].head == :BYROW
        colsidx = outidx[sel.first]
        _check_ind_and_add!(outidx, Symbol(sel.second.second))
        return _names(outidx)[outidx[colsidx]] => sel.second.first[1] => Symbol(sel.second.second)
    end
    throw(ArgumentError("only byrow operation is supported for cols => fun => :name"))
end
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:MultiColumnIndex,
                                <:Pair{<:Expr,
                                    <:Union{Symbol, AbstractString}}}))
    if sel.second.first.head == :BYROW
        colsidx = outidx[sel.first]
        _check_ind_and_add!(outidx, Symbol(sel.second.second))
        return _names(outidx)[outidx[colsidx]] => sel.second.first => Symbol(sel.second.second)
    end
    throw(ArgumentError("only byrow operation is supported for cols => fun => :name"))
end

# cols => fun => names normalise as cols .=> fun .=> names
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:MultiColumnIndex,
                                <:Pair{<:Union{Function},
                                    <:AbstractVector{<:Union{Symbol, AbstractString}}}}))
    colsidx = idx[sel.first]
    if !(length(colsidx) == length(sel.second.second))
        throw(ArgumentError("The input number of columns and the length of the output names should match"))
    end
    normalize_combine!(outidx, idx, colsidx .=> sel.second.first .=> sel.second.second)
end

# handling special case
function normalize_combine!(outidx::Index, idx,
                            @nospecialize(sel::Pair{<:Union{ColumnIndex, MultiColumnIndex}, <:AbstractVector}))
    normalize_combine!(outidx, idx, Ref(sel.first) .=> sel.second)
end

function normalize_combine!(outidx::Index, idx, arg::AbstractVector)
    res = Any[]
    for i in 1:length(arg)
        _res = normalize_combine!(outidx::Index, idx, arg[i])
        if _res isa AbstractVector
            for j in 1:length(_res)
                push!(res, _res[j])
            end
        else
            push!(res, _res)
        end
    end
    return res
end

function normalize_combine_multiple!(outidx::Index, idx, @nospecialize(args...))
    res = Any[]
    for i in 1:length(args)
        _res = normalize_combine!(outidx, idx, args[i])
        if typeof(_res) <: Pair
            push!(res, _res)
        else
            for j in 1:length(_res)
                push!(res, _res[j])
            end
        end
    end
    res
end

function _is_byrow_valid(idx, ms)
    righthands = Int[]
    lookupdict = idx.lookup
    if ms[1].second.first isa Expr
        return false
    end
    for i in 1:length(ms)
        if (ms[i].second.first isa Expr) && ms[i].second.first.head == :BYROW

            byrow_vars = idx[ms[i].first]
            !all(byrow_vars .∈ Ref(righthands)) && return false
        end
        if haskey(idx, ms[i].second.second)
            push!(righthands, idx[ms[i].second.second])
        end
    end
    return true
end

function _check_mutliple_rows_for_each_group(ds, ms)
    for i in 1:length(ms)
        # byrow are not checked since they are not going to modify the number of rows
        if ms[i].first isa Tuple
            T = return_type(ms[i].second.first, ntuple(j-> ds[!, ms[i].first[j]].val, length(ms[i].first)))
            if T <: AbstractVector && T !== Union{}
                return i
            end
        elseif !(ms[i].second.first isa Expr) &&
                 haskey(index(ds), ms[i].first) #&&
                    #!(ms[i].first ∈ map(x->x.second.second, view(ms, 1:(i-1)))) #TODO monitor this for any unseen problem
            T = return_type(ms[i].second.first, ds[!, ms[i].first].val)
            if T <: AbstractVector && T !== Union{}
                return i
            end
        end
    end
    return 0
end

function _is_groupingcols_modifed(ds, ms)
    groupcols::Vector{Int} = _groupcols(ds)
    idx = index(ds)
    all_names = _names(ds)
    for i in 1:length(ms)
        if (ms[i].second.second ∈ all_names) && (idx[ms[i].second.second] ∈ groupcols)
            return true
        end
    end
    return false
end

function _compute_the_mutli_row_trans!(special_res, new_lengths, x, nrows, _f, _first_vector_res, starts, ngroups, threads)
    @_threadsfor threads for g in 1:ngroups
        lo = starts[g]
        g == ngroups ? hi = nrows : hi = starts[g + 1] - 1
        special_res[g] = _f(view(x, lo:hi))
        new_lengths[g] = length(special_res[g])
    end
end
function _compute_the_mutli_row_trans_tuple!(special_res, new_lengths, x, nrows, _f, _first_vector_res, starts, ngroups, threads)
    @_threadsfor threads for g in 1:ngroups
        lo = starts[g]
        g == ngroups ? hi = nrows : hi = starts[g + 1] - 1
        special_res[g] = do_call(_f, x, lo:hi)
        new_lengths[g] = length(special_res[g])
    end
end

# this returns lookup dictionary and names for the new ds
function _create_index_for_newds(ds, ms, groupcols)
    all_names = _names(ds)
    nm = Symbol[]
    lookup = Dict{Symbol, Int}()
    cnt = 1
    for j in 1:length(groupcols)
        new_name = all_names[groupcols[j]]
        if !haskey(lookup, new_name)
            push!(nm, new_name)
            push!(lookup, new_name => cnt)
            cnt += 1
        end
    end
    for i in 1:length(ms)
        new_name = ms[i].second.second
         if !haskey(lookup, new_name)
            push!(nm, new_name)
            push!(lookup, new_name => cnt)
            cnt += 1
        end
    end
    return (lookup, nm)
end

function _push_groups_to_res_pa!(res, _tmpres, x, starts, new_lengths, total_lengths, j, groupcols, ngroups, threads)
    y = DataAPI.refarray(x)
    @_threadsfor threads for i in 1:ngroups
        counter::UnitRange{Int} = 1:1
        i == 1 ? (counter = 1:new_lengths[1]) : (counter = (new_lengths[i - 1] + 1):new_lengths[i])
        fill!(view(_tmpres.refs, (new_lengths[i] - length(counter) + 1):(new_lengths[i])),  y[starts[i]])
    end
    push!(res, _tmpres)
end

function _push_groups_to_res!(res, _tmpres, x, starts, new_lengths, total_lengths, j, groupcols, ngroups, threads)
    @_threadsfor threads for i in 1:ngroups
        counter::UnitRange{Int} = 1:1
        i == 1 ? (counter = 1:new_lengths[1]) : (counter = (new_lengths[i - 1] + 1):new_lengths[i])
        fill!(view(_tmpres, (new_lengths[i] - length(counter) + 1):(new_lengths[i])),  x[starts[i]])
    end
    push!(res, _tmpres)
end
function _check_the_output_type(x, mssecond)
    CT = return_type(mssecond, x)
    # TODO check other possibilities:
    # the result can be
    # * AbstractVector{T} where T
    # * Vector{T}
    # * not a Vector
    CT == Union{} && throw(ArgumentError("compiler cannot assess the return type of calling `$(mssecond)` on input, you may want to try using `byrow`."))
    if CT <: AbstractVector
        if hasproperty(CT, :var)
            T = Union{Missing, CT.var.ub}
        else
            T = Union{Missing, eltype(CT)}
        end
    else
        T = Union{Missing, CT}
    end
    return T
end

function _update_one_col_combine!(res, _res, x, _f, ngroups, new_lengths, total_lengths, col, threads)
    # make sure lo and hi are not defined any where outside the following loop
    @_threadsfor threads for g in 1:ngroups
        counter::UnitRange{Int} = 1:1
        g == 1 ? (counter = 1:new_lengths[1]) : (counter = (new_lengths[g - 1] + 1):new_lengths[g])
        lo = new_lengths[g] - length(counter) + 1
        hi = new_lengths[g]
        _tmp_res = _f(view(x, counter))
        check_scalar = _is_scalar(_tmp_res, length(lo:hi))
        if check_scalar
            fill!(view(_res,lo:hi), _tmp_res)
        else
            copy!(view(_res, lo:hi), _tmp_res)
        end
    end
    res[col] = _res
    return _res
end

function _add_one_col_combine!(res, _res, in_x, _f, starts, ngroups, new_lengths, total_lengths, nrows, threads)
    # make sure lo and hi are not defined any where outside the following loop
    @_threadsfor threads for g in 1:ngroups
        counter::UnitRange{Int} = 1:1
        g == 1 ? (counter = 1:new_lengths[1]) : (counter = (new_lengths[g - 1] + 1):new_lengths[g])
        lo = starts[g]
        g == ngroups ? hi = nrows : hi = starts[g + 1] - 1
        l1 = new_lengths[g] - length(counter) + 1
        h1 = new_lengths[g]
        _tmp_res = _f(view(in_x, lo:hi))
        check_scalar = _is_scalar(_tmp_res, length(l1:h1))
        if check_scalar
            fill!(view(_res,l1:h1), _tmp_res)
        else
            copy!(view(_res, l1:h1), _tmp_res)
        end
    end
    push!(res, _res)
    return _res
end
function _add_one_col_combine_tuple!(res, _res, in_x, _f, starts, ngroups, new_lengths, total_lengths, nrows, threads)
    # make sure lo and hi are not defined any where outside the following loop
    @_threadsfor threads for g in 1:ngroups
        counter::UnitRange{Int} = 1:1
        g == 1 ? (counter = 1:new_lengths[1]) : (counter = (new_lengths[g - 1] + 1):new_lengths[g])
        lo = starts[g]
        g == ngroups ? hi = nrows : hi = starts[g + 1] - 1
        l1 = new_lengths[g] - length(counter) + 1
        h1 = new_lengths[g]
        _tmp_res = do_call(_f, in_x, lo:hi)
        check_scalar = _is_scalar(_tmp_res, length(l1:h1))
        if check_scalar
            fill!(view(_res,l1:h1), _tmp_res)
        else
            copy!(view(_res, l1:h1), _tmp_res)
        end
    end
    push!(res, _res)
    return _res
end
function _update_one_col_combine!(res, _res, in_x, _f, starts, ngroups, new_lengths, total_lengths, nrows, col, threads)
    # make sure lo and hi are not defined any where outside the following loop
    @_threadsfor threads for g in 1:ngroups
        counter::UnitRange{Int} = 1:1
        g == 1 ? (counter = 1:new_lengths[1]) : (counter = (new_lengths[g - 1] + 1):new_lengths[g])
        lo = starts[g]
        g == ngroups ? hi = nrows : hi = starts[g + 1] - 1
        l1 = new_lengths[g] - length(counter) + 1
        h1 = new_lengths[g]
        _tmp_res = _f(view(in_x, lo:hi))
        check_scalar = _is_scalar(_tmp_res, length(l1:h1))
        if check_scalar
            fill!(view(_res,l1:h1), _tmp_res)
        else
            copy!(view(_res, l1:h1), _tmp_res)
        end
    end
    res[col] = _res
    return _res
end

function _special_res_fill_barrier!(_res, vals, nl_g, l_cnt)
    for k in 1:l_cnt
        _res[nl_g - l_cnt + k] = vals[k]
    end
end

# special_res cannot be based on previous columns of the combined data set
function _fill_res_with_special_res!(res, _res, special_res, ngroups, new_lengths, total_lengths, threads)
     @_threadsfor threads for g in 1:ngroups
        counter::UnitRange{Int} = 1:1
        g == 1 ? (counter = 1:new_lengths[1]) : (counter = (new_lengths[g - 1] + 1):new_lengths[g])
        # this is not optimized for pooled arrays
        # for k in 1:length(counter)
        #     _res[new_lengths[g] - length(counter) + k] = special_res[g][k]
        # end
        _special_res_fill_barrier!(_res, special_res[g], new_lengths[g], length(counter))
    end
    empty!(special_res)
    GC.safepoint()
    push!(res, _res)
end
# special_res cannot be based on previous columns of the combined data set
function _update_res_with_special_res!(res, _res, special_res, ngroups, new_lengths, total_lengths, col, threads)
     @_threadsfor threads for g in 1:ngroups
        counter::UnitRange{Int} = 1:1
        g == 1 ? (counter = 1:new_lengths[1]) : (counter = (new_lengths[g - 1] + 1):new_lengths[g])
        # this is not optimized for pooled arrays
        # for k in 1:length(counter)
        #     _res[new_lengths[g] - length(counter) + k] = special_res[g][k]
        # end
        _special_res_fill_barrier!(_res, special_res[g], new_lengths[g], length(counter))
    end
    res[col] = _res
    return _res
end

function _combine_f_barrier_special(special_res, fromds, newds, msfirst, mssecond, mslast, newds_lookup, _first_vector_res, ngroups, new_lengths, total_lengths, threads)
        T = _check_the_output_type(fromds, mssecond)
        _res = allocatecol(Union{Missing, T}, total_lengths)
        _fill_res_with_special_res!(_columns(newds), _res, special_res, ngroups, new_lengths, total_lengths, threads)
end
function _combine_f_barrier_special_tuple(special_res, fromds, newds, msfirst, mssecond, mslast, newds_lookup, _first_vector_res, ngroups, new_lengths, total_lengths, threads)
        T = _check_the_output_type(fromds, mssecond)
        _res = allocatecol(Union{Missing, T}, total_lengths)
        _fill_res_with_special_res!(_columns(newds), _res, special_res, ngroups, new_lengths, total_lengths, threads)
end

function _combine_f_barrier(fromds, newds, msfirst, mssecond, mslast, newds_lookup, starts, ngroups, new_lengths, total_lengths, threads)

    if !(mssecond isa Expr)
        if !haskey(newds_lookup, mslast)
            T = _check_the_output_type(fromds, mssecond)
            _res = allocatecol(Union{Missing, T}, total_lengths)
            _add_one_col_combine!(_columns(newds), _res, fromds, mssecond, starts, ngroups, new_lengths, total_lengths, length(fromds), threads)
        else
            T = _check_the_output_type(fromds, mssecond)
            _res = allocatecol(Union{Missing, T}, total_lengths)
            _update_one_col_combine!(_columns(newds), _res, fromds, mssecond, starts, ngroups, new_lengths, total_lengths, length(fromds), newds_lookup[mslast], threads)
            # _update_one_col_combine!(_columns(newds), _res, fromds, mssecond, ngroups, new_lengths, total_lengths, newds_lookup[mslast])
        end

    elseif (mssecond isa Expr) && mssecond.head == :BYROW
        push!(_columns(newds), byrow(newds, mssecond.args[1], msfirst; mssecond.args[2]...))
    else
        throw(ArgumentError("`combine` doesn't support $(msfirst=>mssecond=>mslast) combination"))
    end
end
function _combine_f_barrier_tuple(fromds, newds, msfirst, mssecond, mslast, newds_lookup, starts, ngroups, new_lengths, total_lengths, threads)
    T = _check_the_output_type(fromds, mssecond)
    _res = allocatecol(Union{Missing, T}, total_lengths)
    _add_one_col_combine_tuple!(_columns(newds), _res, fromds, mssecond, starts, ngroups, new_lengths, total_lengths, length(fromds[1]), threads)

end

function combine(ds::Dataset, @nospecialize(args...); dropgroupcols = false, threads = true)
    !isgrouped(ds) &&  return combine_ds(ds, args...)#throw(ArgumentError("`combine` is only for grouped data sets, use `modify` instead"))
    idx_cpy::Index = Index(Dict{Symbol, Int}(), Symbol[], Dict{Int, Function}())
    if !dropgroupcols
        for i in _sortedcols(ds)
            push!(idx_cpy, Symbol(names(ds)[i]))
        end
    end
    ms = normalize_combine_multiple!(idx_cpy, index(ds), args...)
    # the rule is that in combine, byrow must only be used for already aggregated columns
    # so, we should check every thing pass to byrow has been assigned in args before it
    # if this is not the case, throw ArgumentError and ask user to use modify instead
    newlookup, new_nm = _create_index_for_newds(ds, ms, index(ds).sortedcols)
    !(_is_byrow_valid(Index(newlookup, new_nm, Dict{Int, Function}()), ms)) && throw(ArgumentError("`byrow` must be used for aggregated columns, use `modify` otherwise"))
    # _check_mutliple_rows_for_each_group return the first transformation which causes multiple
    # rows or 0 if all transformations return scalar for each group
    # the transformation returning multiple rows must not be based on the previous columns in combine
    # result (which seems reasonable ??)
    _first_vector_res = _check_mutliple_rows_for_each_group(ds, ms)

    _is_groupingcols_modifed(ds, ms) && throw(ArgumentError("`combine` cannot modify the grouping or sorting columns, use a different name for the computed column"))

    groupcols = index(ds).sortedcols
    starts = index(ds).starts
    ngroups::Int = index(ds).ngroups[]

    # we will use new_lengths later for assigning the grouping info of the new ds
    if _first_vector_res == 0
        new_lengths = ones(Int, ngroups)
        cumsum!(new_lengths, new_lengths)
        total_lengths = ngroups
    else
        if ms[_first_vector_res].first isa Tuple
            CT = return_type(ms[_first_vector_res].second.first,
                 ntuple(i->_columns(ds)[index(ds)[ms[_first_vector_res].first[i]]], length(ms[_first_vector_res].first)))
        else
            CT = return_type(ms[_first_vector_res].second.first,
                 ds[!, ms[_first_vector_res].first].val)
        end
        special_res = _our_vect_alloc(CT, ngroups)
        new_lengths = _our_vect_alloc(Int, ngroups)
        # _columns(ds)[ms[_first_vector_res].first]
        if  ms[_first_vector_res].first isa Tuple
            _compute_the_mutli_row_trans_tuple!(special_res, new_lengths, ntuple(i->_columns(ds)[index(ds)[ms[_first_vector_res].first[i]]], length(ms[_first_vector_res].first)), nrow(ds), ms[_first_vector_res].second.first, _first_vector_res, starts, ngroups, threads)
        else
            _compute_the_mutli_row_trans!(special_res, new_lengths, _columns(ds)[index(ds)[ms[_first_vector_res].first]], nrow(ds), ms[_first_vector_res].second.first, _first_vector_res, starts, ngroups, threads)
        end
        # special_res, new_lengths = _compute_the_mutli_row_trans(ds, ms, _first_vector_res, starts, ngroups)
        cumsum!(new_lengths, new_lengths)
        total_lengths = new_lengths[end]
    end
    all_names = _names(ds)

    newds_idx = Index(Dict{Symbol, Int}(), Symbol[], Dict{Int, Function}(), Int[], Bool[], false, [], Int[], 1, false)

    newds = Dataset([], newds_idx)
    newds_lookup = index(newds).lookup
    var_cnt = 1
    if !dropgroupcols

        for j in 1:length(groupcols)
            _tmpres = allocatecol(ds[!, groupcols[j]].val, total_lengths)
            if DataAPI.refpool(_tmpres) !== nothing
                _push_groups_to_res_pa!(_columns(newds), _tmpres, _columns(ds)[groupcols[j]], starts, new_lengths, total_lengths, j, groupcols, ngroups, threads)
            else
                _push_groups_to_res!(_columns(newds), _tmpres, _columns(ds)[groupcols[j]], starts, new_lengths, total_lengths, j, groupcols, ngroups, threads)
            end
            push!(index(newds), new_nm[var_cnt])
            setformat!(newds, new_nm[var_cnt] => get(index(ds).format, groupcols[j], identity))
            var_cnt += 1

        end
    end
    for i in 1:length(ms)
        if i == _first_vector_res
            if ms[i].first isa Tuple
                _combine_f_barrier_special_tuple(special_res, ntuple(j->_columns(ds)[index(ds)[ms[i].first[j]]], length(ms[i].first)), newds, ms[i].first, ms[i].second.first, ms[i].second.second, newds_lookup, _first_vector_res,ngroups, new_lengths, total_lengths, threads)
            else
                _combine_f_barrier_special(special_res, ds[!, ms[i].first].val, newds, ms[i].first, ms[i].second.first, ms[i].second.second, newds_lookup, _first_vector_res,ngroups, new_lengths, total_lengths, threads)
            end
        else
            if ms[i].first isa Tuple
                _combine_f_barrier_tuple(ntuple(j->_columns(ds)[index(ds)[ms[i].first[j]]], length(ms[i].first)), newds, ms[i].first, ms[i].second.first, ms[i].second.second, newds_lookup, starts, ngroups, new_lengths, total_lengths, threads)
            else
                _combine_f_barrier(haskey(index(ds).lookup, ms[i].first) ? _columns(ds)[index(ds)[ms[i].first]] : _columns(ds)[1], newds, ms[i].first, ms[i].second.first, ms[i].second.second, newds_lookup, starts, ngroups, new_lengths, total_lengths, threads)
            end
        end
        if !haskey(index(newds), ms[i].second.second)
            push!(index(newds), ms[i].second.second)
        end

    end
    # grouping information for the output dataset
    # append!(index(newds).sortedcols, index(newds)[index(ds).names[index(ds).sortedcols]])
    # append!(index(newds).rev, index(ds).rev)
    # append!(index(newds).perm, collect(1:total_lengths))
    # # index(newds).grouped[] = true
    # index(newds).ngroups[] = ngroups
    # append!(index(newds).starts, collect(1:total_lengths))
    # for i in 2:(length(new_lengths))
    #     index(newds).starts[i] = new_lengths[i - 1]+1
    # end
    newds
end

function combine_ds(ds::Dataset, @nospecialize(args...); threads = true)
    idx_cpy::Index = Index(Dict{Symbol, Int}(), Symbol[], Dict{Int, Function}())
    ms = normalize_combine_multiple!(idx_cpy, index(ds), args...)
    newlookup, new_nm = _create_index_for_newds(ds, ms, index(ds).sortedcols)
    !(_is_byrow_valid(Index(newlookup, new_nm, Dict{Int, Function}()), ms)) && throw(ArgumentError("`byrow` must be used for aggregated columns, use `modify` otherwise"))
    _first_vector_res = _check_mutliple_rows_for_each_group(ds, ms)

    starts = [1]
    ngroups::Int = 1

    # we will use new_lengths later for assigning the grouping info of the new ds
    if _first_vector_res == 0
        new_lengths = ones(Int, ngroups)
        cumsum!(new_lengths, new_lengths)
        total_lengths = ngroups
    else
        if ms[_first_vector_res].first isa Tuple
            CT = return_type(ms[_first_vector_res].second.first,
                 ntuple(i->_columns(ds)[index(ds)[ms[_first_vector_res].first[i]]], length(ms[_first_vector_res].first)))
        else
            CT = return_type(ms[_first_vector_res].second.first,
                 ds[!, ms[_first_vector_res].first].val)
        end
        special_res = _our_vect_alloc(CT, ngroups)
        new_lengths = _our_vect_alloc(Int, ngroups)
        # _columns(ds)[ms[_first_vector_res].first]
        if  ms[_first_vector_res].first isa Tuple
            _compute_the_mutli_row_trans_tuple!(special_res, new_lengths, ntuple(i->_columns(ds)[index(ds)[ms[_first_vector_res].first[i]]], length(ms[_first_vector_res].first)), nrow(ds), ms[_first_vector_res].second.first, _first_vector_res, starts, ngroups, threads)
        else
            _compute_the_mutli_row_trans!(special_res, new_lengths, _columns(ds)[index(ds)[ms[_first_vector_res].first]], nrow(ds), ms[_first_vector_res].second.first, _first_vector_res, starts, ngroups, threads)
        end
        # special_res, new_lengths = _compute_the_mutli_row_trans(ds, ms, _first_vector_res, starts, ngroups)
        cumsum!(new_lengths, new_lengths)
        total_lengths = new_lengths[end]
    end
    all_names = _names(ds)

    newds_idx = Index(Dict{Symbol, Int}(), Symbol[], Dict{Int, Function}(), Int[], Bool[], false, [], Int[], 1, false)

    newds = Dataset([], newds_idx)
    newds_lookup = index(newds).lookup
    var_cnt = 1

    for i in 1:length(ms)
        if i == _first_vector_res
            if ms[i].first isa Tuple
                _combine_f_barrier_special_tuple(special_res, ntuple(j->_columns(ds)[index(ds)[ms[i].first[j]]], length(ms[i].first)), newds, ms[i].first, ms[i].second.first, ms[i].second.second, newds_lookup, _first_vector_res,ngroups, new_lengths, total_lengths, threads)
            else
                _combine_f_barrier_special(special_res, ds[!, ms[i].first].val, newds, ms[i].first, ms[i].second.first, ms[i].second.second, newds_lookup, _first_vector_res,ngroups, new_lengths, total_lengths, threads)
            end
        else
            if ms[i].first isa Tuple
                _combine_f_barrier_tuple(ntuple(j->_columns(ds)[index(ds)[ms[i].first[j]]], length(ms[i].first)), newds, ms[i].first, ms[i].second.first, ms[i].second.second, newds_lookup, starts, ngroups, new_lengths, total_lengths, threads)
            else
                _combine_f_barrier(haskey(index(ds).lookup, ms[i].first) ? _columns(ds)[index(ds)[ms[i].first]] : _columns(ds)[1], newds, ms[i].first, ms[i].second.first, ms[i].second.second, newds_lookup, starts, ngroups, new_lengths, total_lengths, threads)
            end
        end
        if !haskey(index(newds), ms[i].second.second)
            push!(index(newds), ms[i].second.second)
        end

    end
    newds
end


text:
using CairoMakie
using naumer_Dimensionality_2022
using LinearAlgebra: tr, svd, dot, det, Diagonal, pinv, eigen
using CSV
using Random: MersenneTwister

################################################
########### Plot Parameters ####################
################################################

mrng = MersenneTwister(1234)

max_steps = 1000
state = 2*randn(mrng, 2)
state_cp = copy(state)
crlb = [4.0 0; 0 4.0]

τ = 0.01
γ = 0.99
actionSpace = [[sin(θ), cos(θ)] for θ in 0:.1:π]

λ = 1
psdSampleCount = [1000]
trajectorySampleCount = 1
timestepSampleCount = 1
σ² = 1
max_iterations = 1000
d_max = 1

#################################################
################# Data Generation ###############
#################################################

system = StaticSystem(2)

optimal_sampling_trace = zeros(max_steps)
random_sampling_trace  = zeros(max_steps)

optimal_sampling_trace[1] = 8
random_sampling_trace[1]  = 8

@info("Approximating Value Function")

values, psdSamples, stateSamples = ValueFunctionApproximation_LocalAverage_precompute( mrng,
                                                                                system,
                                                                            τ,
                                                                            γ,
                                                                            actionSpace,
                                                                            λ,
                                                                            psdSampleCount[1],
                                                                            trajectorySampleCount,
                                                                            timestepSampleCount,
                                                                            σ²,
                                                                            max_iterations,
                                                                            d_max)

@info("Solving Optimal Sampling Problem")

NN_Policy_tuple = (system, σ², τ, values, psdSamples, stateSamples, actionSpace, d_max)

for i in 2:max_steps
    global state
    global crlb
    global NN_Policy_tuple
    action, index, state, crlb = LocalAverage_OptimalPolicy(    state,
                                                                crlb,
                                                                NN_Policy_tuple...)
    ~, S, ~ = svd(crlb)
    optimal_sampling_trace[i] = sum(S)
end

crlb = [4.0 0; 0 4.0]
state = state_cp

for i in 2:max_steps
    global state
    global crlb
    action = actionSpace[rand(mrng, 1:length(actionSpace))]

    jacobian = flowJacobian(state, τ, system)
    crlb = updateCRLB(crlb, action, jacobian, σ²)

    state = flow(state, τ, system)
    ~, S, ~ = svd(crlb)
    random_sampling_trace[i] = sum(S)
end

#################################################
################ Save Data CSV ##################
#################################################

CSV.write("StaticSampling.csv", ( random_sampling_trace = random_sampling_trace,
                                  optimal_sampling_trace = optimal_sampling_trace,
                                  samples = Array(1:max_steps)))


text:
using RoadRunner
using Test

ant_str = """
    model rabbit()

    k1 = 1.1; k2 = 2.2; k3 = 3.3; k4 = 4.4
    k5 = 5.5; k6 = 6.6; k7 = 7.7; k8 = 8.8
    end
"""

rr = RoadRunner.loada(ant_str)

@testset "parameters" begin
    @test RoadRunner.getNumberOfGlobalParameters(rr) == 8
    @test RoadRunner.getGlobalParameterIds(rr) == ["k1", "k2", "k3", "k4", "k5", "k6", "k7", "k8"]
    @test RoadRunner.getGlobalParameterValues(rr) == [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8]
end


text:
################################################################################

# Setup

# Note: This version is against the true solution, so adaptive timestepping is
# used. This means that the option which is set to get the correct timepoints
# is saveat, not tstops!

using DifferentialEquations, RecursiveArrayTools
using NLopt
using BlackBoxOptim

Xiang2015Bounds = Tuple{Float64, Float64}[(9, 11), (20, 30), (2, 3)] # for local optimizations
xlow_bounds = [9.0,20.0,2.0]
xhigh_bounds = [11.0,30.0,3.0]
LooserBounds = Tuple{Float64, Float64}[(0, 22), (0, 60), (0, 6)] # for global optimization
GloIniPar = [0.0, 0.5, 0.1] # for global optimizations
LocIniPar = [9.0, 20.0, 2.0] # for local optimization

g1 = @ode_def_nohes LorenzExample begin
  dx = σ*(y-x)
  dy = x*(ρ-z) - y
  dz = x*y - β*z
end σ=>10.0 ρ=>28.0 β=>2.6666

r0 = [0.1;0.0;0.0]
tspan = (0.0,30.0)
prob = ODEProblem(g1,r0,tspan)
tspan2 = (0.0,3.0)
prob_short = ODEProblem(g1,r0,tspan2)

dt = 30.0/3000
tf = 30.0
tinterval = 0:dt:tf
t  = collect(tinterval)

h = 0.01
M = 300
tstart = 0.0
tstop = tstart + M * h
tinterval_short = 0:h:tstop
t_short = collect(tinterval_short)

# Generate Data
data_sol_short = solve(prob_short,Vern7(),saveat=t_short,reltol=1e-12,abstol=1e-12)
data_short = convert(Array,data_sol_short)
data_sol = solve(prob,Vern7(),saveat=t,reltol=1e-12,abstol=1e-12)
data = convert(Array,data_sol)

# Plot the data

# using Plots
# plot(data_sol_short) # the short solution
# plot(data_sol) # the longer solution

################################################################################

# Using BlackBoxOptim

# Note: Euler uses tstops to hit the estimation timepoints exactly since it's not adaptive
obj_short = build_loss_objective(prob_short,Euler(),L2Loss(t_short,data_short),tstops=t_short)
res1 = bboptimize(obj_short;SearchRange = Xiang2015Bounds, MaxSteps = 11e3)
# Euler could not recover the correct results since its error is too high!

obj_short = build_loss_objective(prob_short,Tsit5(),L2Loss(t_short,data_short))
res1 = bboptimize(obj_short;SearchRange = Xiang2015Bounds, MaxSteps = 11e3)
# Tolernace is still too high to get close enough

obj_short = build_loss_objective(prob_short,Tsit5(),L2Loss(t_short,data_short),reltol=1e-9)
res1 = bboptimize(obj_short;SearchRange = Xiang2015Bounds, MaxSteps = 11e3)
# With the tolerance lower, it achieves the correct solution in 4.5 seconds.

obj_short = build_loss_objective(prob_short,Vern7(),L2Loss(t_short,data_short),reltol=1e-12,abstol=1e-12)
res1 = bboptimize(obj_short;SearchRange = Xiang2015Bounds, MaxSteps = 11e3)
# But too much accuracy in the numerical solution of the ODE actually leads to
# slower convergence, since each step takes longer!

# Fastest BlackBoxOptim: 4.5 seconds

################################################################################

# using NLopt
obj_short = build_loss_objective(prob_short,Tsit5(),L2Loss(t_short,data_short),reltol=1e-14)

opt = Opt(:GN_ORIG_DIRECT_L, 3)
lower_bounds!(opt,LocIniPar)
upper_bounds!(opt,[11.0,30.0,3.0])
min_objective!(opt, obj_short)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)

opt = Opt(:GN_CRS2_LM, 3)
lower_bounds!(opt,LocIniPar)
upper_bounds!(opt,[11.0,30.0,3.0])
min_objective!(opt, obj_short)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)

opt = Opt(:GN_ISRES, 3)
lower_bounds!(opt,LocIniPar)
upper_bounds!(opt,[11.0,30.0,3.0])
min_objective!(opt, obj_short)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)

opt = Opt(:LN_BOBYQA, 3)
lower_bounds!(opt,LocIniPar)
upper_bounds!(opt,[11.0,30.0,3.0])
min_objective!(opt, obj_short)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)
# This one took 0.04 seconds! Wow!

opt = Opt(:LN_NELDERMEAD, 3)
lower_bounds!(opt,LocIniPar)
upper_bounds!(opt,[11.0,30.0,3.0])
min_objective!(opt, obj_short)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)

opt = Opt(:LD_SLSQP, 3)
lower_bounds!(opt,LocIniPar)
upper_bounds!(opt,[11.0,30.0,3.0])
min_objective!(opt, obj_short)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)

################################################################################

# Longer version

obj = build_loss_objective(prob,Euler(),L2Loss(t,data),tstops = t)
# res1 = bboptimize(obj;SearchRange = Xiang2015Bounds, MaxSteps = 8e3)
# Once again, Euler fails to convergence its error is too high

obj = build_loss_objective(prob,Vern7(),L2Loss(t,data),reltol=1e-14)
# res1 = bboptimize(obj;SearchRange = Xiang2015Bounds, MaxSteps = 8e3)
# BB with Tsit5 converges just fine in 14.5 seconds

opt = Opt(:GN_ORIG_DIRECT_L, 3)
lower_bounds!(opt,zeros(3))
upper_bounds!(opt,[22.0,60.0,6.0])
min_objective!(opt, obj)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,GloIniPar)

opt = Opt(:GN_CRS2_LM, 3)
lower_bounds!(opt,zeros(3))
upper_bounds!(opt,[22.0,60.0,6.0])
min_objective!(opt, obj)
xtol_rel!(opt,1e-12)
maxeval!(opt, Int(1e5))
@time (minf,minx,ret) = NLopt.optimize(opt,GloIniPar)

opt = Opt(:GN_ISRES, 3)
lower_bounds!(opt,zeros(3))
upper_bounds!(opt,[22.0,60.0,6.0])
min_objective!(opt, obj)
xtol_rel!(opt,1e-12)
maxeval!(opt, Int(1e6))
@time (minf,minx,ret) = NLopt.optimize(opt,GloIniPar)

opt = Opt(:GN_ESCH, 3)
lower_bounds!(opt,zeros(3))
upper_bounds!(opt,[22.0,60.0,6.0])
min_objective!(opt, obj)
xtol_rel!(opt,1e-12)
maxeval!(opt, Int(1e5))
@time (minf,minx,ret) = NLopt.optimize(opt,GloIniPar)

################################################################

# Local Optimizers fail on the hard problem, as expected
LocIniPar = [11.5312, 26.0192, 2.79983]
opt = Opt(:LN_BOBYQA, 3)
lower_bounds!(opt,xlow_bounds)
upper_bounds!(opt,[13.0,30.0,3.0])
min_objective!(opt, obj)
xtol_rel!(opt,1e-12)
maxeval!(opt, Int(1e5))
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)
# Converges in 0.05 seconds!

opt = Opt(:LN_NELDERMEAD, 3)
lower_bounds!(opt,xlow_bounds)
upper_bounds!(opt,[13.0,30.0,3.0])
min_objective!(opt, obj)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)

opt = Opt(:LD_SLSQP, 3)
lower_bounds!(opt,xlow_bounds)
upper_bounds!(opt,[13.0,30.0,3.0])
min_objective!(opt, obj)
xtol_rel!(opt,1e-12)
maxeval!(opt, 10000)
@time (minf,minx,ret) = NLopt.optimize(opt,LocIniPar)

################################################################################

# Conclusion:

# 1) When doing these experiments, making sure the "data" is correct is key.
#    The lorenz_test.jl results are all skewed because the actual Euler solution
#    that made the data is skewed. But, since it's a fixed timestep, it was able
#    to find the parameters that made the data, given that it's hitting all the
#    same points. That's not the case here. Because there's adaptive timestepping,
#    we're getting rid of the ability to "overfit", i.e. simply match the data
#    we generated. Instead, we have to hit the same solution, albeit at different
#    stepping points, making it less prone to the bias.
# 2) When trying to hit the real data, having a low enough tolerance on the
#    numerical solution is key. If the numerical solution is too rough, then
#    we can never actually hone in on the true parameters since even with the
#    true parameters we will erroneously induce numerical error. Maybe this
#    could be adaptive?
# 3) Too low of a tolerance in the numerical solution is unnecessary and will
#    slow down the convergence of the estimation.
# 4) The estimation method makes a huge difference in the timings. Here, BBO
#    doesn't do very well. Some of the methods in NLopt, like :LN_BOBYQA, solve
#    the problem in <0.05 seconds. This is an algorithm that can scale.
# 5) Fitting shorter timespans is easier... maybe this could be some kind of
#    adaptivity trick as well?


text:
using PhysicalOptics
using Test
using FFTW
using Random
using SpecialFunctions

 # fix seed for reproducibility
Random.seed!(42)

tests = [
    "apertures",
    "convolutions",
    "lenses",
    "physical_conversions",
    "propagations",
    "psf",
    "resolution_equations",
    "utils",
    "utils_view",
]

@testset "Point test" begin
    p = Point(1.0, 2.0, 3.0)

    @test typeof(p) === Point{Float64}
    @test p.x == 1.0
    @test p.y == 2.0
    @test p.z == 3.0

    p = Point(1, 2, 3)
    @test typeof(p) === Point{Int64}
    @test p.x == 1
    @test p.y == 2
    @test p.z == 3

end

for t in tests
    @testset "$(t)" begin
        include("$t.jl")
    end
end

max_stars_repo_name: JuliaPhysics/PhysicalOptics

max_issues_repo_name: JuliaPhysics/PhysicalOptics

max_forks_repo_name: JuliaPhysics/PhysicalOptics


text:
using JuMP, EAGO

                     m = Model()

                     EAGO.register_eago_operators!(m)

                     @variable(m, -1 <= x[i=1:3] <= 1)

                     @variable(m, -7.107954588851326 <= q <= 8.439913276188518)

                     add_NL_constraint(m, :(swish(-0.4953615570310643 + -0.5197659840811282*swish(0.7941458217392698 + 0.49653685598476827*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.8533482485058403*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + -0.13525586788633204*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3]))) + -0.07122185861106178*swish(-0.06420985369672083 + 0.9131512287471262*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.03263883937508805*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + 0.0015134736925910275*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3]))) + 0.004105284291951339*swish(-0.8124442669202567 + 0.4437121866105662*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.26832550078458484*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + 0.2804882536336004*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3])))) + swish(0.7344840166166842 + -0.5219551557665656*swish(0.7941458217392698 + 0.49653685598476827*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.8533482485058403*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + -0.13525586788633204*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3]))) + -0.12237894168399821*swish(-0.06420985369672083 + 0.9131512287471262*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.03263883937508805*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + 0.0015134736925910275*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3]))) + 0.5945370485807278*swish(-0.8124442669202567 + 0.4437121866105662*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.26832550078458484*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + 0.2804882536336004*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3])))) + swish(-0.7546740103129919 + -0.5743271753296177*swish(0.7941458217392698 + 0.49653685598476827*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.8533482485058403*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + -0.13525586788633204*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3]))) + 0.959961363428667*swish(-0.06420985369672083 + 0.9131512287471262*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.03263883937508805*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + 0.0015134736925910275*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3]))) + -0.0685664132974031*swish(-0.8124442669202567 + 0.4437121866105662*swish(-0.2081348932235647 + -0.9819593709419507*$(x[1]) + 0.4754911924134797*$(x[2]) + -0.6006888917392823*$(x[3])) + 0.26832550078458484*swish(0.14812040410194793 + -0.8433474401842953*$(x[1]) + -0.1761112323855083*$(x[2]) + 0.6144836466445156*$(x[3])) + 0.2804882536336004*swish(-0.3937152244783322 + 0.36708048782945024*$(x[1]) + 0.138776295405171*$(x[2]) + -0.8492811105063662*$(x[3])))) - $q <= 0.0))

                     @objective(m, Min, q)

                     return m

max_stars_repo_name: PSORLab/RSActivationFunctions

max_issues_repo_name: PSORLab/RSActivationFunctions

max_forks_repo_name: PSORLab/RSActivationFunctions


text:
# ---
# title: 1526. Minimum Number of Increments on Subarrays to Form a Target Array
# id: problem1526
# author: Tian Jun
# date: 2020-10-31
# difficulty: Hard
# categories: Segment Tree
# link: <https://leetcode.com/problems/minimum-number-of-increments-on-subarrays-to-form-a-target-array/description/>
# hidden: true
# ---
#
# Given an array of positive integers `target` and an array `initial` of same
# size with all zeros.
#
# Return the minimum number of operations to form a `target` array from
# `initial` if you are allowed to do the following operation:
#
#   * Choose **any** subarray from `initial` and increment each value by one.
#
# The answer is guaranteed to fit within the range of a 32-bit signed integer.
#
#
#
# **Example 1:**
#
#
#
#     Input: target = [1,2,3,2,1]
#     Output: 3
#     Explanation: We need at least 3 operations to form the target array from the initial array.
#     [0,0,0,0,0] increment 1 from index 0 to 4 (inclusive).
#     [1,1,1,1,1] increment 1 from index 1 to 3 (inclusive).
#     [1,2,2,2,1] increment 1 at index 2.
#     [1,2,3,2,1] target array is formed.
#
#
# **Example 2:**
#
#
#
#     Input: target = [3,1,1,2]
#     Output: 4
#     Explanation: (initial)[0,0,0,0] -> [1,1,1,1] -> [1,1,1,2] -> [2,1,1,2] -> [3,1,1,2] (target).
#
#
# **Example 3:**
#
#
#
#     Input: target = [3,1,5,4,2]
#     Output: 7
#     Explanation: (initial)[0,0,0,0,0] -> [1,1,1,1,1] -> [2,1,1,1,1] -> [3,1,1,1,1]
#                                       -> [3,1,2,2,2] -> [3,1,3,3,2] -> [3,1,4,4,2] -> [3,1,5,4,2] (target).
#
#
# **Example 4:**
#
#
#
#     Input: target = [1,1,1,1]
#     Output: 1
#
#
#
#
# **Constraints:**
#
#   * `1 <= target.length <= 10^5`
#   * `1 <= target[i] <= 10^5`
#
#
## @lc code=start
using LeetCode

## add your code here:
## @lc code=end


text:
CHARACTERS = "abcdefghijklmnopqrstuvwxyz"
N_CHAR = length(CHARACTERS)
CHAR_TO_IDX = Dict([c => i for (i, c) in enumerate(CHARACTERS)])
BagType = Int
Bag = Vector{BagType}
VOCAB_DICT = Dict{Vector{Int},Vector{String}}()

function word_to_bag(word_str)

    bag = zeros(BagType, N_CHAR)
    for char in word_str
        bag[CHAR_TO_IDX[char]] += 1
    end
    return bag
end

function bag_to_str(bag)
    chars = [repeat(CHARACTERS[i], bag[i]) for i=1:length(bag)]
    return string(chars...)
end

function build_vocab_dict(vocab_vec)

    d = Dict{Vector{Int},Vector{String}}()

    for word in vocab_vec
        bag = word_to_bag(word)
        if !haskey(d, bag)
            d[bag] = String[]
        end
        push!(d[bag], word)
    end

    return d
end

function bag_complement(bag, subbag)
    return bag .- subbag
end

function bag_leq(bag1, bag2)
    s1 = sum(bag1)
    s2 = sum(bag2)
    if s1 != s2
        if s1 < s2
            return true
        else
            return false
        end
    end
    return bag1 >= bag2
end

function module_setup(charset=CHARACTERS,
                      vocab_file=string(@__DIR__, "/collins_scrabble_2019.txt"))

    global CHARACTERS = charset
    global N_CHAR = length(CHARACTERS)
    global CHAR_TO_IDX = Dict([c => i for (i, c) in enumerate(CHARACTERS)])

    vocab_vec = open(vocab_file,"r") do f
        return readlines(f)
    end
    vocab_vec = [lowercase(word) for word in vocab_vec]
    global VOCAB_DICT = build_vocab_dict(vocab_vec)

    println(string("Loaded dictionary at ",vocab_file))

    return
end

module_setup()


text:
######################
# Printing Functions #
######################
function dirac_show(io::IO, dirac)
    print(io, summary(dirac)*":")
    pad = "  "
    maxlen = 10
    i = 1
    for label in keys(dict(dirac))
        if i > maxlen
            break
        else
            i += 1
            println(io)
            print(io, labelrepr(dirac, label, pad))
        end
    end
    if length(dirac) > maxlen
        println(io)
        print(io, "$pad$vdots")
    end
end

function dirac_showcompact(io::IO, dirac)
    pad = " "
    maxlen = 4
    i = 1
    for label in keys(dict(dirac))
        if i > maxlen
            break
        else
            print(io, labelrepr(dirac, label, pad))
            i += 1
            if i <= length(dirac)
                print(io, "$pad+")
            end
        end
    end
    if length(dirac) > maxlen
        print(io, "$pad...")
    end
end

function dirac_repr(dirac)
    tempio = IOBuffer()
    showcompact(tempio, dirac)
    return takebuf_string(tempio)
end

# Placing here for now - may be useful for
# subscripting label factors in the future
digit_subscript(i::Int) = Base.REPLCompletions.latex_completions("\\_$i",3)[2][1][1]
subscript(i::Int) = reduce(*, reverse(map(digit_subscript, digits(i))))


text:
using Rotolo
using Base.Markdown
using Gadfly

global_theme = """
	text-align: justify;
	border-style:solid;
	padding: 5px
  """

@session Himmelblau global_theme
@redirect Markdown.MD HTML Katex
sleep(3)

@container header style=>"text-align: center;"

md"# Himmelblau's function"
@style md"*From Wikipedia, the free encyclopedia*" style=>"font-size:x-small"

@container main style=>"display:flex;flex-direction:row;text-align: justify;"

@container main.desc style=>"display:flex;flex-direction:column"

md"### Definition"

md"""
	In mathematical optimization, Himmelblau's function is a multi-modal function,
	used to test the performance of optimization algorithms. The function is defined by:
	"""

Katex("f(x,y)=(x^2+y-11)^2+(x+y^2-7)^2", true)

md"""
	It has one local maximum at x = -0.270845 and y = -0.923039,
	where f(x,y) = 181.617, and four identical local minima:

		- f(3.0, 2.0) = 0.0,
		- f(-2.805118, 3.131312) = 0.0,
		- f(-3.779310, -3.283186) = 0.0,
		- f(3.584428, -1.848126) = 0.0.

	The locations of all the minima can be found analytically.
	However, because they are roots of cubic polynomials, when
	written in terms of radicals, the expressions are somewhat complicated.

	The function is named after **David Mautner Himmelblau** (1924–2011), who introduced it.
	"""

@container main.plot
md"### Contour Plot"

f(x,y) = (x^2+y-11)^2+(x+y^2-7)^2

white_panel = Theme(panel_fill=colorant"white",
                    default_color=colorant"black")

p = plot(z=f, x=linspace(-5,5,150), y=linspace(-5,5,150),
	 	     Geom.contour(levels=logspace(0,3,15) - 1),
		     white_panel);

set_default_plot_size(450*Gadfly.px, 450*Gadfly.px)
HTML(stringmime("text/html", p))

@style md"This is a Graph of *Himmelblau*'s function" style=>"font-size:small"


text:
module JuExample

"""
  test_func(x,y)

  This is a test function documentation
"""
function test_func(x,y)
  x+y
end

end


text:
# Lexicographic
@mesh(HEXMESH, n)

#warm up
solve([q1,q2,q3,q4,q5,q6]);

# Lex. ordering
regtime = Base.Libc.time();
for iter=1:times
    solve([q1,q2,q3,q4,q5,q6]);
end
regtime = Base.Libc.time() - regtime;
regtime /= times;
timings[2] = regtime;
println("Finished Lex.");


text:
mutable struct Scope
    parent::Union{Scope,Nothing}
    expr::EXPR
    names::Dict{String,Binding}
    modules::Union{Nothing,Dict{Symbol,Any}}
    overloaded::Union{Dict,Nothing}
end
Scope(expr) = Scope(nothing, expr, Dict{Symbol,Binding}(), nothing, nothing)
function Base.show(io::IO, s::Scope)
    printstyled(io, headof(s.expr))
    printstyled(io, " ", join(keys(s.names), ","), color=:yellow)
    s.modules isa Dict && printstyled(io, " ", join(keys(s.modules), ","), color=:blue)
end

function overload_method(scope::Scope, b::Binding, vr::SymbolServer.VarRef)
    if scope.overloaded === nothing
        scope.overloaded = Dict()
    end
    if haskey(scope.overloaded, vr)
        # TODO: need to check this hasn't already been done
        push!(scope.overloaded[vr].refs, b.val)
    else
        scope.overloaded[vr] = b
    end
end

"""
scopehasmodule(s::Scope, mname::Symbol)::Bool

Checks whether the module `mname` has been `using`ed in `s`.
"""
scopehasmodule(s::Scope, mname::Symbol) = s.modules !== nothing && haskey(s.modules, mname)

"""
    addmoduletoscope!(s, m, [mname::Symbol])

Adds module `m` to the list of used modules in scope `s`.
"""
function addmoduletoscope!(s::Scope, m, mname::Symbol)
    if s.modules === nothing
        s.modules = Dict{Symbol,Any}()
    end
    s.modules[mname] = m
end
addmoduletoscope!(s::Scope, m::SymbolServer.ModuleStore) = addmoduletoscope!(s, m, m.name.name)
addmoduletoscope!(s::Scope, m::EXPR) =  CSTParser.defines_module(m) && addmoduletoscope!(s, scopeof(m), Symbol(valof(CSTParser.get_name(m))))
addmoduletoscope!(s::Scope, s1::Scope) = CSTParser.defines_module(s1.expr) && addmoduletoscope!(s, s1, Symbol(valof(CSTParser.get_name(s1.expr))))

getscopemodule(s::Scope, m::Symbol) = s.modules[m]

"""
    scopehasbinding(s::Scope, n::String)

Checks whether s has a binding for variable named `n`.
"""
scopehasbinding(s::Scope, n::String) = haskey(s.names, n)

is_soft_scope(scope::Scope) = scope.expr.head == :for || scope.expr.head == :while || scope.expr.head == :try

"""
    introduces_scope(x::EXPR, state)

Does this expression introduce a new scope?
"""
function introduces_scope(x::EXPR, state)
    # TODO: remove unused 2nd argument.
    if CSTParser.isassignment(x) && (CSTParser.is_func_call(x.args[1]) || CSTParser.iscurly(x.args[1]))
        return true
    elseif CSTParser.defines_anon_function(x)
        return true
    elseif CSTParser.iswhere(x)
        # unless in func def signature
        return !_in_func_or_struct_def(x)
    elseif CSTParser.istuple(x) && CSTParser.hastrivia(x) && ispunctuation(x.trivia[1]) && length(x.args) > 0 && isassignment(x.args[1])
        # named tuple
        return true
    elseif headof(x) === :function ||
            headof(x) === :macro ||
            headof(x) === :for ||
            headof(x) === :while ||
            headof(x) === :let ||
            headof(x) === :generator || # and Flatten?
            headof(x) === :try ||
            headof(x) === :do ||
            headof(x) === :module ||
            headof(x) === :abstract ||
            headof(x) === :primitive ||
            headof(x) === :struct
        return true
    end
    return false
end

hasscope(x::EXPR) = hasmeta(x) && hasscope(x.meta)
scopeof(x) = nothing
scopeof(x::EXPR) = scopeof(x.meta)
CSTParser.parentof(s::Scope) = s.parent

function setscope!(x::EXPR, s)
    if !hasmeta(x)
        x.meta = Meta()
    end
    x.meta.scope = s
end

"""
    scopes(x::EXPR, state)

Called when traversing the syntax tree and handles the association of
scopes with expressions. On the first pass this will add scopes as
necessary, on following passes it empties it.
"""
function scopes(x::EXPR, state)
    clear_scope(x)
    if scopeof(x) === nothing && introduces_scope(x, state)
        setscope!(x, Scope(x))
    end
    s0 = state.scope
    if headof(x) === :file
        setscope!(x, state.scope)
        add_eval_method(x, state)
    elseif scopeof(x) isa Scope
        scopeof(x) != s0 && setparent!(scopeof(x), s0)
        state.scope = scopeof(x)
        if headof(x) === :module && headof(x.args[1]) === :TRUE # Add default modules to a new module
            state.scope.modules = Dict{Symbol,Any}() # TODO: only create new Dict if not assigned?
            state.scope.modules[:Base] = getsymbols(state)[:Base]
            state.scope.modules[:Core] = getsymbols(state)[:Core]
            add_eval_method(x, state)
        elseif headof(x) === :module && headof(x.args[1]) === :FALSE
            state.scope.modules = Dict{String,Any}()
            state.scope.modules[:Core] = getsymbols(state)[:Core]
            add_eval_method(x, state)
        end
        if headof(x) === :module && bindingof(x) !== nothing # Add reference to out of scope binding (i.e. itself)
            # state.scope.names[bindingof(x).name] = bindingof(x)
            # TODO: move this to the binding stage
            add_binding(x, state)
        # elseif headof(x) === :flatten && headof(x[1]) === CSTParser.Generator && length(x[1]) > 0 && headof(x[1][1]) === CSTParser.Generator
        #     setscope!(x[1][1], nothing)
        end
    end
    return s0
end

# Add an `eval` method
function add_eval_method(x, state)
    mod = if x.head === :module
        CSTParser.isidentifier(x.args[3]) ? Symbol(valof(x.args[3])) : :unknown
    else
        Symbol("top-level")
    end
    meth = SymbolServer.MethodStore(:eval, mod, "", 0, [:expr => SymbolServer.FakeTypeName(SymbolServer.VarRef(SymbolServer.VarRef(nothing, :Core), :Any), [])], [], Any)
    state.scope.names["eval"] = Binding(x, SymbolServer.FunctionStore(SymbolServer.VarRef(nothing, :nothing), SymbolServer.MethodStore[meth],"", SymbolServer.VarRef(nothing, :nothing), false), getsymbols(state)[:Core][:DataType], [])
end


text:
using TomoForward
using XfromProjections

# using MKLSparse # uncomment if you've installed MKLSparse, which will boost the performance

# test slice by slice
img = zeros(100, 100, 128)
img[40:70, 40:60, 50:70] .= 1.0

nslice = size(img, 3)

nangles = 90
detcount = Int(floor(size(img,1)*1.4))
proj_geom = ProjGeom(1.0, detcount, LinRange(0,pi,nangles+1)[1:nangles])

isdefined_A = @isdefined A
if isdefined_A == false
    A = fp_op_parallel2d_strip(proj_geom, size(img, 1), size(img, 2))
end

p = zeros(nangles, nslice, detcount)
for i=1:nslice
    p[:,i,:] = reshape(A * vec(img[:,:,i]), nangles, detcount)
end

u = zeros(size(img))

include("pals.jl")
recon2d_slices_tvrdart!(u, A, p, 50, 4)

max_stars_repo_name: JuliaTomo/DynamicTomographyWithFlow

max_issues_repo_name: JuliaTomo/DynamicTomographyWithFlow

max_forks_repo_name: JuliaTomo/DynamicTomographyWithFlow