<jupyter_start><jupyter_text>Notebook for submission data to BioSamples<jupyter_code>import subprocess
import json
import os<jupyter_output><empty_output><jupyter_text>1 (required). Please make sure to signup in AAP webpage (https://aai.ebi.ac.uk/registerUser), then provide your username and passwork in next cell<jupyter_code>your_aap_username = ""
your_aap_password = ""<jupyter_output><empty_output><jupyter_text>2 (required). Fetching your token. Keep in mind that the token is valid for 1 hour. After that you need to fetch new one.<jupyter_code>cmd = f"curl -u {your_aap_username}:{your_aap_password} https://api.aai.ebi.ac.uk/auth"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(token, err) = proc.communicate()
token = token.decode("utf-8")<jupyter_output><empty_output><jupyter_text>3 (optional). Create team. Please provide 'description' - a description of your team (optional) and 'centre_name' - name of your institute or project (required)<jupyter_code>description = "My lab group"
centre_name = "An Institute"
team_json = {
    "description": description,
    "centreName": centre_name
}
team_json = json.dumps(team_json)
cmd = f"curl 'https://submission.ebi.ac.uk/api/user/teams' -i -X POST -H 'Content-Type: application/json' -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}' -d '{team_json}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()<jupyter_output><empty_output><jupyter_text>4 (optional). Managing your team<jupyter_code># Find your user reference
cmd = f"curl https://api.aai.ebi.ac.uk/users/{your_aap_username} -H 'Authorization: Bearer {token}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
user_reference = out['userReference']

# Find your domain reference
cmd = f"curl https://api.aai.ebi.ac.uk/my/domains -H 'Authorization: Bearer {token}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
# Here we use 0 to get first team from list, adjust this value to get team you need
domain_reference = out[0]['domainReference']

# Adding user to a team
cmd = f"curl 'https://api.aai.ebi.ac.uk/domains/{domain_reference}/{user_reference}/user' -X PUT -H 'Authorization: Bearer {token}' -H 'Accept: application/hal+json'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
print(out)
# Each token includes all the details about who you are, and which teams you are a member of. You have now created 
# a new team and added yourself to it since you requested your first token, so that first token is out of date. You 
# should request a new one, as you did before. The new one will include the team you just created. You can use this 
# new token to access the submissions API.<jupyter_output>{'domainName': 'subs.team-11', 'domainDesc': 'My lab group', 'users': [{'userName': 'sokolovbiotech', 'email': '<EMAIL>', 'userReference': 'usr-755befc0-b6a4-4666-a6b2-9f04bc598bfc', '_links': {'self': {'href': 'https://api.aai.ebi.ac.uk/users/usr-755befc0-b6a4-4666-a6b2-9f04bc598bfc'}}}], 'domainReference': 'dom-9d66869c-97cc-4a7d-8735-e690fa87aca4', '_links': {'self': {'href': 'https://api.aai.ebi.ac.uk/domains/dom-9d66869c-97cc-4a7d-8735-e690fa87aca4'}, 'deactivate': {'href': 'https://api.aai.ebi.ac.uk/domains/dom-9d66869c-97cc-4a7d-8735-e690fa87aca4'}, 'get users from domain': {'href': 'https://api.aai.ebi.ac.uk/domains/dom-9d66869c-97cc-4a7d-8735-e690fa87aca4/users'}, 'admin': {'href': 'https://api.aai.ebi.ac.uk/domains/dom-033a4e68-2964-4e58-aa63-e4f701b57ff5'}, 'get all domains': {'href': 'https://api.aai.ebi.ac.uk/domains'}}}<jupyter_text>5 (required). Pick a team<jupyter_code># Get user teams link
cmd = f"curl 'https://submission.ebi.ac.uk/api/' -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
user_teams_link = out['_links']['userTeams']['href']

# Fetch user teams
cmd = f"curl {user_teams_link} -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
teams = json.loads(out.decode('utf-8'))['_embedded']['teams']
for team in teams:
    print(team['_links']['submissions:create'])
create_submission_link = "https://submission.ebi.ac.uk/api/teams/subs.team-11/submissions"<jupyter_output>{'href': 'https://submission.ebi.ac.uk/api/teams/subs.team-11/submissions'}
{'href': 'https://submission.ebi.ac.uk/api/teams/subs.team-9/submissions'}
{'href': 'https://submission.ebi.ac.uk/api/teams/subs.team-12/submissions'}
{'href': 'https://submission.ebi.ac.uk/api/teams/subs.team-8/submissions'}
{'href': 'https://submission.ebi.ac.uk/api/teams/subs.team-10/submissions'}<jupyter_text>6 (requred). Creating a submission<jupyter_code>json_to_submit = {}
cmd = f"curl '{create_submission_link}' -X POST -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}' -d '{json_to_submit}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
submission_json = out
print(json.dumps(out, indent=4))<jupyter_output>{
    "id": "d6b406b7-2ac6-4bec-b95e-26191fcaf9de",
    "submitter": {
        "email": "<EMAIL>",
        "name": "<NAME>"
    },
    "team": {
        "name": "subs.team-11",
        "description": "My lab group",
        "profile": {
            "centre name": "An Institute"
        },
        "_links": {
            "submissions": {
                "href": "https://submission.ebi.ac.uk/api/submissions/search/by-team?teamName=subs.team-11"
            },
            "submissions:create": {
                "href": "https://submission.ebi.ac.uk/api/teams/subs.team-11/submissions"
            },
            "items": {
                "href": "https://submission.ebi.ac.uk/api/teams/subs.team-11/items"
            }
        }
    },
    "createdDate": "2019-11-06T14:31:02.830+0000",
    "lastModifiedDate": "2019-11-06T14:31:02.830+0000",
    "createdBy": "usr-755befc0-b6a4-4666-a6b2-9f04bc598bfc",
    "lastModifiedBy": "usr-755befc0-b6a4-4666-a6b2-9f04bc598bfc",
[...]<jupyter_text>7 (required). Adding documents to a submission<jupyter_code>submission_contents_link = submission_json['_links']['contents']['href']
submission_contents_link
cmd = f"curl {submission_contents_link} -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
print(out['_links']['samples:create']['href'])
create_sample_link = out['_links']['samples:create']['href']
create_project_link = out['_links']['projects:create']['href']
sample_to_add = {
    "alias": "D3",
    "title": "NA12878_D3",
    "description": "Material derived from cell line NA12878",
    "attributes": {
        "Cell line type": [
            {
                "value": "EBV-LCL cell line",
                "terms": [{"url": "http://purl.obolibrary.org/obo/BTO_0003335"}]
            }
        ]
    },
    "sampleRelationships": [],
    "taxonId": 9606,
    "taxon": "Homo sapiens",
    "releaseDate": "2017-01-01"
}
sample_to_add = json.dumps(sample_to_add)
cmd = f"curl '{create_sample_link}' -i -X POST -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}' -d '{sample_to_add}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out
cmd
cmd = f"curl {create_sample_link} -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
validation_results_link = out['_embedded']['samples'][2]['_links']['validationResult']['href']
cmd = f"curl {validation_results_link} -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
print(json.dumps(out))<jupyter_output>{"version": 1, "validationStatus": "Complete", "expectedResults": {"Core": [{"validationAuthor": "Core", "validationStatus": "Pass", "entityUuid": "abbd0d8d-187a-4658-8244-a6a5a39bf05c"}], "Ena": [{"validationAuthor": "Ena", "validationStatus": "Pass", "entityUuid": "abbd0d8d-187a-4658-8244-a6a5a39bf05c"}], "JsonSchema": [{"validationAuthor": "JsonSchema", "validationStatus": "Pass", "entityUuid": "abbd0d8d-187a-4658-8244-a6a5a39bf05c"}], "Taxonomy": [{"validationAuthor": "Taxonomy", "validationStatus": "Pass", "entityUuid": "abbd0d8d-187a-4658-8244-a6a5a39bf05c"}], "Biosamples": [{"validationAuthor": "Biosamples", "validationStatus": "Pass", "entityUuid": "abbd0d8d-187a-4658-8244-a6a5a39bf05c"}]}, "overallValidationOutcomeByAuthor": {"Core": "Pass", "Taxonomy": "Pass", "Ena": "Pass", "Biosamples": "Pass", "JsonSchema": "Pass"}, "_links": {"self": {"href": "https://submission.ebi.ac.uk/api/validationResults/4dd9b21b-4975-453a-8bc1-5bed37d47a09"}, "validationResult": {"href": "https://s[...]<jupyter_text>8 (required). Adding project<jupyter_code>create_project_link
project_to_add = {
    "alias": "example-short-unique-name",
    "title": "Example title for our scientific project, at least 50 characters long",
    "description": "Example description for our scientific project, which must also be at least 50 characters long",
    "attributes": {},
    "contacts": [],
    "publications": [],
    "fundings": [],
    "releaseDate": "2019-09-13"
}
project_to_add = json.dumps(project_to_add)
cmd = f"curl '{create_project_link}' -X POST -H 'Content-Type: application/json;charset=UTF-8' -H 'Accept: application/hal+json' -H 'Authorization: Bearer {token}' -d '{project_to_add}'"
proc = subprocess.Popen([cmd], stdout=subprocess.PIPE, shell=True)
(out, err) = proc.communicate()
out = json.loads(out.decode('utf-8'))
print(json.dumps(out))<jupyter_output>{"alias": "example-short-unique-name", "team": {"name": "subs.team-11", "description": "My lab group", "profile": {"centre name": "An Institute"}, "_links": {"submissions": {"href": "https://submission.ebi.ac.uk/api/submissions/search/by-team?teamName=subs.team-11"}, "submissions:create": {"href": "https://submission.ebi.ac.uk/api/teams/subs.team-11/submissions"}, "items": {"href": "https://submission.ebi.ac.uk/api/teams/subs.team-11/items"}}}, "title": "Example title for our scientific project, at least 50 characters long", "description": "Example description for our scientific project, which must also be at least 50 characters long", "contacts": [], "publications": [], "fundings": [], "releaseDate": "2019-09-13", "createdDate": "2019-11-06T16:08:58.708+0000", "lastModifiedDate": "2019-11-06T16:08:58.758+0000", "createdBy": "usr-755befc0-b6a4-4666-a6b2-9f04bc598bfc", "lastModifiedBy": "usr-755befc0-b6a4-4666-a6b2-9f04bc598bfc", "_links": {"self": {"href": "https://submission.ebi.ac.uk[...]
{'fraction_non_alphanumeric': 0.1843354430379747, 'fraction_numerical': 0.047116736990154715, 'mean_word_length': 5.179793590439978, 'pattern_counts': {'":': 102, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 28, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '445439', 'license': 'Apache-2.0', 'n_tokens_mistral': 4434, 'n_tokens_neox': 4146, 'n_words': 800, 'path': 'biosamples_submission.ipynb', 'repo_name': 'FAANG/dcc-python-notebooks', 'size': '11418'}