<jupyter_start><jupyter_text>Submitting various things for end of grant.<jupyter_code>import os
import sys
import requests
import pandas
import paramiko
import json
from IPython import display
from curation_common import *
from htsworkflow.submission.encoded import DCCValidator
PANDAS_ODF = os.path.expanduser('~/src/odf_pandas')
if PANDAS_ODF not in sys.path:
    sys.path.append(PANDAS_ODF)
    from pandasodf import ODFReader
import gcat
from htsworkflow.submission.encoded import Document
from htsworkflow.submission.aws_submission import run_aws_cp
# live server & control file
#server = ENCODED('www.encodeproject.org')
spreadsheet_name = "ENCODE_test_miRNA_experiments_01112018"

# test server & datafile
server = ENCODED('test.encodedcc.org')
#spreadsheet_name = os.path.expanduser('~diane/woldlab/ENCODE/C1-encode3-limb-2017-testserver.ods')

server.load_netrc()
validator = DCCValidator(server)
award = 'UM1HG009443'<jupyter_output><empty_output><jupyter_text>Submit Documents Example Document submission<jupyter_code>#atac_uuid = '0fc44318-b802-474e-8199-f3b6d708eb6f'
#atac = Document(os.path.expanduser('~/proj/encode3-curation/Wold_Lab_ATAC_Seq_protocol_December_2016.pdf'),
#                'general protocol',
#                'ATAC-Seq experiment protocol for Wold lab',
#                )
#body = atac.create_if_needed(server, atac_uuid)
#print(body['@id'])<jupyter_output><empty_output><jupyter_text>Submit Annotations<jupyter_code>#sheet = gcat.get_file(spreadsheet_name, fmt='pandas_excel')
#annotations = sheet.parse('Annotations', header=0)
#created = server.post_sheet('/annotations/', annotations, verbose=True, dry_run=True)
#print(len(created))
#if created:
#    annotations.to_excel('/tmp/annotations.xlsx', index=False)<jupyter_output><empty_output><jupyter_text>Register Biosamples<jupyter_code>book = gcat.get_file(spreadsheet_name, fmt='pandas_excel')
biosample = book.parse('Biosamples', header=0)
created = server.post_sheet('/biosamples/', biosample, 
                            verbose=True, 
                            dry_run=True, 
                            validator=validator)
print(len(created))
if created:
    biosample.to_excel('/dev/shm/biosamples.xlsx', index=False)<jupyter_output><empty_output><jupyter_text>Register Libraries<jupyter_code>print(spreadsheet_name)
book = gcat.get_file(spreadsheet_name, fmt='pandas_excel')
libraries = book.parse('Libraries', header=0)
created = server.post_sheet('/libraries/', libraries, verbose=True, dry_run=True, validator=validator)
print(len(created))
if created:
    libraries.to_excel('/dev/shm/libraries.xlsx', index=False)<jupyter_output><empty_output><jupyter_text>Register Experiments<jupyter_code>print(server.server)
book = gcat.get_file(spreadsheet_name, fmt='pandas_excel')
experiments = book.parse('Experiments', header=0)
created = server.post_sheet('/experiments/', experiments, verbose=True, dry_run=False, validator=validator)
print(len(created))
if created:
    experiments.to_excel('/dev/shm/experiments.xlsx', index=False)<jupyter_output><empty_output><jupyter_text>Register Replicates<jupyter_code>print(server.server)
print(spreadsheet_name)
book = gcat.get_file(spreadsheet_name, fmt='pandas_excel')
replicates = book.parse('Replicates', header=0)
created = server.post_sheet('/replicates/', replicates, verbose=True, dry_run=True, validator=validator)
print(len(created))
if created:
    replicates.to_excel('/dev/shm/replicates.xlsx', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12220950200343446, 'fraction_numerical': 0.015168860904407555, 'mean_word_length': 6.483940042826553, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '877', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1211, 'n_words': 178, 'path': 'encode-mirna-2018-01.ipynb', 'repo_name': 'detrout/encode4-curation', 'size': '3494'}