<commit_before># Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import ndio.remote.ndingest as NI
# import mni_dataset


def main():

    ni = NI.NDIngest()

    """
    Edit the below values, type and default information can be found on the ingesting page of the ndio docs page.
    """

    dataset_name='MNI3'        #(type=str, help='Name of Dataset')
    imagesize=(182,218,182)           #(type=int[], help='Image size (X,Y,Z)')
    voxelres=(1000000,1000000,1000000)   #(type=float[], help='Voxel scale (X,Y,Z)')
    offset=(0,0,0)              #(type=int[], default=[0, 0, 0], help='Image Offset in X,Y,Z')
    timerange=(0,33)           #(type=int[], default=[0, 0], help='Time Dimensions')
    scalinglevels=0       #(type=int, default=0, help='Required Scaling levels/ Zoom out levels')
    scaling=0             #(type=int, default=0, help='Type of Scaling - Isotropic or Normal')

    project_name='KKI2009_gk2'        #(type=str, help='Name of Project. Has to be unique in OCP. User Defined')
    token_name='KKI<PASSWORD>'          #(type=str, default='', help='Token Name. User Defined')
    channel_name='815_1'        #(type=str, help='Name of Channel. Has to be unique in the same project. User Defined.')
    datatype='uint8'            #(type=str, help='Channel Datatype')
    channel_type='timeseries'        #(type=enum, help='Type of channel - image, annotation, timeseries, probmap')
    exceptions=0          #(type=int, default=0, help='Exceptions')
    resolution=0          #(type=int, default=0, help='Start Resolution')
    windowrange=(0,0)         #(type=int[], default=[0, 0], help='Window clamp function for 16-bit channels with low max value of pixels')
    readonly=0            #(type=int, default=0, help='Read-only Channel or Not. You can remotely post to channel if it is not readonly and overwrite data')
    data_url= 'http://openconnecto.me/mrdata/share/ingest'           #(type=str, help='This url points to the root directory of the files. Dropbox is not an acceptable HTTP Server.')
    file_format='SLICE'         #(type=str, help='This is overal the file format type. For now we support only Slice stacks and CATMAID tiles.')
    file_type='png'           #(type=str, help='This is the specific file format type (tiff, tif, png))

    public=1              #(type=int, default=0, help='Make your project publicly visible')

    metadata=""            #(type=Any, default='', help='Any metadata as appropriate from the LIMS schema')

    #Adds data set information
    ni.add_dataset(dataset_name, imagesize, voxelres, offset, timerange, scalinglevels, scaling)

    #Adds project information
    ni.add_project(project_name, token_name, public)

    #Adds a channel
    ni.add_channel(channel_name, datatype, channel_type, data_url, file_format, file_type, exceptions,
            resolution, windowrange, readonly)

    #Adds metada
    ni.add_metadata(metadata)

    #Post the data
    ni.post_data()

if __name__ == "__main__":
  main()<commit_msg>ingested full dataset into ndstore :sparkles:<commit_after># Copyright 2014 Open Connectome Project (http://openconnecto.me)
#
# Licensed under the Apache License,  Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,  software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,  either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import ndio.remote.ndingest as NI


def main():

    ni = NI.NDIngest()

    """
    Dataset
    """
    dataset_name = 'KKI2009'
    imagesize = (182, 218, 182)
    voxelres = (1000000, 1000000, 1000000)
    offset = (0, 0, 0)
    timerange = (0, 33)
    scalinglevels = 0
    scaling = 0
    public = 1
    metadata = "Kennedy Kreiger Institute 2009 Dataset"

    # Add data set information to ingest object
    ni.add_dataset(dataset_name, imagesize, voxelres, offset,
                   timerange, scalinglevels, scaling)
    ni.add_metadata(metadata)

    """
    Project
    """
    project_name = 'KKI2009'
    token_name = '<PASSWORD>'

    # Adds project information to ingest object
    ni.add_project(project_name, token_name, public)

    """
    Channel
    """
    # Sets up general channel info
    datatype = 'uint8'
    channel_type = 'timeseries'
    exceptions = 0
    resolution = 0
    windowrange = (0, 0)
    readonly = 0
    data_url = 'http://openconnecto.me/mrdata/share/ingest'
    file_format = 'SLICE'
    file_type = 'png'

    # Lists channel names
    channels = ['113_1', '113_2', '127_1', '127_2', '142_1', '142_2', '239_1',
                '239_2', '346_1', '346_2', '422_1', '422_2', '492_1', '492_2',
                '502_1', '502_2', '505_1', '505_2', '656_1', '656_2', '679_1',
                '679_2', '742_1', '742_2', '800_1', '800_2', '814_1', '814_2',
                '815_1', '815_2', '849_1', '849_2', '906_1', '906_2', '913_1',
                '913_2', '916_1', '916_2', '934_1', '934_2', '959_1', '959_2']

    # Adds each channel's information to ingest object
    for channel_name in channels:
        ni.add_channel(channel_name, datatype, channel_type, data_url,
                       file_format, file_type, exceptions, resolution,
                       windowrange, readonly)

    # Post the data
    ni.post_data()


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.11983739837398374, 'fraction_numerical': 0.05056910569105691, 'mean_word_length': 3.172998643147897, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2947225', 'n_tokens_mistral': 2117, 'n_tokens_neox': 1905, 'n_words': 624}