<commit_before>#!/bin/bash
#

SCRIPT_DIR=/kabob.git/scripts/docker
chmod -R 755 ${SCRIPT_DIR}

# This script takes a single argument specifying the repository name to use/construct
export KB_INSTANCE_NAME=$1
export DOCKER_ENV=${SCRIPT_DIR}/stardog-specific/docker-env.sh

source ${DOCKER_ENV}
source ${SCRIPT_DIR}/common-scripts/ENV.sh

echo "LEININGEN=${LEININGEN}"
echo "KB_PORT=${KB_PORT}"
echo "KB_URL=${KB_URL}"
echo "KB_USER=${KB_USER}"
echo "KB_PASS=${KB_PASS}"
echo "KB_NAME=${KB_NAME}"
echo "DATASOURCE_OWL_DIR=${DATASOURCE_OWL_DIR}"
echo "DATASOURCE_ICE_DIR=${DATASOURCE_ICE_DIR}"
echo "KB_DATA_DIR=${KB_DATA_DIR}"

#### Clean out the directory in which we're going to place our artifacts.
#rm -rvf ${KB_DATA_DIR}
#mkdir -p ${KB_DATA_DIR}
#
#### create a new Stardog database
#${SCRIPT_DIR}/stardog-specific/create-new-database-stardog.sh ${KB_NAME}
#
#### generate lists of RDF files that will be loaded in subsequent steps
#${SCRIPT_DIR}/common-scripts/generate-rdf-file-lists.sh ${KB_NAME} ${DOCKER_ENV}
#
### Load the ontologies (note: they will have been converted from OWL to n-triples prior to loading)
#${SCRIPT_DIR}/stardog-specific/load-list-file-stardog.sh \
#  ${KB_PORT} \
#  ${KB_NAME} \
#  ${KB_DATA_DIR}/file-lists/owl-files.${KB_NAME}.list \
#  "ntriples"

## create ICE records for all ontology concepts
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_a_ontology_to_ice/step_a
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_a_ontology_to_ice/step_b
${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_a_ontology_to_ice/step_c

### Load the ICE RDF - the rules above process the ontologies only, so we have waited to load the ICE RDF until this point
#${SCRIPT_DIR}/stardog-specific/load-list-file-stardog.sh \
#  ${KB_PORT} \
#  ${KB_NAME} \
#  ${KB_DATA_DIR}/file-lists/ice-nt-files.${KB_NAME}.list
#
#${SCRIPT_DIR}/stardog-specific/load-list-file-stardog.sh \
#  ${KB_PORT} \
#  ${KB_NAME} \
#  ${KB_DATA_DIR}/file-lists/ice-owl-files.${KB_NAME}.list \
#  "rdfxml"
#
#### Index optimization -- not currently implemented
##${SCRIPT_DIR}/optimize.sh \
##  ${KB_PORT} \
##  ${KB_NAME}
#
#### create skos:exactMatch links between equivalent identifiers
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_b_id_exact_match/chebi
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_b_id_exact_match/equivalent_class
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_b_id_exact_match/shared_label
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_b_id_exact_match/datasource_xref
#
#### Create the ID sets
#export LEIN_ROOT=true
#IS_VIRTUOSO=false
#cd /kabob.git && { ${LEININGEN} generate-id-sets ${KB_URL} ${KB_NAME} ${KB_USER} ${KB_PASS} ${KB_DATA_DIR}/id_sets/exact/ ${KB_DATA_DIR}/id_sets/graph_dbs/ ${IS_VIRTUOSO} ; cd - ; }
#${SCRIPT_DIR}/stardog-specific/LOAD-STARDOG.sh id_sets/exact
#
##  create bioentity for each id set
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/post_identifier_merge/step_a_entity_generation/reify
#
## connect bioentities based on ontology hierarchies
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/post_identifier_merge/step_b_ontology_to_bio/step_a
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/post_identifier_merge/step_b_ontology_to_bio/step_b
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/post_identifier_merge/step_b_ontology_to_bio/step_c<commit_msg>Activate stardog  ice load<commit_after>#!/bin/bash
#

SCRIPT_DIR=/kabob.git/scripts/docker
chmod -R 755 ${SCRIPT_DIR}

# This script takes a single argument specifying the repository name to use/construct
export KB_INSTANCE_NAME=$1
export DOCKER_ENV=${SCRIPT_DIR}/stardog-specific/docker-env.sh

source ${DOCKER_ENV}
source ${SCRIPT_DIR}/common-scripts/ENV.sh

echo "LEININGEN=${LEININGEN}"
echo "KB_PORT=${KB_PORT}"
echo "KB_URL=${KB_URL}"
echo "KB_USER=${KB_USER}"
echo "KB_PASS=${KB_PASS}"
echo "KB_NAME=${KB_NAME}"
echo "DATASOURCE_OWL_DIR=${DATASOURCE_OWL_DIR}"
echo "DATASOURCE_ICE_DIR=${DATASOURCE_ICE_DIR}"
echo "KB_DATA_DIR=${KB_DATA_DIR}"

#### Clean out the directory in which we're going to place our artifacts.
#rm -rvf ${KB_DATA_DIR}
#mkdir -p ${KB_DATA_DIR}
#
#### create a new Stardog database
#${SCRIPT_DIR}/stardog-specific/create-new-database-stardog.sh ${KB_NAME}
#
#### generate lists of RDF files that will be loaded in subsequent steps
#${SCRIPT_DIR}/common-scripts/generate-rdf-file-lists.sh ${KB_NAME} ${DOCKER_ENV}
#
### Load the ontologies (note: they will have been converted from OWL to n-triples prior to loading)
#${SCRIPT_DIR}/stardog-specific/load-list-file-stardog.sh \
#  ${KB_PORT} \
#  ${KB_NAME} \
#  ${KB_DATA_DIR}/file-lists/owl-files.${KB_NAME}.list \
#  "ntriples"

## create ICE records for all ontology concepts
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_a_ontology_to_ice/step_a
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_a_ontology_to_ice/step_b
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_a_ontology_to_ice/step_c

## Load the ICE RDF - the rules above process the ontologies only, so we have waited to load the ICE RDF until this point
${SCRIPT_DIR}/stardog-specific/load-list-file-stardog.sh \
  ${KB_PORT} \
  ${KB_NAME} \
  ${KB_DATA_DIR}/file-lists/ice-nt-files.${KB_NAME}.list

${SCRIPT_DIR}/stardog-specific/load-list-file-stardog.sh \
  ${KB_PORT} \
  ${KB_NAME} \
  ${KB_DATA_DIR}/file-lists/ice-owl-files.${KB_NAME}.list \
  "rdfxml"

#### Index optimization -- not currently implemented
##${SCRIPT_DIR}/optimize.sh \
##  ${KB_PORT} \
##  ${KB_NAME}
#
#### create skos:exactMatch links between equivalent identifiers
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_b_id_exact_match/chebi
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_b_id_exact_match/equivalent_class
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_b_id_exact_match/shared_label
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/pre_identifier_merge/step_b_id_exact_match/datasource_xref
#
#### Create the ID sets
#export LEIN_ROOT=true
#IS_VIRTUOSO=false
#cd /kabob.git && { ${LEININGEN} generate-id-sets ${KB_URL} ${KB_NAME} ${KB_USER} ${KB_PASS} ${KB_DATA_DIR}/id_sets/exact/ ${KB_DATA_DIR}/id_sets/graph_dbs/ ${IS_VIRTUOSO} ; cd - ; }
#${SCRIPT_DIR}/stardog-specific/LOAD-STARDOG.sh id_sets/exact
#
##  create bioentity for each id set
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/post_identifier_merge/step_a_entity_generation/reify
#
## connect bioentities based on ontology hierarchies
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/post_identifier_merge/step_b_ontology_to_bio/step_a
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/post_identifier_merge/step_b_ontology_to_bio/step_b
#${SCRIPT_DIR}/stardog-specific/RUN_RULES_AND_LOAD-STARDOG.sh rules/post_identifier_merge/step_b_ontology_to_bio/step_c
{'fraction_non_alphanumeric': 0.1365597196009706, 'fraction_numerical': 0.0010784578053383662, 'mean_word_length': 6.562691131498471, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2559378', 'n_tokens_mistral': 3100, 'n_tokens_neox': 2889, 'n_words': 428}