<jupyter_start><jupyter_text>**NOTE: downloaded.wikipedia.short_abstracts.tsv.gz**This file is available to be downloaded from `https://drive.google.com/drive/folders/1UkvFFLWbfjJtSw767IKYPfZiFsqUFu5n`The location on `ckg07` is `/data/amandeep/downloaded.wikipedia.short_abstracts.tsv.gz`This file is required for building the cache file for KGTK Browser.<jupyter_code>input_path = "/data/amandeep"
output_path = "/data/amandeep"
project_name = "create-wikidata-dwd"

kgtk_path= "/data/amandeep/Github/kgtk"
kgtk_browser_path = "/data/amandeep/Github/kgtk-browser"
kernel_name = "kgtk-env-ckg07"
wikipedia_short_abstracts_path = '/data/amandeep/downloaded.wikipedia.short_abstracts.tsv.gz'
%%time
ck = ConfigureKGTK([], kgtk_path=kgtk_path)
ck.configure_kgtk(input_graph_path=input_path,
                  output_path=output_path,
                  project_name=project_name,
                 graph_cache_path=None)
ck.print_env_variables()
!git clone https://github.com/usc-isi-i2/kgtk-browser $OUT/kgtk-browser<jupyter_output><empty_output><jupyter_text>Run the Import Wikidata Notebook<jupyter_code># Parameters for Import Wikidata
json_file_path = "/data/amandeep/wikidata-20220519"
import_wikidata_path = "/data/amandeep/wikidata-20220519"
wikidata_project_name = "import-wikidata"
wikidata_json_file = "latest-all.json.bz2"
sort_command = 'sort'
%%time
pm.execute_notebook(
    "import-wikidata.ipynb",
    os.environ["TEMP"] + "/import-wikidata.out.ipynb",
    kernel_name=kernel_name,
    parameters=dict(
        input_path = json_file_path,
        output_path = import_wikidata_path,
        project_name = wikidata_project_name,
        wikidata_json_file = wikidata_json_file,
        kgtk_path = kgtk_path,
        sort_command = sort_command
    )
)<jupyter_output><empty_output><jupyter_text>Run the Useful Files Notebook to compute `isa` and `p279star` files only<jupyter_code># Parameters for First run on Useful Files
first_useful_files_input_path = f"{import_wikidata_path}/{wikidata_project_name}"
first_useful_files_output_path = import_wikidata_path
first_useful_files_project_name = "useful-files"
first_useful_files = 'claims,label_all,alias_all,description_all'
first_useful_files_for_cache = 'claims'
%%time
pm.execute_notebook(
    "Wikidata-Useful-Files.ipynb",
    os.environ["TEMP"] + "/Wikidata-Useful-Files.out.ipynb",
    kernel_name=kernel_name,
    parameters=dict(
        input_path = first_useful_files_input_path,
        output_path = first_useful_files_output_path,
        project_name = first_useful_files_project_name,
        kgtk_path = kgtk_path,
        files = first_useful_files,
        files_for_cache=first_useful_files_for_cache,
        compute_pagerank=False,
        compute_degrees=False,
        debug=False,
        compute_isa_star=False,
        compute_p31p279_star=False
    )
)
!cp $import_wikidata_path/$first_useful_files_project_name/derived.isa.tsv.gz $import_wikidata_path/$wikidata_project_name
!cp $import_wikidata_path/$first_useful_files_project_name/derived.P279star.tsv.gz $import_wikidata_path/$wikidata_project_name<jupyter_output><empty_output><jupyter_text>Run Wikidata Subsets Notebook The following notebook will run the following notebooks ,1. `./partition-wikidata.ipynb`The output will be at the path (example accoding to the parameters specified in the below cell) ,`/data/amandeep/wikidata-20220519-dwd-v5/parts`2. `./Wikidata-Useful-Files.ipynb`The output will be at the path,`/data/amandeep/wikidata-20220519-dwd-v5/useful-files`We will move the output files from the above 2 notebooks to the path `/data/amandeep/wikidata-20220519-dwd-v5` at the end of execution of the `Wikidata-Subsets.ipynb` notebook.<jupyter_code>subset_input_path = f"{import_wikidata_path}/{wikidata_project_name}"
subset_output_path = "/data/amandeep"


subset_project_name = "wikidata-20220519-dwd-v5"

subset_files = 'isa,p279star'

# Classes to remove
remove_classes = "Q7318358,Q13442814"

languages = "en,ru,es,zh-cn,de,it,nl,pl,fr,pt,sv"
%%time
pm.execute_notebook(
    "Wikidata-Subsets.ipynb",
    os.environ["TEMP"] + "/Wikidata-Subsets.out.ipynb",
    kernel_name=kernel_name,
    parameters=dict(
        input_path = subset_input_path,
        output_path = subset_output_path,
        project_name = subset_project_name,
        kgtk_path = kgtk_path,
        files = subset_files,
        remove_classes = remove_classes,
        languages = languages,
        kernel_name = kernel_name
    )
)
!mv $subset_output_path/$subset_project_name/parts/*tsv.gz $subset_output_path/$subset_project_name
!mv $subset_output_path/$subset_project_name/useful-files/*tsv.gz $subset_output_path/$subset_project_name<jupyter_output><empty_output><jupyter_text>Create JSON file for KGTK-Search The following notebook will create following file ,`/data/amandeep/wikidata-20220519-dwd-v5/kgtk-search/wikidata.dwd.all.kgtk.search.sorted.jl`We will then split the json lines file into 1M line partitions and load it into the ES index<jupyter_code>search_input_path = f"{subset_output_path}/{subset_project_name}"
search_output_path = f"{subset_output_path}/{subset_project_name}"

search_project_name = "kgtk-search"

compute_embeddings = False
generate_triples = False
generate_kgtk_search = True
datatype_property = "datatype"

es_url="http://ckg07:9200"
es_index="wikidata-dwd-kgtk-search-04"
%%time
pm.execute_notebook(
    "Embeddings-Elasticsearch-Triples.ipynb",
    os.environ["TEMP"] + "/Embeddings-Elasticsearch-Triples.out.ipynb",
    kernel_name=kernel_name,
    parameters=dict(
        input_path = search_input_path,
        output_path = search_output_path,
        project_name = search_project_name,
        kgtk_path = kgtk_path,
        compute_embeddings = compute_embeddings,
        generate_triples = generate_triples,
        generate_kgtk_search = generate_kgtk_search,
        datatype_property = datatype_property,
        languages = languages
    )
)<jupyter_output><empty_output><jupyter_text>Split the output json lines file to 1M lines partitions<jupyter_code>!mkdir -p $search_output_path/$search_project_name/es_split/
!split $search_output_path/$search_project_name/wikidata.dwd.all.kgtk.search.sorted.jl \
    -l 1000000 \
    $search_output_path/$search_project_name/es_split/<jupyter_output><empty_output><jupyter_text>Run Properties-for-this-type-notebook This notebook is in the [kgtk-notebooks](https://github.com/usc-isi-i2/kgtk-notebooks) repo.<jupyter_code>p_input_path=f"{subset_output_path}/{subset_project_name}"
p_output_path = f"{subset_output_path}/{subset_project_name}"

# we will re use graph cache from the useful-files notebook
# at this point it already has the `claims` file loaded into cache.
# we will only load the required files into the cache, save time
p_graph_cache_path = f"{subset_output_path}/{subset_project_name}/useful-files/temp.useful-files/wikidata.sqlite3.db"
files_for_cache="item,datatypes,p279,p279star"

p_project_name = "p1963"
debug = False
%%time
pm.execute_notebook(
    "properties-for-this-type.ipynb",
    os.environ["TEMP"] + "/properties-for-this-type.out.ipynb",
    kernel_name=kernel_name,
    parameters=dict(
        input_path = p_input_path,
        output_path = p_output_path,
        project_name = p_project_name,
        graph_cache_path = p_graph_cache_path,
        debug = debug,
        files_for_cache=files_for_cache   
    )
)<jupyter_output><empty_output><jupyter_text>**move the files out into the root folder**<jupyter_code>!mv $p_output_path/$p_project_name/*tsv.gz $p_output_path<jupyter_output><empty_output><jupyter_text>Run class-visualization notebook<jupyter_code>c_input_path = f"{subset_output_path}/{subset_project_name}"
c_output_path = f"{subset_output_path}/{subset_project_name}"
c_project_name = "class-visualization"

# re use the graph cache, at this point the cache has the following files loaded
# claims,item,datatypes,p279,p279star
# we only need to load label

c_graph_cache_path = p_graph_cache_path
files_for_cache = "label"
debug = False
%%time
pm.execute_notebook(
    "class-visualization.ipynb",
    os.environ["TEMP"] + "/class-visualization.out.ipynb",
    kernel_name=kernel_name,
    parameters=dict(
        input_path = c_input_path,
        output_path = c_output_path,
        project_name = c_project_name,
        graph_cache_path = c_graph_cache_path,
        debug = debug,
        files_for_cache=files_for_cache   
    )
)
!mv $c_output_path/$c_project_name/class-visualization.node.tsv.gz $c_output_path
!mv $c_output_path/$c_project_name/class-visualization.edge.tsv.gz $c_output_path<jupyter_output><empty_output><jupyter_text>Run Create-claims-augmented-for-browser notebook<jupyter_code>a_input_path = f"{subset_output_path}/{subset_project_name}"
a_output_path = f"{subset_output_path}/{subset_project_name}"
a_project_name = "browser-claims-file"
!cp $wikipedia_short_abstracts_path $subset_output_path/$subset_project_name
%%time
pm.execute_notebook(
    "Create-claims-augmented-for-browser.ipynb",
    os.environ["TEMP"] + "/Create-claims-augmented-for-browser.out.ipynb",
    kernel_name=kernel_name,
    parameters=dict(
        input_path = a_input_path,
        output_path = a_output_path,
        project_name = a_project_name
    )
)<jupyter_output><empty_output><jupyter_text>Run KGTK-Query-Text-Search-Setup Notebook<jupyter_code>q_input_path = f"{a_output_path}/{a_project_name}"
q_output_path = f"{a_output_path}/{a_project_name}"

q_project_name = "kgtk-browser-files"
!cp $subset_output_path/$subset_project_name/class-visualization.edge.tsv.gz $q_input_path
!cp $subset_output_path/$subset_project_name/class-visualization.node.tsv.gz $q_input_path
!cp $subset_output_path/$subset_project_name/metadata.pagerank.undirected.tsv.gz $q_input_path
%%time
pm.execute_notebook(
    f"{os.environ['OUT']}/kgtk-browser/KGTK-Query-Text-Search-Setup.ipynb",
    os.environ["TEMP"] + "/KGTK-Query-Text-Search-Setup.ipynb",
    kernel_name=kernel_name,
    parameters=dict(
        input_path = q_input_path,
        output_path = q_output_path,
        project_name = q_project_name
    )
)<jupyter_output><empty_output><jupyter_text>LOAD ES Index **Make sure [table-linker](https://github.com/usc-isi-i2/table-linker) is installed**<jupyter_code>!curl -H "Content-Type: application/json" \
-XPUT $es_url/$es_index -d @$search_output_path/$search_project_name/wikidata_dwd_v3_mapping.json
os.environ['search_output_path']=search_output_path
os.environ['search_project_name']=search_project_name
os.environ['es_url']=es_url
os.environ['es_index']=es_index
%%bash
for f in $search_output_path/$search_project_name/es_split/* ;
do
 echo $f 
 tl load-elasticsearch-index --es-url $es_url --es-index $es_index --es-version 7 --kgtk-jl-path $f > $TEMP/load_es.log
 sleep 60
done<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1062257528172917, 'fraction_numerical': 0.013301311657121744, 'mean_word_length': 5.225991949396205, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 50, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '159978', 'license': 'MIT', 'n_tokens_mistral': 4035, 'n_tokens_neox': 3898, 'n_words': 569, 'path': 'use-cases/create_wikidata/create-wikidata-dwd.ipynb', 'repo_name': 'usc-isi-i2/kgtk-tutorial-files', 'size': '10826'}