<commit_before>(ns webapp.models.sparql
(:refer-clojure :exclude [filter concat group-by max min count])
  (:require [compojure.core :refer :all]
            [compojure.handler :as handler]
            [compojure.route :as route]
            [clojure.string :refer [capitalize split]]
            [stencil.core :as tmpl]
            [clj-http.client :as http]
            [boutros.matsu.sparql :refer :all]
            [boutros.matsu.core :refer [register-namespaces]]
            [clojure.tools.logging :as log])
  (:use [hiccup.page :only [html5]])
)

;; see notes/query-ext.clj for matsu and other formats
;; and for pdgmqry-sparql-alt

;; Get bibref(s) and geo/demo URL(s) and TXT
(defn langInfoqry-sparql [language lpref]
  (let [Language (capitalize language)]
    (str
     (tmpl/render-string 
      (str "
	PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
	PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
        PREFIX dc:    <http://purl.org/dc/elements>
        PREFIX dcterms:    <http://purl.org/dc/terms>
	PREFIX aama: <http://id.oi.uchicago.edu/aama/2013/> 
	PREFIX aamas: <http://id.oi.uchicago.edu/aama/2013/schema/> 
	PREFIX aamag:	 <http://oi.uchicago.edu/aama/2013/graph/> 
	PREFIX {{lpref}}:   <http://id.oi.uchicago.edu/aama/2013/{{language}}/> 
	SELECT ?bibref ?lurl ?ldesc
	WHERE
        { 
	 { 
	  GRAPH aamag:{{language}}  
          { 
            aama:{{Language}} aamas:dataSource ?bibref .
            aama:{{Language}} aamas:geodemoURL ?lurl  .
            aama:{{Language}} aamas:geodemoTXT ?ldesc .
          }}}
           ")
      {:lpref lpref
       :language language
       :Language Language}))))

;;generate standard SPARQL query using stencil/render-string. 
;;In this version the "for [value values]" section does not
;;stiplulate that the property to which the value belongs
;;have an rdfs:label. See sparql/pdgmqry-sparql-alt for
;;older version.
(defn pdgmqry-sparql-fv [language lpref valstring]
    (let [;; if assume last value is lex (generalize to other pos?)
          vals (clojure.string/replace valstring #"(.*):.*?$" "$1")
          lex (clojure.string/replace valstring #".*:(.*?)$" "$1")
          values (split vals #",")
          Language (capitalize language)
          ]
      ;;(for [lex lexvals]
      (str
      (tmpl/render-string 
       (str "
	PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
	PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
	PREFIX aama: <http://id.oi.uchicago.edu/aama/2013/> 
	PREFIX aamas: <http://id.oi.uchicago.edu/aama/2013/schema/> 
	PREFIX aamag:	 <http://oi.uchicago.edu/aama/2013/graph/> 
	PREFIX {{lpref}}:   <http://id.oi.uchicago.edu/aama/2013/{{language}}/> 
	SELECT ?num ?pers ?gen  ?token
	WHERE
        { 
	 { 
	  GRAPH aamag:{{language}}  
          { 
	   ?s {{lpref}}:pos {{lpref}}:Verb .  
	   ?s aamas:lang aama:{{Language}} . 
	   ?s aamas:lang ?lang . 
	   ?lang rdfs:label ?langLabel .  ")
       {:lpref lpref
        :language language
        :Language Language})
      (apply str  
             (for [value values]
               (tmpl/render-string 
                (str "
           ?s ?Q{{value}}  {{lpref}}:{{value}} .  ")
                {:value value
                 :lpref lpref})))
      ;; if not multi-lex pdgm
      (if (not (.contains lex ","))
        (tmpl/render-string
         (str "
           ?s aamas:lexeme ?lexeme .
           ?lexeme rdfs:label \"{{lex}}\" .")
         {:lex lex}))
      (tmpl/render-string
       (str "
	   OPTIONAL { ?s {{lpref}}:number ?number .  
	   ?number rdfs:label ?num . } 
	   OPTIONAL { ?s {{lpref}}:pngShapeClass ?pngSC .
           ?pngSC rdfs:label  ?shapeClass}  
	   ?s {{lpref}}:person ?person .  
	   ?person rdfs:label ?pers .  
	   OPTIONAL { ?s {{lpref}}:gender ?gender .  
	   ?gender rdfs:label ?gen . } 
	   ?s {{lpref}}:token ?tkn .
           BIND ((IF(BOUND(?pngSC),
                            CONCAT(?tkn,\"[\",SUBSTR(?shapeClass,4,1),\"]\"),
                            ?tkn))
                 AS ?token 
                ) .
                
	  } 
	 } 
	} 
	ORDER BY DESC(?num) ?pers DESC(?gen) ")
       {:lpref lpref})
       );;str
))

(defn pdgmqry-sparql-fv-note [language lpref valstring]
  "This version, for the moment only called by the single pdgm display option, which is designed to give the most information about an individual paradigm, includes information about input paradigm notes, lex, and token-... . Note info should eventually be displayed in the paradigm-label listing."
    (let [;; if assume last value is lex (generalize to other pos?)
          vals (clojure.string/replace valstring #"(.*):.*?$" "$1")
          ;;lex (clojure.string/replace valstring #".*:(.*?)$" "$1")
          values (split vals #",")
          Language (capitalize language)
          ]
      ;;(for [lex lexvals]
      (str
      (tmpl/render-string 
       (str "
	PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
	PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
	PREFIX aama: <http://id.oi.uchicago.edu/aama/2013/> 
	PREFIX aamas: <http://id.oi.uchicago.edu/aama/2013/schema/> 
	PREFIX aamag:	 <http://oi.uchicago.edu/aama/2013/graph/> 
	PREFIX {{lpref}}:   <http://id.oi.uchicago.edu/aama/2013/{{language}}/> 
	SELECT ?comment ?num ?pers ?gen ?token ?lex
	WHERE
        { 
	 { 
	  GRAPH aamag:{{language}}  
          { 
	   ?s {{lpref}}:pos {{lpref}}:Verb .  
	   ?s aamas:lang aama:{{Language}} . 
	   ?s aamas:lang / rdfs:label ?langLabel .  
           ?s aamas:lexeme ?lexeme .
           ?lexeme rdfs:label ?lex . ")
       {:lpref lpref
        :language language
        :Language Language})
      (apply str  
             (for [value values]
               (tmpl/render-string 
                (str "
           ?s ?Q{{value}}  {{lpref}}:{{value}} .  ")
                {:value value
                 :lpref lpref})))
      (tmpl/render-string
       (str "
	   OPTIONAL { ?s {{lpref}}:number / rdfs:label ?num . } 
	   OPTIONAL { ?s {{lpref}}:pngShapeClass ?pngSC .
             ?pngSC rdfs:label  ?shapeClass .}  
	   ?s {{lpref}}:person ?person .  
	   ?person rdfs:label ?pers .  
	   OPTIONAL { ?s {{lpref}}:gender / rdfs:label ?gen . } 
	   OPTIONAL { ?s aamas:memberOf / rdfs:comment ?comment . } 
	   ?s {{lpref}}:token ?tkn .
           BIND ((IF(BOUND(?pngSC),
                     CONCAT(?tkn,\"[\",SUBSTR(?shapeClass,4,1),\"]\"),
                     ?tkn))
                 AS ?token) .
	  } 
	 } 
	} 
	ORDER BY ?lex DESC(?num) ?pers DESC(?gen) ")
       {:lpref lpref})
       );;str
))<commit_msg>everything marked token-note(-...) will be displayed in square brackets beside form in single pdgm display<commit_after>(ns webapp.models.sparql
(:refer-clojure :exclude [filter concat group-by max min count])
  (:require [compojure.core :refer :all]
            [compojure.handler :as handler]
            [compojure.route :as route]
            [clojure.string :refer [capitalize split]]
            [stencil.core :as tmpl]
            [clj-http.client :as http]
            [boutros.matsu.sparql :refer :all]
            [boutros.matsu.core :refer [register-namespaces]]
            [clojure.tools.logging :as log])
  (:use [hiccup.page :only [html5]])
)

;; see notes/query-ext.clj for matsu and other formats
;; and for pdgmqry-sparql-alt

;; Get bibref(s) and geo/demo URL(s) and TXT
(defn langInfoqry-sparql [language lpref]
  (let [Language (capitalize language)]
    (str
     (tmpl/render-string 
      (str "
	PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
	PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
        PREFIX dc:    <http://purl.org/dc/elements>
        PREFIX dcterms:    <http://purl.org/dc/terms>
	PREFIX aama: <http://id.oi.uchicago.edu/aama/2013/> 
	PREFIX aamas: <http://id.oi.uchicago.edu/aama/2013/schema/> 
	PREFIX aamag:	 <http://oi.uchicago.edu/aama/2013/graph/> 
	PREFIX {{lpref}}:   <http://id.oi.uchicago.edu/aama/2013/{{language}}/> 
	SELECT ?bibref ?lurl ?ldesc
	WHERE
        { 
	 { 
	  GRAPH aamag:{{language}}  
          { 
            aama:{{Language}} aamas:dataSource ?bibref .
            aama:{{Language}} aamas:geodemoURL ?lurl  .
            aama:{{Language}} aamas:geodemoTXT ?ldesc .
          }}}
           ")
      {:lpref lpref
       :language language
       :Language Language}))))

;;generate standard SPARQL query using stencil/render-string. 
;;In this version the "for [value values]" section does not
;;stiplulate that the property to which the value belongs
;;have an rdfs:label. See sparql/pdgmqry-sparql-alt for
;;older version.
(defn pdgmqry-sparql-fv [language lpref valstring]
    (let [;; if assume last value is lex (generalize to other pos?)
          vals (clojure.string/replace valstring #"(.*):.*?$" "$1")
          lex (clojure.string/replace valstring #".*:(.*?)$" "$1")
          values (split vals #",")
          Language (capitalize language)
          ]
      ;;(for [lex lexvals]
      (str
      (tmpl/render-string 
       (str "
	PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
	PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
	PREFIX aama: <http://id.oi.uchicago.edu/aama/2013/> 
	PREFIX aamas: <http://id.oi.uchicago.edu/aama/2013/schema/> 
	PREFIX aamag:	 <http://oi.uchicago.edu/aama/2013/graph/> 
	PREFIX {{lpref}}:   <http://id.oi.uchicago.edu/aama/2013/{{language}}/> 
	SELECT ?num ?pers ?gen  ?token
	WHERE
        { 
	 { 
	  GRAPH aamag:{{language}}  
          { 
	   ?s {{lpref}}:pos {{lpref}}:Verb .  
	   ?s aamas:lang aama:{{Language}} . 
	   ?s aamas:lang ?lang . 
	   ?lang rdfs:label ?langLabel .  ")
       {:lpref lpref
        :language language
        :Language Language})
      (apply str  
             (for [value values]
               (tmpl/render-string 
                (str "
           ?s ?Q{{value}}  {{lpref}}:{{value}} .  ")
                {:value value
                 :lpref lpref})))
      ;; if not multi-lex pdgm
      (if (not (.contains lex ","))
        (tmpl/render-string
         (str "
           ?s aamas:lexeme ?lexeme .
           ?lexeme rdfs:label \"{{lex}}\" .")
         {:lex lex}))
      (tmpl/render-string
       (str "
	   OPTIONAL { ?s {{lpref}}:number ?number .  
	   ?number rdfs:label ?num . } 
	   OPTIONAL { ?s {{lpref}}:pngShapeClass ?pngSC .
           ?pngSC rdfs:label  ?shapeClass}  
	   ?s {{lpref}}:person ?person .  
	   ?person rdfs:label ?pers .  
	   OPTIONAL { ?s {{lpref}}:gender ?gender .  
	   ?gender rdfs:label ?gen . } 
	   ?s {{lpref}}:token ?tkn .
           BIND ((IF(BOUND(?pngSC),
                            CONCAT(?tkn,\"[\",SUBSTR(?shapeClass,4,1),\"]\"),
                            ?tkn))
                 AS ?token 
                ) .
                
	  } 
	 } 
	} 
	ORDER BY DESC(?num) ?pers DESC(?gen) ")
       {:lpref lpref})
       );;str
))

(defn pdgmqry-sparql-fv-note [language lpref valstring]
  "This version, for the moment only called by the single pdgm display option, which is designed to give the most information about an individual paradigm, includes information about input paradigm notes, lex, and token-... . Note info should eventually be displayed in the paradigm-label listing."
    (let [;; if assume last value is lex (generalize to other pos?)
          vals (clojure.string/replace valstring #"(.*):.*?$" "$1")
          ;;lex (clojure.string/replace valstring #".*:(.*?)$" "$1")
          values (split vals #",")
          Language (capitalize language)
          ]
      ;;(for [lex lexvals]
      (str
      (tmpl/render-string 
       (str "
	PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
	PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
	PREFIX aama: <http://id.oi.uchicago.edu/aama/2013/> 
	PREFIX aamas: <http://id.oi.uchicago.edu/aama/2013/schema/> 
	PREFIX aamag:	 <http://oi.uchicago.edu/aama/2013/graph/> 
	PREFIX {{lpref}}:   <http://id.oi.uchicago.edu/aama/2013/{{language}}/> 
	SELECT ?comment ?num ?pers ?gen ?token ?lex
	WHERE
        { 
	 { 
	  GRAPH aamag:{{language}}  
          { 
	   ?s {{lpref}}:pos {{lpref}}:Verb .  
	   ?s aamas:lang aama:{{Language}} . 
	   ?s aamas:lang / rdfs:label ?langLabel .  
           ?s aamas:lexeme ?lexeme .
           ?lexeme rdfs:label ?lex . ")
       {:lpref lpref
        :language language
        :Language Language})
      (apply str  
             (for [value values]
               (tmpl/render-string 
                (str "
           ?s ?Q{{value}}  {{lpref}}:{{value}} .  ")
                {:value value
                 :lpref lpref})))
      (tmpl/render-string
       (str "
	   OPTIONAL { ?s {{lpref}}:number / rdfs:label ?num . } 
	   OPTIONAL { ?s {{lpref}}:pngShapeClass ?pngSC .
             ?pngSC rdfs:label  ?shapeClass .}  
	   ?s {{lpref}}:person ?person .  
	   ?person rdfs:label ?pers .  
	   OPTIONAL { ?s {{lpref}}:gender / rdfs:label ?gen . } 
	   OPTIONAL { ?s aamas:memberOf / rdfs:comment ?comment . } 
	   ?s {{lpref}}:token ?tkn .
           OPTIONAL { ?s ?t ?o . FILTER (CONTAINS(str(?t), \"token-note\"))}
           BIND((IF(BOUND(?o),
                    CONCAT(?tkn, \"  [\", ?o, \"]\"),
                    ?tkn))
                   AS ?token) .
	  } 
	 } 
	} 
	ORDER BY ?lex DESC(?num) ?pers DESC(?gen) ")
       {:lpref lpref})
       );;str
))
{'fraction_non_alphanumeric': 0.16293953768541525, 'fraction_numerical': 0.015661471274753407, 'mean_word_length': 2.057550644567219, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7199312', 'n_tokens_mistral': 5081, 'n_tokens_neox': 4609, 'n_words': 1021}