<commit_before>(ns lazada-quest.core
  (:require [ring.middleware.resource :refer :all]
            [ring.middleware.content-type :refer :all]
            [ring.middleware.not-modified :refer :all]
            [ring.adapter.jetty :as jetty]
            [ring.util.response :refer :all]
            [cheshire.core :refer :all]
            [lazada-quest.scrapper :as scrapper]
            [clojure.pprint :refer [pprint]])
  (:gen-class :main true))

;; define max products
(def MAX_PRODUCTS 5)

;; define filename for categories
(def FILE_NAME "main-site-categories-top.edn")

(defn save-to-file
  "Save category tree to file"
  [category-tree]
  (spit FILE_NAME (with-out-str (pprint category-tree))))

(defn generate-category-tree-file
  "If category tree file exists in filesystem, do nothing, otherwise
   scrape lazada website and save the generated category tree into file"
  [site-key priority-key]
  (when (not (.exists (clojure.java.io/as-file FILE_NAME)))
    (-> (scrapper/create-category-tree site-key priority-key MAX_PRODUCTS)
        (save-to-file))))

(defn handler
  "Basic handler function, for every incoming request, the response is
   category tree from edn file converted to json format"
  [request]
  (let [categories (read-string (slurp FILE_NAME))]
    (-> (response (generate-string categories))
        (content-type "application/json"))))

(def webapp
  (-> handler
      (wrap-resource "public")
      (wrap-content-type)
      (wrap-not-modified)))

(defn start-server
  "Starts jetty-webserver on port 3000"
  [app]
  (jetty/run-jetty app {:port 3000 :join? false}))

(defn -main
  "Main entry point for the program"
  [& args]
  (do (generate-category-tree-file :main :top)
      (start-server webapp)))
<commit_msg>Use when-not instead of (when (not...<commit_after>(ns lazada-quest.core
  (:require [ring.middleware.resource :refer :all]
            [ring.middleware.content-type :refer :all]
            [ring.middleware.not-modified :refer :all]
            [ring.adapter.jetty :as jetty]
            [ring.util.response :refer :all]
            [cheshire.core :refer :all]
            [lazada-quest.scrapper :as scrapper]
            [clojure.pprint :refer [pprint]])
  (:gen-class :main true))

;; define max products
(def MAX_PRODUCTS 5)

;; define filename for categories
(def FILE_NAME "main-site-categories-top.edn")

(defn save-to-file
  "Save category tree to file"
  [category-tree]
  (spit FILE_NAME (with-out-str (pprint category-tree))))

(defn generate-category-tree-file
  "If category tree file exists in filesystem, do nothing, otherwise
   scrape lazada website and save the generated category tree into file"
  [site-key priority-key]
  (when-not (.exists (clojure.java.io/as-file FILE_NAME))
    (-> (scrapper/create-category-tree site-key priority-key MAX_PRODUCTS)
        (save-to-file))))

(defn handler
  "Basic handler function, for every incoming request, the response is
   category tree from edn file converted to json format"
  [request]
  (let [categories (read-string (slurp FILE_NAME))]
    (-> (response (generate-string categories))
        (content-type "application/json"))))

(def webapp
  (-> handler
      (wrap-resource "public")
      (wrap-content-type)
      (wrap-not-modified)))

(defn start-server
  "Starts jetty-webserver on port 3000"
  [app]
  (jetty/run-jetty app {:port 3000 :join? false}))

(defn -main
  "Main entry point for the program"
  [& args]
  (do (generate-category-tree-file :main :top)
      (start-server webapp)))

{'fraction_non_alphanumeric': 0.1282707622298066, 'fraction_numerical': 0.005119453924914676, 'mean_word_length': 3.1968973747016705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7288366', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1150, 'n_words': 322}