<commit_before>#lang racket
(require net/url
         json
         html-parsing
         webscraperhelper
         sxml)

;return the list of all related url for a city
;either near by or similar. Try paris or mumbai.
;if you want hotels from city you are looking for
; use first function to front element of list
(define (top-url-list city)
  (let* ([doc  (read-json
                (get-pure-port
                 (string->url
                  (string-append "https://www.tripadvisor.in/TypeAheadJson?query="
                                 city
                                 "&action=API&types=geo,theme_park&link_type=hotel&details=false"))))]
         [hot (hash-ref doc 'results)]
         [all-link (map (λ(x) (hash-ref x 'url)) hot)]
         [ret (map (λ(x) (string-append "https://www.tripadvisor.in" x)) all-link)])
    ret))


;you pass the url of city to fetch all the url links
;related to city (number of pages on tripadvisor)
(define (city-url-list url)
  (let* ([page (html->xexp
                (get-pure-port
                 (string->url url)))]
         [doc ((sxpath "(//div[@class=\"pageNumbers\"]/a[@href])") page)]
         [page-num (map (λ(x) (sxml:attr x 'data-page-number)) doc)])
         (match page-num
           ['() (cons url empty)]
           [page-num-list
            (let*
                ([link (range (string->number (first page-num-list))
                              (+ 1 (string->number (last page-num-list))))]
                 [ret (map (λ(x) (string-append "https://www.tripadvisor.in"
                                                "/Hotels-g304554-oa"
                                                (number->string (* 30 (- x 1)))
                                                "-Mumbai_Bombay_Maharashtra-Hotels.html"))
                           link)])
              (cons url ret))])))
         
(define doc
  (city-url-list "https://www.tripadvisor.in/Hotels-g3458458-Portland_Parish_Jamaica-Hotels.html"))

;write a function that fetch the name of hotel and its review url
;(struct <commit_msg>removed bugs and added comments<commit_after>#lang racket
(require net/url
         json
         html-parsing
         webscraperhelper
         sxml)

;return the list of all related url for a city
;either near by or similar. Try paris or mumbai.
;if you want hotels from city you are looking for
; use first function to front element of list.
;top-url-list "mumbai"
(define (top-url-list city)
  (let* ([doc  (read-json
                (get-pure-port
                 (string->url
                  (string-append "https://www.tripadvisor.in/TypeAheadJson?query="
                                 city
                                 "&action=API&types=geo,theme_park&link_type=hotel&details=false"))))]
         [hot (hash-ref doc 'results)]
         [all-link (map (λ(x) (hash-ref x 'url)) hot)]
         [ret (map (λ(x) (string-append "https://www.tripadvisor.in" x)) all-link)])
    ret))

;helper function for city-url-list
(define (create-url url n)
  (let*-values
      ([(p q) (split-at (string-split url "-") 2)]
       [(tlist) (append p (cons (string-append "oa"
                                               (number->string (* 30 (- n 1)))) empty) q)]
       [(ret) (string-join tlist "-")]) ret))


;you pass the url of city to fetch all the url link of each page
;related to city (number of pages on tripadvisor for the city)
(define (city-url-list url)
  (let* ([page (html->xexp
                (get-pure-port
                 (string->url url)))]
         [doc ((sxpath "(//div[@class=\"pageNumbers\"]/a[@href])") page)]
         [page-num (map (λ(x) (sxml:attr x 'data-page-number)) doc)])
    (match page-num
      ['() (cons url empty)]
      [page-num-list
       (let*
           ([link (range (string->number (first page-num-list))
                         (+ 1 (string->number (last page-num-list))))]
            [ret (map (λ(x) (create-url url x)) link)])
         (cons url ret))])))


;write a function that fetch the name of hotel, review count, rank, and url
;function takes a url of tripadvisor page and return above mentioned data.
;This one is bit complecated because it executes java script so think for a
;method which executes java script racket.
(define (hotel-info url)
  (html->xexp
   (get-pure-port
    (string->url url))))


(hotel-info "https://www.tripadvisor.in/Hotels-g187147-oa1770-Paris_Ile_de_France-Hotels.html")
{'fraction_non_alphanumeric': 0.13598373616444545, 'fraction_numerical': 0.007228371357578495, 'mean_word_length': 1.905511811023622, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 7, 'lorem ipsum': 0, 'www.': 7, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3624566', 'n_tokens_mistral': 1399, 'n_tokens_neox': 1325, 'n_words': 405}