<commit_before>(* Ocsigen
 * http://www.ocsigen.org
 * Module userconf.ml
 * Copyright (C) 2007 <NAME>
 * Laboratoire PPS - CNRS Universit Paris Diderot
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, with linking exception;
 * either version 2.1 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *)
(*****************************************************************************)
(*****************************************************************************)
(* Ocsigen module to allow local (users) config files                        *)
(*****************************************************************************)
(*****************************************************************************)


open Lwt
open Ocsigen_lib
open Ocsigen_extensions

exception NoConfFile


(*****************************************************************************)

let gen hostpattern sitepath charset (regexp, conf, url, prefix, localpath) req_state =
  match req_state with
  | Ocsigen_extensions.Req_found (_, r) -> Lwt.return (Ocsigen_extensions.Ext_found r)
(*VVV not possible to set a filter for now *)
  | Ocsigen_extensions.Req_not_found (previous_extension_err, ri) ->
      let path = ri.ri_sub_path_string in
      match Netstring_pcre.string_match regexp path 0 with
      | None -> Lwt.return (Ext_next previous_extension_err)
      | Some _ -> (* Matching regexp found! *)
          try
            Ocsigen_messages.debug2 "--Userconf: Using user configuration";
            let conf = Ocsigen_extensions.replace_user_dir regexp conf path in
            let url = Netstring_pcre.global_replace regexp url path in
            let prefix = Netstring_pcre.global_replace regexp prefix path in
            let userconf_options = {
              Ocsigen_extensions.localfiles_root =
                Ocsigen_extensions.replace_user_dir regexp localpath path
            } in
            let user_parse_host =
              Ocsigen_extensions.parse_user_site_item userconf_options hostpattern in
            let user_parse_site =
              Ocsigen_extensions.make_parse_site
                (sitepath@[prefix]) charset user_parse_host
            in
            let xmllist =
              try Simplexmlparser.xmlparser_file conf
              with Sys_error _ -> raise NoConfFile
            in
            Lwt.return
              (Ext_sub_result
                 (fun awake cookies_to_set rs ->
                   let path =
                     Ocsigen_lib.remove_slash_at_end
                       (Ocsigen_lib.remove_slash_at_beginning
                          (Ocsigen_lib.remove_dotdot (Neturl.split_path url)))
                   in
                   user_parse_site
                     xmllist
                     awake
                     cookies_to_set
                     (Ocsigen_extensions.Req_not_found
                        (previous_extension_err,
                         {ri with
                          ri_sub_path = path;
                          ri_sub_path_string = url}))
(*VVV We do not want to continue to search in the same <site>
      if the page has not been found (???) *)
(*VVV Filters won't be applied. Is it the right behaviour? *)
(*VVV v v v *)
                   >>=
                   let rec aux ((answer, cts) as r) =
                     match answer with
                     | Ext_found _
                     | Ext_stop_all _
                     | Ext_stop_site _
                     | Ext_stop_host _
                     | Ext_retry_with _ -> Lwt.return r
                     | Ext_next err ->
                         Lwt.return
(*VVV was: Ext_stop_all *)
                           (Ext_stop_site (Ocsigen_http_frame.Cookies.empty, err),
                            cts)
                     | Ext_continue_with (_, cts2, err) ->
(*VVV ??? *)
                         Lwt.return (Ext_stop_site (cts2, err), cts)
                     | Ext_sub_result sr ->
                         sr awake cookies_to_set req_state
                         >>= aux
                   in aux
 (*VVV ^ ^ ^ *)
                 )
              )
          with
          | NoConfFile
          | Unix.Unix_error (Unix.EACCES,_,_)
          | Unix.Unix_error (Unix.ENOENT, _, _) ->
              Lwt.return (Ocsigen_extensions.Ext_next previous_extension_err)
          | Ocsigen_extensions.Error_in_user_config_file s ->
              Ocsigen_messages.errlog
                (Printf.sprintf "Userconf error for url %s: %s" url s);
              Lwt.return
                (Ocsigen_extensions.Ext_stop_site
                   (Ocsigen_http_frame.Cookies.empty, 500))



(*****************************************************************************)
(** Parsing of config file *)
open Simplexmlparser

let parse_config hostpattern path charset =
  fun _ _ ->
    let rec parse_attrs_local ((regexp, conf, url, prefix, path) as res) = function
      | [] -> res
      | ("regexp", s)::l when regexp = None ->
          (try
            parse_attrs_local<commit_msg>Changed the semantics of Ext_ in userconf<commit_after>(* Ocsigen
 * http://www.ocsigen.org
 * Module userconf.ml
 * Copyright (C) 2007 <NAME>
 * Laboratoire PPS - CNRS Universit Paris Diderot
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, with linking exception;
 * either version 2.1 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *)
(*****************************************************************************)
(*****************************************************************************)
(* Ocsigen module to allow local (users) config files                        *)
(*****************************************************************************)
(*****************************************************************************)


open Lwt
open Ocsigen_lib
open Ocsigen_extensions

exception NoConfFile


(*****************************************************************************)

let gen hostpattern sitepath charset (regexp, conf, url, prefix, localpath) req_state =
  match req_state with
  | Ocsigen_extensions.Req_found (_, r) -> Lwt.return (Ocsigen_extensions.Ext_found r)
(*VVV not possible to set a filter for now *)
  | Ocsigen_extensions.Req_not_found (previous_extension_err, ri) ->
      let path = ri.ri_sub_path_string in
      match Netstring_pcre.string_match regexp path 0 with
      | None -> Lwt.return (Ext_next previous_extension_err)
      | Some _ -> (* Matching regexp found! *)
          try
            Ocsigen_messages.debug2 "--Userconf: Using user configuration";
            let conf = Ocsigen_extensions.replace_user_dir regexp conf path in
            let url = Netstring_pcre.global_replace regexp url path in
            let prefix = Netstring_pcre.global_replace regexp prefix path in
            let userconf_options = {
              Ocsigen_extensions.localfiles_root =
                Ocsigen_extensions.replace_user_dir regexp localpath path
            } in
            let user_parse_host =
              Ocsigen_extensions.parse_user_site_item userconf_options hostpattern in
            let user_parse_site =
              Ocsigen_extensions.make_parse_site
                (sitepath@[prefix]) charset user_parse_host
            in
            let xmllist =
              try Simplexmlparser.xmlparser_file conf
              with Sys_error _ -> raise NoConfFile
            in
            Lwt.return
              (Ext_sub_result
                 (fun awake cookies_to_set rs ->
                   let path =
                     Ocsigen_lib.remove_slash_at_end
                       (Ocsigen_lib.remove_slash_at_beginning
                          (Ocsigen_lib.remove_dotdot (Neturl.split_path url)))
                   in
                   user_parse_site
                     xmllist
                     awake
                     cookies_to_set
                     (Ocsigen_extensions.Req_not_found
                        (previous_extension_err,
                         {ri with
                          ri_sub_path = path;
                          ri_sub_path_string = url}))
(*VVV Do we want to continue to search in the same <site>
      if the page has not been found (???) *)
(*VVV Filters wouldn't be applied. *)
                   >>=
                   let rec aux ((answer, cts) as r) =
                     match answer with
                     | Ext_sub_result sr ->
                         sr awake cookies_to_set req_state
                         >>= aux
                     | _ -> Lwt.return r
                   in aux
 (*VVV ^ ^ ^ *)
                 )
              )
          with
          | NoConfFile
          | Unix.Unix_error (Unix.EACCES,_,_)
          | Unix.Unix_error (Unix.ENOENT, _, _) ->
              Lwt.return (Ocsigen_extensions.Ext_next previous_extension_err)
          | Ocsigen_extensions.Error_in_user_config_file s ->
              Ocsigen_messages.errlog
                (Printf.sprintf "Userconf error for url %s: %s" url s);
              Lwt.return
                (Ocsigen_extensions.Ext_stop_site
                   (Ocsigen_http_frame.Cookies.empty, 500))



(*****************************************************************************)
(** Parsing of config file *)
open Simplexmlparser

let parse_config hostpattern path charset =
  fun _ _ ->
    let rec parse_attrs_local ((regexp, conf, url, prefix, path) as res) = function
      | [] -> res
      | ("regexp", s)::l when regexp = None ->
          (try
            parse_attrs_local
{'fraction_non_alphanumeric': 0.1476679104477612, 'fraction_numerical': 0.004850746268656716, 'mean_word_length': 2.029386832438542, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '622952', 'n_tokens_mistral': 2826, 'n_tokens_neox': 2644, 'n_words': 809}