<commit_before>(* Js_of_ocaml compiler
 * http://www.ocsigen.org/js_of_ocaml/
 * Copyright (C) 2010 <NAME>
 * Laboratoire PPS - CNRS Université Paris Diderot
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, with linking exception;
 * either version 2.1 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *)


open Util

let parse_annot loc s =
  let buf = Lexing.from_string s in
  try
    match Annot_parser.annot Annot_lexer.initial buf with
      | `Requires (_,l) -> Some (`Requires (Some loc,l))
      | `Provides (_,n,k) -> Some (`Provides (Some loc,n,k))
      | `Version (_,l) -> Some (`Version  (Some loc, l))
  with
    | Not_found -> None
    | exc ->
    (* Format.eprintf "Not found for %s : %s @." (Printexc.to_string exc) s; *)
    None

let error s = Format.kprintf (fun s -> failwith s) s

let split_dir =
  let reg = Str.regexp_string Filename.dir_sep in
  Str.split reg

let parse_file f =
  let file =
    try
      match Util.path_require_findlib f with
        | Some f ->
          let pkg,f' = match split_dir f with
            | [] -> assert false
            | [f] -> "js_of_ocaml",f
            | pkg::l -> pkg, List.fold_left Filename.concat "" l in
          Filename.concat (Util.find_pkg_dir pkg)  f'
        | None -> f
    with
      | Not_found ->
        error "cannot find file '%s'. @." f
      | Sys_error s ->
        error "%s@." s
  in

  let lex = Parse_js.lexer_from_file ~rm_comment:false file in
  let status,lexs = Parse_js.lexer_fold (fun (status,lexs) t ->
    match t with
      | Js_token.TComment (info,str) -> begin
        match parse_annot info str with
          | None -> (status,lexs)
          | Some a ->
            match status with
              | `Annot annot -> `Annot (a::annot),lexs
              | `Code (an,co) -> `Annot [a], ((List.rev an,List.rev co)::lexs)
      end
      | _ when Js_token.is_comment t -> (status,lexs)
      | c -> match status with
          | `Code (annot,code) -> `Code (annot,c::code),lexs
          | `Annot (annot) -> `Code(annot,[c]),lexs
  ) (`Annot [],[]) lex in
  let lexs = match status with
    | `Annot _ -> lexs
    | `Code(annot,code) -> (List.rev annot,List.rev code)::lexs in

  let res = List.rev_map (fun (annot,code) ->
    let lex = Parse_js.lexer_from_list code in
    try
      annot,Parse_js.parse lex
    with Parse_js.Parsing_error pi ->
      error "cannot parse file %S (orig:%S from l:%d, c:%d)@." f pi.Parse_info.name pi.Parse_info.line pi.Parse_info.col)
    lexs in
  res

let last_code_id = ref 0
let provided = Hashtbl.create 31
let provided_rev = Hashtbl.create 31
let code_pieces = Hashtbl.create 31
let always_included = ref []

let version_match =
  let v = Sys.ocaml_version in
  let split_plus =
    let reg = Str.regexp_string "+" in
    Str.split reg in
  let split_dot =
    let reg = Str.regexp_string "." in
    Str.split reg in
  let cano v = match split_plus v with
    | [] -> assert false
    | x::_ -> List.map int_of_string (split_dot x) in
  let v' = cano v in
  let rec compute op v v' = match v,v' with
    | [x],[y] -> op x y
    | [],[] -> op 0 0
    | [],y::_ -> op 0 y
    | x::_,[] -> op x 0
    | x::xs,y::ys ->
      if x = y
      then compute op xs ys
      else op x y
  in
  List.for_all (fun (op,str) ->
      compute op v' (cano str)
    )

let loc pi = match pi with
  | None -> "unknown location"
  | Some pi -> Printf.sprintf "%s:%d" pi.Parse_info.name pi.Parse_info.line

let add_file f =

  List.iter
    (fun (annot,code) ->
      incr last_code_id;
      let id = !last_code_id in

      let req,has_provide,vmatch = List.fold_left (fun (req,has_provide,vmatch) a -> match a with
        | `Provides (pi,name,kind) ->
          req,Some (name,pi,kind),vmatch
        | `Requires (_,mn) -> (mn@req),has_provide,vmatch
        | `Version (_,l) ->
          let vmatch = match vmatch with
            | Some true -> vmatch
            | _ -> Some (version_match l) in
          req,has_provide,vmatch
        ) ([],None,None) annot in

      (match vmatch with
       | Some false -> ()<commit_msg>COMPILER: no Str at toplevel<commit_after>(* Js_of_ocaml compiler
 * http://www.ocsigen.org/js_of_ocaml/
 * Copyright (C) 2010 <NAME>
 * Laboratoire PPS - CNRS Université Paris Diderot
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, with linking exception;
 * either version 2.1 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *)


open Util

let parse_annot loc s =
  let buf = Lexing.from_string s in
  try
    match Annot_parser.annot Annot_lexer.initial buf with
      | `Requires (_,l) -> Some (`Requires (Some loc,l))
      | `Provides (_,n,k) -> Some (`Provides (Some loc,n,k))
      | `Version (_,l) -> Some (`Version  (Some loc, l))
  with
    | Not_found -> None
    | exc ->
    (* Format.eprintf "Not found for %s : %s @." (Printexc.to_string exc) s; *)
    None

let error s = Format.kprintf (fun s -> failwith s) s

let split_dir =
  let reg = Str.regexp_string Filename.dir_sep in
  Str.split reg

let parse_file f =
  let file =
    try
      match Util.path_require_findlib f with
        | Some f ->
          let pkg,f' = match split_dir f with
            | [] -> assert false
            | [f] -> "js_of_ocaml",f
            | pkg::l -> pkg, List.fold_left Filename.concat "" l in
          Filename.concat (Util.find_pkg_dir pkg)  f'
        | None -> f
    with
      | Not_found ->
        error "cannot find file '%s'. @." f
      | Sys_error s ->
        error "%s@." s
  in

  let lex = Parse_js.lexer_from_file ~rm_comment:false file in
  let status,lexs = Parse_js.lexer_fold (fun (status,lexs) t ->
    match t with
      | Js_token.TComment (info,str) -> begin
        match parse_annot info str with
          | None -> (status,lexs)
          | Some a ->
            match status with
              | `Annot annot -> `Annot (a::annot),lexs
              | `Code (an,co) -> `Annot [a], ((List.rev an,List.rev co)::lexs)
      end
      | _ when Js_token.is_comment t -> (status,lexs)
      | c -> match status with
          | `Code (annot,code) -> `Code (annot,c::code),lexs
          | `Annot (annot) -> `Code(annot,[c]),lexs
  ) (`Annot [],[]) lex in
  let lexs = match status with
    | `Annot _ -> lexs
    | `Code(annot,code) -> (List.rev annot,List.rev code)::lexs in

  let res = List.rev_map (fun (annot,code) ->
    let lex = Parse_js.lexer_from_list code in
    try
      annot,Parse_js.parse lex
    with Parse_js.Parsing_error pi ->
      error "cannot parse file %S (orig:%S from l:%d, c:%d)@." f pi.Parse_info.name pi.Parse_info.line pi.Parse_info.col)
    lexs in
  res

let last_code_id = ref 0
let provided = Hashtbl.create 31
let provided_rev = Hashtbl.create 31
let code_pieces = Hashtbl.create 31
let always_included = ref []

let version_match =
  let v = Sys.ocaml_version in
  let split_plus =
    let reg = Str.regexp_string "+" in
    Str.split reg in
  let split_dot =
    let reg = Str.regexp_string "." in
    Str.split reg in
  let cano v = match split_plus v with
    | [] -> assert false
    | x::_ -> List.map int_of_string (split_dot x) in
  let rec compute op v v' = match v,v' with
    | [x],[y] -> op x y
    | [],[] -> op 0 0
    | [],y::_ -> op 0 y
    | x::_,[] -> op x 0
    | x::xs,y::ys ->
      if x = y
      then compute op xs ys
      else op x y
  in
  (fun l ->
     let v' = cano v in
     List.for_all (fun (op,str) ->
         compute op v' (cano str)
       ) l)

let loc pi = match pi with
  | None -> "unknown location"
  | Some pi -> Printf.sprintf "%s:%d" pi.Parse_info.name pi.Parse_info.line

let add_file f =

  List.iter
    (fun (annot,code) ->
      incr last_code_id;
      let id = !last_code_id in

      let req,has_provide,vmatch = List.fold_left (fun (req,has_provide,vmatch) a -> match a with
        | `Provides (pi,name,kind) ->
          req,Some (name,pi,kind),vmatch
        | `Requires (_,mn) -> (mn@req),has_provide,vmatch
        | `Version (_,l) ->
          let vmatch = match vmatch with
            | Some true -> vmatch
            | _ -> Some (version_match l) in
          req,has_provide,vmatch
        ) ([],None,None) annot in

      (match vmatch with
       | Some false -> ()
{'fraction_non_alphanumeric': 0.11861702127659575, 'fraction_numerical': 0.006595744680851064, 'mean_word_length': 2.3767959770114944, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 96, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2531623', 'n_tokens_mistral': 3274, 'n_tokens_neox': 3141, 'n_words': 1067}