<commit_before>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* Copyright (c) 2009, 2013, 2014 <NAME>                          */
/* Copyright (c) 2016, 2017, 2018, 2019 <NAME>                   */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#include <limits.h>                      // INT_MAX

#include "api-structures.h"
#include "connectors.h"
#include "disjunct-utils.h"
#include "dict-common/dict-common.h"    // contable
#include "linkage/analyze-linkage.h"    // intersect_strings
#include "post-process/post-process.h"
#include "post-process/pp-structures.h"
#include "print/print.h"                // print_disjunct_counts

#include "prune.h"
#include "resources.h"
#include "string-set.h"
#include "tokenize/word-structures.h"   // Word_struct
#include "tokenize/wordgraph.h"
<commit_msg>prune.c: Remove unneded #include<commit_after>/*************************************************************************/
/* Copyright (c) 2004                                                    */
/* <NAME>, <NAME>, and <NAME>                    */
/* Copyright (c) 2009, 2013, 2014 <NAME>                          */
/* Copyright (c) 2016, 2017, 2018, 2019 <NAME>                   */
/* All rights reserved                                                   */
/*                                                                       */
/* Use of the link grammar parsing system is subject to the terms of the */
/* license set forth in the LICENSE file included with this software.    */
/* This license allows free redistribution and use in source and binary  */
/* forms, with or without modification, subject to certain conditions.   */
/*                                                                       */
/*************************************************************************/

#include "api-structures.h"
#include "connectors.h"
#include "disjunct-utils.h"
#include "dict-common/dict-common.h"    // contable
#include "linkage/analyze-linkage.h"    // intersect_strings
#include "post-process/post-process.h"
#include "post-process/pp-structures.h"
#include "print/print.h"                // print_disjunct_counts

#include "prune.h"
#include "resources.h"
#include "string-set.h"
#include "tokenize/word-structures.h"   // Word_struct
#include "tokenize/wordgraph.h"

{'fraction_non_alphanumeric': 0.20378989361702127, 'fraction_numerical': 0.02127659574468085, 'mean_word_length': 1.765625, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4302258', 'n_tokens_mistral': 774, 'n_tokens_neox': 672, 'n_words': 204}