<commit_before>/**
 * 
 * APDPlat - Application Product Development Platform
 * Copyright (c) 2013, 杨尚川, <EMAIL>
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 * 
 */

package org.apdplat.superword.tools;

import java.io.InputStream;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

/**
 * 从 http://www.palosverdes.com/jesse/pvphs/www-freecollege-com-vocab.htm 文章中
 * 提取后缀
 * @author 杨尚川
 */
public class SuffixExtractor {
    private static final String SRC_HTML = "/tools/prefix_suffix.txt";
    
    public static void main(String[] args) throws Exception {
        InputStream in = SuffixExtractor.class.getResourceAsStream(SRC_HTML);
        Document document = Jsoup.parse(in, "utf-8", "");
        document.select("table tbody tr")
                .stream()
                .forEach(SuffixExtractor::extractSuffix);
    }
    
    public static void extractSuffix(Element element){
        List<Element> tds = element.children();
        if(tds==null || tds.size()!=3){
            return ;
        }
        String suffix = tds.get(0).text().trim();
        if(!suffix.startsWith("-")){
            return;
        }
        System.out.println("suffix(wordSet, \""+suffix+"\", \""+tds.get(1).text()+"\");");
    }
}<commit_msg>提取后缀功能增强<commit_after>/**
 *
 * APDPlat - Application Product Development Platform
 * Copyright (c) 2013, 杨尚川, <EMAIL>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package org.apdplat.superword.tools;

import java.io.InputStream;
import java.util.List;
import java.util.stream.Collectors;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

/**
 * 从 http://www.palosverdes.com/jesse/pvphs/www-freecollege-com-vocab.htm 文章中
 * 提取后缀
 * @author 杨尚川
 */
public class SuffixExtractor {
    private static final String SRC_HTML = "/tools/prefix_suffix.txt";

    public static List<SuffixInfo> extract() {
        try(InputStream in = SuffixExtractor.class.getResourceAsStream(SRC_HTML)) {
            Document document = Jsoup.parse(in, "utf-8", "");
            return document.select("table tbody tr")
                    .stream()
                    .map(SuffixExtractor::extractSuffix)
                    .filter(item -> item.getSuffix() != null)
                    .sorted()
                    .distinct()
                    .collect(Collectors.toList());
        }catch (Exception e){
            throw new RuntimeException(e);
        }
    }
    public static SuffixInfo extractSuffix(Element element){
        SuffixInfo suffixInfo = new SuffixInfo();
        List<Element> tds = element.children();
        if(tds==null || tds.size()!=3){
            return suffixInfo;
        }
        String suffix = tds.get(0).text().trim();
        if(!suffix.startsWith("-")){
            return suffixInfo;
        }
        String des = tds.get(1).text();
        return new SuffixInfo(suffix, des);
    }
    public static class SuffixInfo implements Comparable{
        private String suffix;
        private String des;

        public SuffixInfo(){}
        public SuffixInfo(String suffix, String des) {
            this.suffix = suffix;
            this.des = des;
        }

        public String getDes() {
            return des;
        }

        public void setDes(String des) {
            this.des = des;
        }

        public String getSuffix() {
            return suffix;
        }

        public void setSuffix(String suffix) {
            this.suffix = suffix;
        }

        @Override
        public int compareTo(Object o) {
            if(o == null){
                return 1;
            }
            return this.suffix.compareTo(((SuffixInfo)o).getSuffix());
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;
            if (o == null || getClass() != o.getClass()) return false;

            SuffixInfo that = (SuffixInfo) o;

            if (!suffix.equals(that.suffix)) return false;

            return true;
        }

        @Override
        public int hashCode() {
            return suffix.hashCode();
        }
    }
    public static void main(String[] args){
        extract().forEach(suffix ->
                System.out.println("suffix(wordSet, \"" + suffix.getSuffix() + "\", \"" + suffix.getDes() + "\");"));
    }
}
{'fraction_non_alphanumeric': 0.11278195488721804, 'fraction_numerical': 0.003401360544217687, 'mean_word_length': 2.292280494991161, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7486550', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1596, 'n_words': 470}