<commit_before>package io.github.ihongs.util.verify;

import io.github.ihongs.Core;
import io.github.ihongs.CoreConfig;
import io.github.ihongs.HongsExemption;
import io.github.ihongs.action.UploadHelper;
import io.github.ihongs.util.Synt;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLEncoder;
import java.net.URLConnection;
import java.net.MalformedURLException;
import java.net.URLDecoder;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.http.Part;

/**
 * 文件校验
 * <pre>
 * 规则参数:
 *  pass-source yes|no 是否跳过资源链接(存在有"/"的字符)
 *  pass-remote yes|no 是否跳过远程链接(开头为"http://")
 *  down-remote yes|no 是否下载远程文件
 *  drop-origin yes|no 抛弃原始文件, 仅使用 checks 中新创建的
 *  keep-origin yes|no 返回原始路径, 不理会 checks 中新创建的
 *  keep-naming yes|no 保持原文件名, 会对网址末尾的文件名编码
 *  name-digest 命名摘要算法, 如: MD5,SHA-1,SHA-256
 *  temp 上传临时目录, 可用变量 $DATA_PATH, $BASE_PATH 等
 *  path 上传目标目录, 可用变量 $BASE_PATH, $DATA_PATH 等
 *  href 上传文件链接, 可用变量 $BASE_HREF, $SERV_HREF 等, 后者包含域名
 *  type 文件类型限制, 逗号分隔 (Mime-Type)
 *  extn 扩展名称限制, 逗号分隔
 * </pre>
 * @author Hongs
 */
public class IsFile extends Rule {
    @Override
    public Object verify(Value watch) throws Wrong {
        // 跳过空值和空串
        Object value = watch.get();
        if (value  ==  null ) {
            return STAND;
        }
        if (value.equals("")) {
            return STAND;
        }

        if (value instanceof String) {

        // 跳过资源路径
        if (Synt.declare(getParam("pass-source"), false)) {
            String u = value.toString( );
            if (u.indexOf( '/' ) != -1 ) {
                return value;
            }
        }

        // 跳过远程地址
        if (Synt.declare(getParam("pass-remote"), false)) {
            String u = value.toString( );
            if (u.matches("^\\w+://.*")) {
                return value;
            }
        }

        // 下载远程文件
        if (Synt.declare(getParam("down-remote"), false)) {
            String u = value.toString( );
            if (u.matches("^\\w+://.*")) {
            do {
                String x;
                // 如果是本地路径则不再下载
                x = Core.SITE_HREF + "/";
                if (x != null && !"".equals(x)) {
                    if (u.startsWith(x)) {
                        value = u;
                        break ;
                    }
                }
                x = (String) getParam("href");
                if (x != null && !"".equals(x)) {
                    if (u.startsWith(x)) {
                        value = u;
                        break ;
                    }
                }
                // 如果有临时目录则下载到这
                x = (String) getParam("temp");
                if (x == null ||  "".equals(x)) {
                    x = Core.BASE_PATH + "/static/upload/tmp";
                }
                value = stores(value.toString(), x);
                if (value == null) {
                    return   null;
                }
            } while(false);
            }
        }

        } // End If

        UploadHelper hlpr = new UploadHelper();
        String href;
        String path;
        Object para;
        String name;

        para = getParam("temp");
        if (para != null && !"".equals(para)) hlpr.setUploadTemp(Synt.declare(para, String.class));
        para = getParam("path");
        if (para != null && !"".equals(para)) hlpr.setUploadPath(Synt.declare(para, String.class));
        para = getParam("href");
        if (para != null && !"".equals(para)) hlpr.setUploadHref(Synt.declare(para, String.class));
        para = getParam("type");
        if (para != null && !"".equals(para)) hlpr.setAllowTypes(Synt.toArray(para, String.class));
        para = getParam("extn");
        if (para != null && !"".equals(para)) hlpr.setAllowExtns(Synt.toArray(para, String.class));
        para = getParam("name-digest");
        if (para != null && !"".equals(para)) hlpr.setDigestType(Synt.declare(para, String.class));

        if (value instanceof Part ) {
            Part part =(Part) value;
            hlpr.upload(part);
            name = part.getSubmittedFileName();
        } else
        if (value instanceof File ) {<commit_msg>给上传路径末尾追加数据 ID, 以便检测和清理无效文件<commit_after>package io.github.ihongs.util.verify;

import io.github.ihongs.Cnst;
import io.github.ihongs.Core;
import io.github.ihongs.CoreConfig;
import io.github.ihongs.HongsExemption;
import io.github.ihongs.action.UploadHelper;
import io.github.ihongs.util.Syno;
import io.github.ihongs.util.Synt;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URL;
import java.net.URLEncoder;
import java.net.URLConnection;
import java.net.MalformedURLException;
import java.net.URLDecoder;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.http.Part;

/**
 * 文件校验
 * <pre>
 * 规则参数:
 *  pass-source yes|no 是否跳过资源链接(存在有"/"的字符)
 *  pass-remote yes|no 是否跳过远程链接(开头为"http://")
 *  down-remote yes|no 是否下载远程文件
 *  drop-origin yes|no 抛弃原始文件, 仅使用 checks 中新创建的
 *  keep-origin yes|no 返回原始路径, 不理会 checks 中新创建的
 *  keep-naming yes|no 保持原文件名, 会对网址末尾的文件名编码
 *  name-digest 命名摘要算法, 如: MD5,SHA-1,SHA-256
 *  path-add-id 追加表单项ID, 将会从 cleans 提取 id
 *  temp 上传临时目录, 可用变量 $DATA_PATH, $BASE_PATH 等
 *  path 上传目标目录, 可用变量 $BASE_PATH, $DATA_PATH 等
 *  href 上传文件链接, 可用变量 $BASE_HREF, $SERV_HREF 等, 后者包含域名
 *  type 文件类型限制, 逗号分隔 (Mime-Type)
 *  extn 扩展名称限制, 逗号分隔
 * </pre>
 * @author Hongs
 */
public class IsFile extends Rule {
    @Override
    public Object verify(Value watch) throws Wrong {
        // 跳过空值和空串
        Object value = watch.get();
        if (value  ==  null ) {
            return STAND;
        }
        if (value.equals("")) {
            return STAND;
        }

        if (value instanceof String) {

        // 跳过资源路径
        if (Synt.declare(getParam("pass-source"), false)) {
            String u = value.toString( );
            if (u.indexOf( '/' ) != -1 ) {
                return value;
            }
        }

        // 跳过远程地址
        if (Synt.declare(getParam("pass-remote"), false)) {
            String u = value.toString( );
            if (u.matches("^\\w+://.*")) {
                return value;
            }
        }

        // 下载远程文件
        if (Synt.declare(getParam("down-remote"), false)) {
            String u = value.toString( );
            if (u.matches("^\\w+://.*")) {
            do {
                String x;
                // 如果是本地路径则不再下载
                x = Core.SITE_HREF + "/";
                if (x != null && !"".equals(x)) {
                    if (u.startsWith(x)) {
                        value = u;
                        break ;
                    }
                }
                x = (String) getParam("href");
                if (x != null && !"".equals(x)) {
                    if (u.startsWith(x)) {
                        value = u;
                        break ;
                    }
                }
                // 如果有临时目录则下载到这
                x = (String) getParam("temp");
                if (x == null ||  "".equals(x)) {
                    x = Core.BASE_PATH + "/static/upload/tmp";
                }
                value = stores(value.toString(), x);
                if (value == null) {
                    return   null;
                }
            } while(false);
            }
        }

        } // End If

        UploadHelper hlpr = new UploadHelper();
        String href;
        String path;
        Object para;
        String name;

        para = getParam("temp");
        if (para != null && !"".equals(para)) hlpr.setUploadTemp(Synt.declare(para, String.class));
        para = getParam("path");
        if (para != null && !"".equals(para)) hlpr.setUploadPath(Synt.declare(para, String.class));
        para = getParam("href");
        if (para != null && !"".equals(para)) hlpr.setUploadHref(Synt.declare(para, String.class));
        para = getParam("type");
        if (para != null && !"".equals(para)) hlpr.setAllowTypes(Synt.toArray(para, String.class));
        para = getParam("extn");
        if (para != null && !"".equals(para)) hlpr.setAllowExtns(Synt.toArray(para, String.class));
        para = getParam("name-digest");
        if (para != null && !"".equals(para)) hlpr.setDigestType(Synt.declare(para, String.class));

        /**
         * 给当前路径末尾追加表单 ID
         * 以便检测和清理无效文件
         */
        if (Synt.declare(getParam("path-add-id"), false)) {
            String id = Synt.declare(watch.getCleans().get(Cnst.ID_KEY), String.class);
            if (id == null || id.isEmpty()) {
                throw new Wrong(Cnst.ID_KEY +" required for file");
            }
            id   = Syno.splitPath(id); // 避免单层目录数量过多
            path = Synt.declare(getParam("path"), "static/upload");
            href = Synt.declare(getParam("href"), "static/upload");
            hlpr.setUploadPath(path + "/" + id );
            hlpr.setUploadHref(href + "/" + id );
        }

        if (value instanceof Part ) {
            Part part =(Part) value;
            hlpr.upload(part);
            name = part.getSubmittedFileName();
        } else
        if (value instanceof File ) {
{'fraction_non_alphanumeric': 0.1426156771603634, 'fraction_numerical': 0.001267694908092119, 'mean_word_length': 1.7512351060738158, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1115456', 'n_tokens_mistral': 3356, 'n_tokens_neox': 3363, 'n_words': 623}