<commit_before>package com.beiweilingdu.word;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;

import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;

public class WordHandle {
	
	private static final String USER_DIR = System.getProperty("user.dir");
	
	/**
	 * main
	 * @param args
	 * @throws FileNotFoundException
	 * @throws IOException
	 */
	public static void main(String[] args) throws FileNotFoundException, IOException {
		WordHandle wh = new WordHandle();
//		List<Integer> sections = new ArrayList<Integer>();
//		sections.add(1);
//		sections.add(3);
//		sections.add(11);
//		sections.add(12);
//		
//		wh.cutDocument("f:/13_12030084.docx", sections);
		
//		wh.scanIllegalWord("G:\\user\\wuxiaolin\\项目开发\\mspm\\产品手册文档示例\\产品资料--正式文件20141204(测试用)");
		wh.scanIllegalWord(args[0]);
	}
	
	/**
	 * 扫描某路径下所有不规范的word文档，包括word版本、内容段落不全等。
	 * 扫描的结果保存到txt文件中，内容是不规范文档的绝对路径和文件名
	 * @param dir
	 * @throws IOException 
	 */
	public synchronized void scanIllegalWord(String dir) throws IOException {
		
		/**
		 * 1、判断dir路径是否有效
		 * 2、递归扫描dir下所有文件，遇到.doc文件，就直接记录为不规范文档。并把.docx文件的绝对路径保存到一个List<String>中，等着下一步进行文档内容的扫描
		 * 3、循环处理每一个.docx文档，判断其内容是否规范
		 */
		// 参数校验
		if(dir == null || dir == "") {
			log("参数为空，请检查！");
			return;
		}
		// 路径校验
		File dirFile = new File(dir);
		if(!dirFile.exists()) {
			log("扫描路径不存在，请检查！");
			return;
		}
		
		// 递归扫描路径下所有文件
		List<String> result = new ArrayList<String>();
		this.scanAllFile(dir, result);
		
		// 判断内容是否规范
		List<String> keywords = this.getKeywordList();
		if(result != null) {
			
			File outfile = new File(USER_DIR + "/"+UUID.randomUUID()+".txt");
//			if(!outfile.exists()) {
//				outfile.mkdirs();
				outfile.createNewFile();
//			}
			BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile),"utf-8"));
			for(String filepath : result) {
				if(filepath.endsWith(".docx")) {
					File file = new File(filepath);
					if(file.exists()) {
						List<String> lostKeyword = new ArrayList<String>();
						Map<String, String> err = new HashMap<String, String>();
						XWPFDocument wordoc = new XWPFDocument(new FileInputStream(file));
						if(!this.isLegalWord(wordoc, keywords, lostKeyword, err)) {
							// TODO 记录文档内容不规范
							System.out.println("=========================================================");
							bw.write("=========================================================\r\n");
							bw.write(">文件路径：" + filepath + "\r\n");
							System.out.println(">文件路径：" + filepath);
							if(err.get("error") != null && err.get("error") != "") {
								bw.write(">格式错误：" + err.get("error") + "\r\n");
								System.out.println(">格式错误：" + err.get("error"));
							}
							
							if(lostKeyword.size() > 0) {
								bw.write(">缺失标签：\r\n");
								System.out.println(">缺失标签：");
								for(String lostkey : lostKeyword) {
									bw.write("\t" + lostkey + "\r\n");
									System.out.println("\t" + lostkey);
								}
							}
						} else {
							// TODO 规范的文档
//							System.out.println("================ " + filepath + " -> 通过 ===============");
						}
					}
				} else if(filepath.endsWith(".doc")) {
					// TODO 记录文档版本不规范
					bw.write("=========================================================\r\n");
					System.out.println("=========================================================");
					bw.write(">文件路径：" + filepath + "\r\n");
					System.out.println(">文件路径：" + filepath);
					bw.write(">版本错误：word文档版本不是2007及以上版本\r\n");
					System.out.println(">版本错误：word文档版本不是2007及以上版本");
				}
			}
			bw.write("=========================================================");
			bw.close();
			System.out.println("=========================================================");
			System.out.println(">检测结果已经保存到["+outfile.getAbsolutePath()+"]");
		}
	}
	
	/**
	 * 判断文档内容是否规范
	 * @param wordoc word文档
	 * @param keywords 关键字列表
	 * @param lostKeyword 丢失的关键字列表
	 * @return
	 */
	public Boolean isLegalWord(XWPFDocument wordoc, List<String> keywords, List<String> lostKeyword, Map<String, String> err) {
		
		List<XWPFTable> tables = wordoc.getTables();
		if(tables == null || tables.size() > 1) {
			err.put("error", "存在多个表格");
			return false;
		}
		
		// 读取文档表格
		XWPFTable table = tables.get(0);
		
		// 读取表格的行
		List<XWPFTableRow> rows = table.getRows();
		
		boolean isOK = true;
		
		for(String keyword : keywords) {
			boolean ok = false;<commit_msg>增加通过文件的记录<commit_after>package com.beiweilingdu.word;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.UUID;

import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFTable;
import org.apache.poi.xwpf.usermodel.XWPFTableCell;
import org.apache.poi.xwpf.usermodel.XWPFTableRow;

public class WordHandle {
	
	private static final String USER_DIR = System.getProperty("user.dir");
	
	/**
	 * main
	 * @param args
	 * @throws FileNotFoundException
	 * @throws IOException
	 */
	public static void main(String[] args) throws FileNotFoundException, IOException {
		WordHandle wh = new WordHandle();
//		List<Integer> sections = new ArrayList<Integer>();
//		sections.add(1);
//		sections.add(3);
//		sections.add(11);
//		sections.add(12);
//		
//		wh.cutDocument("f:/13_12030084.docx", sections);
		
//		wh.scanIllegalWord("G:\\user\\wuxiaolin\\项目开发\\mspm\\产品手册文档示例\\产品资料--正式文件20141204(测试用)");
		wh.scanIllegalWord(args[0]);
	}
	
	/**
	 * 扫描某路径下所有不规范的word文档，包括word版本、内容段落不全等。
	 * 扫描的结果保存到txt文件中，内容是不规范文档的绝对路径和文件名
	 * @param dir
	 * @throws IOException 
	 */
	public synchronized void scanIllegalWord(String dir) throws IOException {
		
		/**
		 * 1、判断dir路径是否有效
		 * 2、递归扫描dir下所有文件，遇到.doc文件，就直接记录为不规范文档。并把.docx文件的绝对路径保存到一个List<String>中，等着下一步进行文档内容的扫描
		 * 3、循环处理每一个.docx文档，判断其内容是否规范
		 */
		// 参数校验
		if(dir == null || dir == "") {
			log("参数为空，请检查！");
			return;
		}
		// 路径校验
		File dirFile = new File(dir);
		if(!dirFile.exists()) {
			log("扫描路径不存在，请检查！");
			return;
		}
		
		// 递归扫描路径下所有文件
		List<String> result = new ArrayList<String>();
		this.scanAllFile(dir, result);
		
		// 判断内容是否规范
		List<String> keywords = this.getKeywordList();
		if(result != null) {
			
			File outfile = new File(USER_DIR + "/驳回_"+UUID.randomUUID()+".txt");
			File legalFile = new File(USER_DIR + "/通过_"+UUID.randomUUID()+".txt");
//			if(!outfile.exists()) {
//				outfile.mkdirs();
				outfile.createNewFile();
				legalFile.createNewFile();
//			}
			BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outfile),"utf-8"));
			BufferedWriter lbw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(legalFile),"utf-8"));
			int passCount = 1, rejectedCount = 1;
			for(String filepath : result) {
				if(filepath.endsWith(".docx")) {
					File file = new File(filepath);
					if(file.exists()) {
						List<String> lostKeyword = new ArrayList<String>();
						Map<String, String> err = new HashMap<String, String>();
						XWPFDocument wordoc = new XWPFDocument(new FileInputStream(file));
						if(!this.isLegalWord(wordoc, keywords, lostKeyword, err)) {
							// TODO 记录文档内容不规范
							System.out.println("=========================================================");
							bw.write(rejectedCount + " =========================================================\r\n");
							bw.write(">文件路径：" + filepath + "\r\n");
							System.out.println(">文件路径：" + filepath);
							if(err.get("error") != null && err.get("error") != "") {
								bw.write(">格式错误：" + err.get("error") + "\r\n");
								System.out.println(">格式错误：" + err.get("error"));
							}
							
							if(lostKeyword.size() > 0) {
								bw.write(">缺失标签：\r\n");
								System.out.println(">缺失标签：");
								for(String lostkey : lostKeyword) {
									bw.write("\t" + lostkey + "\r\n");
									System.out.println("\t" + lostkey);
								}
							}
							rejectedCount++;
						} else {
							// TODO 规范的文档
//							System.out.println("================ " + filepath + " -> 通过 ===============");
							lbw.write(passCount + "\t" + filepath + "\r\n");
							passCount++;
						}
					}
				} else if(filepath.endsWith(".doc")) {
					// TODO 记录文档版本不规范
					bw.write(rejectedCount + " =========================================================\r\n");
					System.out.println("=========================================================");
					bw.write(">文件路径：" + filepath + "\r\n");
					System.out.println(">文件路径：" + filepath);
					bw.write(">版本错误：word文档版本不是2007及以上版本\r\n");
					System.out.println(">版本错误：word文档版本不是2007及以上版本");
					rejectedCount++;
				}
			}
			bw.write("=========================================================");
			lbw.write("=========================================================");
			bw.close();
			lbw.close();
			System.out.println("=========================================================");
			System.out.println(">驳回结果已经保存到["+outfile.getAbsolutePath()+"]");
			System.out.println(">通过结果已经保存到["+legalFile.getAbsolutePath()+"]");
		}
	}
	
	/**
	 * 判断文档内容是否规范
	 * @param wordoc word文档
	 * @param keywords 关键字列表
	 * @param lostKeyword 丢失的关键字列表
	 * @return
	 */
	public Boolean isLegalWord(XWPFDocument wordoc, List<String> keywords, List<String> lostKeyword, Map<String, String> err) {
		
		List<XWPFTable> tables = wordoc.getTables();
		if(tables == null || tables.size() > 1) {
			err.put("error", "存在多个表格");
			return false;
		}
		
		// 读取文档表格
		XWPFTable table = tables.get(0);
		
		// 读取表格的行
		List<XWPFTableRow> rows = table.getRows();
		
		boolean isOK = true;
		
		for(String keyword : keywords) {
			boolean ok = false;
{'fraction_non_alphanumeric': 0.23374747182895117, 'fraction_numerical': 0.007993836078204758, 'mean_word_length': 3.6233303650934996, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1999317', 'n_tokens_mistral': 4615, 'n_tokens_neox': 4112, 'n_words': 566}