<commit_before>//
//  FetchBot.swift
//  AutoFech
//
//  Created by virus1994 on 2017/9/25.
//  Copyright © 2017年 ascp. All rights reserved.
//

import Foundation

// struct
struct ListItem : Equatable {
    var title : String
    var href : String
    var previewImages : [String]
    init(data: [String:Any]) {
        title = data["title"] as? String ?? ""
        href = data["href"] as? String ?? ""
        previewImages = data["images"] as? [String] ?? []
    }
    
    static func ==(lhs: ListItem, rhs: ListItem) -> Bool {
        return lhs.title == rhs.title && lhs.href == rhs.href
    }
}

/// struct
enum FetchBoard : Int {
    case netDisk = 103
}

/// 列表页面链接信息
struct FetchURL : Equatable {
    var site : String
    var board : FetchBoard
    var page : Int
    var maker : (FetchURL) -> String
    var url : URL {
        get {
            return URL(string: maker(self))!;
        }
    }
    
    static func ==(lhs: FetchURL, rhs: FetchURL) -> Bool {
        return lhs.url == rhs.url
    }
}

/// 抓取内容页面信息模型
struct ContentInfo {
    var title : String
    var page : String
    var msk : String
    var time : String
    var size : String
    var format : String
    var passwod : String
    var downloafLink : [String]
    var imageLink : [String]
    init() {
        page = ""
        title = ""
        msk = ""
        time = ""
        size = ""
        format = ""
        passwod = ""
        downloafLink = [String]()
        imageLink = [String]()
    }
}

/// 内容信息正则规则选项
struct InfoRuleOption {
    /// 是否有码
    static let msk = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "((【是否有码】)|(【有碼無碼】)|(【影片说明】)|(【影片說明】)|(【是否有碼】)){1}[：:]{0,1}((&nbsp;)|(\\s))*", hasSuffix: false, innerRegex: "([^<：:(&nbsp;)])+")
    /// 影片时间
    static let time = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "((【影片时间】)|(【影片時間】)){1}[：:]{0,1}((&nbsp;)|(\\s))*", hasSuffix: false, innerRegex: "([^<(&nbsp;)])+")
    /// 影片大小
    static let size = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "(【影片大小】){1}[：:]{0,1}((&nbsp;)|(\\s))*", hasSuffix: false, innerRegex: "([^<：:(&nbsp;)])+")
    /// 影片格式
    static let format = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "(【影片格式】){1}[：:]{0,1}", hasSuffix: false, innerRegex: "([^<：:])+")
    /// 解压密码
    static let password = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "((【解壓密碼】)|(【解压密码】)|(解壓密碼)){1}[：:]{0,1}((&nbsp;)|(\\s))*", hasSuffix: false, innerRegex: "([^<：:(&nbsp;)])+")
    /// 下载链接
    static let downloadLink = ParserTagRule(tag: "a", isTagPaser: true, attrubutes: [], inTagRegexString: " \\w+=\"\\w+:\\/\\/[\\w+\\.]+[\\/\\-\\w\\.]+\" \\w+=\"\\w+\"", hasSuffix: false, innerRegex: "\\w+:\\/\\/[\\w+\\.]+[\\/\\-\\w\\.]+")
    /// 下载地址2
    static let downloadLinkLi = ParserTagRule(tag: "li", isTagPaser: true, attrubutes: [], inTagRegexString: "", hasSuffix: false, innerRegex: "\\w+:\\/\\/[\\w+\\.]+[\\/\\-\\w\\.]+")
    /// 图片链接
    static let imageLink = ParserTagRule(tag: "img", isTagPaser: true, attrubutes: [ParserAttrubuteRule(key: "file"), ParserAttrubuteRule(key: "href"), ParserAttrubuteRule(key: "src")], inTagRegexString: "( \\w+=[\"']{1}[^<>]*[\"']{1})+ class=\"zoom\"( \\w+=[\"']{1}[^<>]*[\"']{1})+ \\/", hasSuffix: false, innerRegex: nil)
    /// 主内容标签<commit_msg>fix fetch rule<commit_after>//
//  FetchBot.swift
//  AutoFech
//
//  Created by virus1994 on 2017/9/25.
//  Copyright © 2017年 ascp. All rights reserved.
//

import Foundation

// struct
struct ListItem : Equatable {
    var title : String
    var href : String
    var previewImages : [String]
    init(data: [String:Any]) {
        title = data["title"] as? String ?? ""
        href = data["href"] as? String ?? ""
        previewImages = data["images"] as? [String] ?? []
    }
    
    static func ==(lhs: ListItem, rhs: ListItem) -> Bool {
        return lhs.title == rhs.title && lhs.href == rhs.href
    }
}

/// struct
enum FetchBoard : Int {
    case netDisk = 103
}

/// 列表页面链接信息
struct FetchURL : Equatable {
    var site : String
    var board : FetchBoard
    var page : Int
    var maker : (FetchURL) -> String
    var url : URL {
        get {
            return URL(string: maker(self))!;
        }
    }
    
    static func ==(lhs: FetchURL, rhs: FetchURL) -> Bool {
        return lhs.url == rhs.url
    }
}

/// 抓取内容页面信息模型
struct ContentInfo {
    var title : String
    var page : String
    var msk : String
    var time : String
    var size : String
    var format : String
    var passwod : String
    var downloafLink : [String]
    var imageLink : [String]
    init() {
        page = ""
        title = ""
        msk = ""
        time = ""
        size = ""
        format = ""
        passwod = ""
        downloafLink = [String]()
        imageLink = [String]()
    }
}

/// 内容信息正则规则选项
struct InfoRuleOption {
    /// 是否有码
    static let msk = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "((【是否有码】)|(【有碼無碼】)|(【影片说明】)|(【影片說明】)|(【是否有碼】)){1}[：:]{0,1}((&nbsp;)|(\\s))*", hasSuffix: false, innerRegex: "([^<：:(&nbsp;)])+")
    /// 影片时间
    static let time = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "((【影片时间】)|(【影片時間】)|(【视频时间】)){1}[：:]{0,1}((&nbsp;)|(\\s))*", hasSuffix: false, innerRegex: "([^<(&nbsp;)])+")
    /// 影片大小
    static let size = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "((【影片大小】)|(【视频大小】)){1}[：:]{0,1}((&nbsp;)|(\\s))*", hasSuffix: false, innerRegex: "([^<：:(&nbsp;)])+")
    /// 影片格式
    static let format = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "((【影片格式】)|(【视频格式】)){1}[：:]{0,1}", hasSuffix: false, innerRegex: "([^<：:])+")
    /// 解压密码
    static let password = ParserTagRule(tag: "", isTagPaser: false, attrubutes: [], inTagRegexString: "((【解壓密碼】)|(【解压密码】)|(解壓密碼)){1}[：:]{0,1}((&nbsp;)|(\\s))*", hasSuffix: false, innerRegex: "([^<：:])+")
    /// 下载链接
    static let downloadLink = ParserTagRule(tag: "a", isTagPaser: true, attrubutes: [], inTagRegexString: " \\w+=\"\\w+:\\/\\/[\\w+\\.]+[\\/\\-\\w\\.]+\" \\w+=\"\\w+\"", hasSuffix: false, innerRegex: "\\w+:\\/\\/[\\w+\\.]+[\\/\\-\\w\\.]+")
    /// 下载地址2
    static let downloadLinkLi = ParserTagRule(tag: "li", isTagPaser: true, attrubutes: [], inTagRegexString: "", hasSuffix: false, innerRegex: "\\w+:\\/\\/[\\w+\\.]+[\\/\\-\\w\\.]+")
    /// 图片链接
    static let imageLink = ParserTagRule(tag: "img", isTagPaser: true, attrubutes: [ParserAttrubuteRule(key: "file"), ParserAttrubuteRule(key: "href"), ParserAttrubuteRule(key: "src")], inTagRegexString: "( \\w+=[\"']{1}[^<>]*[\"']{1})+ class=\"zoom\"( \\w+=[\"']{1}[^<>]*[\"']{1})+ \\/", hasSuffix: false, innerRegex: nil)
    /// 主内容标签
{'fraction_non_alphanumeric': 0.23923234690887782, 'fraction_numerical': 0.011133899794901847, 'mean_word_length': 2.8203693340794627, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1448597', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2818, 'n_words': 530}