<commit_before>//
//  BlogCrawlingService.swift
//  EbloVaporServer
//
//  Created by <NAME> on 2017-04-28.
//
//

import Foundation

/// The service to start a blog crawling.
class BlogCrawlingService {

  /// The crawledblogs.
  var crawledBlog: [Blog] = []
  
  /// The blogParsers.
  private var blogParsers: [BlogParser] = []
  
  /// Whether or not automatically save crawled blog when crawling finishes.
  private var automaticallySaveWhenCrawlingFinished = false
  
  /// Whether or not blog crawling has finished.
  var crawlingFinished: Bool = false {
    didSet {
      if (crawlingFinished && !oldValue) && automaticallySaveWhenCrawlingFinished {
        // Add in a test yelp blog
//        let testBlog = Blog(title: "Hoooooool lllll",
//                            urlString: "http://yelp.com/test",
//                            companyName: "Yelp")
//        testBlog.publishDate = "May, 27, 2017"
//        testBlog.publishDateInterval = Date().timeIntervalSince1970
//        self.crawledBlog.append(testBlog)
        self.saveCrawledBlog()
      }
    }
  }
  
  /// Start service
  func startService(automaticallySaveWhenCrawlingFinished: Bool) {
    crawlingFinished = false
    let companySource = try! CompanySource.all()
    guard companySource.count > 0,
      let companySourceURLString = companySource.first?.companyDataURLString else {
      return
    }
    self.automaticallySaveWhenCrawlingFinished = automaticallySaveWhenCrawlingFinished
    var url = URL(string: companySourceURLString)
    #if os(OSX)
    if EnviromentManager.local {
      let resourceName = EnviromentManager.newCompany ? "newCompany" : "company"
      let urlPath = Bundle.main.path(forResource: resourceName, ofType: "json")
      url = NSURL.fileURL(withPath: urlPath!)
    }
    #endif
    guard let data = try? Data(contentsOf: url!) else {
      print("\(self) can not load data from URL \(url!)")
      return
    }
    do {
      let json =
        try JSONSerialization.jsonObject(with: data,
                                         options: [.mutableContainers, .allowFragments])
      guard let jsonData = json as? [String: Any],
        let companies = jsonData["company"] as? [[String: Any]] else {
          return
      }
      
      /// The current existed companies.
      var existingCompaniesDict: [String : Company] = [:]
      do {
        let existingCompanies = try Company.all()
        existingCompanies.forEach { company in
          existingCompaniesDict[company.string()] = company
        }
      } catch {
        print("Can not load Company from data base.")
      }
      
      for companyInfo in companies {
        // Create blog parsers.
        if let titlePath = companyInfo["title"] as? String,
          let href = companyInfo["href"] as? String,
          let baseURL = companyInfo["baseURL"] as? String,
          let basedOnBase = companyInfo["basedOnBase"] as? Bool,
          let companyName = companyInfo["name"] as? String {
          
          // Create company instance.
          var company = Company(companyName: companyName, companyUrlString: baseURL)
          
          // Check whether this company exist or not, set company to correct value.
          if let exist = existingCompaniesDict[company.string()],
            company.companyUrlString == exist.companyUrlString {
            company = exist
          } else {
            // Use the new information of a company.
            var toSave = company
            if let exist = existingCompaniesDict[company.string()] {
              // If we already have existing one, we use the existing one's id.
              toSave.id = exist.id
            }
            do {
              try toSave.save()
              existingCompaniesDict[toSave.string()] = toSave
              company = toSave
            } catch {
              print("Some Error \(error)")
            }
          }
          
          let articleInfo = ArticleInfoPath(title: titlePath,<commit_msg>Fix unable to update existing company's company url string bug.<commit_after>//
//  BlogCrawlingService.swift
//  EbloVaporServer
//
//  Created by <NAME> on 2017-04-28.
//
//

import Foundation

/// The service to start a blog crawling.
class BlogCrawlingService {

  /// The crawledblogs.
  var crawledBlog: [Blog] = []
  
  /// The blogParsers.
  private var blogParsers: [BlogParser] = []
  
  /// Whether or not automatically save crawled blog when crawling finishes.
  private var automaticallySaveWhenCrawlingFinished = false
  
  /// Whether or not blog crawling has finished.
  var crawlingFinished: Bool = false {
    didSet {
      if (crawlingFinished && !oldValue) && automaticallySaveWhenCrawlingFinished {
        // Add in a test yelp blog
//        let testBlog = Blog(title: "Hoooooool lllll",
//                            urlString: "http://yelp.com/test",
//                            companyName: "Yelp")
//        testBlog.publishDate = "May, 27, 2017"
//        testBlog.publishDateInterval = Date().timeIntervalSince1970
//        self.crawledBlog.append(testBlog)
        self.saveCrawledBlog()
      }
    }
  }
  
  /// Start service
  func startService(automaticallySaveWhenCrawlingFinished: Bool) {
    crawlingFinished = false
    let companySource = try! CompanySource.all()
    guard companySource.count > 0,
      let companySourceURLString = companySource.first?.companyDataURLString else {
      return
    }
    self.automaticallySaveWhenCrawlingFinished = automaticallySaveWhenCrawlingFinished
    var url = URL(string: companySourceURLString)
    #if os(OSX)
    if EnviromentManager.local {
      let resourceName = EnviromentManager.newCompany ? "newCompany" : "company"
      let urlPath = Bundle.main.path(forResource: resourceName, ofType: "json")
      url = NSURL.fileURL(withPath: urlPath!)
    }
    #endif
    guard let data = try? Data(contentsOf: url!) else {
      print("\(self) can not load data from URL \(url!)")
      return
    }
    do {
      let json =
        try JSONSerialization.jsonObject(with: data,
                                         options: [.mutableContainers, .allowFragments])
      guard let jsonData = json as? [String: Any],
        let companies = jsonData["company"] as? [[String: Any]] else {
          return
      }
      
      /// The current existed companies.
      var existingCompaniesDict: [String : Company] = [:]
      do {
        let existingCompanies = try Company.all()
        existingCompanies.forEach { company in
          existingCompaniesDict[company.string()] = company
        }
      } catch {
        print("Can not load Company from data base.")
      }
      
      for companyInfo in companies {
        // Create blog parsers.
        if let titlePath = companyInfo["title"] as? String,
          let href = companyInfo["href"] as? String,
          let baseURL = companyInfo["baseURL"] as? String,
          let basedOnBase = companyInfo["basedOnBase"] as? Bool,
          let companyName = companyInfo["name"] as? String {
          
          // Create company instance.
          var company = Company(companyName: companyName, companyUrlString: baseURL)
          
          // Check whether this company exist or not, set company to correct value.
          if let exist = existingCompaniesDict[company.string()],
            company.companyUrlString == exist.companyUrlString {
            company = exist
          } else {
            // Use the new information of a company.
            var toSave = company
            if let exist = existingCompaniesDict[company.string()] {
              // If we already have existing one, we use the existing one's id.
              toSave.id = exist.id
              do {
                // We need to delete existing one from data base, because there is no update.
                try exist.delete()
              } catch {
                print("Can not delete existing company")
              }
            }
            do {
              try toSave.save()
              existingCompaniesDict[toSave.string()] = toSave
              company = toSave
            } catch {
              print("Some Error \(error)")
            }
          }
          
          let articleInfo = ArticleInfoPath(title: titlePath,
{'fraction_non_alphanumeric': 0.09398814563928874, 'fraction_numerical': 0.0045965888472239026, 'mean_word_length': 2.0341284403669726, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4821338', 'n_tokens_mistral': 2273, 'n_tokens_neox': 2176, 'n_words': 629}