<commit_before># encoding: utf-8
require 'open-uri'
require 'hpricot'

class Admin::BkToplistsController < ApplicationController
  def toplist
    site = BkHotsite.find(params[:id])
    if site.hot_book_name == 'hongxiu'
      hongxiu_toplist site.hot_book_url 
    elsif site.hot_book_name == 'huanxia'
      huanxia_toplist site.hot_book_url
    elsif site.hot_book_name == 'qidian'
      qidian_toplist site.hot_book_url
    elsif site.hot_book_name == 'xiaoxiang'
      xiaoxiang_toplist site.hot_book_url
    else
      render 'error!'
    end
  end
  
  
  def hongxiu_toplist(url)
    doc = Hpricot(open(url))
    doc.search('#lbox ul').each do |item|
      toplist = BkToplist.new
      toplist.hot_booksite_id = 1
      toplist.book_name = (item/'.tt3 a').inner_html
      toplist.book_order_num = (item/'.t1').inner_html
      toplist.save
    end
  end
  
  def huanxia_toplist(url)
    doc = Hpricot(open(url))
    doc.search('#lbox ul').each do |item|
      toplist = BkToplist.new
      toplist.hot_booksite_id = 1
      toplist.book_name = (item/'.t3 a').inner_html
      toplist.book_order_num = (item/'.t1').inner_html<commit_msg>iconv<commit_after># encoding: utf-8
require 'open-uri'
require 'hpricot'

class Admin::BkToplistsController < ApplicationController
  def toplist
    site = BkHotsite.find(params[:id])
    if site.hot_book_name == 'hongxiu'
      hongxiu_toplist site.hot_book_url 
    elsif site.hot_book_name == 'huanxia'
      huanxia_toplist site.hot_book_url
    elsif site.hot_book_name == 'qidian'
      qidian_toplist site.hot_book_url
    elsif site.hot_book_name == 'xiaoxiang'
      xiaoxiang_toplist site.hot_book_url
    else
      render 'error!'
    end
  end
  
  
  def hongxiu_toplist(url)
    iconv = Iconv.new('UTF-8//IGNORE', 'GBK')

    doc = Hpricot(open(url))
    doc.search('#lbox ul').each do |item|
      toplist = BkToplist.new
      toplist.hot_booksite_id = 1
      
      toplist.book_name = iconv.iconv (item/'.tt3 a').inner_html
      toplist.book_order_num =  (item/'.t1').inner_html
      toplist.save
    end
  end
  
  def huanxia_toplist(url)
    doc = Hpricot(open(url))
    doc.search('#lbox ul').each do |item|
      toplist = BkToplist.new
      toplist.hot_booksite_id = 1
      toplist.book_name = (item/'.t3 a').inner_html
      toplist.book_order_num = (item/'.t1').inner_html
{'fraction_non_alphanumeric': 0.11177220569485763, 'fraction_numerical': 0.006374840628984276, 'mean_word_length': 3.030821917808219, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6952402', 'n_tokens_mistral': 945, 'n_tokens_neox': 894, 'n_words': 155}