<commit_before>#! coding: utf-8
# pylint: disable-msg=W0311, E0611, E1103, E1101
#@PydevCodeAnalysisIgnore
import re
import api
import textwrap
import calendar
import lxml.html
from hashlib import md5
from string import capwords
from pyquery import PyQuery
from urlparse import urljoin 
from simplejson import loads
from datetime import datetime
from lib import cache
from markdown2 import markdown
from lib.wordunmunger import unmungeHtml
from BeautifulSoup import BeautifulSoup, Tag, NavigableString, Comment
from flask_debugtoolbar_lineprofilerpanel.profile import line_profile

months = dict((k,v) for k,v in enumerate(calendar.month_abbr)) 


def last_starred_user(following_users, starred_list):
  user_ids = set([u for u in following_users]).intersection(set(starred_list))
  if user_ids:
    user_id = list(user_ids)[-1]
    return api.get_user_info(user_id)

@line_profile
def lines_truncate(text, lines_count=5):
  
  key = '%s:lines_truncate' % hash(text)
  out = cache.get(key, namespace="filters")
  if out:
    return out
  
  raw = text
  text = _normalize_newlines(text)
  is_html = True if ('<br>' in raw or '</' in raw) else False
  
  # remove blank lines
  lines = [line for line in text.split('\n') if line.strip()]
#  text = '\n'.join(lines)
  
  images = re.compile('<img.*?>', re.IGNORECASE).findall(text)
  for i in images:
    text = text.replace(i, md5(i).hexdigest())
  
  links = re.compile('<a.*?</a>', re.IGNORECASE).findall(text)
  for i in links:
    text = text.replace(i, md5(i).hexdigest())
  
  text = text.replace('<br/>', '<br>')
  text = text.replace('<br>', '8b0f0ea73162b7552dda3c149b6c045d') # md5('<br>').hexdigest() = '8b0f0ea73162b7552dda3c149b6c045d'
  text = text.strip().replace('\n', '<br>')
  
  words_per_line = 15
  longest_line = max(lines[:lines_count], key=len) if len(lines) != 0 else None
  if longest_line and len(longest_line.split()) > words_per_line: 
    lines = textwrap.wrap(text)
  else:
    lines = [line for line in text.split('<br>') if line.strip()]
    
  # skip blank lines (and blank lines quote)
  if len([line for line in lines if line.strip() and line.strip() != '>']) >= lines_count:
    blank_lines = len([line for line in text.split('<br>') if line.strip() in ['', '>']])
    out = ' '.join(lines[:lines_count+blank_lines])
  else:
    out = text
    
  is_truncated = False
  if len(out) < len(text):
    if is_html:
      text = ' '.join(text[:len(out)].split(' ')[0:-1]).rstrip('.')
    else:
      lines = text[:len(out)].split('<br>')
      if len(lines) > 1:
        text = '<br>'.join(lines[0:-1]).rstrip('.')
        
    is_truncated = True
    
    if len(text) / float(len(raw)) > 0.7: # nếu còn 1 ít text thì hiện luôn, không cắt làm gì cho mệt
      text = raw
      is_truncated = False
  
  out = text.replace('<br>', '\n')
  out = out.replace('8b0f0ea73162b7552dda3c149b6c045d', '<br>')
  for i in images:
    out = out.replace(md5(i).hexdigest(), i)
  for i in links:
    out = out.replace(md5(i).hexdigest(), i)
    
  if is_truncated and not out.rstrip().endswith('...</a>'):
    out = out + '...'
    
  cache.set(key, out, namespace="filters")
  return out  


@line_profile
def nl2br(value):
  # Giữ các ký tự <br> hoặc \n là text
  value = value.replace('<br>', '8b0f0ea73162b7552dda3c149b6c045d')
  value = value.replace('"\n"', '2ab3d62a4a147cea43e1e37ba39fa124')
  value = value.replace("'\n'", 'e757d0808bdc16a9047ee997fb8e4d80')
  
  lines = value.split('\n')
  <commit_msg>Bugfix: lines_truncate not working properly<commit_after>#! coding: utf-8
# pylint: disable-msg=W0311, E0611, E1103, E1101
#@PydevCodeAnalysisIgnore
import re
import api
import textwrap
import calendar
import lxml.html
from hashlib import md5
from string import capwords
from pyquery import PyQuery
from urlparse import urljoin 
from simplejson import loads
from datetime import datetime
from lib import cache
from markdown2 import markdown
from lib.wordunmunger import unmungeHtml
from BeautifulSoup import BeautifulSoup, Tag, NavigableString, Comment
from flask_debugtoolbar_lineprofilerpanel.profile import line_profile

months = dict((k,v) for k,v in enumerate(calendar.month_abbr)) 


def last_starred_user(following_users, starred_list):
  user_ids = set([u for u in following_users]).intersection(set(starred_list))
  if user_ids:
    user_id = list(user_ids)[-1]
    return api.get_user_info(user_id)

@line_profile
def lines_truncate(text, lines_count=5):
  
#   key = '%s:lines_truncate' % hash(text)
#   out = cache.get(key, namespace="filters")
#   if out:
#     return out
  
  raw = text
  text = _normalize_newlines(text)
  is_html = True if ('<br>' in raw or '</' in raw) else False
  
  # remove blank lines
  if is_html:
    
    images = re.compile('<img.*?>', re.IGNORECASE).findall(text)
    for i in images:
      text = text.replace(i, md5(i).hexdigest())
    
    links = re.compile('<a.*?</a>', re.IGNORECASE).findall(text)
    for i in links:
      text = text.replace(i, md5(i).hexdigest())
      
    text = text.replace('<br/>', '<br>')
    
    lines = [line for line in text.split('<br>') if line.strip()]
    
  else:
    lines = [line for line in text.split('\n') if line.strip()]
    text = text.strip().replace('\n', '<br>')
  
  words_per_line = 15
  longest_line = max(lines[:lines_count], key=len) if len(lines) != 0 else None
  if longest_line and len(longest_line.split()) > words_per_line: 
    lines = textwrap.wrap(text)
  else:
    lines = [line for line in text.split('<br>') if line.strip()]
    
  # skip blank lines (and blank lines quote)
  if len([line for line in lines if line.strip() and line.strip() != '>']) >= lines_count:
    blank_lines = len([line for line in text.split('<br>') if line.strip() in ['', '>']])
    out = '<br>'.join(lines[:lines_count+blank_lines])
  else:
    out = text
    
  is_truncated = False
  if len(out) < len(text):
    lines = text[:len(out)].split('<br>')
    if len(lines) > 1:
      text = '<br>'.join(lines[0:-1]).rstrip('.')
        
    is_truncated = True
    
    if len(text) / float(len(raw)) > 0.7: # nếu còn 1 ít text thì hiện luôn, không cắt làm gì cho mệt
      text = raw
      is_truncated = False
  
  if not is_html:
    out = text.replace('<br>', '\n')
  
  if is_html:
    for i in images:
      out = out.replace(md5(i).hexdigest(), i)
    for i in links:
      out = out.replace(md5(i).hexdigest(), i)
    
    if is_truncated and not out.rstrip().endswith('...</a>'):
      out = out + '...'
    
#   cache.set(key, out, namespace="filters")
  return out  


@line_profile
def nl2br(value):
  # Giữ các ký tự <br> hoặc \n là text
  value = value.replace('<br>', '8b0f0ea73162b7552dda3c149b6c045d')
  value = value.replace('"\n"', '2ab3d62a4a147cea43e1e37ba39fa124')
  value = value.replace("'\n'", 'e757d0808bdc16a9047ee997fb8e4d80')
  
  lines = value.split('\n')
  
{'fraction_non_alphanumeric': 0.14727325750947798, 'fraction_numerical': 0.0363079615048119, 'mean_word_length': 3.6596467391304346, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3985458', 'n_tokens_mistral': 2681, 'n_tokens_neox': 2550, 'n_words': 634}