# Path: streamcorpus_pipeline/_clean_visible.py
# def cleanse(span, lower=True):
#     '''Convert a unicode string into a lowercase string with no
# punctuation and only spaces for whitespace.
# 
# Replace PennTreebank escaped brackets with ' ':
# -LRB- -RRB- -RSB- -RSB- -LCB- -RCB-
# (The acronyms stand for (Left|Right) (Round|Square|Curly) Bracket.)
# http://www.cis.upenn.edu/~treebank/tokenization.html
# 
# :param span: string
#     '''
#     assert isinstance(span, unicode), \
#         'got non-unicode string %r' % span
#     # lowercase, strip punctuation, and shrink all whitespace
#     span = penn_treebank_brackets.sub(' ', span)
#     if lower:
#         span = span.lower()
#     span = span.translate(strip_punctuation)
#     span = whitespace.sub(' ', span)
#     # trim any leading or trailing whitespace
#     return span.strip()
# 
# def make_clean_visible_from_raw(_html, tag_replacement_char=' '):
#     '''Takes an HTML-like Unicode (or UTF-8 encoded) string as input and
#     returns a Unicode string with all tags replaced by whitespace. In
#     particular, all Unicode characters inside HTML are replaced with a
#     single whitespace character.
# 
#     This *does* detect comments, style, script, link tags and replaces
#     them with whitespace.  This is subtle because these tags can be
#     self-closing or not.
# 
#     It does do anything with HTML-escaped characters.
# 
#     Pre-existing whitespace of any kind *except* newlines (\n) and
#     linefeeds (\r\n) is converted to single spaces ' ', which has the
#     same byte length (and character length).  Newlines and linefeeds
#     are left unchanged.
# 
#     This is a simple state machine iterator without regexes
# 
#     '''
#     if not isinstance(_html, unicode):
#         _html = unicode(_html, 'utf-8')
# 
#     #Strip tags with logic above
#     non_tag = ''.join(non_tag_chars_from_raw(_html))
# 
#     return non_tag.encode('utf-8')
# 
# def make_clean_visible(_html, tag_replacement_char=' '):
#     '''
#     Takes an HTML-like Unicode string as input and returns a UTF-8
#     encoded string with all tags replaced by whitespace. In particular,
#     all Unicode characters inside HTML are replaced with a single
#     whitespace character.
# 
#     This does not detect comments, style, script, link.  It also does
#     do anything with HTML-escaped characters.  All of these are
#     handled by the clean_html pre-cursor step.
# 
#     Pre-existing whitespace of any kind (newlines, tabs) is converted
#     to single spaces ' ', which has the same byte length (and
#     character length).
# 
#     This is a simple state machine iterator without regexes
#     '''
#     def non_tag_chars(html):
#         n = 0
#         while n < len(html):
#             angle = html.find('<', n)
#             if angle == -1:
#                 yield html[n:]
#                 n = len(html)
#                 break
#             yield html[n:angle]
#             n = angle
# 
#             while n < len(html):
#                 nl = html.find('\n', n)
#                 angle = html.find('>', n)
#                 if angle == -1:
#                     yield ' ' * (len(html) - n)
#                     n = len(html)
#                     break
#                 elif nl == -1 or angle < nl:
#                     yield ' ' * (angle + 1 - n)
#                     n = angle + 1
#                     break
#                 else:
#                     yield ' ' * (nl - n) + '\n'
#                     n = nl + 1
#                     # do not break
# 
#     if not isinstance(_html, unicode):
#         _html = unicode(_html, 'utf-8')
# 
#     # Protect emails by substituting with unique key
#     _html = fix_emails(_html)
# 
#     #Strip tags with previous logic
#     non_tag = ''.join(non_tag_chars(_html))
# 
#     return non_tag.encode('utf-8')


from streamcorpus_pipeline._clean_visible import cleanse, \
    make_clean_visible_from_raw, \
    make_clean_visible
example1_1 = '''				<!-- ENLACES_120x90_2015_EI -->
				<ins class="adsbygoogle"
				     style="display:inline-block;width:120px;height:90px"
				     data-ad-client="ca-pub-8118159680005837"
				     data-ad-slot="8414446812"></ins>
				<script>
				(adsbygoogle = window.adsbygoogle || []).push({});
				</script>
				<script async src="//pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
				<!-- ENLACES_120x90_2015_EI -->
				<ins class="adsbygoogle"
				     style="display:inline-block;width:120px;height:90px"
				     data-ad-client="ca-pub-8118159680005837"
				     data-ad-slot="8414446812"></ins>
				<script>
				(adsbygoogle = window.adsbygoogle || []).push({});
				</script>
			</div> 	 

			

			<p style="margin-top:10px;padding-right:10px" align="justify"><div id="HOTWordsTxt" name="HOTWordsTxt" style="text-align:justify;"><!--AUGURE_NOTICIA_INICIO--><b>'''
example1_2 = '''River supo caer para despues levantarse y ganar la Libertadores</b> / <b>El Tiempo</b> / Jonathan Maidana, '''

truth1_1 = ' ' * len(example1_1)
truth1_2   = '''River supo caer para despues levantarse y ganar la Libertadores     /    El Tiempo     / Jonathan Maidana, '''

truth1 = truth1_1 + truth1_2

def test_make_clean_visible_from_raw_example1_1():

# Path: streamcorpus_pipeline/_clean_visible.py
# def cleanse(span, lower=True):
#     '''Convert a unicode string into a lowercase string with no
# punctuation and only spaces for whitespace.
# 
# Replace PennTreebank escaped brackets with ' ':
# -LRB- -RRB- -RSB- -RSB- -LCB- -RCB-
# (The acronyms stand for (Left|Right) (Round|Square|Curly) Bracket.)
# http://www.cis.upenn.edu/~treebank/tokenization.html
# 
# :param span: string
#     '''
#     assert isinstance(span, unicode), \
#         'got non-unicode string %r' % span
#     # lowercase, strip punctuation, and shrink all whitespace
#     span = penn_treebank_brackets.sub(' ', span)
#     if lower:
#         span = span.lower()
#     span = span.translate(strip_punctuation)
#     span = whitespace.sub(' ', span)
#     # trim any leading or trailing whitespace
#     return span.strip()
# 
# def make_clean_visible_from_raw(_html, tag_replacement_char=' '):
#     '''Takes an HTML-like Unicode (or UTF-8 encoded) string as input and
#     returns a Unicode string with all tags replaced by whitespace. In
#     particular, all Unicode characters inside HTML are replaced with a
#     single whitespace character.
# 
#     This *does* detect comments, style, script, link tags and replaces
#     them with whitespace.  This is subtle because these tags can be
#     self-closing or not.
# 
#     It does do anything with HTML-escaped characters.
# 
#     Pre-existing whitespace of any kind *except* newlines (\n) and
#     linefeeds (\r\n) is converted to single spaces ' ', which has the
#     same byte length (and character length).  Newlines and linefeeds
#     are left unchanged.
# 
#     This is a simple state machine iterator without regexes
# 
#     '''
#     if not isinstance(_html, unicode):
#         _html = unicode(_html, 'utf-8')
# 
#     #Strip tags with logic above
#     non_tag = ''.join(non_tag_chars_from_raw(_html))
# 
#     return non_tag.encode('utf-8')
# 
# def make_clean_visible(_html, tag_replacement_char=' '):
#     '''
#     Takes an HTML-like Unicode string as input and returns a UTF-8
#     encoded string with all tags replaced by whitespace. In particular,
#     all Unicode characters inside HTML are replaced with a single
#     whitespace character.
# 
#     This does not detect comments, style, script, link.  It also does
#     do anything with HTML-escaped characters.  All of these are
#     handled by the clean_html pre-cursor step.
# 
#     Pre-existing whitespace of any kind (newlines, tabs) is converted
#     to single spaces ' ', which has the same byte length (and
#     character length).
# 
#     This is a simple state machine iterator without regexes
#     '''
#     def non_tag_chars(html):
#         n = 0
#         while n < len(html):
#             angle = html.find('<', n)
#             if angle == -1:
#                 yield html[n:]
#                 n = len(html)
#                 break
#             yield html[n:angle]
#             n = angle
# 
#             while n < len(html):
#                 nl = html.find('\n', n)
#                 angle = html.find('>', n)
#                 if angle == -1:
#                     yield ' ' * (len(html) - n)
#                     n = len(html)
#                     break
#                 elif nl == -1 or angle < nl:
#                     yield ' ' * (angle + 1 - n)
#                     n = angle + 1
#                     break
#                 else:
#                     yield ' ' * (nl - n) + '\n'
#                     n = nl + 1
#                     # do not break
# 
#     if not isinstance(_html, unicode):
#         _html = unicode(_html, 'utf-8')
# 
#     # Protect emails by substituting with unique key
#     _html = fix_emails(_html)
# 
#     #Strip tags with previous logic
#     non_tag = ''.join(non_tag_chars(_html))
# 
#     return non_tag.encode('utf-8')


# Path: streamcorpus_pipeline/tests/test_clean_visible.py
from streamcorpus_pipeline._clean_visible import cleanse, \
    make_clean_visible_from_raw, \
    make_clean_visible

example1_1 = '''				<!-- ENLACES_120x90_2015_EI -->
				<ins class="adsbygoogle"
				     style="display:inline-block;width:120px;height:90px"
				     data-ad-client="ca-pub-8118159680005837"
				     data-ad-slot="8414446812"></ins>
				<script>
				(adsbygoogle = window.adsbygoogle || []).push({});
				</script>
				<script async src="//pagead2.googlesyndication.com/pagead/js/adsbygoogle.js"></script>
				<!-- ENLACES_120x90_2015_EI -->
				<ins class="adsbygoogle"
				     style="display:inline-block;width:120px;height:90px"
				     data-ad-client="ca-pub-8118159680005837"
				     data-ad-slot="8414446812"></ins>
				<script>
				(adsbygoogle = window.adsbygoogle || []).push({});
				</script>
			</div> 	 

			

			<p style="margin-top:10px;padding-right:10px" align="justify"><div id="HOTWordsTxt" name="HOTWordsTxt" style="text-align:justify;"><!--AUGURE_NOTICIA_INICIO--><b>'''
example1_2 = '''River supo caer para despues levantarse y ganar la Libertadores</b> / <b>El Tiempo</b> / Jonathan Maidana, '''

truth1_1 = ' ' * len(example1_1)
truth1_2   = '''River supo caer para despues levantarse y ganar la Libertadores     /    El Tiempo     / Jonathan Maidana, '''

truth1 = truth1_1 + truth1_2

def test_make_clean_visible_from_raw_example1_1():

    u = make_clean_visible_from_raw(example1_1)