<jupyter_start><jupyter_text><jupyter_code># chapter 2
# 문자 하나 찾기

import re
text = """
sales1.xls
order3.xls
sales2.xls
na1.xls
na2.xls
sa1.xls
ja4.doc
"""

re_pat = re.compile(r'sales.')

print(re_pat.findall(text))

re_pat1 = re.compile(r'.a.\.')
print(re_pat1.findall(text))


re_pat2 = re.compile(r'.a.\.xls')
print(re_pat2.findall(text))
# chapter 3
# 미국/캐나다 우편번호 데이터베이스
import re

post_number = """
11213
A1C2E3
48075
48345
M1B3F2
90046
H1H2H3
"""

re_pat_us = re.compile("\w\d\w\d\w\d")
print(re_pat_us.findall(post_number))

re_pat_ca = re.compile("\d{5}")
print(re_pat_ca.findall(post_number))
import re
text = """
Send personal email to <EMAIL> or 
<EMAIL>. For questions about a book use
<EMAIL>. If your message is
urgent try <EMAIL>.
Feel free to send unsolicited email to 
<EMAIL> (simple)
"""

re_pat = re.compile(r'[\w.]+@[\w.]+\.\w+')
print(re_pat.findall(text))
re_pat2 = re.compile(r'[\w\.]+@[\w\.]+\.\w+')
print(re_pat2.findall(text))
# get url
import re
text = """
The URL is http://www.forta.com, to connect
securely us https://www.forta.com/ instead
"""

re_pat = re.compile(r"https?:\/\/[\w.\/]+")
print(re_pat.findall(text))
# 5장 반복찾기
# date
text = """
4/8/17
10-6-2018
2/2/2
01-01-01
"""

re_date = re.compile(r"\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4}")
print(re_date.findall(text))
# find minimum period
text = """
1001: $496.80
1002: $1290.69
1003: $26.43
1004: $613.42
1005: $7.61
1006: $414.90
1007: $25.00
"""

re_pat = re.compile(r"\d+: \$\d{3,}\.\d{2}") # find cases over 100 dollars
print(re_pat.findall(text))
# to prevent too much fit
text = """
This offer is not available to customers
living in <b>AK</b> and <B>HI</B>.
"""

re_pat = re.compile(r'<[Bb]>.*<\/[Bb]>')
print(re_pat.findall(text))   ## ['<b>AK</b> and <B>'] Greedy search

## Need to make lazy quantifier, not greed
## * -> *?
## + -> +?
## {n,} -> {n,}?

re_pat_lazy = re.compile(r'<[Bb]>.*?<\/[Bb]>')
print(re_pat_lazy.findall(text))  ## ['<b>AK</b>', '<B>HI</B>'] Lazy search 


# Chapter 6
# Position matching
# Designate boundary

text = "The cat scattered his food all over the room."
re_cat = re.compile(r'cat')
print(re_cat.findall(text))

# word boundary \b
re_cat_b = re.compile(r'\bcat\b')
print(re_cat_b.findall(text))

# 단어 경계를 일치시키고 싶지 않을 때는 \B를 사용
text = """
Please enter the nine-digit id as it
appears on your color - coded pass-key.
"""

re_pat_B = re.compile(r"(\B-\B)")
re_pat_b = re.compile(r"(\b-\b)")
print(re_pat_B.findall(text))
print(re_pat_B.search(text))
print(re_pat_b.findall(text))
print(re_pat_b.search(text))



# Chapter 7
# 하위표현식
text = "Regular&nbsp;&nbsp;Expressions, and other subjects."
re_pat1 = re.compile(r'&nbsp;{2,}')
re_pat2 = re.compile(r'(&nbsp;){2,}')

print(re_pat1.findall(text)) #[]
print(re_pat1.search(text)) # None
print(re_pat2.findall(text)) #['&nbsp;']
print(re_pat2.search(text).group()) # &nbsp;&nbsp;
# Find IP
text = "Pinging hog.forta.com [192.168.3.11]"
re_ip = re.compile(r"(\d{1,3}\.){3}\d{1,3}")

print(re_ip.search(text).group()) # 192.168.3.11


# find year
import re

text = "DOB: 1967-08-17"
re_year = re.compile(r'19|20\d{2}')  # which means 19 or 20\d{2}
re_year_paren = re.compile(r'(19|20)\d{2}') # which means 19 or 20

print(re_year.search(text).group())  # 19
print(re_year_paren.search(text).group()) # 1967


# Nested parenthesis
# Find IP
# Criteria
# 1. every 1-digit or 2-digits number
# 2. every 3-digit numbers which start with 1
# 3. 3-digit number which start with 2 and the second digit b/w 0 and 4
# 4. 25+[0~5]

text = """
Pinging hog.forta.com [192.168.3.11]
Pinging hog.forta.com [260.567.190.892]
"""
re_ip = re.compile(r"(\d{1,3}\.){3}\d{1,3}")

print(re_ip.finditer(text)) # <callable_iterator object at 0x7fa0c985bb10>
print([x.group() for x in re_ip.finditer(text)]) 
#['192.168.3.11', '260.567.190.892']
# invalid ip is also searched

re_ip_nested = re.compile(r"(((25[0-5])|(2[0-4]\d)|(1\d{2})|(\d{1,2}))\.){3}((25[0-5])|(2[0-4]\d)|(1\d{2})|(\d{1,2}))")
print([x.group() for x in re_ip_nested.finditer(text)])




# Chapter 8
# Back Reference

# check repeated word
text = """
this is a block of of text,
serveral words here are are
repeated, and and they
should not be
"""

re_pat = re.compile(r"\s+(\w+)\s+\1")
print([x.group() for x in re_pat.finditer(text)]) #[' of of', ' are are', ' and and']

# find header tag
# exclude unmatched pair such as <h1> </h2>
text = """
<body>
<h1>Welcome to my homepage</h1>
blah blah blah <br/>
<h2>SQL</h2>
blah blah
<h2>RegEx</h2>
Infromation about Regex
<h2> This is not vaild HTML</h3>
</body>
"""

re_pat = re.compile(r"<([hH][1-6])>.*?<\/\1>")
print([x.group() for x in re_pat.finditer(text)]) 
# ['<h1>Welcome to my homepage</h1>', '<h2>SQL</h2>', '<h2>RegEx</h2>']
# <h2> This is not vaild HTML</h3> is excluded
# substitution using back reference
# e-mail -> a href
# ex. <EMAIL> -> <a href="mailto:<EMAIL>"><EMAIL></a>

import re
text = "Hello, <EMAIL> is my email address"

re_email = re.compile(r"(?P<mail>\w+[\w\.]*@[\w\.]+\.\w+)")
print(re_email.sub(r'<a href="mailto:\1">\1</a>',text))
# Hello, <a href="<EMAIL>"><EMAIL></a> is my email address
# substitution using back reference
# phone number
# 313-555-1234 --> (313) 555-1234
import re

text = """
313-555-1234
248-846-8753
820-784-9687
"""

re_phone = re.compile(r"(?P<n1>\d{3})-(?P<n2>\d{3})-(?P<n3>\d{4})")
print(re_phone.sub(r"(\1) \2-\3", text))
#(313) 555-1234
#(248) 846-8753
#(820) 784-9687
print(re_phone.sub(r"(\g<n1>) \g<n2>-\g<n3>", text))
#(313) 555-1234
#(248) 846-8753
#(820) 784-9687

# Chapter 9
# Lookaround
# Look ahead ?=

# task: extract protocol from url
# ex. http://www.forta.com -> http
# ex. http -> no extraction

import re

text = """
http://www.forta.com
http
https
https://mail.forta.com/
ftp://ftp.forta.com/
"""
re_protocol = re.compile(r".+(?=:)")

print([x.group() for x in re_protocol.finditer(text)]) # ['http', 'https', 'ftp']
# Look behind ?<=

text = """
ABC01: $23.45
HGG42: $5.31
CFMX1: $899.00
XTC99: $69.96
Total items found: 4
"""

# Extract dollars without dollar mark
re_dollar = re.compile(r"(?<=\$)\d+\.\d+")
re_dollar.findall(text) # ['23.45', '5.31', '899.00', '69.96']
# use look ahead and look behind together
import re

text = """
<head>
<title><NAME>'s Homepage</title>
</head>
"""

re_title = re.compile(r"(?<=\<(title|TITLE)>).+(?=\</(title|TITLE)\>)")
print([x.group() for x in re_title.finditer(text)])
# Negative look ahead & look behind
# Extract price only
import re

text = """
I paid $30 for 100 apples,
50 oranges, and 60 pears.
I saved $5 on this order.
"""

re_price = re.compile(r"(?<=\$)\d+")
print(re_price.findall(text))   # ['30', '5']
print([x.group() for x in re_price.finditer(text)]) # ['30', '5']

# Task 2: Extract quantity
re_quant1 = re.compile(r"(?<!\$)\d+")
print([x.group() for x in re_quant1.finditer(text)]) # ['0', '100', '50', '60']

re_quant2 = re.compile(r"\b(?<!\$)\d+\b")
print([x.group() for x in re_quant2.finditer(text)]) # ['100', '50', '60']



# Chapter 10
# make condition

text = """
123-456-7890
(123)456-7890
(123)-456-7890
(123-456-7890 
1234567890
123 456 7890
"""
# (123-456-7890, (123)-456-7890, 1234567890 is not correct format
# How to extract phone numbers w/o invalid format

re_phone = re.compile(r"\(?\d{3}\)?-?\d{3}-\d{4}")
print([x.group() for x in re_phone.finditer(text)]) 
# ['123-456-7890', '(123)456-7890', '(123)-456-7890', '(123-456-7890']
# ????
# Chapter 11
# General problems using Regex

# North America Telephone Number
text = """
<NAME>: 248-555-1234
<NAME>: (313) 555-1234
<NAME>: (810)555-1234
<NAME>: 734.555.9999
"""

re_ne_number = re.compile(r"(?<=: )[\(\.]?[2-9]\d{2}[\)\.]?[\s\-\.]?[2-9]\d{2}[\-\.]\d{4}")
print([x.group() for x in re_ne_number.finditer(text)])
# USA Post number
# 2 types
# type1. {5 digits}
# ex. 11222
# type2. {5 digits}-{4 digits}
# ex. 48034-1234

text = """
999 1st Avenue, Bigtown, NY, 11222
123 High Street, Any City, MI, 48034-1234
"""

re_post = re.compile(r"\d{5}(-\d{4})?")
print([x.group() for x in re_post.finditer(text)])
# ['11222', '48034-1234']

# UK Post Number
# {outcode} {incode}
# outcode: (1/2 characters) + (1/2 digits)
# incode: 1 digit + 2 characters (No C,I,K,M,O,V)

text = """
171 Kyverdale Road, London N16 6PS
33 Main Street, Portsmouth, P01 3AX
18 High Street, London NW11 8AB
18 High Street, London NW1P 8AB
"""

re_post_uk = re.compile(r"(?P<out>[A-Z]{1,2}\d[A-Z\d]?) (?P<in>\d{1}[ABD-HJLNP-UW-Z]{2})")
print([x.group('out') for x in re_post_uk.finditer(text)])
print([x.group('in') for x in re_post_uk.finditer(text)])
print([x.group() for x in re_post_uk.finditer(text)])



# URL
text = """
http://www.forta.com/blog
https://www.forta.com:80/blog/index.cfm
http://www.forta.com
http://ben:password@www.forta.com/
http://localhost/index.php?ab=1&c=2
http://localhost:8500/
"""

re_url = re.compile(r"https?:\/\/[\w\.]+(\:\d+)?(\/([\w+\/_.]*)?)?")

print([x.group() for x in re_url.finditer(text)])
# ['http://www.forta.com/blog', 'https://www.forta.com:80/blog/index.cfm', 'http://www.forta.com', 'http://ben', 'http://localhost/index.php', 'http://localhost:8500/']

# Register Number 주민등록번호 

text = """
790814-1234567
135-600
799999-1234567
791231-1234567
080601-3456789
830105-5678123
100406-4567890
"""

#re_reg = re.compile(r"\d{6}-\d{7}")
re_reg = re.compile(r"\d{2}(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])-\d{7}")

print([x.group() for x in re_reg.finditer(text)])
# ['790814-1234567', '791231-1234567', '080601-3456789', '830105-5678123', '100406-4567890']

# date change
# mm/dd/yyyy -> yyyy-mm-dd

text = "08/14/1979"

re_date = re.compile(r"(?P<month>\d{2})\/(?P<date>\d{2})\/(?P<year>\d{4})")
print(re_date.sub(r"\g<year>-\g<month>-\g<date>", text))<jupyter_output>1979-08-14
{'fraction_non_alphanumeric': 0.236151302190988, 'fraction_numerical': 0.10272840016535759, 'mean_word_length': 4.3024657534246575, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 95, 'https://': 4, 'lorem ipsum': 0, 'www.': 11, 'xml': 0}, 'pii_count': 5, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '474576', 'license': 'MIT', 'n_tokens_mistral': 4940, 'n_tokens_neox': 4377, 'n_words': 892, 'path': 'Learning_Regular_Expressions.ipynb', 'repo_name': 'HyuksuRyu/regex_tutorial', 'size': '9771'}