<jupyter_start><jupyter_text>Identify Job Ads Post Structure last update: 2020-03-20 Assumption:- Sentences which belong the same section are in sequence.- Each section only occur exactly only one time. Advantages:1. Ordering of different sections does not matter.2. May apply to different job ads websites. Limitataion:1. Some sections may have overlapping issues. E.g. Responsibilities and Requirements.2. Chinese Characters have not yet been dealt with. What's next:1. Expand the tagging to sentence-wise or paragraph-wise2. Tokenizr Chinese Characters to support job ads in Chinese  Training Stage: Load training data<jupyter_code>df = get_df('./jobsdb_dump_first100.json')
idx = 2
text = df["jobDetail"][idx]["jobDescription"]
pt = PrepText(text)
ajson = pt.get_variables()
jds = [PrepText(x["jobDescription"]).get_variables()["lower_text"] for x in df["jobDetail"]]<jupyter_output>[Warn]<set_lower_text>: Contains non latin characters.
[Warn]<set_lower_text>: Contains non latin characters.
[Warn]<set_lower_text>: Contains non latin characters.
[Warn]<set_lower_text>: Contains non latin characters.
[Warn]<set_lower_text>: Contains non latin characters.<jupyter_text>Training Stage: NGram of all texts<jupyter_code># Ngram
sentence = ' '.join(jds)
sentence = sentence.replace("   ", " ").replace("  ", " ").replace("  ", " ").replace("  ", " ").strip("\n\r\t ")
sentence = sentence.replace("...", ".").replace(". . ", ". ").replace(". . ", ". ").replace(". . ", ". ")
n = 2
sixgrams = ngrams(sentence.split(), n)
gram_arr = list(set(sixgrams))

print(f'Token counts: {len(list(set(gram_arr)))}')

result = []
for idx in range(len(gram_arr)):
    word = ' '.join(gram_arr[idx])
    aobj = (word, sentence.count(word))
    
    result.append(aobj)

all_2grams = sorted(result, key=lambda x: x[1], reverse=True)
all_2grams<jupyter_output>Token counts: 11873<jupyter_text>Training Stage: split into Sentence Array, doing one item<jupyter_code>def lookfor(line, arr):
    for item in arr:
         if line.lower().find(item) != -1:
                return True
    return False
idx = 2
text = df["jobDetail"][idx]["jobDescription"]
pt = PrepText(text)
ajson = pt.get_variables()
ajson["sentence_arr"]
s_arr = ajson["sentence_arr"]
collection = []
# -A requirement
# -B responsibility
# -C benefit
# -D apply resume
# -E company description
# -F others
curr_section = ""
for line in s_arr:
    if curr_section != "":
        matched = [curr_section]
    else:
        matched = []

    if lookfor(line, ["equirement", "or above", "or equivalent"]):
        curr_section = "A"
        matched.append(curr_section)

    if lookfor(line, ["description", "work on", "the role", "responsib"]):
        curr_section = "B"
        matched.append(curr_section)

    if lookfor(line, ["attractive", "competitive", "day work", "mpf", "medical", "remuneration"]):
        curr_section = "C"
        matched.append(curr_section)

    if lookfor(line, ["interested party", "resume", "apply", "more detail"]):
        curr_section = "D"
        matched.append(curr_section)

    if lookfor(line, ["our client"]):
        curr_section = "E"
        matched.append(curr_section)

    if lookfor(line, ["personal data", "personal information"]):
        curr_section = "F"
        matched.append(curr_section)

    matched = list(set(matched))

    collection.append({
        "cate": matched,
        "line": line
    })

collection<jupyter_output><empty_output><jupyter_text>Training Stage: split into Sentence Array, doing ALL training items<jupyter_code>gcoll = []
for idx in range(len(df["jobDetail"]))[:]:
    text = df["jobDetail"][idx]["jobDescription"]
    pt = PrepText(text)
    ajson = pt.get_variables()
    ajson["sentence_arr"]
    
    s_arr = ajson["sentence_arr"]
    collection = []
    # -A requirement
    # -B responsibility
    # -C benefit
    # -D apply resume
    # -E company description
    # -F others
    curr_section = ""
    for line in s_arr:
        if curr_section != "":
            matched = [curr_section]
        else:
            matched = []

        if lookfor(line, ["equirement", "or above", "or equivalent"]):
            curr_section = "A"
            matched.append(curr_section)

        if lookfor(line, ["description", "work on", "the role", "responsib"]):
            curr_section = "B"
            matched.append(curr_section)

        if lookfor(line, ["attractive", "competitive", "day work", "mpf", "medical", "remuneration"]):
            curr_section = "C"
            matched.append(curr_section)

        if lookfor(line, ["interested party", "resume", "apply", "more detail"]):
            curr_section = "D"
            matched.append(curr_section)

        if lookfor(line, ["our client"]):
            curr_section = "E"
            matched.append(curr_section)

        if lookfor(line, ["personal data", "personal information"]):
            curr_section = "F"
            matched.append(curr_section)

        matched = list(set(matched))

        collection.append({
            "cate": matched,
            "line": line
        })

    # collection
    gcoll.append({
    "#A requirement": ' '.join([x["line"] for x in collection if "A" in x["cate"] and len(x["cate"]) == 1]),
    "#B responsibility": ' '.join([x["line"] for x in collection if "B" in x["cate"] and len(x["cate"]) == 1]),
    "#C benefit": ' '.join([x["line"] for x in collection if "C" in x["cate"] and len(x["cate"]) == 1]),
    "#D apply resume": ' '.join([x["line"] for x in collection if "D" in x["cate"] and len(x["cate"]) == 1]),
    "#E company description": ' '.join([x["line"] for x in collection if "E" in x["cate"] and len(x["cate"]) == 1]),
    "#F others": ' '.join([x["line"] for x in collection if "F" in x["cate"] and len(x["cate"]) == 1]),
    "#U unclass": ' '.join([x["line"] for x in collection if len(x["cate"]) == 0]),
    })
df_all = pd.DataFrame(gcoll)
df_all<jupyter_output>[Warn]<set_lower_text>: Contains non latin characters.
[Warn]<set_lower_text>: Contains non latin characters.
[Warn]<set_lower_text>: Contains non latin characters.
[Warn]<set_lower_text>: Contains non latin characters.
[Warn]<set_lower_text>: Contains non latin characters.<jupyter_text>Training Stage: Build Sets of Tokens<jupyter_code>def build_sets(df, key):
    sentence = ' '.join(df[key]).lower()
    sixgrams = ngrams(sentence.split(), n)
    # for grams in sixgrams:
    #     print(grams)
    gram_arr = list(set(sixgrams))
#     result = []
#     for idx in range(len(gram_arr)):
#         word = ' '.join(gram_arr[idx])
#         aobj = (word, sentence.count(word))

#         result.append(aobj)
#     sorted(result, key=lambda x: x[1], reverse=True)
    return gram_arr
mapping = {
"A": "#A requirement",
"B": "#B responsibility",
"C": "#C benefit",
"D": "#D apply resume",
"E": "#E company description",
"F": "#F others",
"U": "#U unclass",
}
sets = {}
sets["A"] = build_sets(df_all, mapping["A"])
sets["B"] = build_sets(df_all, mapping["B"])
sets["C"] = build_sets(df_all, mapping["C"])
sets["D"] = build_sets(df_all, mapping["D"])
sets["E"] = build_sets(df_all, mapping["E"])
def wanted(sets, wanted_set):
    mapping = {
    "A": "#A requirement",
    "B": "#B responsibility",
    "C": "#C benefit",
    "D": "#D apply resume",
    "E": "#E company description",
    "F": "#F others",
    "U": "#U unclass",
    }
    all_sets = set(sets["A"]).union(set(sets["B"]), set(sets["C"]), set(sets["D"]), set(sets["E"]))
    all_common = set(sets["A"]).intersection(set(sets["B"]), set(sets["C"]), set(sets["D"]), set(sets["E"]))
    exclu_wanted = all_sets - set(sets[wanted_set])
    exclu_set = exclu_wanted.intersection(set(sets[wanted_set]))
    new_gram_arr = list(set(sets[wanted_set]) - exclu_set - all_common)

    exclu_set = set([])
    if wanted_set != "A":
        exclu_set = exclu_set.union(set(sets[wanted_set]).intersection(set(sets["A"])))
    if wanted_set != "B":
        exclu_set = exclu_set.union(set(sets[wanted_set]).intersection(set(sets["B"])))
    if wanted_set != "C":
        exclu_set = exclu_set.union(set(sets[wanted_set]).intersection(set(sets["C"])))
    if wanted_set != "D":
        exclu_set = exclu_set.union(set(sets[wanted_set]).intersection(set(sets["D"])))
    if wanted_set != "E":
        exclu_set = exclu_set.union(set(sets[wanted_set]).intersection(set(sets["E"])))
    new_gram_arr = list(set(sets[wanted_set]) - exclu_set)

    sentence = ' '.join(df_all[mapping[wanted_set]]).lower()
    result = []
    for idx in range(len(new_gram_arr)):
        word = ' '.join(new_gram_arr[idx])
        aobj = (word, sentence.count(word))

        result.append(aobj)
    return sorted(result, key=lambda x: x[1], reverse=True)

wanted(sets, "E")<jupyter_output><empty_output><jupyter_text>Training Stage: testing mutually exclusive of token in different sets<jupyter_code>token = wanted(sets, "E")[15][0]
list(filter(lambda x: x[0] == token, all_2grams))
len(all_2grams)
{
    "word": token,
    "A": tuple(token.split()) in sets["A"],
    "B": tuple(token.split()) in sets["B"],
    "C": tuple(token.split()) in sets["C"],
    "D": tuple(token.split()) in sets["D"],
    "E": tuple(token.split()) in sets["E"],
}<jupyter_output><empty_output><jupyter_text>Training Stage: Build token dictionary of each section<jupyter_code>wA = wanted(sets, "A")
wB = wanted(sets, "B")
wC = wanted(sets, "C")
wD = wanted(sets, "D")
wE = wanted(sets, "E")<jupyter_output><empty_output><jupyter_text>Testing Stage: Labelling a Job Ads with guessing section using testing Datasets<jupyter_code>df400 = get_df("./jobsdb_dump_400-500.json")

idx = 64
text = df400["jobDetail"][idx]["jobDescription"]
text = PrepText(text).get_variables()["lower_text"]
text
total_token_count = len(all_2grams)
collection = []
for i in range(len(text.split())-2):
    token = ' '.join(text.split()[i:i+2])
    # print(token)
    term_in_section = []
    probability = []
    lsA = list(filter(lambda x: x[0] == token, wA))
    lsB = list(filter(lambda x: x[0] == token, wB))
    lsC = list(filter(lambda x: x[0] == token, wC))
    lsD = list(filter(lambda x: x[0] == token, wD))
    lsE = list(filter(lambda x: x[0] == token, wE))

    if len(lsA) > 0:
        term_in_section.append({"key": "A", "prob": lsA[0][1] / total_token_count})
    if len(lsB) > 0:
        term_in_section.append({"key": "B", "prob": lsB[0][1] / total_token_count})
    if len(lsC) > 0:
        term_in_section.append({"key": "C", "prob": lsC[0][1] / total_token_count})
    if len(lsD) > 0:
        term_in_section.append({"key": "D", "prob": lsD[0][1] / total_token_count})
    if len(lsE) > 0:
        term_in_section.append({"key": "E", "prob": lsE[0][1] / total_token_count})
    collection.append({
        "token": token,
        "in": term_in_section,
    })
ajson = json.dumps(collection)
with open("tagged_section.json", mode="w") as f:
    f.write(ajson)
# collection<jupyter_output><empty_output><jupyter_text>Testing Stage: Output HTML of section tagging<jupyter_code>ajson = json.dumps(collection)

with open("tagged_section.html", mode="w", encoding="utf8") as f:
    mapping_color = {
        "A": "red",
        "B": "orange",
        "C": "blue",
        "D": "green",
        "E": "#FF00FF",
    }
    resultHTML = ""
    for legend in mapping_color.keys():
        resultHTML = resultHTML + f'<p><span style="color: {mapping_color[legend]};">{mapping[legend]}</span></p>'
    resultHTML = resultHTML + '<hr>'
        
    for o in collection:
        word = o["token"].split(" ")[0];
        color = ""
        if len(o["in"]) > 0:
            if True or o["in"][0]["prob"] > (1/total_token_count):
                color = mapping_color[o["in"][0]["key"]]
            
        print(o)

        resultHTML = resultHTML + f'<span style="color: {color};">{word}</span> '
        if word.find(".") != -1:
            resultHTML = resultHTML + '<br>'
            
    lastItem = collection[len(collection) -1 ]
    word = lastItem["token"].split(" ")[1];
    color = ""
    if len(lastItem["in"]) > 0:
        color = mapping_color[lastItem["in"][0]["key"]]
    resultHTML = resultHTML + f'<span style="color: {color};">{word}</span> '

    
    js_version = """<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>Tagged Job Ads section</title>
<script>
window.ajson = """ + ajson + """;
</script>
</head>

<body>
<h1>Tagged Job Ads section</h1>
<div id="tags">

</div>

<script>
let tagsEl = document.querySelector("#tags");
let resultHTML = "";
let mapping_color = {
    "A": "red",
    "B": "orange",
    "C": "blue",
    "D": "green",
    "E": "#FF00FF",
    "Other": ""
}

window.ajson.map((o) => {
    let word = o["token"].split(" ")[0];
    let color = ""
    if (o["in"].length > 0) {
        if (true || o["in"][0]["prob"] > (1/""" + str(total_token_count) + """)) {
            color = mapping_color[o["in"][0]["key"]];
        }
    }
    resultHTML += `<span style="color: ${color};">${word}</span> `
    if (word.indexOf(".") != -1) {
        resultHTML += `<br>`
    }
})


let lastItem = window.ajson[window.ajson.length-1]
word = lastItem["token"].split(" ")[1];
color = ""
if (lastItem["in"].length > 0) {
    color = mapping_color[lastItem["in"][0]["key"]];
}
resultHTML += `<span style="color: ${color};">${word}</span> `

tagsEl.innerHTML = resultHTML;
</script>
</body>
</html>
    """
    pre_gen_version = """
    <div id="tags">
    """ + resultHTML + """
</div>
    """
    f.write(js_version)
display(HTML(pre_gen_version))
# End<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18557692307692308, 'fraction_numerical': 0.008062130177514793, 'mean_word_length': 3.5833898305084744, 'pattern_counts': {'":': 61, '<': 79, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '373932', 'license': 'MIT', 'n_tokens_mistral': 4817, 'n_tokens_neox': 4690, 'n_words': 1069, 'path': 'notebooks/a03a.Job_Ads_Post_Structure_Identification.ipynb', 'repo_name': 'ok524/clap_fulfillment', 'size': '13520'}