<commit_before>package ch.fhnw.jobannotations.skills;

import ch.fhnw.jobannotations.JobOffer;
import ch.fhnw.jobannotations.Main;
import ch.fhnw.jobannotations.utils.FileUtils;
import ch.fhnw.jobannotations.utils.IntStringPair;
import edu.stanford.nlp.ling.HasWord;
import edu.stanford.nlp.ling.IndexedWord;
import edu.stanford.nlp.ling.TaggedWord;
import edu.stanford.nlp.parser.nndep.DependencyParser;
import edu.stanford.nlp.process.DocumentPreprocessor;
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
import edu.stanford.nlp.trees.EnglishGrammaticalRelations;
import edu.stanford.nlp.trees.GrammaticalStructure;
import edu.stanford.nlp.trees.TypedDependency;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.StringReader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author Hoang
 */
public class JobSkillsExtractor {

    private static MaxentTagger tagger;
    private static DependencyParser parser;
    private static final List<String> DETERMINERS_YOUR = Arrays.asList("ihre", "ihr", "deine", "dein");
    private static final List<String> DETERMINERS_OUR = Arrays.asList("unsere", "unser");
    private static final List<String> SUBJECTS_YOU = Arrays.asList("sie", "du");
    private static final List<String> SUBJECTS_WE = Arrays.asList("wir", "ich");
    private static final List<String> NOUNS_SKILL_SYNONYM = Arrays.asList(
            "qualifikation",
            "profil",
            "erfahrung",
            "voraussetzung",
            "anforderung",
            "kompetenz",
            "fähigkeit",
            "kenntniss",
            "eigenschaft"
    );
    private static final List<String> NOUNS_JOB_SYNONYM = Arrays.asList(
            "aufgabe",
            "herausforderung",
            "wirkungsfeld",
            "job",
            "funktion",
            "tätigkeit",
            "arbeit"
    );
    private static final List<String> NOUNS_EXPECTATION_SYNONYM = Arrays.asList("anforderung", "erwartung");
    private static final List<String> NOUNS_OFFER_SYNONYM = Arrays.asList("angebot");
    private static final List<String> NOUNS_POSSIBILITY_SYNONYM = Arrays.asList("möglichkeit", "perspektive");
    private static final List<String> VERBS_OFFER = Arrays.asList(
            "bringen",
            "bringst",
            "bieten",
            "bietest",
            "mitbringen",
            "mitbringst"
    );
    private static final List<String> VERBS_EXPECT = Arrays.asList(
            "erwarten",
            "verlangen"
    );

    public String parseJobSkills(JobOffer jobOffer) {

        if (Main.DEBUG) {
            System.out.println("\n" + StringUtils.repeat("-", 80));
            System.out.println("[skills-indicator]\t" + "Started to parse skills from offer");
        }

        Element body = jobOffer.getBodyElement();
        Map<IntStringPair, List<String>> ratedSkills = new HashMap<>();

        Elements listElements = new Elements();

        // TODO check for sub lists
        // add ul elements
        listElements.addAll(body.getElementsByTag("ul"));

        // add ol elements
        listElements.addAll(body.getElementsByTag("ol"));

        for (Element listElement : listElements) {
            Elements listItems = listElement.getElementsByTag("li");
            String listTitle = getListTitle(jobOffer, listItems);
            int rating = 100 + calculateHtmlSkillListItemsRating(listItems);

            List<String> skills = new ArrayList<>();
            for (Element listItem : listItems) {
                skills.add(listItem.text());
            }

            IntStringPair ratedSkillListTitle = new IntStringPair(rating, listTitle);
            ratedSkills.put(ratedSkillListTitle, skills);
        }


        // Find non valid html lists
        String lastBulletPoint = null;
        String lastLine = null;
        String lastListTitle = null;
        List<String> lastListItems = new ArrayList<>();
        String[] lines = jobOffer.getPlainText().split("\\n");
        for (String line : lines) {
            line = line.trim();

            boolean addSkillList = false;

            if (line.isEmpty()) {
                if (lastListTitle != null) {
                    addSkillList = true;
                }
            } else {
                // check for potential list
                Matcher matcher = Pattern.compile("^([^a-zA-ZäöüÄÖÜ])\\s+.*$").matcher(line);
                if (matcher.find()) {
                    String bulletPoint = matcher.group(1);

                    if (lastBulletPoint == null) {
                        // first list item
                        lastBulletPoint = bulletPoint;
                        lastListTitle = lastLine;

                        line = line.substring(1, line.length()).trim();
                        lastListItems.add(line);

                    } else if (!bulletPoint.equals(lastBulletPoint)) {
                        if (lastListTitle != null) {
                            addSkillList = true;
                        }
                    } else if (bulletPoint.equals(lastBulletPoint)) {
                        // additional list item
                        line = line.substring(1, line.length()).trim();
                        lastListItems.add(line);

                    }
                } else {
                    if (lastListTitle != null) {
                        addSkillList = true;
                    }
                }
            }

            if (addSkillList) {
                IntStringPair ratedSkillListTitle = new IntStringPair(100, lastListTitle);
                ratedSkills.put(ratedSkillListTitle, lastListItems);
                lastListItems = new ArrayList<>();

                lastListTitle = null;
                lastBulletPoint = null;
            }

            // keep last non empty line
            lastLine = line;
        }


        // calculate rating for skill lists
        List<IntStringPair> ratedListTitles = new ArrayList<>();
        for (IntStringPair ratedListTitle : ratedSkills.keySet()) {
            ratedListTitles.add(ratedListTitle);
            List<String> listItems = ratedSkills.get(ratedListTitle);
            calculateSkillListProbability(jobOffer, ratedListTitle, listItems);
        }

        ratedListTitles.sort((o1, o2) -> o2.getInt() - o1.getInt());

        // TODO remove skill lists with bad ratings

        if (ratedSkills.isEmpty()) {
            return null;
        }

        boolean isFirstLine = true;<commit_msg>HT: fixed skills extractor (remove skill lists with bad rating)<commit_after>package ch.fhnw.jobannotations.skills;

import ch.fhnw.jobannotations.JobOffer;
import ch.fhnw.jobannotations.Main;
import ch.fhnw.jobannotations.utils.FileUtils;
import ch.fhnw.jobannotations.utils.IntStringPair;
import edu.stanford.nlp.ling.HasWord;
import edu.stanford.nlp.ling.IndexedWord;
import edu.stanford.nlp.ling.TaggedWord;
import edu.stanford.nlp.parser.nndep.DependencyParser;
import edu.stanford.nlp.process.DocumentPreprocessor;
import edu.stanford.nlp.tagger.maxent.MaxentTagger;
import edu.stanford.nlp.trees.EnglishGrammaticalRelations;
import edu.stanford.nlp.trees.GrammaticalStructure;
import edu.stanford.nlp.trees.TypedDependency;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.StringReader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author Hoang
 */
public class JobSkillsExtractor {

    private static MaxentTagger tagger;
    private static DependencyParser parser;
    private static final List<String> DETERMINERS_YOUR = Arrays.asList("ihre", "ihr", "deine", "dein");
    private static final List<String> DETERMINERS_OUR = Arrays.asList("unsere", "unser");
    private static final List<String> SUBJECTS_YOU = Arrays.asList("sie", "du");
    private static final List<String> SUBJECTS_WE = Arrays.asList("wir", "ich");
    private static final List<String> NOUNS_SKILL_SYNONYM = Arrays.asList(
            "qualifikation",
            "profil",
            "erfahrung",
            "voraussetzung",
            "anforderung",
            "kompetenz",
            "fähigkeit",
            "kenntniss",
            "eigenschaft"
    );
    private static final List<String> NOUNS_JOB_SYNONYM = Arrays.asList(
            "aufgabe",
            "herausforderung",
            "wirkungsfeld",
            "job",
            "funktion",
            "tätigkeit",
            "arbeit"
    );
    private static final List<String> NOUNS_EXPECTATION_SYNONYM = Arrays.asList("anforderung", "erwartung");
    private static final List<String> NOUNS_OFFER_SYNONYM = Arrays.asList("angebot");
    private static final List<String> NOUNS_POSSIBILITY_SYNONYM = Arrays.asList("möglichkeit", "perspektive");
    private static final List<String> VERBS_OFFER = Arrays.asList(
            "bringen",
            "bringst",
            "bieten",
            "bietest",
            "mitbringen",
            "mitbringst"
    );
    private static final List<String> VERBS_EXPECT = Arrays.asList(
            "erwarten",
            "verlangen"
    );

    public String parseJobSkills(JobOffer jobOffer) {

        if (Main.DEBUG) {
            System.out.println("\n" + StringUtils.repeat("-", 80));
            System.out.println("[skills-indicator]\t" + "Started to parse skills from offer");
        }

        Element body = jobOffer.getBodyElement();
        Map<IntStringPair, List<String>> ratedSkills = new HashMap<>();

        Elements listElements = new Elements();

        // TODO check for sub lists
        // add ul elements
        listElements.addAll(body.getElementsByTag("ul"));

        // add ol elements
        listElements.addAll(body.getElementsByTag("ol"));

        for (Element listElement : listElements) {
            Elements listItems = listElement.getElementsByTag("li");
            String listTitle = getListTitle(jobOffer, listItems);
            int rating = 100 + calculateHtmlSkillListItemsRating(listItems);

            List<String> skills = new ArrayList<>();
            for (Element listItem : listItems) {
                skills.add(listItem.text());
            }

            IntStringPair ratedSkillListTitle = new IntStringPair(rating, listTitle);
            ratedSkills.put(ratedSkillListTitle, skills);
        }


        // Find non valid html lists
        String lastBulletPoint = null;
        String lastLine = null;
        String lastListTitle = null;
        List<String> lastListItems = new ArrayList<>();
        String[] lines = jobOffer.getPlainText().split("\\n");
        for (String line : lines) {
            line = line.trim();

            boolean addSkillList = false;

            if (line.isEmpty()) {
                if (lastListTitle != null) {
                    addSkillList = true;
                }
            } else {
                // check for potential list
                Matcher matcher = Pattern.compile("^([^a-zA-ZäöüÄÖÜ])\\s+.*$").matcher(line);
                if (matcher.find()) {
                    String bulletPoint = matcher.group(1);

                    if (lastBulletPoint == null) {
                        // first list item
                        lastBulletPoint = bulletPoint;
                        lastListTitle = lastLine;

                        line = line.substring(1, line.length()).trim();
                        lastListItems.add(line);

                    } else if (!bulletPoint.equals(lastBulletPoint)) {
                        if (lastListTitle != null) {
                            addSkillList = true;
                        }
                    } else if (bulletPoint.equals(lastBulletPoint)) {
                        // additional list item
                        line = line.substring(1, line.length()).trim();
                        lastListItems.add(line);

                    }
                } else {
                    if (lastListTitle != null) {
                        addSkillList = true;
                    }
                }
            }

            if (addSkillList) {
                IntStringPair ratedSkillListTitle = new IntStringPair(100, lastListTitle);
                ratedSkills.put(ratedSkillListTitle, lastListItems);
                lastListItems = new ArrayList<>();

                lastListTitle = null;
                lastBulletPoint = null;
            }

            // keep last non empty line
            lastLine = line;
        }


        // calculate rating for skill lists
        List<IntStringPair> ratedListTitles = new ArrayList<>();
        for (IntStringPair ratedListTitle : ratedSkills.keySet()) {
            ratedListTitles.add(ratedListTitle);
            List<String> listItems = ratedSkills.get(ratedListTitle);
            calculateSkillListProbability(jobOffer, ratedListTitle, listItems);
        }

        ratedListTitles.sort((o1, o2) -> o2.getInt() - o1.getInt());

        // only keep titles with high ratings
        ratedListTitles.removeIf(t -> t.getInt() < 100);

        if (ratedSkills.isEmpty()) {
            return null;
        }

        boolean isFirstLine = true;
{'fraction_non_alphanumeric': 0.10518006164950004, 'fraction_numerical': 0.002631381099165476, 'mean_word_length': 2.0006767426122263, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3633806', 'n_tokens_mistral': 3860, 'n_tokens_neox': 3695, 'n_words': 758}