<jupyter_start><jupyter_text>Import raw data from crawler<jupyter_code>file_name = 'sample_full.json'
df_sample = pd.read_json(join('..', 'data', 'crawler', file_name))<jupyter_output><empty_output><jupyter_text>NER extraction  Objects with structured description<jupyter_code>regexps = {
    'name': compile('име обект: (?P<name>.*?) вид обект'),
    'category': compile('вид обект: (?P<category>.*?) град'),
    'city': compile('град: (?P<city>.*?) адрес'),
    'address': compile('адрес: (?P<address>.*?) описание')
}

def match(prop, text):
    m = regexps[prop].search(text)
    if m is not None:
        return m.group(prop).strip()
    else:
        return None
df_sample['matched_name'] = df_sample['description'].apply(lambda x: match('name', x))
df_sample['matched_category'] = df_sample['description'].apply(lambda x: match('category', x))
df_sample['matched_city'] = df_sample['description'].apply(lambda x: match('city', x))
df_sample['matched_address'] = df_sample['description'].apply(lambda x: match('address', x))
df_sample['matched_name'].count()
df_sample['matched_city'].count()
df_sample['matched_address'].count()
df_sample['matched_category'].count()<jupyter_output><empty_output><jupyter_text>Categories based on matched_category<jupyter_code>categories = list(filter(lambda x: x is not None, df_sample['matched_category'].unique()))<jupyter_output><empty_output><jupyter_text>Parsing to get set of unique categories<jupyter_code>def parse_category(category):
    splitted = re.split(r'[-\\\/,\.:]', category)
    return [w.strip().lower() for w in splitted]

def flatten(l):
    return sum(l, [])

all_cats = flatten([ parse_category(category) for category in categories])
unique_cats = set([cat for cat in all_cats if len(cat) > 2])
cat_group = "|".join(unique_cats)<jupyter_output><empty_output><jupyter_text>Objects with non structured description<jupyter_code>import nltk
from nltk.tokenize import word_tokenize, TweetTokenizer
from nltk.tag import pos_tag, RegexpTagger
from nltk.chunk.regexp import RegexpChunkParser, RegexpChunkRule
from nltk.tree import Tree<jupyter_output><empty_output><jupyter_text>Custom pos-tags<jupyter_code>regexp_tagger = RegexpTagger([
    ("^{}$".format(cat_group), 'PLACE_MARKER'),
    (r'^[,]$', 'DT'),
    (r'^в$', 'BEG'),
    (r'^[A-Za-z0-9 &]+$', 'LATIN_WORD'),
    (r'^[а-яА-Я0-9 &]+$', 'CYRILLIC_WORD')
])<jupyter_output><empty_output><jupyter_text>Tokenize with word_tokenize and TweetTokenizer<jupyter_code>tweet_token = TweetTokenizer().tokenize

def tokenize(title, tokenizer):
    words = tokenizer(title)
    tokens = regexp_tagger.tag(words)
    return [token for token in tokens if token[1] is not None]<jupyter_output><empty_output><jupyter_text>Chunkparser to extract name.<jupyter_code>chuncker = RegexpChunkParser([
        RegexpChunkRule.fromstring(r'{<BEG> (<LATIN_WORD>|<CYRILLIC_WORD>)+ <PLACE_MARKER>+ (<LATIN_WORD>|<CYRILLIC_WORD>)* (?=<DT>|<BEG>)*}'),
        RegexpChunkRule.fromstring(r'{(<PLACE_MARKER>+ (<LATIN_WORD>|<CYRILLIC_WORD>)+)+ <PLACE_MARKER>* (?=<DT>|<BEG>)*}'),
        RegexpChunkRule.fromstring(r'{<PLACE_MARKER>+ (<LATIN_WORD>|<CYRILLIC_WORD>)+ (?=<DT>|<BEG>)*}'),
    ],
    chunk_label='Name'
)


def get_words(tree):
    return [word for word in tree.leaves()]

def find_name(title, tokenizer, visual=False):
    if title is "":
        return None
    
    title = title.lower()
    tokens = tokenize(title, tokenizer)
    tree = chuncker.parse(tokens)
    chunk_tree = [t.leaves() for t in tree if type(t) is Tree]
    flattened = sum(chunk_tree, [])
    filtered = [(w.strip(), m) for w, m in flattened if w != "в"]
    if visual:
        return tree
    return filtered<jupyter_output><empty_output><jupyter_text>Examples<jupyter_code>tree = find_name('Пушене на закрито в бар Sweet bar&grill Фестивална', tweet_token, visual=True)

tree
tree = find_name('Пушене в Bar "Korner", град София (м)', tweet_token, visual=True)

tree
tree = find_name('Пушене на ел.цигара в мотрисите на Софийското метро, град София', tweet_token, visual=True)

tree<jupyter_output><empty_output><jupyter_text>Apply functions to extract name for all objects<jupyter_code>df_sample['matched_title'] = df_sample['title'].apply(lambda x: find_name(x, tweet_token))
df_sample['matched_title_2'] = df_sample['title'].apply(lambda x: find_name(x, word_tokenize))
def concat_title(tuples):
    if tuples == [] or tuples is None:
        return None
    name = []
    for word, marker in tuples:
        name.append(word)
        
    return ' '.join(name)
df_sample['matched_title_str'] = df_sample['matched_title'].apply(concat_title)
df_sample['matched_title_2_str'] = df_sample['matched_title_2'].apply(concat_title)<jupyter_output><empty_output><jupyter_text>Final DF and Export<jupyter_code>df_sample
df_sample.to_json(join('..', 'data', 'ner', 'sample_full_names_tokens.json'), orient='split')
pd.read_json(join('..', 'data', 'ner', 'sample_full_names_tokens.json'), orient='split').count()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16074281150159744, 'fraction_numerical': 0.0017971246006389776, 'mean_word_length': 6.248914616497829, 'pattern_counts': {'":': 1, '<': 68, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '37760', 'license': 'MIT', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1846, 'n_words': 340, 'path': 'ner/Extract from Description and Title.ipynb', 'repo_name': 'ViktorMarinov/no-smoke-search', 'size': '5008'}