<jupyter_start><jupyter_text>Python for linguistic reasearch Project: extracting terminology1. Upload and examine a .vert text (output of the TreeTagger)- Orwell 1984 for En and De are available on https://heibox.uni-heidelberg.de/d/d65daff8341e467c82b1/- You can generate such file from your own texts with the TreeTagger2. We will create a pattern-based dictionary of terminological candidates, learning fundamentals of data preparation in Python<jupyter_code>import os, re, sys
# example for downloading files for different languages, e.g., German file in  from the https://heibox.uni-heidelberg.de/f/ea06aa47fe2d49959a62/
# remove / insert comments to choose the language you want to work with

# German
# 1984
# !wget https://heibox.uni-heidelberg.de/f/32856950701c444d8e21/?dl=1

# Georgian
# 1984
!wget https://heibox.uni-heidelberg.de/f/04dc02c96a3746e3a8f1/?dl=1
# Constitution
# !wget https://heibox.uni-heidelberg.de/f/6d2bb00b1f324d0aaad5/?dl=1
# Geo_Brown corpus
# !wget https://heibox.uni-heidelberg.de/f/2c3f3084b5594ac58c9b/?dl=1

# English
# 1984
# !wget https://heibox.uni-heidelberg.de/f/46798c5ec146485b8d96/?dl=1


# renaming file
!mv index.html?dl=1 go1984en-vert.txt
class clProcCorpus(object):
    ''' we will read a text file and return a dictionary
    this will be done on the line by line basis
    The dictionary can be sorted later...
    '''
    # this is a class for processing a corpus

    def __init__(self, FileIN):
        self.DictFrq = {}
        self.processCorpus(FileIN)

    def processCorpus(self, FileIN):
        LTerm = []
        for Line in FileIN:
            Line = Line.strip()
            LLine = re.split('\t', Line)
            try:
                Word = LLine[0]
                PoS = LLine[1]
                Lemma = LLine[2]
            except:
                Word = ""
                PoS = ""
                Lemma = ""
            
            # if re.match('N\..*', PoS) or re.match('ADJ.*', PoS) or re.match('von', Word):
            if re.match('N.*', PoS) or re.match('J.*', PoS):
                LTerm.append(Word)
            else:
                STerm = ' '.join(LTerm)
                LTerm = []

                try:
                    self.DictFrq[STerm] += 1
                except:
                    self.DictFrq[STerm] = 1        

        return


FileIN = open('go1984en-vert.txt', 'r')
FileOut = open('go1984en-terms.txt', 'w')
# step 5: save the frequency dictionary into file, by decreasing frequencies
# FileOutput.write( str( DictionaryFrq ) + '\n' )

OCorpus = clProcCorpus(FileIN)
DictionaryFrq = OCorpus.DictFrq

for Word, Frq in sorted( DictionaryFrq.items() , key=lambda x: x[1], reverse=True):
    if re.search(' ', Word):
        FileOut.write(Word + '\t' + str(Frq) + '\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11794138670478914, 'fraction_numerical': 0.048248749106504646, 'mean_word_length': 2.434355828220859, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '114177', 'license': 'MIT', 'n_tokens_mistral': 1033, 'n_tokens_neox': 920, 'n_words': 242, 'path': 'Session01python2linguisticResearch.ipynb', 'repo_name': 'iued-uni-heidelberg/DAAD-Training-2021', 'size': '2798'}