<commit_before>from __future__ import unicode_literals
import sys
from collections import defaultdict
from mimetypes import guess_type

from sqlalchemy.orm import joinedload
import attr
from clldutils.dsv import reader
from clldutils.misc import slug, nfilter
from clldutils.path import Path
from clldutils import jsonlib

from clld.scripts.util import initializedb, Data, bibtex2source, add_language_codes
from clld.db.meta import DBSession
from clld.db.models import common
try:
    from pyglottolog.api import Glottolog
except ImportError:
    Glottolog = None

import dogonlanguages
from dogonlanguages import models
from dogonlanguages.scripts import util
from dogonlanguages.scripts.data import LANGUAGES


def main(args):
    for f in util.iter_files(args):
        DBSession.add(models.File(**attr.asdict(f)))

    lexicon = list(util.iter_lexicon(args))
    villages = util.get_villages(args)
    ff_images = list(util.ff_images(args))
    bib = list(util.get_bib(args))

    files_dir = args.data_file('files')
    if not files_dir.exists():
        files_dir.mkdir(parents=True)
    data = Data()

    dataset = common.Dataset(
        id=dogonlanguages.__name__,
        name="Dogon and Bangime Linguistics",
        publisher_name="Max Planck Institute for the Science of Human History",
        publisher_place="Jena",
        publisher_url="http://shh.mpg.de",
        license="http://creativecommons.org/licenses/by/4.0/",
        domain='dogonlanguages.clld.org',
        jsondata={
            'license_icon': 'cc-by.png',
            'license_name': 'Creative Commons Attribution 4.0 International License'}
    )
    DBSession.add(dataset)
<commit_msg>added contact email<commit_after>from __future__ import unicode_literals
import sys
from collections import defaultdict
from mimetypes import guess_type

from sqlalchemy.orm import joinedload
import attr
from clldutils.dsv import reader
from clldutils.misc import slug, nfilter
from clldutils.path import Path
from clldutils import jsonlib

from clld.scripts.util import initializedb, Data, bibtex2source, add_language_codes
from clld.db.meta import DBSession
from clld.db.models import common
try:
    from pyglottolog.api import Glottolog
except ImportError:
    Glottolog = None

import dogonlanguages
from dogonlanguages import models
from dogonlanguages.scripts import util
from dogonlanguages.scripts.data import LANGUAGES


def main(args):
    for f in util.iter_files(args):
        DBSession.add(models.File(**attr.asdict(f)))

    lexicon = list(util.iter_lexicon(args))
    villages = util.get_villages(args)
    ff_images = list(util.ff_images(args))
    bib = list(util.get_bib(args))

    files_dir = args.data_file('files')
    if not files_dir.exists():
        files_dir.mkdir(parents=True)
    data = Data()

    dataset = common.Dataset(
        id=dogonlanguages.__name__,
        name="Dogon and Bangime Linguistics",
        contact="<EMAIL>",
        publisher_name="Max Planck Institute for the Science of Human History",
        publisher_place="Jena",
        publisher_url="http://shh.mpg.de",
        license="http://creativecommons.org/licenses/by/4.0/",
        domain='dogonlanguages.clld.org',
        jsondata={
            'license_icon': 'cc-by.png',
            'license_name': 'Creative Commons Attribution 4.0 International License'}
    )
    DBSession.add(dataset)

{'fraction_non_alphanumeric': 0.08942848682262364, 'fraction_numerical': 0.002961208172934557, 'mean_word_length': 3.784702549575071, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6423870', 'n_tokens_mistral': 1053, 'n_tokens_neox': 1028, 'n_words': 261}