<commit_before>#!/usr/bin/env python
"""Search over a lexicon for Ganong effect items."""

# Copyright 2014 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from lingtools.corpus.cmudictreader import CMUDict
from lingtools.phon import syllabify
from lingtools.phon.arpabet import remove_stress

VOICING_PAIRS = {
    'K': 'G',
    'T': 'D',
    'P': 'B',
    }

BAD_CHARACTERS = set((
    "'",
    '-',
    '"',
    '.',
    '(',
    ')',
    '}',
    '{',
    ))

VOWELS = syllabify.vowels


def find_ganong(cmudict_path, max_sylls):
    """Find Ganong effect items a dictionary with <= max_sylls syllables."""
    word_prons = CMUDict(cmudict_path)

    # Add the reverse mapping for voicing pairs
    for key, val in VOICING_PAIRS.items():
        VOICING_PAIRS[val] = key

    # Filter to words that start with the right segment and have a
    # simplex onset
    eligible_word_prons = {word: pron for word, pron in word_prons.iteritems()
                           if (pron[0] in VOICING_PAIRS and
                               remove_stress(pron[1]) in VOWELS)}

    # Count number of syllables in every word, deleting any that
    # cannot be syllabified.
    word_nsylls = {}
    for word, pron in word_prons.items():
        try:
            nsylls = len(syllabify.syllabify(pron))
        except ValueError:
            del word_prons[word]
            continue
        else:
            word_nsylls[word] = nsylls

    # Set all of prons for checking if a pron is in the lexicon
    all_prons = set(tuple(value) for value in word_prons.itervalues())

    # Find the pairs. If max_sylls is None, skip the syllable length check.
    ganong_words = [word for word, pron in word_prons.iteritems()
                    if (word_nsylls[word] <= max_sylls and True and
                        not _exclude_word(word) and
                        is_ganong_pron(pron, all_prons, VOICING_PAIRS))]
                    
    return ganong_words


def _exclude_word(word):
    """Return whether a word should be excluded."""
    return any(char in BAD_CHARACTERS for char in word)


def is_ganong_pron(pron, all_prons, voicing_pairs):
    """Return whether an item is a Ganong effect item."""
    first = pron[0]
    # Skip things that don't start with the right segment
    if first not in voicing_pairs:
        return False
    # Make the new pronunciation
    new_pron = [voicing_pairs[first]] + pron[1:]
    # Return true if the new pron is not a word
    return tuple(new_pron) not in all_prons


def main():
    """Parse arguments and call the extractor."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('cmudict', help='input CSV file')
    parser.add_argument('-s', '--syll', nargs='?', default=None, type=int,
                        help='maximum number of syllables')
    args = parser.parse_args()
    items = find_ganong(args.cmudict, args.syll)
    for word in sorted(items):
        print word


if __name__ == "__main__":
    main()<commit_msg>Improve Ganong effect items finder.<commit_after>#!/usr/bin/env python
"""
Search over a lexicon for Ganong effect items.

"""

# Copyright 2014 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import csv

from lingtools.corpus.cmudictreader import CMUDict
from lingtools.corpus.elp import ELP
from lingtools.corpus.subtlexreader import SubtlexDict
from lingtools.phon import syllabify
from lingtools.phon.arpabet import remove_stress

VOICING_PAIRS = {
    'K': 'G',
    'T': 'D',
    'P': 'B',
    }

BAD_CHARACTERS = set((
    "'",
    '-',
    '"',
    ))

VOWELS = syllabify.vowels


def find_ganong(elp_path, cmudict_path, subtlex_path, max_sylls, out_path):
    """Find Ganong effect items a dictionary with <= max_sylls syllables."""
    # Open corpora
    elp = ELP(elp_path)
    cmudict = CMUDict(cmudict_path)
    subtlex = SubtlexDict(subtlex_path)

    # Collect all monomorphemic words with few enough syllables and no
    # excluded characters
    words = set(word.text for word in elp.itervalues()
                if (word.nsyll <= max_sylls and word.monomorph and
                    not any(char in BAD_CHARACTERS for char in word.text)))

    # Make sure items can be syllabified and record an authoritative syllable count
    word_sylls = {}
    word_prons = {}
    # Iterate over a copy to allow for modification
    for word in words.copy():
        try:
            pron = cmudict[word]
            sylls = syllabify.syllabify(pron)
        except (KeyError, ValueError):
            # Not in CMUDict or could not be syllabified
            words.remove(word)
            continue
        else:
            if len(sylls) > max_sylls:
                words.remove(word)
            else:
                word_prons[word] = pron
                word_sylls[word] = sylls

    # Add the reverse mapping for voicing pairs
    for key, val in VOICING_PAIRS.items():
        VOICING_PAIRS[val] = key

    # Filter to words that start with the right segment and have a
    # simplex onset
    eligible_words = {word for word, pron in word_prons.iteritems()
                      if (pron[0] in VOICING_PAIRS and
                          remove_stress(pron[1]) in VOWELS)}

    # Set all of prons for checking if a pron is in the lexicon
    all_prons = set(tuple(value) for value in cmudict.itervalues())

    # Find the pairs
    ganong_words = set(word for word in eligible_words
                       if is_ganong_pron(word_prons[word], all_prons, VOICING_PAIRS))

    # Get frequency information
    word_freqs = {}
    for word in ganong_words:
        try:
            word_freqs[word] = subtlex[word].freq_count_low
        except KeyError:
            pass

    # Write the output
    with open(out_path, 'wb') as out_file:
        fields = ['word', 'onset', 'nucleus', 'coda', 'sbtlx.freq_low']
        writer = csv.writer(out_file)
        writer.writerow(fields)
        for word, freq in word_freqs.iteritems():
            # Decompose the first syllable
            onset, nucleus, coda = [''.join(item) for item in word_sylls[word][0]]
            # Strip stress off nucleus
            nucleus = remove_stress(nucleus)
            writer.writerow([word, onset, nucleus, coda, freq])


def _exclude_word(word):
    """Return whether a word should be excluded."""
    return any(char in BAD_CHARACTERS for char in word)


def is_ganong_pron(pron, all_prons, voicing_pairs):
    """Return whether an item is a Ganong effect item."""
    first = pron[0]
    # Skip things that don't start with the right segment
    if first not in voicing_pairs:
        return False
    # Make the new pronunciation
    new_pron = [voicing_pairs[first]] + pron[1:]
    # Return true if the new pron is not a word
    return tuple(new_pron) not in all_prons


def main():
    """Parse arguments and call the extractor."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('elp_path', help='ELP item CSV file')
    parser.add_argument('cmudict_path', help='CMUDict file')
    parser.add_argument('subtlex_path', help='SUBTLEX file')
    parser.add_argument('out_path', help='output CSV file')
    parser.add_argument('-s', '--syll', nargs='?', default=None, type=int,
                        help='maximum number of syllables')
    args = parser.parse_args()
    find_ganong(args.elp_path, args.cmudict_path, args.subtlex_path,
                args.syll, args.out_path)


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.09247261161503122, 'fraction_numerical': 0.0029449876310519494, 'mean_word_length': 2.7367957746478875, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4185762', 'n_tokens_mistral': 2744, 'n_tokens_neox': 2597, 'n_words': 852}