import string

book_order = ["Genesis",
              "Exodus",
              "Leviticus",
              "Numbers",
              "Deuteronomy",
              "Joshua",
              "Judges",
              "Ruth",
              "1 Samuel",
              "2 Samuel",
              "1 Kings",
              "2 Kings",
              "1 Chronicles",
              "2 Chronicles",
              "Ezra",
              "Nehemiah",
              "Esther",
              "Job",
              "Psalms",
              "Proverbs",
              "Ecclesiastes",
              "Song of Solomon",
              "Isaiah",
              "Jeremiah",
              "Lamentations",
              "Ezekiel",
              "Daniel",
              "Hosea",
              "Joel",
              "Amos",
              "Obadiah",
              "Jonah",
              "Micah",
              "Nahum",
              "Habakkuk",
              "Zephaniah",
              "Haggai",
              "Zechariah",
              "Malachi",
              "Matthew",
              "Mark",
              "Luke",
              "John",
              "Acts",
              "Romans",
              "1 Corinthians",
              "2 Corinthians",
              "Galatians",
              "Ephesians",
              "Philippians",
              "Colossians",
              "1 Thessalonians",
              "2 Thessalonians",
              "1 Timothy",
              "2 Timothy",
              "Titus",
              "Philemon",
              "Hebrews",
              "James",
              "1 Peter",
              "2 Peter",
              "1 John",
              "2 John",
              "3 John",
              "Jude",
              "Revelation"]

books = {'Judges': [36, 23, 31, 24, 31, 40, 25, 35, 57, 18, 40, 15, 25, 20, 20, 31, 13, 31, 30, 48, 25],
         'Micah': [16, 13, 12, 13, 15, 16, 20],
         'Proverbs': [33, 22, 35, 27, 23, 35, 27, 36, 18, 32, 31, 28, 25, 35, 33, 33, 28, 24, 29, 30, 31, 29, 35, 34, 28, 28, 27, 28, 27, 33, 31],
         'Revelation': [20, 29, 22, 11, 14, 17, 17, 13, 21, 11, 19, 17, 18, 20, 8, 21, 18, 24, 21, 15, 27, 21],
         'Deuteronomy': [46, 37, 29, 49, 33, 25, 26, 20, 29, 22, 32, 32, 18, 29, 23, 22, 20, 22, 21, 20, 23, 30, 25, 22, 19, 19, 26, 68, 29, 20, 30, 52, 29, 12],
         'Haggai': [15, 23],
         '3 John': [15],
         '1 Kings': [53, 46, 28, 34, 18, 38, 51, 66, 28, 29, 43, 33, 34, 31, 34, 34, 24, 46, 21, 43, 29, 53],
         'Mark': [45, 28, 35, 41, 43, 56, 37, 38, 50, 52, 33, 44, 37, 72, 47, 20],
         'Matthew': [25, 23, 17, 25, 48, 34, 29, 34, 38, 42, 30, 50, 58, 36, 39, 28, 27, 35, 30, 34, 46, 46, 39, 51, 46, 75, 66, 20],
         '1 Thessalonians': [10, 20, 13, 18, 28],
         'Daniel': [21, 49, 30, 37, 31, 28, 28, 27, 27, 21, 45, 13],
         'Malachi': [14, 17, 18, 6],
         'Colossians': [29, 23, 25, 18],
         'Ruth': [22, 23, 18, 22],
         'Genesis': [31, 25, 24, 26, 32, 22, 24, 22, 29, 32, 32, 20, 18, 24, 21, 16, 27, 33, 38, 18, 34, 24, 20, 67, 34, 35, 46, 22, 35, 43, 55, 32, 20, 31, 29, 43, 36, 30, 23, 23, 57, 38, 34, 34, 28, 34, 31, 22, 33, 26],
         '2 Samuel': [27, 32, 39, 12, 25, 23, 29, 18, 13, 19, 27, 31, 39, 33, 37, 23, 29, 33, 43, 26, 22, 51, 39, 25],
         'Obadiah': [21],
         'Esther': [22, 23, 15, 17, 14, 14, 10, 17, 32, 3],
         'Exodus': [22, 25, 22, 31, 23, 30, 25, 32, 35, 29, 10, 51, 22, 31, 27, 36, 16, 27, 25, 26, 36, 31, 33, 18, 40, 37, 21, 43, 46, 38, 18, 35, 23, 35, 35, 38, 29, 31, 43, 38],
         'Jeremiah': [19, 37, 25, 31, 31, 30, 34, 22, 26, 25, 23, 17, 27, 22, 21, 21, 27, 23, 15, 18, 14, 30, 40, 10, 38, 24, 22, 17, 32, 24, 40, 44, 26, 22, 19, 32, 21, 28, 18, 16, 18, 22, 13, 30, 5, 28, 7, 47, 39, 46, 64, 34],
         'Ephesians': [23, 22, 21, 32, 33, 24],
         'Habakkuk': [17, 20, 19],
         'Luke': [80, 52, 38, 44, 39, 49, 50, 56, 62, 42, 54, 59, 35, 35, 32, 31, 37, 43, 48, 47, 38, 71, 56, 53],
         'Song of Solomon': [17, 17, 11, 16, 16, 13, 13, 14],
         'Jonah': [17, 10, 10, 11],
         'Acts': [26, 47, 26, 37, 42, 15, 60, 40, 43, 48, 30, 25, 52, 28, 41, 40, 34, 28, 41, 38, 40, 30, 35, 27, 27, 32, 44, 31],
         'Job': [22, 13, 26, 21, 27, 30, 21, 22, 35, 22, 20, 25, 28, 22, 35, 22, 16, 21, 29, 29, 34, 30, 17, 25, 6, 14, 23, 28, 25, 31, 40, 22, 33, 37, 16, 33, 24, 41, 30, 24, 34, 17],
         'Titus': [16, 15, 15],
         '2 Timothy': [18, 26, 17, 22],
         'James': [27, 26, 18, 17, 20],
         '2 John': [13],
         'Isaiah': [31, 22, 26, 6, 30, 13, 25, 22, 21, 34, 16, 6, 22, 32, 9, 14, 14, 7, 25, 6, 17, 25, 18, 23, 12, 21, 13, 29, 24, 33, 9, 20, 24, 17, 10, 22, 38, 22, 8, 31, 29, 25, 28, 28, 25, 13, 15, 22, 26, 11, 23, 15, 12, 17, 13, 12, 21, 14, 21, 22, 11, 12, 19, 12, 25, 24],
         '2 Thessalonians': [12, 17, 18],
         '1 Chronicles': [54, 55, 24, 43, 26, 81, 40, 40, 44, 14, 47, 40, 14, 17, 29, 43, 27, 17, 19, 8, 30, 19, 32, 31, 31, 32, 34, 21, 30],
         '1 Timothy': [20, 15, 16, 16, 25, 21],
         'Leviticus': [17, 16, 17, 35, 19, 30, 38, 36, 24, 20, 47, 8, 59, 57, 33, 34, 16, 30, 37, 27, 24, 33, 44, 23, 55, 46, 34],
         '1 Peter': [25, 25, 22, 19, 14],
         'Psalms': [6, 12, 8, 8, 12, 10, 17, 9, 20, 18, 7, 8, 6, 7, 5, 11, 15, 50, 14, 9, 13, 31, 6, 10, 22, 12, 14, 9, 11, 12, 24, 11, 22, 22, 28, 12, 40, 22, 13, 17, 13, 11, 5, 26, 17, 11, 9, 14, 20, 23, 19, 9, 6, 7, 23, 13, 11, 11, 17, 12, 8, 12, 11, 10, 13, 20, 7, 35, 36, 5, 24, 20, 28, 23, 10, 12, 20, 72, 13, 19, 16, 8, 18, 12, 13, 17, 7, 18, 52, 17, 16, 15, 5, 23, 11, 13, 12, 9, 9, 5, 8, 28, 22, 35, 45, 48, 43, 13, 31, 7, 10, 10, 9, 8, 18, 19, 2, 29, 176, 7, 8, 9, 4, 8, 5, 6, 5, 6, 8, 8, 3, 18, 3, 3, 21, 26, 9, 8, 24, 13, 10, 7, 12, 15, 21, 10, 20, 14, 9, 6],
         'Zephaniah': [18, 15, 20],
         'Joel': [20, 32, 21],
         'Nahum': [15, 13, 19],
         'Jude': [25],
         '2 Chronicles': [17, 18, 17, 22, 14, 42, 22, 18, 31, 19, 23, 16, 22, 15, 19, 14, 19, 34, 11, 37, 20, 12, 21, 27, 28, 23, 9, 27, 36, 27, 21, 33, 25, 33, 27, 23],
         'Hosea': [11, 23, 5, 19, 15, 11, 16, 14, 17, 15, 12, 14, 16, 9],
         'Zechariah': [21, 13, 10, 14, 11, 15, 14, 23, 17, 12, 17, 14, 9, 21],
         'Nehemiah': [11, 20, 32, 23, 19, 19, 73, 18, 38, 39, 36, 47, 31],
         'John': [51, 25, 36, 54, 47, 71, 53, 59, 41, 42, 57, 50, 38, 31, 27, 33, 26, 40, 42, 31, 25],
         '1 John': [10, 29, 24, 21, 21],
         'Lamentations': [22, 22, 66, 22, 22],
         'Amos': [15, 16, 15, 13, 27, 14, 17, 14, 15],
         '1 Samuel': [28, 36, 21, 22, 12, 21, 17, 22, 27, 27, 15, 25, 23, 52, 35, 23, 58, 30, 24, 42, 15, 23, 29, 22, 44, 25, 12, 25, 11, 31, 13],
         'Joshua': [18, 24, 17, 24, 15, 27, 26, 35, 27, 43, 23, 24, 33, 15, 63, 10, 18, 28, 51, 9, 45, 34, 16, 33],
         '1 Corinthians': [31, 16, 23, 21, 13, 20, 40, 13, 27, 33, 34, 31, 13, 40, 58, 24],
         'Ezra': [11, 70, 13, 24, 17, 22, 28, 36, 15, 44],
         'Romans': [32, 29, 31, 25, 21, 23, 25, 39, 33, 21, 36, 21, 14, 23, 33, 27],
         '2 Corinthians': [24, 17, 18, 18, 21, 18, 16, 24, 15, 18, 33, 21, 14],
         'Hebrews': [14, 18, 19, 16, 14, 20, 28, 13, 28, 39, 40, 29, 25],
         'Ezekiel': [28, 10, 27, 17, 17, 14, 27, 18, 11, 22, 25, 28, 23, 23, 8, 63, 24, 32, 14, 49, 32, 31, 49, 27, 17, 21, 36, 26, 21, 26, 18, 32, 33, 31, 15, 38, 28, 23, 29, 49, 26, 20, 27, 31, 25, 24, 23, 35],
         '2 Peter': [21, 22, 18],
         'Philippians': [30, 30, 21, 23],
         'Numbers': [54, 34, 51, 49, 31, 27, 89, 26, 23, 36, 35, 16, 33, 45, 41, 50, 13, 32, 22, 29, 35, 41, 30, 25, 18, 65, 23, 31, 40, 16, 54, 42, 56, 29, 34, 13],
         'Philemon': [25],
         'Galatians': [24, 21, 29, 31, 26, 18],
         '2 Kings': [18, 25, 27, 44, 27, 33, 20, 29, 37, 36, 21, 21, 25, 29, 38, 20, 41, 37, 37, 21, 26, 20, 37, 20, 30],
         'Ecclesiastes': [18, 26, 22, 16, 20, 12, 29, 17, 18, 20, 10, 14]}

abbrevs = {'rom': 'Romans',
           'ge': 'Genesis',
           'ga': 'Galatians',
           'gn': 'Genesis',
           'i thes': '1 Thessalonians',
           'iith': '2 Thessalonians',
           'iiti': '2 Timothy',
           'iitm': '2 Timothy',
           '2chronicles': '2 Chronicles',
           'ti': 'Titus',
           'iij': '2 John',
           'esther': 'Esther',
           'song': 'Song of Solomon',
           'ii co': '2 Corinthians',
           'ii tim': '2 Timothy',
           'ii ch': '2 Chronicles',
           '1pe': '1 Peter',
           '2 corinthians': '2 Corinthians',
           'genesis': 'Genesis',
           'isaiah': 'Isaiah',
           'ii corinthians': '2 Corinthians',
           '1pt': '1 Peter',
           '1thes': '1 Thessalonians',
           '1 chronicles': '1 Chronicles',
           '2samuel': '2 Samuel',
           'joel': 'Joel',
           'i kings': '1 Kings',
           'ezra': 'Ezra',
           'judg': 'Judges',
           'iiijn': '3 John',
           'iiijo': '3 John',
           'jude': 'Jude',
           '2chron': '2 Chronicles',
           'iitimothy': '2 Timothy',
           '1 pe': '1 Peter',
           'neh': 'Nehemiah',
           '3j': '3 John',
           '1 thessalonians': '1 Thessalonians',
           '1 kg': '1 Kings',
           '1 ki': '1 Kings',
           'ithessalonians': '1 Thessalonians',
           'jonah': 'Jonah',
           '2kg': '2 Kings',
           'lev': 'Leviticus',
           '2ki': '2 Kings',
           '1 timothy': '1 Timothy',
           'micah': 'Micah',
           'iikings': '2 Kings',
           'psalms': 'Psalms',
           'psalm': 'Psalms',
           'ii sam': '2 Samuel',
           'joshua': 'Joshua',
           '1sa': '1 Samuel',
           '2 pt': '2 Peter',
           'ipe': '1 Peter',
           '1sm': '1 Samuel',
           '2 timothy': '2 Timothy',
           'ipt': '1 Peter',
           '2 pe': '2 Peter',
           '2thes': '2 Thessalonians',
           'iipt': '2 Peter',
           'revelation': 'Revelation',
           'iipe': '2 Peter',
           'colossians': 'Colossians',
           'isam': '1 Samuel',
           'iip': '2 Peter',
           '2king': '2 Kings',
           '2kg': '2 Kings',
           'iic': '2 Corinthians',
           '1tm': '1 Timothy',
           '1th': '1 Thessalonians',
           '1ti': '1 Timothy',
           'ii sm': '2 Samuel',
           'ml': 'Malachi',
           'jdg': 'Judges',
           'song of sol': 'Song of Solomon',
           'mi': 'Micah',
           'deu': 'Deuteronomy',
           'mk': 'Mark',
           'mt': 'Matthew',
           '1 john': '1 John',
           '1 samuel': '1 Samuel',
           '1jn': '1 John',
           '1jo': '1 John',
           'ithes': '1 Thessalonians',
           '2 chron': '2 Chronicles',
           '1corinthians': '1 Corinthians',
           'ithess': '1 Thessalonians',
           'numbers': 'Numbers',
           'ecclesiastes': 'Ecclesiastes',
           'acts': 'Acts',
           'acts of the apostles': 'Acts',
           'iiij': '3 John',
           '1 cor': '1 Corinthians',
           'hos': 'Hosea',
           'lam': 'Lamentations',
           'mar': 'Mark',
           'mat': 'Matthew',
           'iisamuel': '2 Samuel',
           '1 peter': '1 Peter',
           'mal': 'Malachi',
           'ss': 'Song of Solomon',
           '2 tim': '2 Timothy',
           'so': 'Song of Solomon',
           'sg': 'Song of Solomon',
           'zec': 'Zechariah',
           '1 corinthians': '1 Corinthians',
           'itm': '1 Timothy',
           'ith': '1 Thessalonians',
           'iti': '1 Timothy',
           '2sa': '2 Samuel',
           'zep': 'Zephaniah',
           'daniel': 'Daniel',
           'jnh': 'Jonah',
           'ichron': '1 Chronicles',
           'iitim': '2 Timothy',
           'la': 'Lamentations',
           'lk': 'Luke',
           'lv': 'Leviticus',
           'lu': 'Luke',
           '1 sam': '1 Samuel',
           'dan': 'Daniel',
           '1 thess': '1 Thessalonians',
           'ii ki': '2 Kings',
           'exod': 'Exodus',
           '2p': '2 Peter',
           'james': 'James',
           'nehemiah': 'Nehemiah',
           '2c': '2 Corinthians',
           '2j': '2 John',
           '2 samuel': '2 Samuel',
           '1 ch': '1 Chronicles',
           '1 co': '1 Corinthians',
           'obadiah': 'Obadiah',
           'iiijohn': '3 John',
           'phi': 'Philippians',
           'ec': 'Ecclesiastes',
           'i th': '1 Thessalonians',
           'exodus': 'Exodus',
           'ex': 'Exodus',
           'ez': 'Ezekiel',
           'ep': 'Ephesians',
           'es': 'Esther',
           'ru': 'Ruth',
           'rv': 'Revelation',
           '2 thes': '2 Thessalonians',
           'phm': 'Philemon',
           'rev': 'Revelation',
           're': 'Revelation',
           '2 jn': '2 John',
           '2 jo': '2 John',
           'rm': 'Romans',
           'ro': 'Romans',
           'ecc': 'Ecclesiastes',
           'jeremiah': 'Jeremiah',
           '1john': '1 John',
           'romans': 'Romans',
           '2 kings': '2 Kings',
           'i cor': '1 Corinthians',
           'ii chron': '2 Chronicles',
           'zech': 'Zechariah',
           '2 tm': '2 Timothy',
           '2 th': '2 Thessalonians',
           '2 ti': '2 Timothy',
           'i samuel': '1 Samuel',
           'ism': '1 Samuel',
           'isa': '1 Samuel',
           'i john': '1 John',
           'iichronicles': '2 Chronicles',
           'jer': 'Jeremiah',
           '1p': '1 Peter',
           'iii john': '3 John',
           '1c': '1 Corinthians',
           'ii kings': '2 Kings',
           '1j': '1 John',
           'i sam': '1 Samuel',
           'iich': '2 Chronicles',
           'iico': '2 Corinthians',
           'john': 'John',
           'icorinthians': '1 Corinthians',
           '2 co': '2 Corinthians',
           '2 ch': '2 Chronicles',
           '1 jo': '1 John',
           '1 jn': '1 John',
           'psalm': 'Psalms',
           'iithes': '2 Thessalonians',
           'i ch': '1 Chronicles',
           '1tim': '1 Timothy',
           'iijn': '2 John',
           'iijo': '2 John',
           '1thessalonians': '1 Thessalonians',
           'pro': 'Proverbs',
           'jerem': 'Jeremiah',
           'ii cor': '2 Corinthians',
           'prv': 'Proverbs',
           '1pet': '1 Peter',
           'i pet': '1 Peter',
           'mic': 'Micah',
           '2corinthians': '2 Corinthians',
           'deut': 'Deuteronomy',
           'lamen': 'Lamentations',
           'icor': '1 Corinthians',
           '2tm': '2 Timothy',
           '2ti': '2 Timothy',
           '2th': '2 Thessalonians',
           'philem': 'Philemon',
           '2cor': '2 Corinthians',
           'exo': 'Exodus',
           'phil': 'Philippians',
           '3 john': '3 John',
           '2sm': '2 Samuel',
           '1 pt': '1 Peter',
           '2 thess': '2 Thessalonians',
           'philippians': 'Philippians',
           'philemon': 'Philemon',
           '1thess': '1 Thessalonians',
           'zeph': 'Zephaniah',
           'judges': 'Judges',
           'jas': 'James',
           'i thess': '1 Thessalonians',
           'jam': 'James',
           'ii sa': '2 Samuel',
           'ijohn': '1 John',
           'dn': 'Daniel',
           'luke': 'Luke',
           'de': 'Deuteronomy',
           'da': 'Daniel',
           '1samuel': '1 Samuel',
           'dt': 'Deuteronomy',
           'du': 'Deuteronomy',
           'gen': 'Genesis',
           '1king': '1 Kings',
           '3 jo': '3 John',
           '3 jn': '3 John',
           'titus': 'Titus',
           'i peter': '1 Peter',
           'hosea': 'Hosea',
           'ii timothy': '2 Timothy',
           'iichron': '2 Chronicles',
           'i timothy': '1 Timothy',
           'ipet': '1 Peter',
           '2 chronicles': '2 Chronicles',
           'num': 'Numbers',
           'eccles': 'Ecclesiastes',
           'iithessalonians': '2 Thessalonians',
           'zechariah': 'Zechariah',
           '3jn': '3 John',
           '3jo': '3 John',
           '2timothy': '2 Timothy',
           'luk': 'Luke',
           'i pe': '1 Peter',
           'jos': 'Joshua',
           '2pt': '2 Peter',
           '2thessalonians': '2 Thessalonians',
           'ju': 'Jude',
           'i chron': '1 Chronicles',
           'jl': 'Joel',
           'jn': 'John',
           'i pt': '1 Peter',
           '2pe': '2 Peter',
           'joe': 'Joel',
           'joh': 'John',
           'je': 'Jeremiah',
           'iki': '1 Kings',
           'ii peter': '2 Peter',
           'ja': 'James',
           'jb': 'Job',
           'i corinthians': '1 Corinthians',
           'job': 'Job',
           'i co': '1 Corinthians',
           'jo': 'John',
           'ikg': '1 Kings',
           'col': 'Colossians',
           'co': 'Colossians',
           'song of solomon': 'Song of Solomon',
           'ii chronicles': '2 Chronicles',
           'ps': 'Psalms',
           'pv': 'Proverbs',
           '1sam': '1 Samuel',
           'mark': 'Mark',
           '1timothy': '1 Timothy',
           '2 cor': '2 Corinthians',
           'iikg': '2 Kings',
           'jon': 'Jonah',
           'iiki': '2 Kings',
           '2pet': '2 Peter',
           'i ki': '1 Kings',
           '2tim': '2 Timothy',
           'esth': 'Esther',
           'i kg': '1 Kings',
           'gal': 'Galatians',
           'leviticus': 'Leviticus',
           '2 peter': '2 Peter',
           '2 ki': '2 Kings',
           'ii thess': '2 Thessalonians',
           'ii thes': '2 Thessalonians',
           'iicor': '2 Corinthians',
           '2 pet': '2 Peter',
           'ip': '1 Peter',
           'is': 'Isaiah',
           'iithess': '2 Thessalonians',
           'ij': '1 John',
           'ic': '1 Corinthians',
           'isamuel': '1 Samuel',
           '2 thessalonians': '2 Thessalonians',
           'iisam': '2 Samuel',
           'i chronicles': '1 Chronicles',
           'matthew': 'Matthew',
           '3john': '3 John',
           'ruth': 'Ruth',
           'i ti': '1 Timothy',
           '2sam': '2 Samuel',
           'i tm': '1 Timothy',
           'ephesians': 'Ephesians',
           'habakkuk': 'Habakkuk',
           'galatians': 'Galatians',
           '1chron': '1 Chronicles',
           '1 thes': '1 Thessalonians',
           'php': 'Philippians',
           'ii thessalonians': '2 Thessalonians',
           '2kings': '2 Kings',
           'zephaniah': 'Zephaniah',
           '1 ti': '1 Timothy',
           '1 th': '1 Thessalonians',
           '1 tm': '1 Timothy',
           'eph': 'Ephesians',
           '1ch': '1 Chronicles',
           '1co': '1 Corinthians',
           '1kings': '1 Kings',
           'nahum': 'Nahum',
           'ikings': '1 Kings',
           '2jo': '2 John',
           '2jn': '2 John',
           '2 sam': '2 Samuel',
           'ezek': 'Ezekiel',
           'iijohn': '2 John',
           'hab': 'Habakkuk',
           'josh': 'Joshua',
           'hag': 'Haggai',
           'ichronicles': '1 Chronicles',
           'nehem': 'Nehemiah',
           'amo': 'Amos',
           'psa': 'Psalms',
           'pss': 'Psalms',
           'i thessalonians': '1 Thessalonians',
           'ii samuel': '2 Samuel',
           'ii john': '2 John',
           'amos': 'Amos',
           'i sa': '1 Samuel',
           'hebrews': 'Hebrews',
           'ezr': 'Ezra',
           'i sm': '1 Samuel',
           'revel': 'Revelation',
           'ezk': 'Ezekiel',
           'eze': 'Ezekiel',
           'iiking': '2 Kings',
           'ijn': '1 John',
           'ijo': '1 John',
           'est': 'Esther',
           'ico': '1 Corinthians',
           'prov': 'Proverbs',
           '1 kings': '1 Kings',
           '2thess': '2 Thessalonians',
           'itimothy': '1 Timothy',
           'i tim': '1 Timothy',
           'jgs': 'Judges',
           '2john': '2 John',
           'haggai': 'Haggai',
           '1 tim': '1 Timothy',
           'iicorinthians': '2 Corinthians',
           '1 sm': '1 Samuel',
           '1 sa': '1 Samuel',
           'rut': 'Ruth',
           'ii pt': '2 Peter',
           'ob': 'Obadiah',
           'oba': 'Obadiah',
           'le': 'Leviticus',
           '2co': '2 Corinthians',
           '2ch': '2 Chronicles',
           'ii pe': '2 Peter',
           'act': 'Acts',
           '1chronicles': '1 Chronicles',
           'deuteronomy': 'Deuteronomy',
           'son': 'Song of Solomon',
           'sos': 'Song of Solomon',
           'iipet': '2 Peter',
           'i jo': '1 John',
           'i jn': '1 John',
           'heb': 'Hebrews',
           'iii jo': '3 John',
           'iii jn': '3 John',
           'jud': 'Jude',
           'iking': '1 Kings',
           'ho': 'Hosea',
           '2 sa': '2 Samuel',
           'hb': 'Habakkuk',
           '2 sm': '2 Samuel',
           'hg': 'Haggai',
           'he': 'Hebrews',
           'proverbs': 'Proverbs',
           'itim': '1 Timothy',
           'iism': '2 Samuel',
           'iisa': '2 Samuel',
           'ac': 'Acts',
           'am': 'Amos',
           'ezekiel': 'Ezekiel',
           '2 john': '2 John',
           'ii jo': '2 John',
           'ii jn': '2 John',
           'obad': 'Obadiah',
           'nm': 'Numbers',
           'tit': 'Titus',
           'na': 'Nahum',
           'nah': 'Nahum',
           '1 pet': '1 Peter',
           'ne': 'Nehemiah',
           'malachi': 'Malachi',
           'nu': 'Numbers',
           '1ki': '1 Kings',
           'ii tm': '2 Timothy',
           '1cor': '1 Corinthians',
           '1kg': '1 Kings',
           'matt': 'Matthew',
           'ii ti': '2 Timothy',
           'ii th': '2 Thessalonians',
           'ich': '1 Chronicles',
           '1 chron': '1 Chronicles',
           'ii pet': '2 Peter',
           'lamentations': 'Lamentations'}

class ReferenceError(Exception):
    """Any error while parsing a range or verse reference"""
    pass

class ReferenceValidator(object):
    def __init__(self):
        pass
    def regularize_verse(self, verse):
        try:
            verse.book = abbrevs[verse.book.lower()]
        except KeyError:
            raise ReferenceError("Unrecognized book: %s" % verse.book)

    def regularize_range(self, rng):
        self.regularize_verse(rng.from_)

        if not rng.from_.verse:
            rng.from_.verse = 1

        if not rng.from_.chapter:
            rng.from_.chapter = 1

        self.regularize_verse(rng.to)

        if not rng.to.chapter:
            rng.to.chapter = len(books[rng.to.book])

        if not rng.to.verse:
            rng.to.verse = books[rng.to.book][rng.to.chapter - 1]

    def validate_verse(self, verse):
        book = None

        try:
            book = abbrevs[verse.book.lower()]
        except KeyError:
            return False

        if verse.chapter:
            if not (0 < verse.chapter <= len(books[book])):
                return False

        if verse.verse:
            if not (0 < verse.verse <= books[book][verse.chapter - 1]):
                return False

        return True

    def validate_range(self, rng):
        math = VerseMath()
        lt = math.lt(rng.from_, rng.to)

        if not lt:
            raise ReferenceError(
                "End of range precedes start of range: %s" % rng)

        return (self.validate_verse(rng.from_) and
                self.validate_verse(rng.to) and
                lt)

class VerseMath(object):
    def __init__(self):
        pass
    def next_book(self, book):
        return book_order[book_order.index(book) + 1]
    def previous_book(self, book):
        return book_order[book_order.index(book) - 1]
    def add(self, verse, verses):
        book = verse.book
        chapter = verse.chapter
        remaining = verses + verse.verse

        while remaining > 0:
            remaining = remaining - books[book][chapter - 1]
            if remaining > 0:
                if chapter == len(books[book]):
                    book = self.next_book(book)
                    chapter = 1
                else:
                    chapter += 1

        return Verse(book, chapter, books[book][chapter - 1] + remaining)

    def lt(self, verse1, verse2):
        return ((book_order.index(verse1.book) <
                 book_order.index(verse2.book)) or
                ((verse1.book == verse2.book) and
                 (verse1.chapter < verse2.chapter)) or
                ((verse1.book == verse2.book) and
                 (verse1.chapter == verse2.chapter) and
                 (verse1.verse < verse2.verse)))
    def eq(self, verse1, verse2):
        return ((verse1.book == verse2.book) and
                (verse1.chapter == verse2.chapter) and
                (verse1.verse == verse2.verse))
    def gt(self, verse1, verse2):
        return not (self.lt(verse1, verse2) or
                    self.eq(verse1, verse2))

class Verse(object):
    """Represents a verse of the Bible"""
    def __init__(self, book, chapter, verse):
        self.book = book
        self.chapter = chapter
        self.verse = verse
        self.math = VerseMath()
    def __cmp__(self, other):
        if self.math.lt(self, other):
            return -1
        elif self.math.eq(self, other):
            return 0
        else:
            return 1
    def __repr__(self):
        if self.verse:
            return "%s %s:%s" % (self.book,
                                 self.chapter,
                                 self.verse)
        elif self.chapter:
            return "%s %s" % (self.book,
                              self.chapter)
        else:
            return self.book

class Range(object):
    """Represents a range of Bible text from one verse to another"""
    def __init__(self, from_, to):
        self.from_ = from_
        self.to = to

    def __repr__(self):
        return "%s - %s" % (self.from_, self.to)

    def contains(self, ref):
        if type(ref) == Verse:
            return self.from_ <= ref <= self.to
        elif type(ref) == Range:
            return self.contains(ref.from_) and self.contains(ref.to)
    def __cmp__(self, other):
        if type(other) == Range:
            other = other.from_

        return self.from_.__cmp__(other)

class VerseParser(object):
    """A parser for Bible verse references"""
    def __init__(self):
        self.validator = ReferenceValidator()
    def is_digit(self):
        try:
            return self.verse[self.index].isdigit()
        except IndexError:
            return False
    def parse(self, verse):
        self.verse = verse.strip()
        self.index = 0

        # skip any leading numeric characters
        while self.is_digit():
            self.index += 1

        # skip up to a number (or the end)


#-----------------------------------------------------------------------------
# Copyright (c) 2013-2016, PyInstaller Development Team.
#
# Distributed under the terms of the GNU General Public License with exception
# for distributing bootloader.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

import os
import sys

from PyInstaller.compat import is_win, is_darwin, is_unix, is_venv, base_prefix
from PyInstaller.compat import modname_tkinter
from PyInstaller.depend.bindepend import selectImports, getImports
from PyInstaller.building.datastruct import Tree
from PyInstaller.utils.hooks import exec_statement, logger

def _handle_broken_tcl_tk():
    """
    When freezing from a Windows venv, overwrite the values of the standard
    `${TCL_LIBRARY}`, `${TK_LIBRARY}`, and `${TIX_LIBRARY}` environment
    variables.

    This is a workaround for broken Tcl/Tk detection in Windows virtual
    environments. Older versions of `virtualenv` set such variables erroneously,
    preventing PyInstaller from properly detecting Tcl/Tk. This issue has been
    noted for `virtualenv` under Python 2.4 and Windows 7.

    See Also
    -------
    https://github.com/pypa/virtualenv/issues/93
    """
    if is_win and is_venv:
        basedir = os.path.join(base_prefix, 'tcl')
        files = os.listdir(basedir)

        # Detect Tcl/Tk paths.
        for f in files:
            abs_path = os.path.join(basedir, f)
            if f.startswith('tcl') and os.path.isdir(abs_path):
                os.environ['TCL_LIBRARY'] = abs_path
            elif f.startswith('tk') and os.path.isdir(abs_path):
                os.environ['TK_LIBRARY'] = abs_path
            elif f.startswith('tix') and os.path.isdir(abs_path):
                os.environ['TIX_LIBRARY'] = abs_path

def _warn_if_activetcl_or_teapot_installed(tcl_root, tcltree):
    """
    If the current Tcl installation is a Teapot-distributed version of ActiveTcl
    *and* the current platform is OS X, log a non-fatal warning that the
    resulting executable will (probably) fail to run on non-host systems.

    PyInstaller does *not* freeze all ActiveTcl dependencies -- including
    Teapot, which is typically ignorable. Since Teapot is *not* ignorable in
    this case, this function warns of impending failure.

    See Also
    -------
    https://github.com/pyinstaller/pyinstaller/issues/621
    """
    from PyInstaller.lib.macholib import util

    # System libraries do not experience this problem.
    if util.in_system_path(tcl_root):
        return

    # Absolute path of the "init.tcl" script.
    try:
        init_resource = [r[1] for r in tcltree if r[1].endswith('init.tcl')][0]
    # If such script could not be found, silently return.
    except IndexError:
        return

    mentions_activetcl = False
    mentions_teapot = False
    with open(init_resource, 'r') as init_file:
        for line in init_file.readlines():
            line = line.strip().lower()
            if line.startswith('#'):
                continue
            if 'activetcl' in line:
                mentions_activetcl = True
            if 'teapot' in line:
                mentions_teapot = True
            if mentions_activetcl and mentions_teapot:
                break

    if mentions_activetcl and mentions_teapot:
        logger.warning(
            """
You appear to be using an ActiveTcl build of Tcl/Tk, which PyInstaller has
difficulty freezing. To fix this, comment out all references to "teapot" in:

     %s

See https://github.com/pyinstaller/pyinstaller/issues/621 for more information.
            """ % init_resource)

def _find_tcl_tk_darwin_frameworks(binaries):
    """
    Get an OS X-specific 2-tuple of the absolute paths of the top-level
    external data directories for both Tcl and Tk, respectively.

    Under OS X, Tcl and Tk are installed as Frameworks requiring special care.

    Returns
    -------
    list
        2-tuple whose first element is the value of `${TCL_LIBRARY}` and whose
        second element is the value of `${TK_LIBRARY}`.
    """
    tcl_root = tk_root = None
    for nm, fnm in binaries:
        if nm == 'Tcl':
            tcl_root = os.path.join(os.path.dirname(fnm), 'Resources/Scripts')
        elif nm == 'Tk':
            tk_root =  os.path.join(os.path.dirname(fnm), 'Resources/Scripts')
    return tcl_root, tk_root

def _find_tcl_tk_dir():
    """
    Get a platform-agnostic 2-tuple of the absolute paths of the top-level
    external data directories for both Tcl and Tk, respectively.

    Returns
    -------
    list
        2-tuple whose first element is the value of `${TCL_LIBRARY}` and whose
        second element is the value of `${TK_LIBRARY}`.
    """
    # Python code to get path to TCL_LIBRARY.
    tcl_root = exec_statement(
        'from %s import Tcl; print(Tcl().eval("info library"))' % modname_tkinter)
    tk_version = exec_statement(
        'from _tkinter import TK_VERSION; print(TK_VERSION)')

    # TK_LIBRARY is in the same prefix as Tcl.
    tk_root = os.path.join(os.path.dirname(tcl_root), 'tk%s' % tk_version)
    return tcl_root, tk_root

def _find_tcl_tk(hook_api):
    """
    Get a platform-specific 2-tuple of the absolute paths of the top-level
    external data directories for both Tcl and Tk, respectively.

    Returns
    -------
    list
        2-tuple whose first element is the value of `${TCL_LIBRARY}` and whose
        second element is the value of `${TK_LIBRARY}`.
    """
    bins = selectImports(hook_api.__file__)

    if is_darwin:
        # _tkinter depends on system Tcl/Tk frameworks.
        # For example this is the case of Python from homebrew.
        if not bins:
            # 'hook_api.binaries' can't be used because on Mac OS X _tkinter.so
            # might depend on system Tcl/Tk frameworks and these are not
            # included in 'hook_api.binaries'.
            bins = getImports(hook_api.__file__)
            # Reformat data structure from
            #     set(['lib1', 'lib2', 'lib3'])
            # to
            #     [('Tcl', '/path/to/Tcl'), ('Tk', '/path/to/Tk')]
            mapping = {}
            for l in bins:
                mapping[os.path.basename(l)] = l
            bins = [
                ('Tcl', mapping['Tcl']),
                ('Tk', mapping['Tk']),
            ]

        # _tkinter depends on Tcl/Tk compiled as frameworks.
        path_to_tcl = bins[0][1]
        if 'Library/Frameworks' in path_to_tcl:
            tcl_tk = _find_tcl_tk_darwin_frameworks(bins)
        # Tcl/Tk compiled as on Linux other Unixes.
        # For example this is the case of Tcl/Tk from macports.
        else:
            tcl_tk = _find_tcl_tk_dir()

    else:
        tcl_tk = _find_tcl_tk_dir()

    return tcl_tk

def _collect_tcl_tk_files(hook_api):
    """
    Get a list of TOC-style 3-tuples describing all external Tcl/Tk data files.

    Returns
    -------
    Tree
        Such list.
    """
    # Workaround for broken Tcl/Tk detection in virtualenv on Windows.
    _handle_broken_tcl_tk()

    tcl_root, tk_root = _find_tcl_tk(hook_api)

    # TODO Shouldn't these be fatal exceptions?
    if not tcl_root:
        logger.error('Tcl/Tk improperly installed on this system.')
        return []
    if not os.path.isdir(tcl_root):
        logger.error('Tcl data directory "%s" not found.', tcl_root)
        return []


# Grammar-Guided Feature Extraction (GGFE)
#
# Author:   Damian Eads
#
# File:     image_features.py
#
# Purpose:  This module contains image processing primitives and
#           a grammar for generating signal processing programs for
#           feature extraction.
#
# Date:     January 13, 2010

import numpy as np
import scipy as sp
import scipy.ndimage as ndi
from core import variables, functions, Variable, Grammar, Environment
import scipy.stats

def smean(X):
    return X.mean(axis=1)

def var(X):
    return X.var(axis=1)

def stddev(X):
    return X.std(axis=1)

def skewness(X):
    """
    deviation = (X - X.sum(axis=1))
    g1 = (deviation ** 3.0).mean(axis=1)
    g1 /= (deviation ** 2.0).mean(axis=1) ** (3./2.)
    n = X.shape[1]
    k3 = np.sqrt(n * (n - 1));
    k2_ = n - 2
    """
    return sp.stats.skew(X, axis=1)

def erode(I, struct):
    "Performs a morphological erosion."
    return ndi.grey_erosion(I, footprint=struct)

def dilate(I, struct):
    "Performs a morphological dilation."
    return ndi.grey_dilation(I, footprint=struct)

def open(I, struct):
    "Performs a morphological opening."
    return ndi.grey_opening(I, footprint=struct)

def close(I, struct):
    "Performs a morpphological closing."
    return ndi.grey_closing(I, footprint=struct)

def kurtosis(X):
    return sp.stats.kurtosis(X, axis=1)

def window(X, a, b):
    n = X.shape[1]
    if b < a:
        a, b = swap(a, b)
    a = np.floor(n * a)
    b = np.floor(n * b)
    Xw = X[:,a:b]
    return Xw

def trapz(X):
    return np.trapz(X)

def swap(a, b):
    return b, a

def ptile(X, percentile, struct):
    "Computes the kth percentile filter with a particular structuring element."
    return ndi.percentile_filter(X, percentile, footprint=struct)

def line_selem(n):
    return np.ones((1, n))

def rand1n(n):
    "Generates a random integer between 1 and n (exclusive)."
    return np.random.randint(n)+1

def rand0n(n):
    "Generates a random integer between 0 and n (exclusive)."
    if n == 0:
        return 0
    else:
        return np.random.randint(n)

def sigmoid(I, a, b):
    "Soft maximum with shape parameters a and b."
    return  np.arctan(b * (I + a)) / b

def normDiff(X, Y):
    "Takes the normalized difference, i.e. r=(a-b)/(a+b)."
    sum = X + Y
    result = (X - Y) / sum
    result[sum == 0.0] = 0.0
    return result

def mult(X, Y):
    "Multiplies two images, i.e. r=a*b."
    return X * Y

def scaledSub(X, Y):
    "Takes the difference and scales, i.e. r=abs((a-b)/2+0.5)."
    result = (X - Y) / 2.0 + 0.5
    result[result < 0.0] = 0.0
    return result

def blend(*args):
    "Takes the average of several images, i.e. r=(a_1+a_2+...+a_n)/n"
    result = args[0]
    for arg in args[1:]:
        result = result + arg
    return result / float(len(args))

def get_signal_grammar():
    import signal_features
    reload(signal_features)
    grammar = Grammar("main_sig")

    Feature, SignalFeature, Morph, Compound, RandomSE, Statistic, Window = \
       grammar.productions(
        ["Feature", "SignalFeature", "Morph", "Compound", "RandomSE", "Statistic", "Window"]
        )

    X, Y, S = variables(["X", "Y", "S"])

    Binary, Unary, NLBinary, LBinary = \
        grammar.productions(
        ["Binary", "Unary", "NLBinary", "LBinary"]
        )
    erode, dilate, open, close, sigmoid, ptile = \
          functions(["erode", "dilate", "open", "close", "sigmoid", "ptile"],
                     module=signal_features)

    smean, stddev, var, skewness, kurtosis, trapz = \
        functions(["smean", "stddev", "var", "skewness", "kurtosis", "trapz"], module=signal_features);

    mult, normDiff, scaledSub, blend, line_selem, rand1n, window = \
          functions(["mult", "normDiff", "scaledSub", "blend", "line_selem", "rand1n", "window"],
                    module=signal_features)

    normal, random, randint, transpose, dot = \
            functions(["normal", "random", "randint", "transpose", "dot"],
                      module=np.random)

    [nan_to_num] = functions(["nan_to_num"])

    Feature[X] = (Statistic[SignalFeature[Window[X]]]
                  | nan_to_num(Statistic[SignalFeature[Window[X]]] /
                               Statistic[SignalFeature[Window[X]]]))

    SignalFeature[X] = (Binary[Unary[X], Unary[X]]
                        | Unary[X]
                        | Unary[Unary[X]]
                        | NLBinary[Unary[X], Unary[X]]
                        | Compound[X])

    Statistic[X] = (smean(X)
                    | stddev(X)
                    | var(X)
                    | skewness(X)
                    | kurtosis(X)
                    | trapz(X))

    Binary[X, Y] = (mult(X, Y)
                   | normDiff(X, Y)
                   | scaledSub(X, Y)
                   | blend(X, Y))

    Unary[X] = (sigmoid(X, ~normal(), 10 ** (~random() * 2 - 1))
                  | ptile(X, ~(random() * 100), RandomSE[...])
                  | Morph[X, RandomSE[...]])

    Morph[X,S] = (erode(X, S) |
                  dilate(X, S) |
                  open(X, S) |
                  close(X, S))

    Compound[X] = (Unary[X]
                   | Binary[X, Compound[X]])

    NLBinary[X, Y] = (mult(X, Y)
                      | normDiff(X, Y))

    LBinary[X, Y] = (scaledSub(X, Y)
                     | blend(X, Y))

    Window[X] = X | window(X, ~random(), ~random())

    RandomSE[...] = line_selem(rand1n(100))
    return grammar

def generate_and_evaluate(grammar, SIG):
    """
    Generate a random feature from the grammar passed and evaluate it
    on the image ``SIG``.
    """
    from numpy import pi, sin, cos, dot, transpose, nan_to_num
    feature = grammar.Feature(Variable('SIG'))
    print 'Evaluating random feature: ', str(feature)
    return eval(str(feature))

def eval_feature(program_string, SIG):
    from numpy import pi, sin, cos, dot, transpose, nan_to_num
    return eval(program_string)

def eval_stump(X, thresh, dir):
    if dir:
        return X >= thresh
    else:
        return X < thresh

def compute_werror(Yh, Y, Dt):
    return np.dot(Yh == Y, Dt).sum()

def dstump_find(X, Y, Dt):
    n = X.shape
    uX = np.unique(X)
    best_werror = np.dot(Y == 0, Dt)
    best_thresh = np.inf
    best_dir = True
    best_prev_thresh = None
    prev_thresh = None
    nT = uX.shape[0]
    for i in xrange(0, nT):
        thresh = uX[i]
        #print thresh
        if i+1 < nT:
            next_thresh = uX[i+1]
        else:
            next_thresh = None
        preds = eval_stump(X, thresh, True)
        werror = np.dot(preds != Y, Dt).sum()
        if werror < best_werror:
            best_thresh = thresh
            best_prev_thresh = prev_thresh
            best_werror = werror
            best_dir = True
        preds = eval_stump(X, thresh, False)
        werror = np.dot(preds != Y, Dt).sum()
        if werror < best_werror:
            best_thresh = thresh
            best_prev_thresh = next_thresh
            best_werror = werror
            best_dir = False
        prev_thresh = thresh
    if best_prev_thresh is not None:
        best_thresh = (best_thresh + best_prev_thresh) / 2.0
        if best_dir:
            best_werror = np.dot((X >= best_thresh) != Y, Dt).sum()
        else:
            best_werror = np.dot((X < best_thresh) != Y, Dt).sum()
    hyp = {}
    hyp["thresh"] = best_thresh
    hyp["dir"] = best_dir
    hyp["werror"] = best_werror
    return hyp

def get_alpha(werror):
    return .5 * np.log((1. - werror) / werror)

def adaboost_learn(grammar, X, Y, iterations, features_per_iteration):
    hypotheses = []
    n, m = X.shape
    Dt = np.ones((n,)) / n
    for it in xrange(0, iterations):
        best_werror = 1.0
        best_hypothesis = None
        best_result = None
        fnum = 0
        while best_werror >= 0.5 or fnum < features_per_iteration:
            feature = str(grammar.Feature(Variable('SIG')))
            result = eval_feature(feature, X)
            whyp = dstump_find(result, Y, Dt)
            print "It %d Feature %d Program %s WError: %5.8f Best WError: %5.8f" % (it, fnum, feature, whyp["werror"], best_werror)
            if whyp["werror"] < best_werror:
                best_werror = whyp["werror"]
                alpha = get_alpha(best_werror)
                best_hypothesis = whyp
                whyp["alpha"] = alpha
                best_result = result
            fnum += 1
        hypotheses.append(best_hypothesis)
        Yh = eval_stump(best_result, best_hypothesis["thresh"], best_hypothesis["dir"])


#!/usr/bin/env python3
# -*- coding: utf8 -*-
################################################################################
##
## This file is part of BetterPonymotes.
## Copyright (c) 2012-2015 Typhos.
##
## This program is free software: you can redistribute it and/or modify it
## under the terms of the GNU Affero General Public License as published by
## the Free Software Foundation, either version 3 of the License, or (at your
## option) any later version.
##
## This program is distributed in the hope that it will be useful, but WITHOUT
## ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
## FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License
## for more details.
##
## You should have received a copy of the GNU Affero General Public License
## along with this program.  If not, see <http://www.gnu.org/licenses/>.
##
################################################################################

import re

import bplib
import bplib.css
import bplib.objects

def filter_ponyscript_ignores(css_rules):
    # Takes list of CssRule's and sets .ignore properties on them as appropriate.
    #
    # This function also warns if any rules are split up as a result.
    visible, hidden = set(), set()
    ignoring = False
    for rule in css_rules:
        # NOTE: As a very weird edge case, supposing this is combined with
        # other selectors, the entire block would be hidden. That's probably
        # the best thing to do, though.
        if "START-PONYSCRIPT-IGNORE" in rule.selectors:
            ignoring = True

        # Split-rule detection
        add_to, refuse_in = (hidden, visible) if ignoring else (visible, hidden)
        for selector in rule.selectors:
            add_to.add(selector)
            if selector in refuse_in:
                # FIXME: Since this code runs before emote extraction, this
                # warning goes off for non-emote rules. We should fix that.
                #
                # Maybe run an extraction phase that just edits rules a bit,
                # but doesn't combine them? Then this, and *then* combine things.
                print("WARNING: Selector %r split across PONYSCRIPT-IGNORE block!" % (selector))

        rule.ignore = ignoring

        # This goes down here, so that the END-PONYSCRIPT-IGNORE rule is itself
        # filtered out.
        if "END-PONYSCRIPT-IGNORE" in rule.selectors:
            ignoring = False

def extract_emote_blocks(css_rules):
    # Extracts partial emotes from a list of CSS rules.
    for rule in css_rules:
        alias_pairs = [_parse_emote_selector(s) for s in rule.selectors]
        if any(ap is None for ap in alias_pairs) and any(ap is not None for ap in alias_pairs):
            print("WARNING: CSS rule has both emotes and non-emote selectors")

        for pair in alias_pairs:
            if pair is None:
                continue

            name, suffix = pair

            # Copy CSS so it's not read-only
            yield bplib.objects.EmoteCSSBlock(name, suffix, rule.properties.copy(), rule.ignore)

_selector_regexp = re.compile(r"""
    a(?P<pc1>:[\w\-()]+)?
    \[href[|^]?=(?P<quote>["'])(?P<name>\/[\w:!#\/]+)(?P=quote)\] # Match quotes
    (?P<pc2>:[\w:\-()]+)?\s* # Accept : to get basically any non-space thing
    (?P<sel>[\w\s:\-()]*)$
    """, re.VERBOSE)

def _parse_emote_selector(selector):
    match = re.match(_selector_regexp, selector)
    if match is None:
        return None

    pc1 = match.groupdict()["pc1"] or ""
    name = match.groupdict()["name"]
    pc2 = match.groupdict()["pc2"] or ""
    sel = match.groupdict()["sel"].strip()

    # Generally, pc1 will be empty, but some emotes put :hover there. pc2 is
    # what we're chiefly interested in, and sel is used by image macros and
    # such.

    def verify_psuedo_class(pc):
        if pc == ":nth-of-type(n)":
            return "" # Drop this
        elif pc and pc not in (":hover", ":active"):
            print("WARNING: Unknown psuedo-class on %r" % (selector))
        return pc

    pc1 = verify_psuedo_class(pc1)
    pc2 = verify_psuedo_class(pc2)
    suffix = pc1 + pc2
    if sel:
        suffix += " " + sel

    return (name, suffix.strip())

def combine_emote_blocks(emote_blocks):
    # Combines properties in partial emotes, producing a single emote map.
    emotes = {}
    for block in emote_blocks:
        if block.name not in emotes:
            # Newly seen emote
            emotes[block.name] = (block.ignore, {block.suffix: block})
        else:
            ignore, variants = emotes[block.name]

            if ignore ^ block.ignore:
                print("WARNING: Emote %s split across PONYSCRIPT-IGNORE block" % (block.name))
            ignore |= block.ignore

            if block.suffix not in variants:
                # New suffix for an existing emote
                variants[block.suffix] = block
            else:
                # Existing emote. Check for property overwrites
                base = variants[block.suffix]
                for (prop, value) in block.css.items():
                    if prop in base.css and bplib.css.prop(base.css[prop]) != bplib.css.prop(value):
                        base_important = "!important" in base.css[prop]
                        block_important = "!important" in value

                        if base_important and not block_important:
                            print("WARNING: emote %r redefined property %r from base value %r to %r, keeping base" % (
                                bplib.combine_name_pair(block.name, block.suffix), prop, base.css[prop], value))
                        else:
                            print("WARNING: emote %r redefined property %r from base value %r to %r, using new value" % (
                                bplib.combine_name_pair(block.name, block.suffix), prop, base.css[prop], value))
                            base.css[prop] = value
                    else:
                        base.css[prop] = value

            # Update ignore
            emotes[block.name] = (ignore, variants)
    return emotes

def classify_emotes(emote_data):
    # Sorts emotes based on whether or not they are "normal" emotes belonging to
    # a spritesheet, or "custom" ones possessing only arbitrary CSS.

    emotes = {}
    for (name, (ignore, variants)) in emote_data.items():
        vardata = {}
        for (suffix, block) in variants.items():
            # Required properties (background-position is semi-required). One
            # subreddit uses background instead of background-image.
            if all(k in block.css for k in ("width", "height")) and ("background-image" in block.css or "background" in block.css):
                # Probably an emote. We could check for expected values of display/
                # clear/float, but they're broken in a lot of places, and not worth
                # the resulting warning spam.
                vardata[suffix] = _convert_emote(name, suffix, block)
            else:
                # Replace one class with another, essentially
                vardata[suffix] = bplib.objects.CustomVariant(name, suffix, block.css)
        emotes[name] = bplib.objects.Emote(name, vardata, {}, ignore)
    return emotes

IGNORED_PROPERTIES = [
    "transform",
    "-moz-transform",
    "-webkit-transform",
    "-ms-transform",
    "-o-transform",

    "animation",
    "-moz-animation",
    "-webkit-animation",
    "-o-animation",

    "cursor"
]

def _convert_emote(name, suffix, block):
    css = block.css.copy()

    for (prop, expected_value) in [("float", "left")]:
        if prop not in css:
            print("WARNING: %r is missing %r property" % (bplib.combine_name_pair(name, suffix), prop))
        else:
            if css[prop] != expected_value:
                print("WARNING: %r has unexpected value %r for property %r (expected %r)" % (
                    bplib.combine_name_pair(name, suffix), css[prop], prop, expected_value))

            del css[prop]

    width = bplib.css.as_size(css.pop("width"))
    height = bplib.css.as_size(css.pop("height"))


#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
""" buildbase.py.

provides a base class for fx desktop builds
author: Jordan Lund

"""
import json

import os
import pprint
import subprocess
import time
import uuid
import copy
import glob
from itertools import chain

# import the power of mozharness ;)
import sys
from datetime import datetime
import re
from mozharness.base.config import BaseConfig, parse_config_file
from mozharness.base.log import ERROR, OutputParser, FATAL, WARNING
from mozharness.base.script import PostScriptRun
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.buildbot import BuildbotMixin, TBPL_STATUS_DICT, \
    TBPL_EXCEPTION, TBPL_RETRY, EXIT_STATUS_DICT, TBPL_WARNING, TBPL_SUCCESS, \
    TBPL_WORST_LEVEL_TUPLE, TBPL_FAILURE
from mozharness.mozilla.purge import PurgeMixin
from mozharness.mozilla.mock import MockMixin
from mozharness.mozilla.signing import SigningMixin
from mozharness.mozilla.mock import ERROR_MSGS as MOCK_ERROR_MSGS
from mozharness.mozilla.testing.errors import TinderBoxPrintRe
from mozharness.mozilla.testing.unittest import tbox_print_summary
from mozharness.mozilla.updates.balrog import BalrogMixin

AUTOMATION_EXIT_CODES = EXIT_STATUS_DICT.values()
AUTOMATION_EXIT_CODES.sort()

MISSING_CFG_KEY_MSG = "The key '%s' could not be determined \
Please add this to your config."

ERROR_MSGS = {
    'undetermined_repo_path': 'The repo could not be determined. \
Please make sure that either "repo" is in your config or, if \
you are running this in buildbot, "repo_path" is in your buildbot_config.',
    'comments_undetermined': '"comments" could not be determined. This may be \
because it was a forced build.',
    'src_mozconfig_path_not_found': '"abs_src_mozconfig" path could not be \
determined. Please make sure it is a valid path off of "abs_src_dir"',
    'tooltool_manifest_undetermined': '"tooltool_manifest_src" not set, \
Skipping run_tooltool...',
}
ERROR_MSGS.update(MOCK_ERROR_MSGS)

### Output Parsers

TBPL_UPLOAD_ERRORS = [
    {
        'regex': re.compile("Connection timed out"),
        'level': TBPL_RETRY,
    },
    {
        'regex': re.compile("Connection reset by peer"),
        'level': TBPL_RETRY,
    },
    {
        'regex': re.compile("Connection refused"),
        'level': TBPL_RETRY,
    }
]

class MakeUploadOutputParser(OutputParser):
    tbpl_error_list = TBPL_UPLOAD_ERRORS
    # let's create a switch case using name-spaces/dict
    # rather than a long if/else with duplicate code
    property_conditions = [
        # key: property name, value: condition
        ('symbolsUrl', "m.endswith('crashreporter-symbols.zip') or "
                       "m.endswith('crashreporter-symbols-full.zip')"),
        ('testsUrl', "m.endswith(('tests.tar.bz2', 'tests.zip'))"),
        ('unsignedApkUrl', "m.endswith('apk') and "
                           "'unsigned-unaligned' in m"),
        ('robocopApkUrl', "m.endswith('apk') and 'robocop' in m"),
        ('jsshellUrl', "'jsshell-' in m and m.endswith('.zip')"),
        ('partialMarUrl', "m.endswith('.mar') and '.partial.' in m"),
        ('completeMarUrl', "m.endswith('.mar')"),
    ]

    def __init__(self, **kwargs):
        super(MakeUploadOutputParser, self).__init__(**kwargs)
        self.matches = {}
        self.tbpl_status = TBPL_SUCCESS

    def parse_single_line(self, line):
        prop_assigned = False
        pat = r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$'''
        m = re.compile(pat).match(line)
        if m:
            m = m.group(1)
            for prop, condition in self.property_conditions:
                if eval(condition):
                    self.matches[prop] = m
                    prop_assigned = True
                    break
            if not prop_assigned:
                # if we found a match but haven't identified the prop then this
                # is the packageURL. Let's consider this the else block
                self.matches['packageUrl'] = m

        # now let's check for retry errors which will give log levels:
        # tbpl status as RETRY and mozharness status as WARNING
        for error_check in self.tbpl_error_list:
            if error_check['regex'].search(line):
                self.num_warnings += 1
                self.warning(line)
                self.tbpl_status = self.worst_level(
                    error_check['level'], self.tbpl_status,
                    levels=TBPL_WORST_LEVEL_TUPLE
                )
                break
        else:
            self.info(line)

class CheckTestCompleteParser(OutputParser):
    tbpl_error_list = TBPL_UPLOAD_ERRORS

    def __init__(self, **kwargs):
        self.matches = {}
        super(CheckTestCompleteParser, self).__init__(**kwargs)
        self.pass_count = 0
        self.fail_count = 0
        self.leaked = False
        self.harness_err_re = TinderBoxPrintRe['harness_error']['full_regex']

    def parse_single_line(self, line):
        # Counts and flags.
        # Regular expression for crash and leak detections.
        if "TEST-PASS" in line:
            self.pass_count += 1
            return self.info(line)
        if "TEST-UNEXPECTED-" in line:
            # Set the error flags.
            # Or set the failure count.
            m = self.harness_err_re.match(line)
            if m:
                r = m.group(1)
                if r == "missing output line for total leaks!":
                    self.leaked = None
                else:
                    self.leaked = True
            else:
                self.fail_count += 1
            return self.warning(line)
        self.info(line)  # else

    def evaluate_parser(self):
        # Return the summary.
        summary = tbox_print_summary(self.pass_count,
                                     self.fail_count,
                                     self.leaked)
        self.info("TinderboxPrint: check<br/>%s\n" % summary)

class BuildingConfig(BaseConfig):
    # TODO add nosetests for this class
    def get_cfgs_from_files(self, all_config_files, parser):
        """ create a config based upon config files passed

        This is class specific. It recognizes certain config files
        by knowing how to combine them in an organized hierarchy
        """
        # override from BaseConfig

        # this is what we will return. It will represent each config
        # file name and its associated dict
        # eg ('builds/branch_specifics.py', {'foo': 'bar'})
        all_config_dicts = []
        # important config files
        variant_cfg_file = branch_cfg_file = pool_cfg_file = ''

        # we want to make the order in which the options were given
        # not matter. ie: you can supply --branch before --build-pool
        # or vice versa and the hierarchy will not be different

        #### The order from highest precedence to lowest is:
        ## There can only be one of these...
        # 1) build_pool: this can be either staging, pre-prod, and prod cfgs
        # 2) branch: eg: mozilla-central, cedar, cypress, etc
        # 3) build_variant: these could be known like asan and debug
        #                   or a custom config
        ##
        ## There can be many of these
        # 4) all other configs: these are any configs that are passed with
        #                       --cfg and --opt-cfg. There order is kept in
        #                       which they were passed on the cmd line. This
        #                       behaviour is maintains what happens by default
        #                       in mozharness
        ##
        ####

        # so, let's first assign the configs that hold a known position of
        # importance (1 through 3)
        for i, cf in enumerate(all_config_files):
            if parser.build_pool:
                if cf == BuildOptionParser.build_pools[parser.build_pool]:
                    pool_cfg_file = all_config_files[i]

            if cf == BuildOptionParser.branch_cfg_file:
                branch_cfg_file = all_config_files[i]

            if cf == parser.build_variant:
                variant_cfg_file = all_config_files[i]

        # now remove these from the list if there was any.
        # we couldn't pop() these in the above loop as mutating a list while
        # iterating through it causes spurious results :)
        for cf in [pool_cfg_file, branch_cfg_file, variant_cfg_file]:
            if cf:
                all_config_files.remove(cf)

        # now let's update config with the remaining config files.
        # this functionality is the same as the base class
        all_config_dicts.extend(
            super(BuildingConfig, self).get_cfgs_from_files(all_config_files,
                                                            parser)
        )

        # stack variant, branch, and pool cfg files on top of that,
        # if they are present, in that order
        if variant_cfg_file:
            # take the whole config
            all_config_dicts.append(
                (variant_cfg_file, parse_config_file(variant_cfg_file))
            )
        if branch_cfg_file:
            # take only the specific branch, if present
            branch_configs = parse_config_file(branch_cfg_file)
            if branch_configs.get(parser.branch or ""):
                all_config_dicts.append(
                    (branch_cfg_file, branch_configs[parser.branch])
                )
        if pool_cfg_file:
            # take only the specific pool. If we are here, the pool
            # must be present
            build_pool_configs = parse_config_file(pool_cfg_file)
            all_config_dicts.append(
                (pool_cfg_file, build_pool_configs[parser.build_pool])
            )
        return all_config_dicts

# noinspection PyUnusedLocal
class BuildOptionParser(object):
    # TODO add nosetests for this class
    platform = None
    bits = None
    config_file_search_path = [
        '.', os.path.join(sys.path[0], '..', 'configs'),
        os.path.join(sys.path[0], '..', '..', 'configs')
    ]

    # add to this list and you can automagically do things like
    # --custom-build-variant asan
    # and the script will pull up the appropriate path for the config
    # against the current platform and bits.
    # *It will warn and fail if there is not a config for the current
    # platform/bits
    build_variants = {
        'asan': 'builds/releng_sub_%s_configs/%s_asan.py',
        'debug': 'builds/releng_sub_%s_configs/%s_debug.py',
        'asan-and-debug': 'builds/releng_sub_%s_configs/%s_asan_and_debug.py',
        'stat-and-debug': 'builds/releng_sub_%s_configs/%s_stat_and_debug.py',
        'mulet': 'builds/releng_sub_%s_configs/%s_mulet.py',
        'code-coverage': 'builds/releng_sub_%s_configs/%s_code_coverage.py',
    }
    build_pools = {
        'staging': 'builds/build_pool_specifics.py',
        'production': 'builds/build_pool_specifics.py',
    }
    branch_cfg_file = 'builds/branch_specifics.py'

    @classmethod
    def _query_pltfrm_and_bits(cls, target_option, options):
        """ determine platform and bits

        This can be from either from a supplied --platform and --bits
        or parsed from given config file names.
        """
        error_msg = (
            'Whoops!\nYou are trying to pass a shortname for '
            '%s. \nHowever, I need to know the %s to find the appropriate '
            'filename. You can tell me by passing:\n\t"%s" or a config '
            'filename via "--config" with %s in it. \nIn either case, these '
            'option arguments must come before --custom-build-variant.'
        )
        current_config_files = options.config_files or []
        if not cls.bits:
            # --bits has not been supplied
            # lets parse given config file names for 32 or 64
            for cfg_file_name in current_config_files:
                if '32' in cfg_file_name:
                    cls.bits = '32'
                    break
                if '64' in cfg_file_name:
                    cls.bits = '64'
                    break
            else:
                sys.exit(error_msg % (target_option, 'bits', '--bits',
                                      '"32" or "64"'))

        if not cls.platform:
            # --platform has not been supplied
            # lets parse given config file names for platform
            for cfg_file_name in current_config_files:
                if 'windows' in cfg_file_name:
                    cls.platform = 'windows'
                    break
                if 'mac' in cfg_file_name:
                    cls.platform = 'mac'
                    break
                if 'linux' in cfg_file_name:
                    cls.platform = 'linux'
                    break
            else:
                sys.exit(error_msg % (target_option, 'platform', '--platform',
                                      '"linux", "windows", or "mac"'))
        return cls.bits, cls.platform

    @classmethod
    def set_build_variant(cls, option, opt, value, parser):
        """ sets an extra config file.

        This is done by either taking an existing filepath or by taking a valid
        shortname coupled with known platform/bits.
        """

        valid_variant_cfg_path = None
        # first let's see if we were given a valid short-name
        if cls.build_variants.get(value):
            bits, pltfrm = cls._query_pltfrm_and_bits(opt, parser.values)
            prospective_cfg_path = cls.build_variants[value] % (pltfrm, bits)
        else:
            # this is either an incomplete path or an invalid key in
            # build_variants
            prospective_cfg_path = value

        if os.path.exists(prospective_cfg_path):
            # now let's see if we were given a valid pathname
            valid_variant_cfg_path = value
        else:
            # let's take our prospective_cfg_path and see if we can
            # determine an existing file
            for path in cls.config_file_search_path:
                if os.path.exists(os.path.join(path, prospective_cfg_path)):
                    # success! we found a config file
                    valid_variant_cfg_path = os.path.join(path,
                                                          prospective_cfg_path)
                    break

        if not valid_variant_cfg_path:
            # either the value was an indeterminable path or an invalid short
            # name
            sys.exit("Whoops!\n'--custom-build-variant' was passed but an "
                     "appropriate config file could not be determined. Tried "
                     "using: '%s' but it was either not:\n\t-- a valid "
                     "shortname: %s \n\t-- a valid path in %s \n\t-- a "
                     "valid variant for the given platform and bits." % (
                         prospective_cfg_path,
                         str(cls.build_variants.keys()),
                         str(cls.config_file_search_path)))
        parser.values.config_files.append(valid_variant_cfg_path)
        setattr(parser.values, option.dest, value)  # the pool

    @classmethod
    def set_build_pool(cls, option, opt, value, parser):
        if cls.build_pools.get(value):
            # first let's add the build pool file where there may be pool
            # specific keys/values. Then let's store the pool name
            parser.values.config_files.append(cls.build_pools[value])
            setattr(parser.values, option.dest, value)  # the pool
        else:
            sys.exit(
                "Whoops!\n--build-pool was passed with '%s' but only "
                "'%s' are valid options" % (value, str(cls.build_pools.keys()))
            )

    @classmethod
    def set_build_branch(cls, option, opt, value, parser):
        # first let's add the branch_specific file where there may be branch
        # specific keys/values. Then let's store the branch name we are using
        parser.values.config_files.append(cls.branch_cfg_file)
        setattr(parser.values, option.dest, value)  # the branch name

    @classmethod
    def set_platform(cls, option, opt, value, parser):
        cls.platform = value
        setattr(parser.values, option.dest, value)

    @classmethod
    def set_bits(cls, option, opt, value, parser):
        cls.bits = value
        setattr(parser.values, option.dest, value)

# this global depends on BuildOptionParser and therefore can not go at the
# top of the file
BUILD_BASE_CONFIG_OPTIONS = [
    [['--developer-run', '--skip-buildbot-actions'], {
        "action": "store_false",
        "dest": "is_automation",
        "default": True,
        "help": "If this is running outside of Mozilla's build"
                "infrastructure, use this option. It ignores actions"
                "that are not needed and adds config checks."}],
    [['--platform'], {
        "action": "callback",
        "callback": BuildOptionParser.set_platform,
        "type": "string",
        "dest": "platform",
        "help": "Sets the platform we are running this against"
                " valid values: 'windows', 'mac', 'linux'"}],
    [['--bits'], {
        "action": "callback",
        "callback": BuildOptionParser.set_bits,
        "type": "string",
        "dest": "bits",
        "help": "Sets which bits we are building this against"
                " valid values: '32', '64'"}],
    [['--custom-build-variant-cfg'], {
        "action": "callback",
        "callback": BuildOptionParser.set_build_variant,
        "type": "string",
        "dest": "build_variant",
        "help": "Sets the build type and will determine appropriate"
                " additional config to use. Either pass a config path"
                " or use a valid shortname from: "
                "%s" % (BuildOptionParser.build_variants.keys(),)}],
    [['--build-pool'], {
        "action": "callback",
        "callback": BuildOptionParser.set_build_pool,
        "type": "string",
        "dest": "build_pool",
        "help": "This will update the config with specific pool"
                " environment keys/values. The dicts for this are"
                " in %s\nValid values: staging or"
                " production" % ('builds/build_pool_specifics.py',)}],
    [['--branch'], {
        "action": "callback",
        "callback": BuildOptionParser.set_build_branch,
        "type": "string",
        "dest": "branch",
        "help": "This sets the branch we will be building this for."
                " If this branch is in branch_specifics.py, update our"
                " config with specific keys/values from that. See"
                " %s for possibilites" % (
                    BuildOptionParser.branch_cfg_file,
                )}],
    [['--enable-pgo'], {
        "action": "store_true",
        "dest": "pgo_build",
        "default": False,
        "help": "Sets the build to run in PGO mode"}],
    [['--enable-nightly'], {
        "action": "store_true",
        "dest": "nightly_build",
        "default": False,
        "help": "Sets the build to run in nightly mode"}],
    [['--who'], {
        "dest": "who",
        "default": '',
        "help": "stores who made the created the buildbot change."}],

]

def generate_build_ID():
    return time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))

def generate_build_UID():
    return uuid.uuid4().hex

class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
                  SigningMixin, MercurialScript):
    def __init__(self, **kwargs):
        # objdir is referenced in _query_abs_dirs() so let's make sure we
        # have that attribute before calling BaseScript.__init__
        self.objdir = None
        super(BuildScript, self).__init__(**kwargs)
        # epoch is only here to represent the start of the buildbot build
        # that this mozharn script came from. until I can grab bbot's
        # status.build.gettime()[0] this will have to do as a rough estimate
        # although it is about 4s off from the time it would be if it was
        # done through MBF.
        # TODO find out if that time diff matters or if we just use it to
        # separate each build
        self.epoch_timestamp = int(time.mktime(datetime.now().timetuple()))
        self.branch = self.config.get('branch')
        self.stage_platform = self.config.get('stage_platform')
        if not self.branch or not self.stage_platform:
            if not self.branch:
                self.error("'branch' not determined and is required")
            if not self.stage_platform:
                self.error("'stage_platform' not determined and is required")
            self.fatal("Please add missing items to your config")
        self.repo_path = None
        self.buildid = None
        self.builduid = None
        self.query_buildid()  # sets self.buildid
        self.query_builduid()  # sets self.builduid
        self.generated_build_props = False

    def _pre_config_lock(self, rw_config):
        c = self.config
        cfg_files_and_dicts = rw_config.all_cfg_files_and_dicts
        build_pool = c.get('build_pool', '')
        build_variant = c.get('build_variant', '')
        variant_cfg = ''
        if build_variant:
            variant_cfg = BuildOptionParser.build_variants[build_variant] % (
                BuildOptionParser.platform,
                BuildOptionParser.bits
            )
        build_pool_cfg = BuildOptionParser.build_pools.get(build_pool)
        branch_cfg = BuildOptionParser.branch_cfg_file

        cfg_match_msg = "Script was ran with '%(option)s %(type)s' and \
'%(type)s' matches a key in '%(type_config_file)s'. Updating self.config with \
items from that key's value."
        pf_override_msg = "The branch '%(branch)s' has custom behavior for the \
platform '%(platform)s'. Updating self.config with the following from \
'platform_overrides' found in '%(pf_cfg_file)s':"

        for i, (target_file, target_dict) in enumerate(cfg_files_and_dicts):
            if branch_cfg and branch_cfg in target_file:
                self.info(
                    cfg_match_msg % {
                        'option': '--branch',
                        'type': c['branch'],
                        'type_config_file': BuildOptionParser.branch_cfg_file
                    }
                )
            if build_pool_cfg and build_pool_cfg in target_file:
                self.info(
                    cfg_match_msg % {
                        'option': '--build-pool',
                        'type': build_pool,
                        'type_config_file': BuildOptionParser.build_pools[
                            build_pool
                        ]
                    }
                )
            if variant_cfg and variant_cfg in target_file:
                self.info(
                    cfg_match_msg % {
                        'option': '--custom-build-variant-cfg',
                        'type': build_variant,
                        'type_config_file': variant_cfg,
                    }
                )
        if c.get("platform_overrides"):
            if c['stage_platform'] in c['platform_overrides'].keys():
                self.info(
                    pf_override_msg % {
                        'branch': c['branch'],
                        'platform': c['stage_platform'],
                        'pf_cfg_file': BuildOptionParser.branch_cfg_file
                    }
                )
                branch_pf_overrides = c['platform_overrides'][
                    c['stage_platform']
                ]
                self.info(pprint.pformat(branch_pf_overrides))
                c.update(branch_pf_overrides)
        self.info('To generate a config file based upon options passed and '
                  'config files used, run script as before but extend options '
                  'with "--dump-config"')
        self.info('For a diff of where self.config got its items, '
                  'run the script again as before but extend options with: '
                  '"--dump-config-hierarchy"')
        self.info("Both --dump-config and --dump-config-hierarchy don't "
                  "actually run any actions.")

    def _assert_cfg_valid_for_action(self, dependencies, action):
        """ assert dependency keys are in config for given action.

        Takes a list of dependencies and ensures that each have an
        assoctiated key in the config. Displays error messages as
        appropriate.

        """
        # TODO add type and value checking, not just keys
        # TODO solution should adhere to: bug 699343
        # TODO add this to BaseScript when the above is done
        # for now, let's just use this as a way to save typing...
        c = self.config
        undetermined_keys = []
        err_template = "The key '%s' could not be determined \
and is needed for the action '%s'. Please add this to your config \
or run without that action (ie: --no-{action})"
        for dep in dependencies:
            if dep not in c:
                undetermined_keys.append(dep)
        if undetermined_keys:
            fatal_msgs = [err_template % (key, action)
                          for key in undetermined_keys]
            self.fatal("".join(fatal_msgs))
        # otherwise:
        return  # all good

    def _query_build_prop_from_app_ini(self, prop, app_ini_path=None):
        dirs = self.query_abs_dirs()
        print_conf_setting_path = os.path.join(dirs['abs_src_dir'],
                                               'config',
                                               'printconfigsetting.py')
        if not app_ini_path:
            # set the default
            app_ini_path = dirs['abs_app_ini_path']
        if (os.path.exists(print_conf_setting_path) and
                os.path.exists(app_ini_path)):
            cmd = [
                'python', print_conf_setting_path, app_ini_path,
                'App', prop
            ]
            return self.get_output_from_command(cmd, cwd=dirs['base_work_dir'])
        else:
            return None

    def query_builduid(self):
        c = self.config
        if self.builduid:
            return self.builduid

        builduid = None
        if c.get("is_automation"):
            if self.buildbot_config['properties'].get('builduid'):
                self.info("Determining builduid from buildbot properties")
                builduid = self.buildbot_config['properties']['builduid'].encode(
                    'ascii', 'replace'
                )

        if not builduid:
            self.info("Creating builduid through uuid hex")
            builduid = generate_build_UID()

        if c.get('is_automation'):
            self.set_buildbot_property('builduid',
                                       builduid,
                                       write_to_file=True)
        self.builduid = builduid
        return self.builduid

    def query_buildid(self):
        c = self.config
        if self.buildid:
            return self.buildid

        buildid = None
        if c.get("is_automation"):
            if self.buildbot_config['properties'].get('buildid'):
                self.info("Determining buildid from buildbot properties")
                buildid = self.buildbot_config['properties']['buildid'].encode(
                    'ascii', 'replace'
                )

        if not buildid:
            self.info("Creating buildid through current time")
            buildid = generate_build_ID()

        if c.get('is_automation'):
            self.set_buildbot_property('buildid',
                                       buildid,
                                       write_to_file=True)

        self.buildid = buildid
        return self.buildid

    def _query_objdir(self):
        if self.objdir:
            return self.objdir

        if not self.config.get('objdir'):
            return self.fatal(MISSING_CFG_KEY_MSG % ('objdir',))
        self.objdir = self.config['objdir']
        return self.objdir

    def _query_repo(self):
        if self.repo_path:
            return self.repo_path
        c = self.config

        # unlike b2g, we actually supply the repo in mozharness so if it's in
        #  the config, we use that (automation does not require it in
        # buildbot props)
        if not c.get('repo_path'):
            repo_path = 'projects/%s' % (self.branch,)
            self.info(
                "repo_path not in config. Using '%s' instead" % (repo_path,)
            )
        else:
            repo_path = c['repo_path']
        self.repo_path = '%s/%s' % (c['repo_base'], repo_path,)
        return self.repo_path

    def _skip_buildbot_specific_action(self):
        """ ignore actions from buildbot's infra."""
        self.info("This action is specific to buildbot's infrastructure")
        self.info("Skipping......")
        return

    def query_build_env(self, replace_dict=None, **kwargs):
        c = self.config

        if not replace_dict:
            replace_dict = {}
        # now let's grab the right host based off staging/production
        # symbol_server_host is defined in build_pool_specifics.py
        replace_dict.update({"symbol_server_host": c['symbol_server_host']})

        # let's evoke the base query_env and make a copy of it
        # as we don't always want every key below added to the same dict
        env = copy.deepcopy(
            super(BuildScript, self).query_env(replace_dict=replace_dict,
                                               **kwargs)
        )

        # first grab the buildid
        env['MOZ_BUILD_DATE'] = self.query_buildid()

        if self.query_is_nightly():
            env["IS_NIGHTLY"] = "yes"
            # in branch_specifics.py we might set update_channel explicitly
            if c.get('update_channel'):
                env["MOZ_UPDATE_CHANNEL"] = c['update_channel']
            else:  # let's just give the generic channel based on branch
                env["MOZ_UPDATE_CHANNEL"] = "nightly-%s" % (self.branch,)

        if self.config.get('pgo_build') or self._compile_against_pgo():
            env['MOZ_PGO'] = '1'

        if c.get('enable_signing'):
            moz_sign_cmd = subprocess.list2cmdline(
                self.query_moz_sign_cmd(formats=None)
            )
            # windows fix. This is passed to mach build env and we call that
            # with python, not with bash so we need to fix the slashes here
            env['MOZ_SIGN_CMD'] = moz_sign_cmd.replace('\\', '\\\\\\\\')

        # we can't make env an attribute of self because env can change on
        # every call for reasons like MOZ_SIGN_CMD
        return env

    def query_mach_build_env(self):
        c = self.config
        mach_env = {}
        if c.get('upload_env'):
            mach_env.update(c['upload_env'])
            mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % {
                'stage_server': c['stage_server']
            }
            mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % {
                'stage_username': c['stage_username']
            }
            mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % {
                'stage_ssh_key': c['stage_ssh_key']
            }

        if self.query_is_nightly():
            mach_env['LATEST_MAR_DIR'] = c['latest_mar_dir'] % {
                'branch': self.branch
            }

        # _query_post_upload_cmd returns a list (a cmd list), for env sake here
        # let's make it a string
        pst_up_cmd = ' '.join([str(i) for i in self._query_post_upload_cmd()])
        mach_env['POST_UPLOAD_CMD'] = pst_up_cmd

        return mach_env

    def _compile_against_pgo(self):
        """determines whether a build should be run with pgo even if it is
        not a classified as a 'pgo build'.

        requirements:
        1) must be a platform that can run against pgo
        2) either:
            a) must be a nightly build
            b) must be on a branch that runs pgo if it can everytime
        """
        c = self.config
        if self.stage_platform in c['pgo_platforms']:
            if c.get('branch_uses_per_checkin_strategy') or self.query_is_nightly():
                return True
        return False

    def query_check_test_env(self):
        c = self.config
        dirs = self.query_abs_dirs()
        check_test_env = {}
        if c.get('check_test_env'):
            for env_var, env_value in c['check_test_env'].iteritems():
                check_test_env[env_var] = env_value % dirs
        return check_test_env

    def _query_moz_symbols_buildid(self):
        # this is a bit confusing but every platform that make
        # uploadsymbols may or may not include a
        # MOZ_SYMBOLS_EXTRA_BUILDID in the env and the value of this
        # varies.
        # logic goes:
        #   If it's the release branch, we only include it for
        # 64bit platforms and we use just the platform as value.
        #   If it's a project branch off m-c, we include only the branch
        # for the value on 32 bit platforms and we include both the
        # platform and branch for 64 bit platforms
        c = self.config
        moz_symbols_extra_buildid = ''
        if c.get('use_platform_in_symbols_extra_buildid'):
            moz_symbols_extra_buildid += self.stage_platform
        if c.get('use_branch_in_symbols_extra_buildid'):
            if moz_symbols_extra_buildid:
                moz_symbols_extra_buildid += '-%s' % (self.branch,)
            else:
                moz_symbols_extra_buildid = self.branch
        return moz_symbols_extra_buildid

    def _query_who(self):
        """ looks for who triggered the build with a change.

        This is used for things like try builds where the upload dir is
        associated with who pushed to try. First it will look in self.config
        and failing that, will poll buildbot_config
        If nothing is found, it will default to returning "nobody@example.com"
        """
        _who = "nobody@example.com"
        if self.config.get('who'):
            _who = self.config['who']
        else:
            try:
                if self.buildbot_config:
                    _who = self.buildbot_config['sourcestamp']['changes'][0]['who']
            except (KeyError, IndexError):
                # KeyError: "sourcestamp" or "changes" or "who" not in buildbot_config
                # IndexError: buildbot_config['sourcestamp']['changes'] is empty
                # "who" is not available, using the default value
                pass
        return _who

    def _query_post_upload_cmd(self):
        c = self.config
        post_upload_cmd = ["post_upload.py"]
        buildid = self.query_buildid()
        revision = self.query_revision()
        platform = self.stage_platform
        who = self._query_who()
        if c.get('pgo_build'):
            platform += '-pgo'

        if c.get('tinderbox_build_dir'):
            # TODO find out if we should fail here like we are
            if not who and not revision:
                self.fatal("post upload failed. --tinderbox-builds-dir could "
                           "not be determined. 'who' and/or 'revision' unknown")
            # branches like try will use 'tinderbox_build_dir
            tinderbox_build_dir = c['tinderbox_build_dir'] % {
                'who': who,
                'got_revision': revision
            }
        else:
            # the default
            tinderbox_build_dir = "%s-%s" % (self.branch, platform)

        if who and self.branch == 'try':
            post_upload_cmd.extend(["--who", who])
        if c.get('include_post_upload_builddir'):
            post_upload_cmd.extend(
                ["--builddir", "%s-%s" % (self.branch, platform)]
            )
        post_upload_cmd.extend(["--tinderbox-builds-dir", tinderbox_build_dir])
        post_upload_cmd.extend(["-p", c['stage_product']])
        post_upload_cmd.extend(['-i', buildid])
        if revision:
            post_upload_cmd.extend(['--revision', revision])
        if c.get('to_tinderbox_dated'):
            post_upload_cmd.append('--release-to-tinderbox-dated-builds')
            post_upload_cmd.append('--release-to-latest-tinderbox-builds')
        if c.get('release_to_try_builds'):
            post_upload_cmd.append('--release-to-try-builds')
        if self.query_is_nightly():
            post_upload_cmd.extend(['-b', self.branch])
            post_upload_cmd.append('--release-to-dated')
            if c['platform_supports_post_upload_to_latest']:
                post_upload_cmd.append('--release-to-latest')
        return post_upload_cmd

    def _ccache_z(self):
        """clear ccache stats."""
        dirs = self.query_abs_dirs()
        env = self.query_build_env()
        self.run_command(command=['ccache', '-z'],
                         cwd=dirs['base_work_dir'],
                         env=env)

    def _ccache_s(self):
        """print ccache stats. only done for unix like platforms"""
        dirs = self.query_abs_dirs()
        env = self.query_build_env()
        cmd = ['ccache', '-s']
        self.run_command(cmd, cwd=dirs['abs_src_dir'], env=env)

    def _rm_old_package(self):
        """rm the old package."""
        c = self.config
        dirs = self.query_abs_dirs()
        old_package_paths = []
        old_package_patterns = c.get('old_packages')

        self.info("removing old packages...")
        if os.path.exists(dirs['abs_obj_dir']):
            for product in old_package_patterns:
                old_package_paths.extend(
                    glob.glob(product % {"objdir": dirs['abs_obj_dir']})
                )
        if old_package_paths:
            for package_path in old_package_paths:
                self.rmtree(package_path)
        else:
            self.info("There wasn't any old packages to remove.")

    def _get_mozconfig(self):
        """assign mozconfig."""
        c = self.config
        dirs = self.query_abs_dirs()
        if c.get('src_mozconfig'):
            self.info('Using in-tree mozconfig')
            abs_src_mozconfig = os.path.join(dirs['abs_src_dir'],
                                             c.get('src_mozconfig'))
            if not os.path.exists(abs_src_mozconfig):
                self.info('abs_src_mozconfig: %s' % (abs_src_mozconfig,))
                self.fatal(ERROR_MSGS['src_mozconfig_path_not_found'])
            self.copyfile(abs_src_mozconfig,
                          os.path.join(dirs['abs_src_dir'], '.mozconfig'))
            self.info("mozconfig content:")
            with open(abs_src_mozconfig) as mozconfig:
                next(mozconfig)
                for line in mozconfig:
                    self.info(line)
        else:
            self.fatal("To build, you must supply a mozconfig from inside the "
                       "tree to use use. Please provide the path in your "
                       "config via 'src_mozconfig'")

    # TODO add this or merge to ToolToolMixin
    def _run_tooltool(self):
        self._assert_cfg_valid_for_action(
            ['tooltool_script', 'tooltool_bootstrap', 'tooltool_url'],
            'build'
        )
        c = self.config
        dirs = self.query_abs_dirs()
        if not c.get('tooltool_manifest_src'):
            return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
        fetch_script_path = os.path.join(dirs['abs_tools_dir'],
                                         'scripts/tooltool/tooltool_wrapper.sh')
        tooltool_manifest_path = os.path.join(dirs['abs_src_dir'],
                                              c['tooltool_manifest_src'])
        cmd = [
            'sh',
            fetch_script_path,
            tooltool_manifest_path,
            c['tooltool_url'],
            c['tooltool_bootstrap'],
        ]
        cmd.extend(c['tooltool_script'])
        self.info(str(cmd))
        self.run_command(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True)

    def query_revision(self, source_path=None):
        """ returns the revision of the build

         first will look for it in buildbot_properties and then in
         buildbot_config. Failing that, it will actually poll the source of
         the repo if it exists yet.
        """
        # this is basically a copy from b2g_build.py
        # TODO get b2g_build.py to use this version of query_revision
        revision = None
        if 'revision' in self.buildbot_properties:
            revision = self.buildbot_properties['revision']
        elif (self.buildbot_config and
                  self.buildbot_config.get('sourcestamp', {}).get('revision')):
            revision = self.buildbot_config['sourcestamp']['revision']
        elif self.buildbot_config and self.buildbot_config.get('revision'):
            revision = self.buildbot_config['revision']
        else:
            if not source_path:
                dirs = self.query_abs_dirs()
                source_path = dirs['abs_src_dir']  # let's take the default

            # Look at what we have checked out
            if os.path.exists(source_path):
                hg = self.query_exe('hg', return_type='list')
                revision = self.get_output_from_command(
                    hg + ['parent', '--template', '{node|short}'], cwd=source_path
                )
        return revision[0:12].encode('ascii', 'replace') if revision else None

    def _checkout_source(self):
        """use vcs_checkout to grab source needed for build."""
        # TODO make this method its own action
        c = self.config
        dirs = self.query_abs_dirs()
        repo = self._query_repo()
        vcs_checkout_kwargs = {
            'repo': repo,
            'dest': dirs['abs_src_dir'],
            'revision': self.query_revision(),
            'env': self.query_build_env()
        }
        if c.get('clone_by_revision'):
            vcs_checkout_kwargs['clone_by_revision'] = True

        if c.get('clone_with_purge'):
            vcs_checkout_kwargs['clone_with_purge'] = True
        rev = self.vcs_checkout(**vcs_checkout_kwargs)
        if c.get('is_automation'):