<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import os
import re
import webapp2
import jinja2
import logging
import StringIO
import json
from apirdflib import load_graph, getNss, getRevNss

from markupsafe import Markup, escape # https://pypi.python.org/pypi/MarkupSafe

import parsers
import threading
import itertools
import datetime, time
from time import gmtime, strftime

from google.appengine.ext import ndb
from google.appengine.ext import blobstore
from google.appengine.api import users
from google.appengine.ext.webapp import blobstore_handlers
from google.appengine.api import modules
from google.appengine.api import runtime

from api import inLayer, read_file, full_path, read_schemas, read_extensions, read_examples, namespaces, DataCache, PageStore, HeaderStore
from api import Unit, GetTargets, GetSources, GetComments, GetsoftwareVersions
from api import GetComment, all_terms, GetAllTypes, GetAllProperties, GetAllEnumerationValues, GetAllTerms, LoadExamples
from api import GetParentList, GetImmediateSubtypes, HasMultipleBaseTypes
from api import GetJsonLdContext, ShortenOnSentence, StripHtmlTags, MD
from api import setInTestHarness, getInTestHarness, setAllLayersList

logging.basicConfig(level=logging.INFO) # dev_appserver.py --log_level debug .
log = logging.getLogger(__name__)

SCHEMA_VERSION=3.1

FEEDBACK_FORM_BASE_URL='https://docs.google.com/a/google.com/forms/d/1krxHlWJAO3JgvHRZV9Rugkr9VYnMdrI10xbGsWt733c/viewform?entry.1174568178&entry.41124795={0}&entry.882602760={1}'
# {0}: term URL, {1} category of term.

sitemode = "mainsite" # whitespaced list for CSS tags,
            # e.g. "mainsite testsite" when off expected domains
            # "extensionsite" when in an extension (e.g. blue?)

releaselog = { "2.0": "2015-05-13", "2.1": "2015-08-06", "2.2": "2015-11-05", "3.0": "2016-05-04", "3.1": "2016-06-09" }

silent_skip_list =  [ "favicon.ico" ] # Do nothing for now

all_layers = {}
ext_re = re.compile(r'([^\w,])+')

#TODO: Modes:
# mainsite
# webschemadev
# known extension (not skiplist'd, eg. demo1 on schema.org)

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
    extensions=['jinja2.ext.autoescape'], autoescape=True, cache_size=0)

ENABLE_JSONLD_CONTEXT = True
ENABLE_CORS = True
ENABLE_HOSTED_EXTENSIONS = True

#INTESTHARNESS = True #Used to indicate we are being called from tests - use setInTestHarness() & getInTestHarness() to manage value

EXTENSION_SUFFIX = "" # e.g. "*"

ENABLED_EXTENSIONS = ['auto', 'bib', 'health-lifesci', 'pending', 'meta', 'iot'  ]
ALL_LAYERS = ['core',""]

ALL_LAYERS += ENABLED_EXTENSIONS
setAllLayersList(ALL_LAYERS)


FORCEDEBUGGING = False
# FORCEDEBUGGING = True

SHAREDSITEDEBUG = True
if getInTestHarness():
    SHAREDSITEDEBUG = False

############# Warmup Control ########
WarmedUp = False
WarmupState = "Auto"
if "WARMUPSTATE" in os.environ:
    WarmupState = os.environ["WARMUPSTATE"]
log.info("WarmupState: %s" % WarmupState)
<commit_msg>Removed draft iot.schema.org<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import os
import re
import webapp2
import jinja2
import logging
import StringIO
import json
from apirdflib import load_graph, getNss, getRevNss

from markupsafe import Markup, escape # https://pypi.python.org/pypi/MarkupSafe

import parsers
import threading
import itertools
import datetime, time
from time import gmtime, strftime

from google.appengine.ext import ndb
from google.appengine.ext import blobstore
from google.appengine.api import users
from google.appengine.ext.webapp import blobstore_handlers
from google.appengine.api import modules
from google.appengine.api import runtime

from api import inLayer, read_file, full_path, read_schemas, read_extensions, read_examples, namespaces, DataCache, PageStore, HeaderStore
from api import Unit, GetTargets, GetSources, GetComments, GetsoftwareVersions
from api import GetComment, all_terms, GetAllTypes, GetAllProperties, GetAllEnumerationValues, GetAllTerms, LoadExamples
from api import GetParentList, GetImmediateSubtypes, HasMultipleBaseTypes
from api import GetJsonLdContext, ShortenOnSentence, StripHtmlTags, MD
from api import setInTestHarness, getInTestHarness, setAllLayersList

logging.basicConfig(level=logging.INFO) # dev_appserver.py --log_level debug .
log = logging.getLogger(__name__)

SCHEMA_VERSION=3.1

FEEDBACK_FORM_BASE_URL='https://docs.google.com/a/google.com/forms/d/1krxHlWJAO3JgvHRZV9Rugkr9VYnMdrI10xbGsWt733c/viewform?entry.1174568178&entry.41124795={0}&entry.882602760={1}'
# {0}: term URL, {1} category of term.

sitemode = "mainsite" # whitespaced list for CSS tags,
            # e.g. "mainsite testsite" when off expected domains
            # "extensionsite" when in an extension (e.g. blue?)

releaselog = { "2.0": "2015-05-13", "2.1": "2015-08-06", "2.2": "2015-11-05", "3.0": "2016-05-04", "3.1": "2016-06-09" }

silent_skip_list =  [ "favicon.ico" ] # Do nothing for now

all_layers = {}
ext_re = re.compile(r'([^\w,])+')

#TODO: Modes:
# mainsite
# webschemadev
# known extension (not skiplist'd, eg. demo1 on schema.org)

JINJA_ENVIRONMENT = jinja2.Environment(
    loader=jinja2.FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')),
    extensions=['jinja2.ext.autoescape'], autoescape=True, cache_size=0)

ENABLE_JSONLD_CONTEXT = True
ENABLE_CORS = True
ENABLE_HOSTED_EXTENSIONS = True

#INTESTHARNESS = True #Used to indicate we are being called from tests - use setInTestHarness() & getInTestHarness() to manage value

EXTENSION_SUFFIX = "" # e.g. "*"

ENABLED_EXTENSIONS = ['auto', 'bib', 'health-lifesci', 'pending', 'meta'  ]
ALL_LAYERS = ['core',""]

ALL_LAYERS += ENABLED_EXTENSIONS
setAllLayersList(ALL_LAYERS)


FORCEDEBUGGING = False
# FORCEDEBUGGING = True

SHAREDSITEDEBUG = True
if getInTestHarness():
    SHAREDSITEDEBUG = False

############# Warmup Control ########
WarmedUp = False
WarmupState = "Auto"
if "WARMUPSTATE" in os.environ:
    WarmupState = os.environ["WARMUPSTATE"]
log.info("WarmupState: %s" % WarmupState)

{'fraction_non_alphanumeric': 0.11723117395593555, 'fraction_numerical': 0.03288391976323578, 'mean_word_length': 5.583333333333333, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5337999', 'n_tokens_mistral': 2272, 'n_tokens_neox': 2145, 'n_words': 511}