<commit_before>#!/usr/bin/env python3

"""
twloadworld: Copyright (c) 2013, <NAME>
(Available under the MIT License; see LICENSE file.)

This script reads a world definition file and pushes it into the database.
This is an administrator tool; it does no permission checking and can
modify or overwrite any world.

I built this as a temporary measure, awaiting a full-fledged world-creation
interface. However, I suspect it will remain useful for various cases
(wiping and rebuilding a Tworld database, etc).
"""

import sys
import os
import datetime
import ast

import bson
from bson.objectid import ObjectId
import pymongo

import tornado.options

# Set up all the options. (Generally found in the config file.)

# Clever hack to parse a config file off the command line.
tornado.options.define(
    'config', type=str,
    help='configuration file',
    callback=lambda path: tornado.options.parse_config_file(path, final=False))

tornado.options.define(
    'python_path', type=str,
    help='Python modules directory (optional)')

tornado.options.define(
    'mongo_database', type=str, default='tworld',
    help='name of mongodb database')

###
#tornado.options.define(
#    'removeworld', type=bool,
#    help='remove world completely')

tornado.options.define(
    'remove', type=bool,
    help='only remove the named room or room.prop')

tornado.options.define(
    'display', type=bool,
    help='only display the named room or room.prop')

tornado.options.define(
    'check', type=bool,
    help='only check consistency of the file')

# Parse 'em up.
args = tornado.options.parse_command_line()
opts = tornado.options.options

# But tornado options don't support options after arguments. Hack to work
# around this.
if '--check' in args:
    args.remove('--check')
    opts.check = True
if '--display' in args:
    args.remove('--display')
    opts.display = True
if '--remove' in args:
    args.remove('--remove')
    opts.remove = True
if '--removeworld' in args:
    args.remove('--removeworld')
    opts.removeworld = True

if opts.python_path:
    sys.path.insert(0, opts.python_path)

import two.interp
from two.interp import sluggify

if not args:
    print('usage: twloadworld.py worldfile [ room ... or room.prop ... ]')
    sys.exit(-1)

class World(object):
    portlist_define_order = 0    # yes, a hack
    
    def __init__(self):
        self.creator = 'Admin'
        self.wid = None
        self.name = None
        self.copyable = True
        self.instancing = 'standard'
        self.props = {}
        self.proplist = []
        self.locations = {}
        self.locationlist = []
        self.portals = {}

    def check_symbols_used(self):
        self.symbolsused = set()
        all_interptext_props = []
        all_code_props = []
        for (key, propval) in self.props.items():
            if is_interp_text(propval):
                all_interptext_props.append( (propval['text'], None) )
            if is_code(propval):
                all_code_props.append( (key, propval['text'], None) )
            if is_move(propval):
                if propval['loc'] not in self.locations:
                    print('Warning: move prop "%s" in %s goes to undefined loc: %s' % (key, None, propval['loc']))
        for (lockey, loc) in self.locations.items():
            for (key, propval) in loc.props.items():
                if is_interp_text(propval):
                    all_interptext_props.append( (propval['text'], lockey) )
                if is_code(propval):
                    all_code_props.append( (key, propval['text'], lockey) )
                if is_move(propval):
                    if propval['loc'] not in self.locations:
                        print('Warning: move prop "%s" in %s goes to undefined loc: %s' %(key, lockey, propval['loc']))
            
        for (key, text, lockey) in all_code_props:
            try:
                ast.parse(text, filename='%s.%s' % (lockey, key,))
            except Exception as ex:
                print('Warning: code prop "%s" in %s does not parse: %s' % (key, lockey, ex))

        for (text, lockey) in all_interptext_props:
            for nod in two.interp.parse(text):
                if isinstance(nod, two.interp.Link):
                    self.symbolsused.add( (nod.target, lockey) )
                if isinstance(nod, two.interp.Interpolate):
                    self.symbolsused.add( (nod.expr, lockey) )

        for (symbol, lockey) in self.symbolsused:
            if lockey is None:
                loc = None
            else:
                loc = self.locations[lockey]
            if loc and symbol in loc.props:
                continue
            if symbol in self.props:
                continue<commit_msg>Check for keyword-incompatible property names!<commit_after>#!/usr/bin/env python3

"""
twloadworld: Copyright (c) 2013, <NAME>
(Available under the MIT License; see LICENSE file.)

This script reads a world definition file and pushes it into the database.
This is an administrator tool; it does no permission checking and can
modify or overwrite any world.

I built this as a temporary measure, awaiting a full-fledged world-creation
interface. However, I suspect it will remain useful for various cases
(wiping and rebuilding a Tworld database, etc).
"""

import sys
import os
import datetime
import ast
import keyword

import bson
from bson.objectid import ObjectId
import pymongo

import tornado.options

# Set up all the options. (Generally found in the config file.)

# Clever hack to parse a config file off the command line.
tornado.options.define(
    'config', type=str,
    help='configuration file',
    callback=lambda path: tornado.options.parse_config_file(path, final=False))

tornado.options.define(
    'python_path', type=str,
    help='Python modules directory (optional)')

tornado.options.define(
    'mongo_database', type=str, default='tworld',
    help='name of mongodb database')

###
#tornado.options.define(
#    'removeworld', type=bool,
#    help='remove world completely')

tornado.options.define(
    'remove', type=bool,
    help='only remove the named room or room.prop')

tornado.options.define(
    'display', type=bool,
    help='only display the named room or room.prop')

tornado.options.define(
    'check', type=bool,
    help='only check consistency of the file')

# Parse 'em up.
args = tornado.options.parse_command_line()
opts = tornado.options.options

# But tornado options don't support options after arguments. Hack to work
# around this.
if '--check' in args:
    args.remove('--check')
    opts.check = True
if '--display' in args:
    args.remove('--display')
    opts.display = True
if '--remove' in args:
    args.remove('--remove')
    opts.remove = True
if '--removeworld' in args:
    args.remove('--removeworld')
    opts.removeworld = True

if opts.python_path:
    sys.path.insert(0, opts.python_path)

import two.interp
from two.interp import sluggify

if not args:
    print('usage: twloadworld.py worldfile [ room ... or room.prop ... ]')
    sys.exit(-1)

class World(object):
    portlist_define_order = 0    # yes, a hack
    
    def __init__(self):
        self.creator = 'Admin'
        self.wid = None
        self.name = None
        self.copyable = True
        self.instancing = 'standard'
        self.props = {}
        self.proplist = []
        self.locations = {}
        self.locationlist = []
        self.portals = {}

    def check_symbols_used(self):
        self.symbolsused = set()
        all_interptext_props = []
        all_code_props = []
        for (key, propval) in self.props.items():
            if keyword.iskeyword(key):
                print('Warning: prop "%s" in %s is a keyword' % (key, None))
            if is_interp_text(propval):
                all_interptext_props.append( (propval['text'], None) )
            if is_code(propval):
                all_code_props.append( (key, propval['text'], None) )
            if is_move(propval):
                if propval['loc'] not in self.locations:
                    print('Warning: move prop "%s" in %s goes to undefined loc: %s' % (key, None, propval['loc']))
        for (lockey, loc) in self.locations.items():
            for (key, propval) in loc.props.items():
                if keyword.iskeyword(key):
                    print('Warning: prop "%s" in %s is a keyword' % (key, lockey))
                if is_interp_text(propval):
                    all_interptext_props.append( (propval['text'], lockey) )
                if is_code(propval):
                    all_code_props.append( (key, propval['text'], lockey) )
                if is_move(propval):
                    if propval['loc'] not in self.locations:
                        print('Warning: move prop "%s" in %s goes to undefined loc: %s' %(key, lockey, propval['loc']))
            
        for (key, text, lockey) in all_code_props:
            try:
                ast.parse(text, filename='%s.%s' % (lockey, key,))
            except Exception as ex:
                print('Warning: code prop "%s" in %s does not parse: %s' % (key, lockey, ex))

        for (text, lockey) in all_interptext_props:
            for nod in two.interp.parse(text):
                if isinstance(nod, two.interp.Link):
                    self.symbolsused.add( (nod.target, lockey) )
                if isinstance(nod, two.interp.Interpolate):
                    self.symbolsused.add( (nod.expr, lockey) )

        for (symbol, lockey) in self.symbolsused:
            if lockey is None:
                loc = None
            else:
                loc = self.locations[lockey]
            if loc and symbol in loc.props:
                continue
            if symbol in self.props:
                continue
{'fraction_non_alphanumeric': 0.1138695116947066, 'fraction_numerical': 0.0016413623307345096, 'mean_word_length': 2.3617241379310343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3253764', 'n_tokens_mistral': 2964, 'n_tokens_neox': 2854, 'n_words': 879}