<commit_before>#!/usr/bin/env python
"""
Simple tornado app to list and read files.

Start server:
python tornado.py -port=8001

(Default port is 8000)

Using service

/svc
/svc/list/path/to/dir
/svc/read/path/to/file
/svc/filter/path/to/file?ids=1,2

I'm not really sure what filter is supposed to do so i may have gotten the matching wrong.

All services return -1 if there is any error.

"""
import tornado.ioloop
from tornado.options import define, options, logging
import tornado.web
import qedconf
import os


define("port", default=8000, help="run on the given port", type=int)

settings = {
    "debug": True,
}

server_settings = {
    "xheaders" : True,
    "address" : "0.0.0.0"
}

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.write("Service Root")

class ReadHandler(tornado.web.RequestHandler):
    def get(self,filepath):
        try:
            rfile = open(qedconf.BASE_PATH + filepath)
            self.write(rfile.read())
            rfile.close()
        except:
            self.write("-1")

class FilterHandler(tornado.web.RequestHandler):
    def get(self,filepath):
        try:
            ids=self.get_argument("ids")
           
            if not ids is None: 
                ids = ids.split(",")

            rfile = open(qedconf.BASE_PATH + filepath)
            for idx, line in enumerate(rfile):
                if idx == 0:
                    self.write(line.rstrip())
                else:
                    for id in ids:
                        if id in line:
                            self.write(line.rstrip())
                            break
        except:
            self.write("-1")

class ListHandler(tornado.web.RequestHandler):
    def get(self,filepath):
        try:
            dirs=[]
            files=[]
            for f in os.listdir(qedconf.BASE_PATH + filepath):
                if not f.startswith("."):
                    label = os.path.basename(f)
                    item = '{"label":"%s", "uri":"%s"}' % (label, os.path.join(filepath,f))

                    if os.path.isdir(qedconf.BASE_PATH + filepath + f):
                        dirs.append(item)
                    else:
                        files.append(item)

            self.write('{ "directories": [' + ",".join(dirs) + '], "files": [' + ",".join(files) + '] }')
        except:
            self.write("-1")



def main():
    tornado.options.parse_command_line()
    logging.info("Starting Tornado web server on http://localhost:%s" % options.port)
    application = tornado.web.Application([
         (r"/", MainHandler),
        (r"/list?(.*)", ListHandler),
        (r"/filter?(.*)", FilterHandler),
        (r"/read?(.*)", ReadHandler),
    ], **settings)
    application.listen(options.port, **server_settings)
    tornado.ioloop.IOLoop.instance().start()

if __name__ == "__main__":
    main()<commit_msg>Simplified interface to tornado service<commit_after>#!/usr/bin/env python
"""
Simple tornado app to list and read files.

Start server:
python tornado.py -port=8001

(Default port is 8000)

Using service

/
List:
/data/path/to/dir
Read:
/data/path/to/file
Filter:
/data/path/to/file?ids=1,2

I'm not really sure what filter is supposed to do so i may have gotten the matching wrong.

All services return -1 if there is any error.

"""
import tornado.ioloop
from tornado.options import define, options, logging
import tornado.web
import qedconf
import os


define("port", default=8000, help="run on the given port", type=int)

settings = {
    "debug": True,
}

server_settings = {
    "xheaders" : True,
    "address" : "0.0.0.0"
}

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.write("Service Root")


class FilterHandler(tornado.web.RequestHandler):
    def get(self,filepath):
        try:
            ids=self.get_arguments("ids")

           
            if len(ids) > 0: 
                ids = ids.split(",")

                rfile = open(qedconf.BASE_PATH + filepath)
                for idx, line in enumerate(rfile):
                    if idx == 0:
                        self.write(line.rstrip())
                    else:
                        for id in ids:
                            if id in line:
                                self.write(line.rstrip())
                                break
            elif os.path.isdir(qedconf.BASE_PATH + filepath):
                dirs=[]
                files=[]
                for f in os.listdir(qedconf.BASE_PATH + filepath):
                    if not f.startswith("."):
                        label = os.path.basename(f)
                        item = '{"label":"%s", "uri":"%s"}' % (label, os.path.join(filepath,f))

                        if os.path.isdir(qedconf.BASE_PATH + filepath + f):
                            dirs.append(item)
                        else:
                            files.append(item)

                self.write('{ "directories": [' + ",".join(dirs) + '], "files": [' + ",".join(files) + '] }')
            else:
                rfile = open(qedconf.BASE_PATH + filepath)
                self.write(rfile.read())
                rfile.close()

        except:
            self.write("-1")




def main():
    tornado.options.parse_command_line()
    logging.info("Starting Tornado web server on http://localhost:%s" % options.port)
    application = tornado.web.Application([
        (r"/", MainHandler),
        (r"/data?(.*)", FilterHandler),
    ], **settings)
    application.listen(options.port, **server_settings)
    tornado.ioloop.IOLoop.instance().start()

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.13398576512455515, 'fraction_numerical': 0.00800711743772242, 'mean_word_length': 1.8119059529764883, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6590400', 'n_tokens_mistral': 1765, 'n_tokens_neox': 1672, 'n_words': 393}