<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:        sfcli
# Purpose:     Command Line Interface for SpiderFoot.
#
# Author:      <NAME> <<EMAIL>>
#
# Created:     03/05/2017
# Copyright:   (c) <NAME> 2017
# Licence:     GPL
# -------------------------------------------------------------------------------

import sys
import os
import re
import json
import requests
import shlex
import codecs
import time
import readline
import cmd
import argparse
from os.path import expanduser

# Colors to make things purty
class bcolors:
    GREYBLUE = '\x1b[38;5;25m'
    GREY = '\x1b[38;5;243m'
    DARKRED = '\x1b[38;5;124m'
    DARKGREEN = '\x1b[38;5;30m'
    BOLD = '\033[1m'
    ENDC = '\033[0m'
    GREYBLUE_DARK = '\x1b[38;5;24m'

class SpiderFootCli(cmd.Cmd):
    version = "2.10"
    pipecmd = None
    output = None
    modules = []
    types = []
    prompt = "sf> "
    nohelp = "[!] Unknown command '%s'."
    knownscans = []
    ownopts = {
        "cli.debug": False,
        "cli.silent": False,
        "cli.color": True,
        "cli.output": "pretty",
        "cli.history": True,
        "cli.history_file": "",
        "cli.spool": False,
        "cli.spool_file": "",
        "cli.username": "",
        "cli.password": "",
        "cli.server_baseurl": "http://127.0.0.1:5001"
    }

    def default(self, line):
        if line.startswith('#'):
            return

        self.edprint("Unknown command")

    # Auto-complete for these commands
    def complete_start(self, text, line, startidx, endidx):
        return self.complete_default(text, line, startidx, endidx)

    def complete_find(self, text, line, startidx, endidx):
        return self.complete_default(text, line, startidx, endidx)

    def complete_data(self, text, line, startidx, endidx):
        return self.complete_default(text, line, startidx, endidx)

    # Command completion for arguments
    def complete_default(self, text, line, startidx, endidx):
        ret = list()
        if "-m" in line and line.find("-m") > line.find("-t"):
            for m in self.modules:
                if m.startswith(text):
                    ret.append(m)

        if "-t" in line and line.find("-t") > line.find("-m"):
            for t in self.types:
                if t.startswith(text):
                    ret.append(t)
        return ret

    def dprint(self, msg, err=False, deb=False, plain=False, color=None):
        cout = ""
        sout = ""
        pfx = ""
        if err:
            pfx = "[!]"
            if self.ownopts['cli.color']:
                col = bcolors.DARKRED
        else:
            pfx = "[*]"
            if self.ownopts['cli.color']:
                col = bcolors.DARKGREEN
        if deb:
            if not self.ownopts["cli.debug"]:
                return
            pfx = "[+]"
            if self.ownopts['cli.color']:
                col = bcolors.GREY

        if color:
            pfx = ""
            col = color

        if err or not self.ownopts["cli.silent"]:
            if not plain or color:
                cout = col + bcolors.BOLD + pfx + " " + bcolors.ENDC + \
                       col +  msg + bcolors.ENDC
                # Never include color in the spool
                sout = pfx + " " + msg
            else:
                cout = msg
                sout = msg

            print cout

        if self.ownopts['cli.spool']:
            f = codecs.open(self.ownopts['cli.spool_file'], "a", encoding="utf-8")
            f.write(sout)
            f.write('\n')
            f.close()

    # Shortcut commands
    def do_debug(self, line):
        """debug
        Short-cut command for set cli.debug = 1"""
        if self.ownopts['cli.debug']:
            val = "0"
        else:
            val = "1"
        return self.do_set("cli.debug = " + val)

    def do_spool(self, line):
        """spool
        Short-cut command for set cli.spool = 1/0"""
        if self.ownopts['cli.spool']:
            val = "0"
        else:
            val = "1"
        if self.ownopts['cli.spool_file']:
            return self.do_set("cli.spool = " + val)
        else:
            self.edprint("You haven't set cli.spool_file. Set that before enabling spooling.")

    def do_history(self, line):
        """history [-l]
        Short-cut command for set cli.history = 1/0. Add -l to just list the history."""
        c = self.myparseline(line)

        if '-l' in c[0]:
            i = 0
            while i < readline.get_current_history_length():
                self.dprint(readline.get_history_item(i), plain=True)
                i += 1
            return
        if self.ownopts['cli.history']:
            val = "0"
        else:
            val = "1"
        return self.do_set("cli.history = " + val)

    # Run before all commands to handle history and spooling
    def precmd(self, line):
        if self.ownopts['cli.history'] and line != "EOF":
            f = codecs.open(self.ownopts["cli.history_file"], "a", encoding="utf-8")
            f.write(line)
            f.write('\n')
            f.close()
        if self.ownopts['cli.spool']:
            f = codecs.open(self.ownopts["cli.spool_file"], "a", encoding="utf-8")
            f.write(self.prompt + line)
            f.write('\n')
            f.close()

        return line

    # Debug print
    def ddprint(self, msg):
        self.dprint(msg, deb=True)

    # Error print
    def edprint(self, msg):
        self.dprint(msg, err=True)

    # Print nice tables.
    def pretty(self, data, titlemap=None):
        out = list()
        # Get the column titles
        maxsize = dict()
        if type(data[0]) == dict:
            cols = data[0].keys()
        else:
            # for lists, use the index numbers as titles
            cols = map(str, range(0, len(data[0])))

        # Strip out columns that don't have titles
        if titlemap:
            nc = list()
            for c in cols:
                if c in titlemap:
                    nc.append(c)
            cols = nc

        spaces = 2
        # Find the maximum column sizes
        for r in data:
            i = 0
            for c in r:
                if type(r) == list:
                    # we have  list index
                    cn = str(i)
                    if type(c) == int:
                        v = str(c)
                    if type(c) == str:
                        v = unicode(c, errors='replace')
                    if type(c) == unicode:
                        v = c
                else:
                    # we have a dict key
                    cn = c
                    v = r[c]
                #print str(cn) + ", " + str(c) + ", " + str(v)
                if len(v) > maxsize.get(cn, 0):
                    maxsize[cn] = len(v)
                i += 1


        # Adjust for long titles
        if titlemap:
            for c in maxsize:
                if len(titlemap.get(c, c)) > maxsize[c]:
                    maxsize[c] = len(titlemap.get(c, c))

        # Display the column titles
        i = 0
        for c in cols:
            if titlemap:
                t = titlemap.get(c, c)
            else:
                t = c
            #out += t
            out.append(t)
            sdiff = maxsize[c] - len(t) + 1
            #out += " " * spaces
            out.append(" " * spaces)
            if sdiff > 0 and i < len(cols)-1:
                #out += " " * sdiff
                out.append(" " * sdiff)
            i += 1
        #out += "\n"
        out.append('\n')

        # Then the separator
        i = 0
        for c in cols:
            #out += "-" * ((maxsize[c]+spaces))
            out.append("-" * ((maxsize[c]+spaces)))
            if i < len(cols)-1:
                #out += "+"
                out.append("+")
            i += 1
        #out += "\n"
        out.append("\n")

        # Then the actual data
        #ts = time.time()
        for r in data:
            i = 0
            di = 0
            tr = type(r)
            for c in r:
                if tr == list:
                    # we have  list index
                    cn = str(i)
                    tc = type(c)
                    if tc == int:
                        v = str(c)
                    if tc == str:
                        v = unicode(c, errors='replace')
                    if tc == unicode:
                        v = c
                else:
                    # we have a dict key
                    cn = c
                    v = r[c]
                if cn not in cols:
                    i += 1
                    continue

                out.append(v)
                lv = len(v)
                # there is a preceeding space if this is after the
                # first column
                # sdiff = number of spaces between end of word and |
                if di == 0:
                    sdiff = (maxsize[cn] - lv) + spaces
                else:
                    sdiff = (maxsize[cn] - lv) + spaces - 1
                if di < len(cols)-1:
                    #out += " " * sdiff
                    out.append(" " * sdiff)
                if di < len(cols)-1:
                    #out += "| "
                    out.append("| ")
                di += 1
                i += 1
            #out += "\n"
            out.append("\n")

        #print "time: " + str(time.time() - ts)
        #return ""
        return ''.join(out)

    # Make a request to the SpiderFoot server
    def request(self, url, post=None):
        #logging.basicConfig()
        #logging.getLogger().setLevel(logging.DEBUG)
        #requests_log = logging.getLogger("requests.packages.urllib3")
        #requests_log.setLevel(logging.DEBUG)
        #requests_log.propagate = True
        try:
            headers = {
                "User-agent": "SpiderFoot-CLI/" + self.version
            }

            self.ddprint("Fetching: " + url)
            self.ddprint("Posting: " + str(post))
            if not post:
                r = requests.get(url, 
                             headers = headers,
                             auth = requests.auth.HTTPDigestAuth(
                                        self.ownopts['cli.username'], 
                                        self.ownopts['cli.password']
                             )
                    )
            else:
                r = requests.post(url,
                             headers = headers,
                             auth = requests.auth.HTTPDigestAuth(
                                        self.ownopts['cli.username'],
                                        self.ownopts['cli.password']
                             ),
                             data = post
                    )
            self.ddprint("Response: " + str(r))
            if r.status_code == requests.codes.ok:
                return r.text
            else:
                r.raise_for_status()
        except BaseException as e:
            self.edprint("Failed communicating with server: " + str(e))
            return None

    def emptyline(self):
        return

    def completedefault(self, text, line, begidx, endidx):
        return []

    # Parse the command line, returns a list of lists:
    # sf> scans "blahblah test" | top 10 | grep foo ->
    # [[ 'blahblah test' ], [[ 'top', '10' ], [ 'grep', 'foo']]]
    def myparseline(self, cmdline, replace=True):
        ret = [list(),list()]
        s = shlex.split(cmdline.decode('string_escape'))
        for c in s:
            if c == '|':
                break
            if replace and c.startswith("$") and c in self.ownopts:
                ret[0].append(self.ownopts[c])
            else:
                ret[0].append(c)

        if s.count('|') == 0:
            return ret

        # Handle any pipe commands at the end
        ret[1] = list()
        i = 0
        ret[1].append(list())
        for t in s[(s.index('|')+1):]:
            if t == '|':
                i += 1
                ret[1].append(list())
                continue
            # Replace variables
            if t.startswith("$") and t in self.ownopts:
                ret[1][i].append(self.ownopts[t])
            else:
                ret[1][i].append(t)

        return ret

    # Send the command output to the user, proessing the pipes
    # that may have been used.
    def send_output(self, data, cmd, titles=None, total=True, raw=False):
        totalrec = 0
        out = None
        try:
            if not raw:
                j = json.loads(data)
                totalrec = len(j)
            else:
                j = data
        except BaseException as e:
            self.edprint("Unable to parse data from server: " + str(e))
            return

        if not raw:
            if self.ownopts['cli.output'] == "json":
                out = json.dumps(j, indent=4, separators=(',', ': '))

            if self.ownopts['cli.output'] == "pretty":
                out = self.pretty(j, titlemap=titles)

            if not out:
                self.edprint("Unknown output format '" + self.ownopts['cli.output'] + "'.")
                return
        else:
            out = data

        c = self.myparseline(cmd)

        # If no pipes, just disply the output
        if len(c[1]) == 0:
            self.dprint(out, plain=True)
            if total:
                self.dprint("Total records: " + str(totalrec))
            return
        
        for pc in c[1]:
            newout = ""
            if len(pc) == 0:
                self.edprint("Invalid syntax.")
                return
            pipecmd = pc[0]
            pipeargs = " ".join(pc[1:])
            if pipecmd not in [ "str", "regex", "file", "grep", "top", "last" ]:
                self.edprint("Unrecognised pipe command.")
                return

            if pipecmd == "regex":
                p = re.compile(pipeargs, re.IGNORECASE)
                for r in out.split("\n"):
                    if re.match(p, r.strip()):
                        newout += r + "\n"

            if pipecmd == "str" or pipecmd == "grep":
                for r in out.split("\n"):
                    if pipeargs.lower() in r.strip().lower():
                        newout += r + "\n"

            if pipecmd == "top":
                if not pipeargs.isdigit():
                    self.edprint("Invalid syntax.")
                    return
                newout = "\n".join(out.split("\n")[0:int(pipeargs)])
                
            if pipecmd == "last":
                if not pipeargs.isdigit():
                    self.edprint("Invalid syntax.")
                    return
                tot = len(out.split("\n"))
                i = tot - int(pipeargs)
                newout = "\n".join(out.split("\n")[i:])

            if pipecmd == "file":
                try:
                    f = codecs.open(pipeargs, "w", encoding="utf-8")
                    f.write(out)
                    f.close()
                except BaseException as e:
                    self.edprint("Unable to write to file: " + str(e))
                    return
                self.dprint("Successfully wrote to file '" + str(pipeargs) + "'.")
                return

            out = newout

        self.dprint(newout, plain=True)

    # Run SQL against the DB.
    def do_query(self, line):
        """query <SQL query>
        Run an <SQL query> against the database."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return
        query = ' '.join(c[0])
        d = self.request(self.ownopts['cli.server_baseurl'] + "/query",
                         post={"query": query})
        if not d:
            return
        j = json.loads(d)
        if j[0] == "ERROR":
            self.edprint("Error running your query: " + j[1])
            return
        self.send_output(d, line)

    # Ping the server.
    def do_ping(self, line):
        """ping
        Ping the SpiderFoot server to ensure it's responding."""
        d = self.request(self.ownopts['cli.server_baseurl'] + "/ping")
        if not d:
            return

        s = json.loads(d)
        if s[0] == "SUCCESS":
            self.dprint("Server " + self.ownopts['cli.server_baseurl'] + " responding.")
            self.do_modules("", cacheonly=True)
            self.do_types("", cacheonly=True)
        else:
            self.dprint("Something odd happened: " + str(d))

        if s[1] != self.version:
            self.edprint("Server and CLI version are not the same (" + s[1] + \
                        " / " + self.version + "). This could lead to unpredictable results!")

    # List all SpiderFoot modules.
    def do_modules(self, line, cacheonly=False):
        """modules
        List all available modules and their descriptions."""
        d = self.request(self.ownopts['cli.server_baseurl'] + "/modules")
        if not d:
            return
        if cacheonly:
            j = json.loads(d)
            for m in j:
                self.modules.append(m['name'])
            return
        self.send_output(d, line, titles={"name": "Module name",
                                          "descr": "Description"})

    # List all SpiderFoot data element types.
    def do_types(self, line, cacheonly=False):
        """types
        List all available element types and their descriptions."""
        d = self.request(self.ownopts['cli.server_baseurl'] + "/eventtypes")
        if not d:
            return
        if cacheonly:
            j = json.loads(d)
            for t in j:
                self.types.append(t[0])
            return
        self.send_output(d, line, titles={ "1": "Element description",
                                           "0": "Element name"})

    # Load commands from a file.
    def do_load(self, line):
        """load <file>
        Execute SpiderFoot CLI commands found in <file>."""
        pass

    # Get scan info and config.
    def do_scaninfo(self, line):
        """scaninfo <sid> [-c]
        Get status information for scan ID <sid>, optionally also its configuration if -c is supplied."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return

        sid = c[0][0]
        d = self.request(self.ownopts['cli.server_baseurl'] + "/scanopts?id=" + sid)
        if not d:
            return
        j = json.loads(d)
        if len(j) == 0:
            self.dprint("No such scan exists.")
            return

        out  = "Name: " + j['meta'][0] + "\n"
        out += "ID: " + sid + "\n"
        out += "Target: " + j['meta'][1] + "\n"
        out += "Started: " + j['meta'][3] + "\n"
        out += "Completed: " + j['meta'][4] + "\n"
        out += "Status: " + j['meta'][5] + "\n"

        if "-c" in c[0]:
            out += "Configuration:\n"
            for k in sorted(j['config']):
                out += "  " + k + " = " + j['config'][k] + "\n"

        self.send_output(out, line, total=False, raw=True)

    # List scans.
    def do_scans(self, line):
        """scans [-x]
        List all scans, past and present. -x for extended view."""
        d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlist")
        if not d:
            return
        j = json.loads(d)
        if len(j) == 0:
            self.dprint("No scans exist.")
            return
   
        c = self.myparseline(line)
        titles = dict()
        if "-x" in c[0]:
            titles = {"0": "ID",
              "1": "Name",
              "2": "Target",
              "4": "Started",
              "5": "Finished",
              "6": "Status",
              "7": "Total Elements"
            }
        else:
            titles = {
              "0": "ID",
              "2": "Target",
              "6": "Status",
              "7": "Total Elements"
            }

        self.send_output(d, line, titles=titles)

    # Show the data from a scan.
    def do_data(self, line):
        """data <sid> [-t type] [-x] [-u]
        Get the scan data for scan ID <sid> and optionally the element type [type] (e.g. EMAILADDR),
        [type]. Use -x for extended format. Use -u for a unique set of results."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return

        post = { "id": c[0][0] }

        if "-t" in c[0]:
            post["eventType"] = c[0][c[0].index("-t")+1]
        else:
            post["eventType"] = "ALL"

        if "-u" in c[0]:
            url = self.ownopts['cli.server_baseurl'] + "/scaneventresultsunique"
        else:
            url = self.ownopts['cli.server_baseurl'] + "/scaneventresults"

        d = self.request(url, post=post)
        if not d:
            return
        j = json.loads(d)
        if len(j) < 1:
            self.dprint("No results.")
            return

        if "-u" in c[0]:
            titles={
                    "0": "Data"
            }
        else:
            titles={
                    "10": "Type",
                    "1": "Data" 
            }
        if "-x" in c[0]:
            titles["0"] = "Last Seen"
            titles["3"] = "Module"
            titles["2"] = "Source Data"

        self.send_output(d, line, titles=titles)

    # Show logs.
    def do_logs(self, line):
        """logs <sid> [-l count] [-w]
        Show the most recent [count] logs for a given scan ID, <sid>. 
        If no count is supplied, all logs are given.
        If -w is supplied, logs will be streamed to the console until Ctrl-C is entered."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return
        sid = c[0][0] 
        limit = None
        if "-l" in c[0]:
            limit = c[0][c[0].index("-l")+1]

        if "-w" not in c[0]:
            d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
                             post={ 'id': sid, 'limit': limit })
            if not d:
                return
            j = json.loads(d)
            if len(j) < 1:
                self.dprint("No results.")
                return

            self.send_output(d, line, titles={"0": "Generated",
                                              "1": "Type",
                                              "2": "Source",
                                              "3": "Message"
                                      }
                            )
            return
        else:
            # Get the rowid of the latest log message
            d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
                             post={'id': sid, 'limit': '1'})
            if not d:
                return
            j = json.loads(d)
            if len(j) < 1:
                self.dprint("No logs (yet?).")
                return
            else:
                rowid = j[0][4]

            try:
                if not limit:
                    limit = 10
                d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
                                 post={'id': sid, 'reverse': '1', 'rowId': rowid-int(limit)})
                if not d:
                    return
                j = json.loads(d)
                for r in j:
                    #self.send_output(str(r), line, total=False, raw=True)
                    if r[2] == "ERROR":
                        self.edprint(r[1] + ": " + r[3])
                    else:
                        self.dprint(r[1] + ": " + r[3])
 
                while True:
                    d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
                                     post={'id': sid, 'reverse': '1', 'rowId': rowid})
                    if not d:
                        return
                    j = json.loads(d)
                    for r in j:
                        if r[2] == "ERROR":
                            self.edprint(r[1] + ": " + r[3])
                        else:
                            self.dprint(r[1] + ": " + r[3])

                        rowid = str(r[4])
                    time.sleep(0.5)
            except KeyboardInterrupt:
                return

    # Start a new scan.
    def do_start(self, line):
        """start <target> (-m m1,... | -t t1,... | -u case) [-n name] [-w]
        Start a scan against <target> using modules m1,... OR looking for types t1,...
        OR by use case ("all", "investigate", "passive" and "footprint").

        Scan be be optionally named [name], without a name the target will be used.
        Use -w to watch the logs from the scan. Ctrl-C to abort the logging (but will not 
        abort the scan).
        """
        mods = ""
        types = ""
        usecase = ""
        c = self.myparseline(line)
        if len(c[0]) < 3:
            self.edprint("Invalid syntax.")
            return

        if "-m" in c[0]:
            mods = c[0][c[0].index("-m")+1]

        if "-t" in c[0]:
            # Scan by type
            types = c[0][c[0].index("-t")+1]

        if "-u" in c[0]:
            # Scan by use case
            usecase = c[0][c[0].index("-u")+1]

        if not mods and not types and not usecase:
            self.edprint("Invalid syntax.")
            return

        target = c[0][0]
        if "-n" in c[0]:
            title = c[0][c[0].index("-n")+1]
        else:
            title = target

        post = { "scanname": title,
                 "scantarget": target,
                 "modulelist": mods,
                 "typelist": types,
                 "usecase": usecase,
                 "cli": "1"
        }
        d = self.request(self.ownopts['cli.server_baseurl'] + "/startscan",
                         post=post)
        if not d:
            return

        s = json.loads(d)
        if s[0] == "SUCCESS":
            self.dprint("Successfully initiated scan.")
            self.dprint("Scan ID: " + s[1])
        else:
            self.dprint("Unable to start scan: " + str(s[1]))

        if "-w" in c[0]:
            return self.do_logs(s[1] + " -w")

    # Stop a running scan.
    def do_stop(self, line):
        """stop <sid>
        Abort the running scan with scan ID, <sid>."""
        c = self.myparseline(line)
        try:
            id = c[0][0]
        except BaseException as e:
            self.edprint("Invalid syntax.")
            return

        d = self.request(self.ownopts['cli.server_baseurl'] + \
                         "/stopscan?id=" + id + "&cli=1")
        if not d:
            return

        s = json.loads(d)
        if s[0] == "SUCCESS":
            self.dprint("Successfully requested scan to stop. This could take some minutes to complete.")
        else:
            self.dprint("Unable to stop scan: " + str(s[1]))

    # Search for data, alias to find
    def do_search(self, line):
        """search (look up 'find')
        """
        return self.do_find(line)

    # Search for data
    def do_find(self, line):
        """find "<string|/regex/>" <[-s sid]|[-t type]> [-x]
        Search for string/regex, limited to the scope of either a scan ID or event type. -x for extended format."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return

        val = c[0][0]
        sid = None
        etype = None
        if "-t" in c[0]:
            etype = c[0][c[0].index("-t")+1]
        if "-s" in c[0]:
            sid = c[0][c[0].index("-s")+1]

        titles={"0": "Last Seen",
                "1": "Data",
                "3": "Module"
        }
        if "-x" in c[0]:
            titles["2"] = "Source Data"

        d = self.request(self.ownopts['cli.server_baseurl'] + "/search",
                         post={'value': val, 'id': sid, 'eventType': etype})
        if not d:
            return
        j = json.loads(d)

        if not j:
            self.dprint("No results found.")
            return
        if len(j) < 1:
            self.dprint("No results found.")
            return

        self.send_output(d, line, titles)

    # Summary of a scan
    def do_summary(self, line):
        """summary <sid> [-t]
        Summarise the results for a scan ID, <sid>. -t to only show the element types."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return

        sid = c[0][0]

        if "-t" in c[0]:
            titles = {"0": "Element Type"}
        else:
            titles={"0": "Element Type",
                    "1": "Element Description",
                    "3": "Total",<commit_msg>Updated formatting.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:        sfcli
# Purpose:     Command Line Interface for SpiderFoot.
#
# Author:      <NAME> <<EMAIL>>
#
# Created:     03/05/2017
# Copyright:   (c) <NAME> 2017
# Licence:     GPL
# -------------------------------------------------------------------------------

import sys
import os
import re
import json
import requests
import shlex
import codecs
import time
import readline
import cmd
import argparse
from os.path import expanduser

# Colors to make things purty
class bcolors:
    GREYBLUE = '\x1b[38;5;25m'
    GREY = '\x1b[38;5;243m'
    DARKRED = '\x1b[38;5;124m'
    DARKGREEN = '\x1b[38;5;30m'
    BOLD = '\033[1m'
    ENDC = '\033[0m'
    GREYBLUE_DARK = '\x1b[38;5;24m'

class SpiderFootCli(cmd.Cmd):
    version = "2.10"
    pipecmd = None
    output = None
    modules = []
    types = []
    prompt = "sf> "
    nohelp = "[!] Unknown command '%s'."
    knownscans = []
    ownopts = {
        "cli.debug": False,
        "cli.silent": False,
        "cli.color": True,
        "cli.output": "pretty",
        "cli.history": True,
        "cli.history_file": "",
        "cli.spool": False,
        "cli.spool_file": "",
        "cli.username": "",
        "cli.password": "",
        "cli.server_baseurl": "http://127.0.0.1:5001"
    }

    def default(self, line):
        if line.startswith('#'):
            return

        self.edprint("Unknown command")

    # Auto-complete for these commands
    def complete_start(self, text, line, startidx, endidx):
        return self.complete_default(text, line, startidx, endidx)

    def complete_find(self, text, line, startidx, endidx):
        return self.complete_default(text, line, startidx, endidx)

    def complete_data(self, text, line, startidx, endidx):
        return self.complete_default(text, line, startidx, endidx)

    # Command completion for arguments
    def complete_default(self, text, line, startidx, endidx):
        ret = list()
        if "-m" in line and line.find("-m") > line.find("-t"):
            for m in self.modules:
                if m.startswith(text):
                    ret.append(m)

        if "-t" in line and line.find("-t") > line.find("-m"):
            for t in self.types:
                if t.startswith(text):
                    ret.append(t)
        return ret

    def dprint(self, msg, err=False, deb=False, plain=False, color=None):
        cout = ""
        sout = ""
        pfx = ""
        if err:
            pfx = "[!]"
            if self.ownopts['cli.color']:
                col = bcolors.DARKRED
        else:
            pfx = "[*]"
            if self.ownopts['cli.color']:
                col = bcolors.DARKGREEN
        if deb:
            if not self.ownopts["cli.debug"]:
                return
            pfx = "[+]"
            if self.ownopts['cli.color']:
                col = bcolors.GREY

        if color:
            pfx = ""
            col = color

        if err or not self.ownopts["cli.silent"]:
            if not plain or color:
                cout = col + bcolors.BOLD + pfx + " " + bcolors.ENDC + \
                       col +  msg + bcolors.ENDC
                # Never include color in the spool
                sout = pfx + " " + msg
            else:
                cout = msg
                sout = msg

            print cout

        if self.ownopts['cli.spool']:
            f = codecs.open(self.ownopts['cli.spool_file'], "a", encoding="utf-8")
            f.write(sout)
            f.write('\n')
            f.close()

    # Shortcut commands
    def do_debug(self, line):
        """debug
        Short-cut command for set cli.debug = 1"""
        if self.ownopts['cli.debug']:
            val = "0"
        else:
            val = "1"
        return self.do_set("cli.debug = " + val)

    def do_spool(self, line):
        """spool
        Short-cut command for set cli.spool = 1/0"""
        if self.ownopts['cli.spool']:
            val = "0"
        else:
            val = "1"
        if self.ownopts['cli.spool_file']:
            return self.do_set("cli.spool = " + val)
        else:
            self.edprint("You haven't set cli.spool_file. Set that before enabling spooling.")

    def do_history(self, line):
        """history [-l]
        Short-cut command for set cli.history = 1/0. 
        Add -l to just list the history."""
        c = self.myparseline(line)

        if '-l' in c[0]:
            i = 0
            while i < readline.get_current_history_length():
                self.dprint(readline.get_history_item(i), plain=True)
                i += 1
            return
        if self.ownopts['cli.history']:
            val = "0"
        else:
            val = "1"
        return self.do_set("cli.history = " + val)

    # Run before all commands to handle history and spooling
    def precmd(self, line):
        if self.ownopts['cli.history'] and line != "EOF":
            f = codecs.open(self.ownopts["cli.history_file"], "a", encoding="utf-8")
            f.write(line)
            f.write('\n')
            f.close()
        if self.ownopts['cli.spool']:
            f = codecs.open(self.ownopts["cli.spool_file"], "a", encoding="utf-8")
            f.write(self.prompt + line)
            f.write('\n')
            f.close()

        return line

    # Debug print
    def ddprint(self, msg):
        self.dprint(msg, deb=True)

    # Error print
    def edprint(self, msg):
        self.dprint(msg, err=True)

    # Print nice tables.
    def pretty(self, data, titlemap=None):
        out = list()
        # Get the column titles
        maxsize = dict()
        if type(data[0]) == dict:
            cols = data[0].keys()
        else:
            # for lists, use the index numbers as titles
            cols = map(str, range(0, len(data[0])))

        # Strip out columns that don't have titles
        if titlemap:
            nc = list()
            for c in cols:
                if c in titlemap:
                    nc.append(c)
            cols = nc

        spaces = 2
        # Find the maximum column sizes
        for r in data:
            i = 0
            for c in r:
                if type(r) == list:
                    # we have  list index
                    cn = str(i)
                    if type(c) == int:
                        v = str(c)
                    if type(c) == str:
                        v = unicode(c, errors='replace')
                    if type(c) == unicode:
                        v = c
                else:
                    # we have a dict key
                    cn = c
                    v = r[c]
                #print str(cn) + ", " + str(c) + ", " + str(v)
                if len(v) > maxsize.get(cn, 0):
                    maxsize[cn] = len(v)
                i += 1


        # Adjust for long titles
        if titlemap:
            for c in maxsize:
                if len(titlemap.get(c, c)) > maxsize[c]:
                    maxsize[c] = len(titlemap.get(c, c))

        # Display the column titles
        i = 0
        for c in cols:
            if titlemap:
                t = titlemap.get(c, c)
            else:
                t = c
            #out += t
            out.append(t)
            sdiff = maxsize[c] - len(t) + 1
            #out += " " * spaces
            out.append(" " * spaces)
            if sdiff > 0 and i < len(cols)-1:
                #out += " " * sdiff
                out.append(" " * sdiff)
            i += 1
        #out += "\n"
        out.append('\n')

        # Then the separator
        i = 0
        for c in cols:
            #out += "-" * ((maxsize[c]+spaces))
            out.append("-" * ((maxsize[c]+spaces)))
            if i < len(cols)-1:
                #out += "+"
                out.append("+")
            i += 1
        #out += "\n"
        out.append("\n")

        # Then the actual data
        #ts = time.time()
        for r in data:
            i = 0
            di = 0
            tr = type(r)
            for c in r:
                if tr == list:
                    # we have  list index
                    cn = str(i)
                    tc = type(c)
                    if tc == int:
                        v = str(c)
                    if tc == str:
                        v = unicode(c, errors='replace')
                    if tc == unicode:
                        v = c
                else:
                    # we have a dict key
                    cn = c
                    v = r[c]
                if cn not in cols:
                    i += 1
                    continue

                out.append(v)
                lv = len(v)
                # there is a preceeding space if this is after the
                # first column
                # sdiff = number of spaces between end of word and |
                if di == 0:
                    sdiff = (maxsize[cn] - lv) + spaces
                else:
                    sdiff = (maxsize[cn] - lv) + spaces - 1
                if di < len(cols)-1:
                    #out += " " * sdiff
                    out.append(" " * sdiff)
                if di < len(cols)-1:
                    #out += "| "
                    out.append("| ")
                di += 1
                i += 1
            #out += "\n"
            out.append("\n")

        #print "time: " + str(time.time() - ts)
        #return ""
        return ''.join(out)

    # Make a request to the SpiderFoot server
    def request(self, url, post=None):
        #logging.basicConfig()
        #logging.getLogger().setLevel(logging.DEBUG)
        #requests_log = logging.getLogger("requests.packages.urllib3")
        #requests_log.setLevel(logging.DEBUG)
        #requests_log.propagate = True
        try:
            headers = {
                "User-agent": "SpiderFoot-CLI/" + self.version
            }

            self.ddprint("Fetching: " + url)
            self.ddprint("Posting: " + str(post))
            if not post:
                r = requests.get(url, 
                             headers = headers,
                             auth = requests.auth.HTTPDigestAuth(
                                        self.ownopts['cli.username'], 
                                        self.ownopts['cli.password']
                             )
                    )
            else:
                r = requests.post(url,
                             headers = headers,
                             auth = requests.auth.HTTPDigestAuth(
                                        self.ownopts['cli.username'],
                                        self.ownopts['cli.password']
                             ),
                             data = post
                    )
            self.ddprint("Response: " + str(r))
            if r.status_code == requests.codes.ok:
                return r.text
            else:
                r.raise_for_status()
        except BaseException as e:
            self.edprint("Failed communicating with server: " + str(e))
            return None

    def emptyline(self):
        return

    def completedefault(self, text, line, begidx, endidx):
        return []

    # Parse the command line, returns a list of lists:
    # sf> scans "blahblah test" | top 10 | grep foo ->
    # [[ 'blahblah test' ], [[ 'top', '10' ], [ 'grep', 'foo']]]
    def myparseline(self, cmdline, replace=True):
        ret = [list(),list()]
        s = shlex.split(cmdline.decode('string_escape'))
        for c in s:
            if c == '|':
                break
            if replace and c.startswith("$") and c in self.ownopts:
                ret[0].append(self.ownopts[c])
            else:
                ret[0].append(c)

        if s.count('|') == 0:
            return ret

        # Handle any pipe commands at the end
        ret[1] = list()
        i = 0
        ret[1].append(list())
        for t in s[(s.index('|')+1):]:
            if t == '|':
                i += 1
                ret[1].append(list())
                continue
            # Replace variables
            if t.startswith("$") and t in self.ownopts:
                ret[1][i].append(self.ownopts[t])
            else:
                ret[1][i].append(t)

        return ret

    # Send the command output to the user, proessing the pipes
    # that may have been used.
    def send_output(self, data, cmd, titles=None, total=True, raw=False):
        totalrec = 0
        out = None
        try:
            if not raw:
                j = json.loads(data)
                totalrec = len(j)
            else:
                j = data
        except BaseException as e:
            self.edprint("Unable to parse data from server: " + str(e))
            return

        if not raw:
            if self.ownopts['cli.output'] == "json":
                out = json.dumps(j, indent=4, separators=(',', ': '))

            if self.ownopts['cli.output'] == "pretty":
                out = self.pretty(j, titlemap=titles)

            if not out:
                self.edprint("Unknown output format '" + self.ownopts['cli.output'] + "'.")
                return
        else:
            out = data

        c = self.myparseline(cmd)

        # If no pipes, just disply the output
        if len(c[1]) == 0:
            self.dprint(out, plain=True)
            if total:
                self.dprint("Total records: " + str(totalrec))
            return
        
        for pc in c[1]:
            newout = ""
            if len(pc) == 0:
                self.edprint("Invalid syntax.")
                return
            pipecmd = pc[0]
            pipeargs = " ".join(pc[1:])
            if pipecmd not in [ "str", "regex", "file", "grep", "top", "last" ]:
                self.edprint("Unrecognised pipe command.")
                return

            if pipecmd == "regex":
                p = re.compile(pipeargs, re.IGNORECASE)
                for r in out.split("\n"):
                    if re.match(p, r.strip()):
                        newout += r + "\n"

            if pipecmd == "str" or pipecmd == "grep":
                for r in out.split("\n"):
                    if pipeargs.lower() in r.strip().lower():
                        newout += r + "\n"

            if pipecmd == "top":
                if not pipeargs.isdigit():
                    self.edprint("Invalid syntax.")
                    return
                newout = "\n".join(out.split("\n")[0:int(pipeargs)])
                
            if pipecmd == "last":
                if not pipeargs.isdigit():
                    self.edprint("Invalid syntax.")
                    return
                tot = len(out.split("\n"))
                i = tot - int(pipeargs)
                newout = "\n".join(out.split("\n")[i:])

            if pipecmd == "file":
                try:
                    f = codecs.open(pipeargs, "w", encoding="utf-8")
                    f.write(out)
                    f.close()
                except BaseException as e:
                    self.edprint("Unable to write to file: " + str(e))
                    return
                self.dprint("Successfully wrote to file '" + str(pipeargs) + "'.")
                return

            out = newout

        self.dprint(newout, plain=True)

    # Run SQL against the DB.
    def do_query(self, line):
        """query <SQL query>
        Run an <SQL query> against the database."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return
        query = ' '.join(c[0])
        d = self.request(self.ownopts['cli.server_baseurl'] + "/query",
                         post={"query": query})
        if not d:
            return
        j = json.loads(d)
        if j[0] == "ERROR":
            self.edprint("Error running your query: " + j[1])
            return
        self.send_output(d, line)

    # Ping the server.
    def do_ping(self, line):
        """ping
        Ping the SpiderFoot server to ensure it's responding."""
        d = self.request(self.ownopts['cli.server_baseurl'] + "/ping")
        if not d:
            return

        s = json.loads(d)
        if s[0] == "SUCCESS":
            self.dprint("Server " + self.ownopts['cli.server_baseurl'] + " responding.")
            self.do_modules("", cacheonly=True)
            self.do_types("", cacheonly=True)
        else:
            self.dprint("Something odd happened: " + str(d))

        if s[1] != self.version:
            self.edprint("Server and CLI version are not the same (" + s[1] + \
                        " / " + self.version + "). This could lead to unpredictable results!")

    # List all SpiderFoot modules.
    def do_modules(self, line, cacheonly=False):
        """modules
        List all available modules and their descriptions."""
        d = self.request(self.ownopts['cli.server_baseurl'] + "/modules")
        if not d:
            return
        if cacheonly:
            j = json.loads(d)
            for m in j:
                self.modules.append(m['name'])
            return
        self.send_output(d, line, titles={"name": "Module name",
                                          "descr": "Description"})

    # List all SpiderFoot data element types.
    def do_types(self, line, cacheonly=False):
        """types
        List all available element types and their descriptions."""
        d = self.request(self.ownopts['cli.server_baseurl'] + "/eventtypes")
        if not d:
            return
        if cacheonly:
            j = json.loads(d)
            for t in j:
                self.types.append(t[0])
            return
        self.send_output(d, line, titles={ "1": "Element description",
                                           "0": "Element name"})

    # Load commands from a file.
    def do_load(self, line):
        """load <file>
        Execute SpiderFoot CLI commands found in <file>."""
        pass

    # Get scan info and config.
    def do_scaninfo(self, line):
        """scaninfo <sid> [-c]
        Get status information for scan ID <sid>, optionally also its 
        configuration if -c is supplied."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return

        sid = c[0][0]
        d = self.request(self.ownopts['cli.server_baseurl'] + "/scanopts?id=" + sid)
        if not d:
            return
        j = json.loads(d)
        if len(j) == 0:
            self.dprint("No such scan exists.")
            return

        out  = "Name: " + j['meta'][0] + "\n"
        out += "ID: " + sid + "\n"
        out += "Target: " + j['meta'][1] + "\n"
        out += "Started: " + j['meta'][3] + "\n"
        out += "Completed: " + j['meta'][4] + "\n"
        out += "Status: " + j['meta'][5] + "\n"

        if "-c" in c[0]:
            out += "Configuration:\n"
            for k in sorted(j['config']):
                out += "  " + k + " = " + j['config'][k] + "\n"

        self.send_output(out, line, total=False, raw=True)

    # List scans.
    def do_scans(self, line):
        """scans [-x]
        List all scans, past and present. -x for extended view."""
        d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlist")
        if not d:
            return
        j = json.loads(d)
        if len(j) == 0:
            self.dprint("No scans exist.")
            return
   
        c = self.myparseline(line)
        titles = dict()
        if "-x" in c[0]:
            titles = {"0": "ID",
              "1": "Name",
              "2": "Target",
              "4": "Started",
              "5": "Finished",
              "6": "Status",
              "7": "Total Elements"
            }
        else:
            titles = {
              "0": "ID",
              "2": "Target",
              "6": "Status",
              "7": "Total Elements"
            }

        self.send_output(d, line, titles=titles)

    # Show the data from a scan.
    def do_data(self, line):
        """data <sid> [-t type] [-x] [-u]
        Get the scan data for scan ID <sid> and optionally the element 
        type [type] (e.g. EMAILADDR), [type]. Use -x for extended format. 
        Use -u for a unique set of results."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return

        post = { "id": c[0][0] }

        if "-t" in c[0]:
            post["eventType"] = c[0][c[0].index("-t")+1]
        else:
            post["eventType"] = "ALL"

        if "-u" in c[0]:
            url = self.ownopts['cli.server_baseurl'] + "/scaneventresultsunique"
        else:
            url = self.ownopts['cli.server_baseurl'] + "/scaneventresults"

        d = self.request(url, post=post)
        if not d:
            return
        j = json.loads(d)
        if len(j) < 1:
            self.dprint("No results.")
            return

        if "-u" in c[0]:
            titles={
                    "0": "Data"
            }
        else:
            titles={
                    "10": "Type",
                    "1": "Data" 
            }
        if "-x" in c[0]:
            titles["0"] = "Last Seen"
            titles["3"] = "Module"
            titles["2"] = "Source Data"

        self.send_output(d, line, titles=titles)

    # Show logs.
    def do_logs(self, line):
        """logs <sid> [-l count] [-w]
        Show the most recent [count] logs for a given scan ID, <sid>. 
        If no count is supplied, all logs are given.
        If -w is supplied, logs will be streamed to the console until 
        Ctrl-C is entered."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return
        sid = c[0][0] 
        limit = None
        if "-l" in c[0]:
            limit = c[0][c[0].index("-l")+1]

        if "-w" not in c[0]:
            d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
                             post={ 'id': sid, 'limit': limit })
            if not d:
                return
            j = json.loads(d)
            if len(j) < 1:
                self.dprint("No results.")
                return

            self.send_output(d, line, titles={"0": "Generated",
                                              "1": "Type",
                                              "2": "Source",
                                              "3": "Message"
                                      }
                            )
            return
        else:
            # Get the rowid of the latest log message
            d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
                             post={'id': sid, 'limit': '1'})
            if not d:
                return
            j = json.loads(d)
            if len(j) < 1:
                self.dprint("No logs (yet?).")
                return
            else:
                rowid = j[0][4]

            try:
                if not limit:
                    limit = 10
                d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
                                 post={'id': sid, 'reverse': '1', 'rowId': rowid-int(limit)})
                if not d:
                    return
                j = json.loads(d)
                for r in j:
                    #self.send_output(str(r), line, total=False, raw=True)
                    if r[2] == "ERROR":
                        self.edprint(r[1] + ": " + r[3])
                    else:
                        self.dprint(r[1] + ": " + r[3])
 
                while True:
                    d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
                                     post={'id': sid, 'reverse': '1', 'rowId': rowid})
                    if not d:
                        return
                    j = json.loads(d)
                    for r in j:
                        if r[2] == "ERROR":
                            self.edprint(r[1] + ": " + r[3])
                        else:
                            self.dprint(r[1] + ": " + r[3])

                        rowid = str(r[4])
                    time.sleep(0.5)
            except KeyboardInterrupt:
                return

    # Start a new scan.
    def do_start(self, line):
        """start <target> (-m m1,... | -t t1,... | -u case) [-n name] [-w]
        Start a scan against <target> using modules m1,... OR looking 
        for types t1,...
        OR by use case ("all", "investigate", "passive" and "footprint").

        Scan be be optionally named [name], without a name the target 
        will be used.
        Use -w to watch the logs from the scan. Ctrl-C to abort the 
        logging (but will not abort the scan).
        """
        mods = ""
        types = ""
        usecase = ""
        c = self.myparseline(line)
        if len(c[0]) < 3:
            self.edprint("Invalid syntax.")
            return

        if "-m" in c[0]:
            mods = c[0][c[0].index("-m")+1]

        if "-t" in c[0]:
            # Scan by type
            types = c[0][c[0].index("-t")+1]

        if "-u" in c[0]:
            # Scan by use case
            usecase = c[0][c[0].index("-u")+1]

        if not mods and not types and not usecase:
            self.edprint("Invalid syntax.")
            return

        target = c[0][0]
        if "-n" in c[0]:
            title = c[0][c[0].index("-n")+1]
        else:
            title = target

        post = { "scanname": title,
                 "scantarget": target,
                 "modulelist": mods,
                 "typelist": types,
                 "usecase": usecase,
                 "cli": "1"
        }
        d = self.request(self.ownopts['cli.server_baseurl'] + "/startscan",
                         post=post)
        if not d:
            return

        s = json.loads(d)
        if s[0] == "SUCCESS":
            self.dprint("Successfully initiated scan.")
            self.dprint("Scan ID: " + s[1])
        else:
            self.dprint("Unable to start scan: " + str(s[1]))

        if "-w" in c[0]:
            return self.do_logs(s[1] + " -w")

    # Stop a running scan.
    def do_stop(self, line):
        """stop <sid>
        Abort the running scan with scan ID, <sid>."""
        c = self.myparseline(line)
        try:
            id = c[0][0]
        except BaseException as e:
            self.edprint("Invalid syntax.")
            return

        d = self.request(self.ownopts['cli.server_baseurl'] + \
                         "/stopscan?id=" + id + "&cli=1")
        if not d:
            return

        s = json.loads(d)
        if s[0] == "SUCCESS":
            self.dprint("Successfully requested scan to stop. This could take some minutes to complete.")
        else:
            self.dprint("Unable to stop scan: " + str(s[1]))

    # Search for data, alias to find
    def do_search(self, line):
        """search (look up 'find')
        """
        return self.do_find(line)

    # Search for data
    def do_find(self, line):
        """find "<string|/regex/>" <[-s sid]|[-t type]> [-x]
        Search for string/regex, limited to the scope of either a scan ID or 
        event type. -x for extended format."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return

        val = c[0][0]
        sid = None
        etype = None
        if "-t" in c[0]:
            etype = c[0][c[0].index("-t")+1]
        if "-s" in c[0]:
            sid = c[0][c[0].index("-s")+1]

        titles={"0": "Last Seen",
                "1": "Data",
                "3": "Module"
        }
        if "-x" in c[0]:
            titles["2"] = "Source Data"

        d = self.request(self.ownopts['cli.server_baseurl'] + "/search",
                         post={'value': val, 'id': sid, 'eventType': etype})
        if not d:
            return
        j = json.loads(d)

        if not j:
            self.dprint("No results found.")
            return
        if len(j) < 1:
            self.dprint("No results found.")
            return

        self.send_output(d, line, titles)

    # Summary of a scan
    def do_summary(self, line):
        """summary <sid> [-t]
        Summarise the results for a scan ID, <sid>. -t to only show 
        the element types."""
        c = self.myparseline(line)
        if len(c[0]) < 1:
            self.edprint("Invalid syntax.")
            return

        sid = c[0][0]

        if "-t" in c[0]:
            titles = {"0": "Element Type"}
        else:
            titles={"0": "Element Type",
                    "1": "Element Description",
                    "3": "Total",
{'fraction_non_alphanumeric': 0.13783572359843546, 'fraction_numerical': 0.009908735332464147, 'mean_word_length': 1.2198811453268503, 'pattern_counts': {'":': 134, '<': 79, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2025612', 'n_tokens_mistral': 17314, 'n_tokens_neox': 16530, 'n_words': 4496}