## ------------------------------------------------------------------------
## iSiLK - A python-based graphical user interface for SiLK
## ------------------------------------------------------------------------
## Copyright (c) 2007-2008 Carnegie Mellon University. All Rights Reserved.
## ------------------------------------------------------------------------
## Authors: Andrew Kompanek (ajk@cert.org)
## ------------------------------------------------------------------------
## GNU General Public License (GPL) Rights pursuant to Version 2, June 1991
## Government Purpose License Rights (GPLR) pursuant to DFARS 252.225-7013
## ------------------------------------------------------------------------
## See the LICENSE file included in this distribution
## ------------------------------------------------------------------------

# Notes:
#
# result argument should support the following methods:
#
# _UpdateStatus(self, loc=None, status=None, percentComplete=None)
# _LogCommand(self, cmd)
#
# FIXME: add utilities for renaming and moving files locally and remotely?
import wx # Needed for PasswordEntryDialog
import os, os.path, sys, copy, tempfile, re, shutil
import pdb
#import config.silk as config
import config.silk
import logging
import traceback
import datetime, time
import remote
import threading

import paramiko
PARAMIKO_INSTALLED = True
    
CUT_FIELDS = [ "nhIP", "sensor", "class", "type", "scc", "dcc", "sval", "dval",
    "sip", "dip", "sport",
    "dport", "icmpTypeCode", "proto", "packets", "bytes", "flags",
    "stime", "dur", "etime" ]
SHELL_PATH = os.getenv("PATH")

# Dictionary mapping private key file names to passwords
_paramiko_password_cache_lock = threading.Lock()
_paramiko_password_cache = {}
_paramiko_dialog_close_event = threading.Event()
_paramiko_dialog_return_code = wx.ID_OK

class SimpleRunContext(object):
    def __init__(self, sshOptions=None, runRemote=True, runDir="."):
        if not sshOptions:
            self.sshOptions = config.silk.getSSHDefaults()
        else:
            self.sshOptions = sshOptions
        self.runRemote = runRemote
        self.runDir = runDir
    def GetSSHOptions(self):
        return self.sshOptions
    def _UpdateStatus(self, loc=None, status=None, percentComplete=None):
        print "Status is %s - percent complete %s" % (status, percentComplete)
    def _LogCommand(self, cmd):
        print "Ran command %s" % cmd
    def _ResultFileAvailable(self, label, filename):
        pass

class CommandError(Exception):
    def __init__(self, summary, details):
        self.summary = summary
        self.details = details

    def __str__(self):
        return "%s - %s" % (self.summary, self.details)
        
def format_log_message(user, host, msg):
    ts = datetime.datetime.now().strftime("%d-%b-%y %H:%M:%S")
    msg = "%s %s@%s %s" % (ts, user, host, msg)
    return msg
        
def _handle_errors(context, f, exec_name, outputFile=None):
    output = []
    while True:
        l = f.readline()
        if not l: break
        output.append(l)
    rv = f.close()
    if rv or (outputFile and not _validate_file_exists(context, outputFile)):
        details = "".join(output)
        raise CommandError("%s invocation failed" % exec_name, details)
    else:
        if context:
            context._UpdateStatus("Successfully ran %s" % exec_name)
client = None

def _paramiko_create_client():
    _paramiko_ssh_client = paramiko.SSHClient()
    _paramiko_ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    _paramiko_ssh_client.load_system_host_keys()
    return _paramiko_ssh_client
    
# Prompt the user for the password corresponding
# to the given encrypted private key
def _paramiko_get_password(key_file, message):
    global _paramiko_password_cache
    global _paramiko_password_cache_lock
    global _paramiko_dialog_close_event
    global _paramiko_dialog_return_code

    # Function to run in the main UI thread.  Displays a password
    # dialog, adds the password to the cache if the user clicks
    # OK and sets an event to indicate the dialog has been closed
    def _showInMainThread(key_file, message=None):
        global _paramiko_dialog_close_event
        global _paramiko_dialog_return_code
        global _paramiko_password_cache
        if message == None:
            message='''Your SSH is password protected.  Please provide a password.
This password will be used for the remainder of this session.'''
        print "Creating dialog"
        dlg = wx.PasswordEntryDialog(wx.GetApp().GetTopWindow(),
                message = message,
                value = "")
        dlg.CenterOnScreen()
        _paramiko_dialog_return_code = dlg.ShowModal()
        if _paramiko_dialog_return_code == wx.ID_OK:
            _paramiko_password_cache[key_file] = dlg.GetValue()
        # We're done
        _paramiko_dialog_close_event.set()
            
    # Only allow one thread at a time
    _paramiko_password_cache_lock.acquire()
    try:
        # Display the dialog and wait for it to close
        print "Calling InvokeLater to display dialog"
        wx.GetApp().InvokeLater(_showInMainThread, key_file, message)
        _paramiko_dialog_close_event.clear()        
        _paramiko_dialog_close_event.wait()
        _paramiko_dialog_close_event.clear()
        #print "Close event fired; done waiting."
        # Return boolean indicating whether user specified a password
        # (i.e., clicked OK)
        if _paramiko_dialog_return_code == wx.ID_OK:
            _paramiko_password_cache_lock.release()
            return True
        else:
            _paramiko_password_cache_lock.release()
            return False
    except:
        # Let the caller handle the exception after we
        # release the lock.
        _paramiko_password_cache_lock.release()
        raise

def _paramiko_connect(context, client, max_tries=3):
    global _paramiko_password_cache
    key_file = context.GetSSHOptions()["RSH_KEY"]
    try_count = 0

    # Try connecting and keep prompting the user for a password
    # until there's a successful connection or max_tries is
    # exceeded.  Raise an AuthenticationRequired exception 
    # ultimately no connection is established.
    while True:
        _password = _paramiko_password_cache.get(key_file, "")
        try:
            try_count = try_count + 1
            failed = False
            if _password != "":
                print "Connecting WITH key password"
                client.connect(context.GetSSHOptions()["REMOTE_HOST"],
                    username =context.GetSSHOptions()["REMOTE_USER"],
                    key_filename=context.GetSSHOptions()["RSH_KEY"],
                    password=_password,
                    port = int(context.GetSSHOptions()["REMOTE_PORT"]))
            else:
                print "Connecting WITHOUT key password"
                client.connect(context.GetSSHOptions()["REMOTE_HOST"],
                    username =context.GetSSHOptions()["REMOTE_USER"],
                    key_filename=context.GetSSHOptions()["RSH_KEY"],
                    port = int(context.GetSSHOptions()["REMOTE_PORT"]))
        except paramiko.PasswordRequiredException, e:
            print "Failed with PasswordRequiredException"
            message='''Please provide a password.  This password will be
used for the remainder of this session.'''
            failed = True
        except paramiko.AuthenticationException, e:
            print "Failed with AuthenticationException"
            failed = True
            message='''Please provide a password.  This password will be
used for the remainder of this session.'''
        # If we couldn't connect then get a password from the user
        if failed:
            result = _paramiko_get_password(key_file, message)
            if not result:
                raise e
            elif try_count > max_tries:
                print "Maximum number of tries exceeded"
                raise e
            else:
                print "Trying again with new password..."
                continue
        else:
            # succeeded
            return

def _paramiko_establish_connection(context):
    "Open a new connection for communication."
    try:
        # Create a new client and connection for each invocation
        # FIXME: Is there a better way to do this that's more efficient
        # (saw some odd issues when I tried to reuse the client object)
        _paramiko_ssh_client = _paramiko_create_client()
        print "Connecting to %s using key file %s" % (
            context.GetSSHOptions()["REMOTE_HOST"],
            context.GetSSHOptions()["RSH_KEY"])
        _paramiko_connect(context, _paramiko_ssh_client)
        print "Connected."
        return _paramiko_ssh_client
    except Exception, e:
        conn_str = "%s@%s" % (context.GetSSHOptions()["REMOTE_USER"],
            context.GetSSHOptions()["REMOTE_HOST"])
        raise CommandError("SSH Error",
            "Couldn't establish ssh connection to %s\n %s" % (conn_str, str(e)))

        
def _paramiko_exec_command(context, cmdLine, combineStderr=False):
    # Simple file-like wrapper for a result.  I return this rather than
    # stdout to work around an issue with the paramiko library. 
    # If you don't keep a reference to the client object around
    # it is garbage collected and the stream becomes invalid.
    class _sshresult_:
        def __init__(self, client, stdin, stdout, stderr):
            self.client = client
            #self.stdin = stdin
            self.stdout = stdout
            #self.stderr = stderr
        def readline(self):
            return self.stdout.readline()
        def readlines(self):
            return self.stdout.readlines()
        # Replicate the behavior of close on popen result
        def close(self):
            exit_code = self.stdout.channel.recv_exit_status() 
            self.stdout.close()
            self.client.close()
            return exit_code
    _paramiko_ssh_client = _paramiko_establish_connection(context)
    # Execute the command (handle paramiko exceptions)
    # NOTE: caller expected to check the exit code returned by
    # closing the stream with the close method
    try:
        (stdin, stdout, stderr) = \
            _paramiko_ssh_client.exec_command(cmdLine)
        stdout.channel.set_combine_stderr(combineStderr)
    except Exception, e:
         raise CommandError("SSH command execution failed",
            "%s\n %s" % (cmdLine, str(e)))
    rv = _sshresult_(_paramiko_ssh_client, stdin, stdout, stderr)
    return rv
    
def _local_exec_command(result, cmdLine, combineStderr=False):
    logging.info("Running command: %s", cmdLine)
    # FIXME: Is there a way to check the exit code early that 
    # will work on windows?  Unfortunately wait and poll
    # are not available on Windows:
    #   http://docs.python.org/lib/module-popen2.html
    # We're stuck checking to see if close() on stdout 
    # returns a non-zero value (see _handle_errors)
    try:
        return os.popen(cmdLine)
    except Exception, e:
         raise CommandError("SSH command execution failed",
            "%s\n %s" % (cmdLine, str(e)))
    return None

def _paramiko_write_remote_file(context, fileName, contents):
    pmk_ssh  = _paramiko_establish_connection(context)
    pmk_sftp = pmk_ssh.open_sftp()
    f = pmk_sftp.file(fileName, 'w')
    try:
        try:
            return f.write(contents)
        except Exception, e:
            raise CommandError("SSH Error",
                "Couldn't write %s remotely\n %s" % (conn_str, str(e)))
    finally:
        try:
            f.close()
        except:
            pass
    
def _build_ssh_command(cmdLine):
    command_line = "\"%s\"" % cmdLine
    return config.silk.buildExternalRemoteShellCommand(command_line)
    
# FIXME: changes here required if local analysis re-enabled
def exec_command(context, cmdLine, remote=True, combineStderr=True):
    logging.info("Running command: %s", cmdLine)
    if context.runDir and context.runDir != "":
        cmdLine = "cd %s && %s" % (context.runDir, cmdLine)
    if config.silk.useExternalSSH():
        cmdLine = _build_ssh_command(cmdLine)
        f = _local_exec_command(context, cmdLine)
    else:
        f = _paramiko_exec_command(context, cmdLine, combineStderr)
    return f
    

def _normalize_args(arg_opts):
    arg_opts = copy.copy(arg_opts)
    for o in arg_opts:
        val = arg_opts[o]
        if o.find("_") != -1:
            new_o = o.replace("_", "-")
            del arg_opts[o]
            arg_opts[new_o] = val
    return arg_opts
    
def _denormalize_args(arg_opts):
    arg_opts = copy.copy(arg_opts)
    for o in arg_opts:
        val = arg_opts[o]
        if o.find("-") != -1:
            new_o = o.replace("-", "_")
            del arg_opts[o]
            arg_opts[new_o] = val
    return arg_opts    
    
    
def _build_commandline(runRemote, cmd, valid_opts=[], arg_opts={},
        inputFileName="", cmdDir=None, ignoreOpts=[],
        commandsOnly=False):
    opts = _normalize_args(arg_opts)
    opts_str = ""
    for arg in valid_opts:
        if arg in opts:
            val = opts[arg];
            del opts[arg]
            # Make sure any "_" are converted back to "-"
            arg = arg.replace("_", "-")
            if val == None or type(val) == bool:
                opts_str += " --%s" % arg
            else:
                opts_str += " --%s=%s" % (arg, val) 
    for arg in opts:
        if not arg in ignoreOpts:
            logging.debug("Unexpected option: %s" % opts.keys()[0])
            raise CommandError("Error building command line: %s" % cmd, 
                        "Unexpected option: %s" % opts.keys()[0])
    if inputFileName:
        cmd = "%s %s%s" % (cmd, inputFileName, opts_str)
    else:
        cmd = "%s%s" % (cmd, opts_str)
    return cmd    

    
# FIXME: validate it's the right type (i.e., RW binary file) too
def _validate_file_exists(context, filename):
    if context.runRemote:
        return True
    else:
        print "Does file exist: runDir=%s, filename=%s" % (
            context.runDir, filename)
        if os.path.exists(os.path.join(context.runDir, filename)): 
            return True
    raise Exception, "Can't find input file: %s" % (os.path.join(context.runDir, filename))
    
def remove_remote_file(context, filename, remoteDir="."):
    cmd = _build_commandline(True, "rm %s" % filename, [], {}, cmdDir=remoteDir)
    f = exec_command(context, cmd)
    output = "".join(f.readlines())
    rv = f.close()
    if not rv:
        context._UpdateStatus(status="Deleted file.")
    else:
        details = "%s" % "\n".join(output)
        raise CommandError("mv (rename) invocation failed", details)
    

def rename_remote_file(context, remoteHost, remoteDir, oldFileName, newFileName):
    cmd = _build_commandline(True, "mv %s %s" % (oldFileName, newFileName), 
        [], {}, cmdDir=remoteDir)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    output = "".join(f.readlines())
    rv = f.close()
    if not rv:
        context._UpdateStatus(status="Completed rename.")
    else:
        details = "%s" % "\n".join(output)
        raise CommandError("mv (rename) invocation failed", details)
        
def copy_remote_file(context, oldFileName, newFileName):
    cmd = _build_commandline(True, "cp %s %s" % (oldFileName, newFileName), 
        [], {})
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    output = "".join(f.readlines())
    rv = f.close()
    if not rv:
        context._UpdateStatus(status="Completed copy.")
    else:
        details = "%s" % "\n".join(output)
        raise CommandError("cp (copy) invocation failed", details)

def write_remote_file(context, fileName, content):
    ''' Write a new file on the remote side. Just a passthrough for
        _paramiko_write_remote_file at the moment. '''
    return _paramiko_write_remote_file(context, fileName, content)
    
        
def old_copy_remote_file(remoteHost, remoteSource, remoteDest):
    context = SimpleRunContext(runRemote = True) 
    cmd = _build_commandline(True, "cp %s %s" % (remoteSource, remoteDest), 
        [], {})
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    output = "".join(f.readlines())
    rv = f.close()
    if not rv:
        context._UpdateStatus(status="Completed copy.")
    else:
        details = "%s" % "\n".join(output)
        raise CommandError("remote copy invocation failed", details)

def remote_mkpath(dirName):
    cmd_str = "mkdir -p %s" % remote.join(dirName, "scratch")
    cmd = _build_commandline(True, cmd_str, [], {})
    exec_command(self, cmd)

# alternative implmenetation
# t = paramiko.Transport((config.opts["REMOTE_HOST"],
#          int(config.opts["REMOTE_PORT"])))
# 
#  pkey= paramiko.DSSKey.from_private_key_file(config.opts["RSH_KEY"])
#  t.connect(username =config.opts["REMOTE_USER"],
#          pkey=pkey)
#  _paramiko_sftp_client = paramiko.SFTPClient.from_transport(t)
#  t.close()
  


def copy_file(context, remoteHost, remotePath, localPath):
    if not config.silk.useExternalSSH():
        _paramiko_ssh_client = _paramiko_create_client()
        _paramiko_connect(context, _paramiko_ssh_client)
        _paramiko_sftp_client = _paramiko_ssh_client.open_sftp()
        print "Trying to get remote file %s" % remotePath
        _paramiko_sftp_client.get(remotePath, localPath)
        _paramiko_sftp_client.close()
        output="(no output)"
    else:
        # Build the command
        cmd = config.silk.buildExternalDownloadCommand(remotePath, localPath)
        # Execute the command
        f = _local_exec_command(context, cmd)
        output = "".join(f.readlines())
        rv = f.close()
    if not os.path.exists(localPath):
        sys.exit()
        details = "Couldn't copy %s:%s to %s\n%s" % (remoteHost, remotePath,
            localPath, output) 
        raise CommandError("Remote file copy failed", details)

# Get record count for ascii file
def get_file_record_count(context, filename):
    cmd_str = "wc -l %s | awk \'{print $1}\'"        
    cmd = _build_commandline(context.runRemote, cmd_str % filename, [], [], "", 
        context.runDir)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    l = f.readline()
#    print "Read get_file_record_count line: '%s'" % l
    _handle_errors(context, f, "wc")
    return int(l.rstrip(os.linesep)) - 1 
    
def create_remote_directory(context, dir):
    cmd_str = "mkdir -p %s" % dir
    cmd = _build_commandline(True, cmd_str)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "mkdir", outputFile=dir)
    
def remote_directory_exists(context, dir):
    cmd_str = "ls %s" % dir
    cmd = _build_commandline(True, cmd_str)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    try:
        _handle_errors(context, f, "ls", outputFile=dir)
    except:
        return False
    return True
    
def get_file_size(context, filename):
    cmd_str = "ls -l %s | awk \'{print $5}\'"
    cmd = _build_commandline(context.runRemote, cmd_str % filename, [], [], "", context.runDir)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    l = f.readline()
    _handle_errors(context, f, "ls")
    return int(l.rstrip(os.linesep))
        
def list_files(context, recurse=False):
    if isinstance(context, str) or isinstance(context, unicode):
        context = SimpleRunContext(runRemote=True, runDir=context)
    if recurse:
        cmd_str = "find . -type f"
    else:
        cmd_str = "ls"
    cmd = _build_commandline(context.runRemote, cmd_str, [], [], "",
        context.runDir)
    context._LogCommand(cmd)
    f = exec_command(context, cmd, combineStderr=True)
    files = []
    while True:
        l = f.readline()
        if not l: break
        files.append(l.rstrip(os.linesep))
    _handle_errors(context, f, "ls")
    return files
        
def upload_file(context, localPath, remoteHost, remotePath):
    # Periodically post update event in update thread
    # Build the command
    cmd = "%s %s %s:%s" % (config.silk.opts["FILE_DOWNLOAD_CMD"], localPath,
                          remoteHost, remotePath)
    # Execute the command
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    output = "".join(f.readlines())
    rv = f.close()
    # FIXME: error checking

def get_default_class(context):
    cmd_str = 'rwfglob --help|grep "^--class Req Arg"'
    cmd = _build_commandline(context.runRemote, cmd_str, cmdDir=context.runDir)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    output = f.readlines()[0]
    default_class = output.split(' ')[-1].split('\n')[0]
    return default_class

def get_sensor_map(context):
    sensor_map = {}
    r = re.compile(r'\s*(\w*)\s*->\s*([\w\-]*)\s*\[([\w\-]*)\]')
    cmd_str = "mapsid --print-classes"
    cmd = _build_commandline(context.runRemote, cmd_str, cmdDir=context.runDir)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    output = sensors = f.readlines()
    for s in sensors:
        s = s.rstrip('\n')
        m = r.match(s) 
        if m:
            (num, name, sclass) = m.groups()
#            print "%s,%s,%s" % (num, name, sclass)
            if not sclass in sensor_map:
                sensor_map[sclass] = []
            sensor_map[sclass].append(name)
    rv = f.close()
    if not rv:
        context._UpdateStatus(status="Completed mapsid")
    else:
        details = "%s" % "\n".join(output)
        raise CommandError("mapsid invocation failed - %s" % cmd_str, "")
    return sensor_map

def build_filter_command(runRemote, outputFileName, opts, inputFileName, cmdDir=None, commandsOnly=False):
    filter_opts = ["any-address", "saddress", "daddress",
                   "aport", "dport", "sport", "proto", "scc", "dcc",
                   "stime", "etime", "dur", "flags-all", "bytes", "packets",
                   "icmp-type", "icmp-code", "dipset", "sipset", "anyset", "sensor",
                   "pmap-file", "pmap-saddress", "pmap-daddress"]
    other_opts = ["pass"]
    valid_opts = filter_opts
    valid_opts.extend(other_opts)
    # If no additional filtering options then stick a proto=0-255 in there
    if len(set(opts.keys()).intersection(set(filter_opts))) == 0:
        opts["proto"] = "0-255"  
    # Initialize output file name argument
    opts["pass"] = outputFileName

    # quote pmap-saddress and pmap-daddress, if present
    if "pmap-saddress" in opts:
        opts["pmap-saddress"] = '"' + opts["pmap-saddress"] + '"'
    if "pmap-daddress" in opts:
        opts["pmap-daddress"] = '"' + opts["pmap-daddress"] + '"'

    cmd = _build_commandline(runRemote, "rwfilter", valid_opts, opts,
        inputFileName, cmdDir, commandsOnly=commandsOnly)
    return cmd    

        
def build_query_command(runRemote, outputFileName, opts, cmdDir=None, commandsOnly=False):
    # FIXME: fail option is special option - hack to indicate to use
    # --fail instaed of --pass
    fglob_opts =  ["class", "type", "sensors", "start-date", "end-date"]
    filter_opts = ["saddress", "daddress", "any-address", "dport", "sport", 
                   "aport", "proto", "scc", "dcc",
                   "stime", "etime", "dur", "flags-all", 
                   "bytes", "packets", "icmp-type", "icmp-code",
                   "dipset", "sipset", "anyset",
                   "pmap-file", "pmap-saddress", "pmap-daddress", "max-pass-records"]
    other_opts = ["pass", "fail"]
    valid_opts = fglob_opts
    valid_opts.extend(filter_opts)
    valid_opts.extend(other_opts)
    # If no additional filtering options then stick a proto=0-255 in there
    if len(set(opts.keys()).intersection(set(filter_opts))) == 0:
        opts["proto"] = "0-255"              
    # Initialize output file name argument
    # FIXME: a bit of a hack--if fail is there then use the fail output
    # switch
    if "fail" in opts:
        opts["fail"] = outputFileName
    else:
        
        opts["pass"] = outputFileName
    # Enforce a maximuum number of records to return
    # FIXME: expose this as part of site-specific configuration
    opts["max-pass-records"] = config.silk.opts["MAX_QUERY_RECORDS"]

    # quote pmap-saddress and pmap-daddress, if present
    if "pmap-saddress" in opts:
        opts["pmap-saddress"] = '"' + opts["pmap-saddress"] + '"'
    if "pmap-daddress" in opts:
        opts["pmap-daddress"] = '"' + opts["pmap-daddress"] + '"'

    # Build the command line
    cmd = _build_commandline(True, "rwfilter", valid_opts, opts,
       cmdDir=cmdDir, commandsOnly=commandsOnly)
    return cmd
     
def _format_remaining_time(time_secs):
    if time_secs < 60:
        return "less than one minute"
    elif time_secs < (60*60):
        minutes = int(time_secs/60)
        if minutes == 1:
            return "%d minute" % (time_secs/60.0)
        else:
            return "%d minutes" % (time_secs/60.0)            
    else:
        hours = int(time_secs/(60.0*60.0))
        minutes = (time_secs - hours*60*60)/60
        if hours == 1:
            return "%d hour %d minutes" % (hours, minutes)
        else:
            return "%d hours %d minutes" % (hours, minutes)            

def query(context, fileCount, outputFile, **kargs):
    start_time = time.time()
    opts = kargs
    cmd = build_query_command(context.runRemote, outputFile, opts,
            cmdDir=context.runDir)
    # Redirect to stderr to stdout so we get --print-filenames output
    cmd = "%s --print-filenames" % (cmd)
    context._UpdateStatus(status="Running rwfilter")
    context._LogCommand(cmd)
    f = exec_command(context, cmd, combineStderr=True)
    i = 0
    # Update status based on output from --print-filenames (directed
    # from stderr to stdout)
    data_dir = config.silk.opts["REMOTE_DATA_DIR"]
    output = []
    while True:
       l = f.readline()
       print "Read line: %s" % l
       if not l: break # done reading
       output.append(l)
       if l[0:len("/")] == "/":
           percentComplete = (float(i+1)/fileCount)*100
           elapsed_time = time.time() - start_time
           recs_per_sec = float(i)/elapsed_time
           if recs_per_sec == 0: 
               recs_per_sec = 1
           remaining_time = (fileCount - i) / recs_per_sec
           print "file %d of %d" % (i, fileCount)
           print "elapsed=%f, files/sec=%f, remaining=%f" % (elapsed_time, recs_per_sec, remaining_time)
           status = "Filtering hourly file %d of %d" % (i+1, fileCount)
           status += " - %s remaining" % _format_remaining_time(remaining_time)
           context._UpdateStatus(status = status, 
                percentComplete=percentComplete)
           i = i + 1
    rv = f.close()
    if not rv and os.path.join(context.runDir, outputFile): 
        context._UpdateStatus(status="Completed rwfilter query")
    else:
        details = "%s" % "".join(output)
        raise CommandError("rwfilter invocation failed", details)
    return i
    
def fileinfo(context, filename):
    _validate_file_exists(context, filename)    
    # Run rwfileinfo to get information about the file
    fileinfo = {}
    cmd = _build_commandline(context.runRemote, "rwfileinfo", [], [], 
        inputFileName=filename,
        cmdDir=context.runDir)
    #cmd = "%s %s" % (cmd, filename)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    header_fields = ["format(id)", "version", "byte-order",
        "compression(id)", "header-length", "record-length",
        "count-records", "file-size"]
    field_count = 0
    output = []
    while True:
         l = f.readline()
         if not l: break
         output.append(l)
         tokens = l.split()
         if tokens[0] in header_fields:
             fileinfo[tokens[0]] = tokens[1]
             field_count += 1
         if tokens[0] == "command-lines": 
             break
    # FIXME: process standard error
    if not field_count == len(header_fields):
        raise CommandError("rwfileinfo invocation failed", "Unexpected output")
    # Read command lines
    fileinfo["command-lines"] = []
    while True:
          l = f.readline()
          if not l: break
          output.append(l.rstrip(os.linesep))
          tokens = str(l).split()
          tokens = tokens[1:len(tokens)]
          l = " ".join(tokens).strip()
          fileinfo["command-lines"].append(l)
    rv = f.close()
    if not rv:
        return fileinfo
    else:
        details = "".join(output)
        raise CommandError("fileinfo invocation failed", details)

def parse_fglob_date(date):
    r_str = r'(\d\d\d\d)/(\d\d)/(\d\d)(:\d\d)?'
    r = re.compile(r_str)
    try:
        (y,m,d,h) = r.match(date).groups()
        if h==None:
            h = 0
        else:
            # Get rid of the colon
            h = h[1:len(h)]
    except:
        raise CommandError("Couldn't parse fglob date", date)
    return (int(y),int(m),int(d),int(h))

# Parse a list of filenames of the form:
#   <ANYTHING>/<type>/2004/12/15/<type>-<sensor>_20041215.<hour>
def parse_fglob_result(files):
    result = []

    # the initial implementation was bugged in the case where the sensor name
    # contained a dash.  I've reduced the reliance on regular expressions in an
    # effort to generalize.
    dt_str = r'(\d\d\d\d)(\d\d)(\d\d)\.(\d\d).*'
    dtexpr = re.compile(dt_str)

    for f in files:
        basename = os.path.basename(f).split('_')
        t, s = basename[0].split('-', 1)
        raw_dt = basename[-1]
        print "Checking %s" % raw_dt
        m = dtexpr.match(raw_dt)
        if not m:
            raise CommandError("couldn't parse fglob result!", f)
        #print m.groups()
        (y, m, d, h) = m.groups()
        y = int(y); m = int(m); d = int(d); h = int(h)
        dt = (y, m, d, h)
        result.append((t,s,dt))
    return result


# Return hours for which there are no data files
# FIXME: more precise information?
def find_missing_fglob_hours(date_list, start_date, end_date):
    missing = []
    counts = {}
    for (t, s, dt) in date_list:
        if not dt in counts:
            #print "Adding %d,%d,%d,%d" % dt
            counts[dt] = 1
        else:
            counts[dt] += 1
    d = start_date
    by = datetime.timedelta(hours=1)
    while d <= end_date:
        val = (d.year, d.month, d.day, d.hour)
        #print "Looking for time %d,%d,%d,%d" % val
        if not val in counts:
            missing.append(val)
        d = d + by
    return missing


import silk
def _test():
    p = os.popen("rwfglob --start=2004/12/15:08 --end=2004/12/15:23")
    files = p.readlines()
    files = files[0:len(files)-1]
    date_list = parse_fglob_result(files)
    missing = silk.find_missing_fglob_hours(date_list,
        datetime.datetime(2004,12,15,7),
        datetime.datetime(2004,12,15,23))
    return (files, missing)

def fglob(context, **kargs):
    opts = kargs
    valid_opts = ["start-date", "end-date", "sensors", "class", "type"]
    ignore_opts = ["any-address", "saddress", "daddress",
                   "aport", "dport", "sport", "proto",
                   "stime", "etime", "dur", "flags-all", "bytes", "packets",
                   "icmp-type", "icmp-code", "dipset", "sipset", "anyset",
                   "pmap-file", "pmap-saddress", "pmap-daddress"]
    cmd = _build_commandline(context.runRemote, "rwfglob", valid_opts, opts,
        ignoreOpts=ignore_opts)
    cmd = cmd + ""
    print "Built fglob command: '%s'" % cmd
    context._UpdateStatus(status="Running fglob")
    f = exec_command(context, cmd)
    context._LogCommand(cmd)
    output=[]
    files = []
    while True:
       l = f.readline()
       if not l: break
       l = l.rstrip(os.linesep)
       print "read fglob line: %s" % l
       output.append(l)
       if l[0] == "/":
           files.append(l)
    rv = f.close()
    if not rv:
        print "Returning %d files" % len(files)
        return files
    else:
        details = "%s" % "\n".join(output)
        raise CommandError("rwfglob invocation failed", details)


def filter(context, inputFile, outputFile, **kargs):
    _validate_file_exists(context, inputFile)        
    opts = kargs
    cmd = build_filter_command(context.runRemote, outputFile, opts,
        inputFile, context.runDir)
    cmd = "%s" % (cmd)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    if context.runDir != None and inputFile != None:
        str_info = os.path.join(context.runDir, inputFile)
    else:
        str_info = ""
    _handle_errors(context, f, "rwfilter", str_info)


def cut(context, inputFile, outputFile, **kargs):
    _validate_file_exists(context, inputFile)  
    opts = kargs  
    valid_opts = ["fields", "output-path", "legacy-timestamps", "pmap-file"]
    opts["output-path"] = outputFile
    # FIXME: don't hardwire this
    opts["legacy-timestamps"] = True

    if not "fields" in opts:
        opts["fields"] = ",".join(CUT_FIELDS)
    print "fields = %s" % opts["fields"]
    field_list = opts["fields"].split(",")
    if not "pmap_file" in opts:
        cmd = "rwcut"
        if "sval" in field_list: field_list.remove("sval")
        if "dval" in field_list: field_list.remove("dval")
        opts["fields"] = ",".join(field_list)
    else:
        # Make sure pmap-file occurs before everything else
        cmd = "rwcut --pmap-file=%s" % opts["pmap_file"]
        # and remove it from the options
        del opts["pmap_file"]
        
    cmd = _build_commandline(context.runRemote, cmd, valid_opts, opts, 
            inputFile, cmdDir=context.runDir) 
    cmd = cmd + ""
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwcut", os.path.join(context.runDir, inputFile))

# FIXME: fieldName and summarize fields are hacks
def rwsetcat(context, inputFile, outputFile, fieldName=None, **kargs):
    opts = kargs
    valid_opts = []
    if fieldName == None:
        if "sip-file" in opts:
            fieldName = "sip"
        else:
            fieldName = "dip"
    cmd_str = "echo '%s|' > %s && rwsetcat %s >> %s" % (
        fieldName, outputFile, inputFile, outputFile)
    opts = {}
    cmd = _build_commandline(context.runRemote,
        cmd_str, valid_opts, {}, cmdDir=context.runDir)
    print "build command line returned '%s'" % cmd
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwsetcat",
        os.path.join(context.runDir, inputFile))    
    os.system("wc -l %s" % os.path.join(context.runDir, outputFile))
    
# FIXME: Hack for purposes of demo.  Unify this    
def rwsetcat_ranges(context, inputFile, outputFile, fieldName=None, **kargs):
    opts = kargs
    valid_opts = []
    if fieldName == None:
        if "sip-file" in opts:
            fieldName = "sip"
        else:
            fieldName = "dip"
    cmd_str = "echo 'start-ip|end-ip' > %s && rwsetcat  --ip-ranges %s  | awk 'BEGIN {FS=\"|\"} {print $2\"|\"$3}' >> %s" % (
        outputFile, inputFile, outputFile)
    opts = {}
    cmd = _build_commandline(context.runRemote,
        cmd_str, valid_opts, {}, cmdDir=context.runDir)
    print "build command line returned '%s'" % cmd
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwsetcat",
        os.path.join(context.runDir, inputFile))    
    os.system("wc -l %s" % os.path.join(context.runDir, outputFile))


def rwsetbuild_ips(context, inputFile, outputFile, **kargs):
    ''' Write an ascii list of IPs (i.e., the text representation of a
        SiLK set)to a file. '''
    ips = kargs['ips']
    outdir = "%s/%s" % (context.runDir, outputFile)
    print "writing ips to %s" % outdir
    write_remote_file(context, outdir, ips)


def rwsetbuild(context, inputFile, outputFile, **kargs):
    cmd = "rwsetbuild %(rundir)s/%(in)s %(rundir)s/%(out)s" % {
        "rundir" : context.runDir,
        "in" : inputFile,
        "out" : outputFile }
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwsetbuild",
        os.path.join(context.runDir, inputFile))


def build_rwset_command(runRemote, runDir, inputFile, commandsOnly=False, **kargs):
    opts = kargs
    valid_opts = ["sip-file", "dip-file"]
    cmd = _build_commandline(runRemote, "rwset", 
        valid_opts, opts, inputFile, cmdDir=runDir, commandsOnly=commandsOnly) 
    cmd = cmd + ""
    return cmd


def rwset(context, inputFile, outputFile, **kargs):
    _validate_file_exists(context, inputFile)    
    print "Building set command: %s" % kargs
    cmd = build_rwset_command(context.runRemote, context.runDir, inputFile, **kargs)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwset", os.path.join(context.runDir, inputFile))

def build_rwsettool_command(runRemote, runDir, inputFile, outputFile, commandsOnly=False, **kargs):
    opts = kargs
    valid_opts = ["union", "intersect", "difference", "output-path"]
    opts["output-path"] = outputFile
    # FIXME: note hack "inputs" for extra inputs
    inputs = opts["inputs"]
    del opts["inputs"]
    
    cmd = _build_commandline(runRemote, "rwsettool", 
        valid_opts, opts, inputFile, cmdDir=runDir, commandsOnly=commandsOnly) 
    cmd = cmd + " %s" % inputs
    return cmd

def rwsettool(context, inputFile, outputFile, **kargs):
    _validate_file_exists(context, inputFile)    
    print "Building set tool command: %s" % kargs
    cmd = build_rwsettool_command(context.runRemote, context.runDir, inputFile, outputFile, **kargs)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwset", os.path.join(context.runDir, inputFile))     
     
def xxxxbuild_rwsetcat_command(runRemote, runDir, inputFile, outputFile, commandsOnly=False, **kargs):
    opts = kargs
    valid_opts = ["cidr-blocks"]
    cmd = _build_commandline(runRemote, "rwsettool", 
        valid_opts, opts, inputFile, cmdDir=runDir, commandsOnly=commandsOnly) 
    cmd = cmd + ""
    return cmd

def xxxrwsetcat(context, inputFile, outputFile, **kargs):
    _validate_file_exists(context, inputFile)    
    print "Building set tool command: %s" % kargs
    cmd = build_rwsetcat_command(context.runRemote, context.runDir, inputFile, outputFile, **kargs)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwset", os.path.join(context.runDir, inputFile))


def build_uniq_command(runRemote, runDir, inputFile, outputFile, commandsOnly=False, **kargs):
    opts = kargs
    valid_opts = ["fields", "output-path", "bytes", "packets", "flows", 
        "sip-distinct", "dip-distinct", "pmap-file"]
    opts["output-path"] = outputFile
    cmd = _build_commandline(runRemote, "rwuniq", 
        valid_opts, opts, inputFile, cmdDir=runDir, commandsOnly=commandsOnly) 
    cmd = cmd + ""
    return cmd


def uniq(context, inputFile, outputFile, **kargs):
    _validate_file_exists(context, inputFile)    
    cmd = build_uniq_command(context.runRemote, context.runDir, inputFile, outputFile, **kargs)
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwuniq", os.path.join(context.runDir, inputFile))


def build_count_command(runRemote, runDir, inputFile, outputFile, commandsOnly=False, **kargs):
    opts = kargs
    valid_opts = ["bin-size", "output-path"]
    opts["output-path"] = outputFile
    cmd = _build_commandline(runRemote, "rwcount", valid_opts, opts,
           inputFile, cmdDir=runDir, commandsOnly=commandsOnly)
    return cmd


def count(context, inputFile, outputFile,  **kargs):
    _validate_file_exists(context, inputFile)
    cmd = build_count_command(context.runRemote, context.runDir, inputFile, outputFile, **kargs)
    cmd = cmd + ""
    context._LogCommand(cmd)
    f = exec_command(context, cmd)
    _handle_errors(context, f, "rwcount", os.path.join(context.runDir, inputFile))

