# @GPL_HEADER_START
# Use of the RAVE and related source code is subject to the terms
# of the following licenses:
# 
# GNU Public License (GPL) Rights pursuant to Version 2, June 1991
# Government Purpose License Rights (GPLR) pursuant to DFARS 252.225-7013
# 
# NO WARRANTY
# 
# ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER
# PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY
# PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN
# "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY
# KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT
# LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE,
# MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE
# OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT,
# SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY
# TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF
# WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES.
# LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF
# CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON
# CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE
# DELIVERABLES UNDER THIS LICENSE.
# 
# Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie
# Mellon University, its trustees, officers, employees, and agents from
# all claims or demands made against them (and any related losses,
# expenses, or attorney's fees) arising out of, or relating to Licensee's
# and/or its sub licensees' negligent use or willful misuse of or
# negligent conduct or willful misconduct regarding the Software,
# facilities, or other rights or assistance granted by Carnegie Mellon
# University under this License, including, but not limited to, any
# claims of product liability, personal injury, death, damage to
# property, or violation of any laws or regulations.
# 
# Carnegie Mellon University Software Engineering Institute authored
# documents are sponsored by the U.S. Department of Defense under
# Contract F19628-00-C-0003. Carnegie Mellon University retains
# copyrights in all material produced under this contract. The U.S.
# Government retains a non-exclusive, royalty-free license to publish or
# reproduce these documents, or allow others to do so, for U.S.
# Government purposes only pursuant to the copyright license under the
# contract clause at 252.227.7013.
# 
# @GPL_HEADER_END
"""
Decorator functions used in creating RAVE analyses.

RAVE uses python decorators[1] to identify functions and metadata
that it can use to cache data. These decorators can be divided into
three categories:


"Top-level" RAVE decorators

    @op
    @op_file

Currently only @op and @op_file. Functions decorated with these
decorators have their data cached and may be exported to other
processes through a service provider interface (like the one raved
provides). These decorators are actually implemented as objects
rather than functions; functions such as op and op_file merely
provide an interface to thses objects that is consistant with the
other decorators.

Modifier decorators

    @typemap
    @use_strategy
    @version
    @mime_type

These decorators pass information to the top-level decorators by
setting attributes on the function they modify, but they do not
directly change the function's behavior. They are implemented as
functions.

Expiration strategy functions and factories

    duration_strategy
    crossover_strategy
    nocache
    forever

Expiration strategies are used to determine a time beyond which the
data from an RAVE operation and should be considered invalid and
expired from the cache. nocache and forever are normal expiration
strategies; they take a request object and return a UNIX time value
(like the one returned by time.time()) beyond which the data should
be considered expired. duration_strategy and crossover_strategy
are expiration strategy factories; they return expiration strategy
functions.

Type converters:
    list_of
    one_of
    any

Converters are used in typemaps to convert values to types more
complex than normal. list_of and one_of are actually converter
factories.  The converter returned by list_of converts its input
into a list of the specified types; the converter returned by any_of
converts its input into the first of the specified sequence of types
that can successfully convert it. any represents that the input can
be of any type; its input is passed through unconverted.

Miscellany

    set_rave_modifier
    RaveOperation

These items are described in detail below.
"""

# TODOCUMENT:
#
# * Typemaps are optional. However, if supplied, they must
#   fully describe the op signature.

import rave.log as rlog
import rave.exceptions as rexcept
import rave.util as rutil
import rave.util.json as rjson
import rave.cache.core as rccore
import rave.plugins.times as rtimes
import rave.plugins.request as rreq
import argmanip

from sets import Set
from datetime import datetime, timedelta
import atexit
import cPickle as pickle
import codecs
import errno
import inspect
import os
import os.path
import pdb
import re
import sys
import tempfile
import threading
import time
import types

get_log = rlog.log_factory("rave.plugins.decorators")

# Should we stop at breakpoints?
trace = False

def set_trace():
    if trace:
        pdb.set_trace()

# TODO: Run typemaps on all arguments, not just those from params=...

#
# Default caches
#

class SimpleCache(rccore.Cache):
    "Cache Mixin for the default caches"
    def __init__(self, local_var, global_var):
        self.using_temp = False
        dir = os.getenv(local_var)
        if dir is None:
            dir = os.getenv(global_var)
        if dir is None:
            dir = tempfile.mkdtemp()
            self.using_temp = True
            should_warn = False
        else:
            should_warn = True
        self.dir = dir
        super(SimpleCache, self).__init__(
              repository=dir
            , strat=forever 
            , warn_on_empty_repository=should_warn)
    def cleanup(self):
        if self.using_temp:
            # TODO: clean up for non-UNIX
            os.system('rm -rf %s' % self.dir)
            

# Maintain a private reference to the original values of ana_cache and
# fname_cache so we can clean them up on exit. (Wrap them in a
# LazyProxy because they may get replaced before they're ever used,
# and we'd like to avoid the side effects of their construction if
# possible.)
_ana_cache = rutil.LazyProxy(SimpleCache, 'RAVE_DATA_CACHE_PATH', 'RAVE_CACHE_PATH')
_fname_cache = rutil.LazyProxy(SimpleCache, 'RAVE_MEDIA_CACHE_PATH', 'RAVE_CACHE_PATH')

def _clean_default_caches():
    global _ana_cache
    global _fname_cache
    try:
        try:
            if _ana_cache.lp_constructed:
                _ana_cache.cleanup()
            if _fname_cache.lp_constructed:
                _fname_cache.cleanup()
        except OSError, e:
            # Ignore file not found
            if e.errno != errno.ENOENT:
                raise
    except:
        # Log everything else...
        get_log().exception("Error cleaning default cache")
        # ...but swallow the exception

atexit.register(_clean_default_caches)

# If you don't want to use the default caches, supply an object with
# the rave.cache.core.Cache interface for either ana_cache (to store
# intermediate data) or fname_cache (to store results, like
# visualizations, that will be consumed as files by other processes.)
#
# If the values of these variables change while analysis processing is
# going on, you might get very strange behavior. Don't do this.
ana_cache = _ana_cache
fname_cache = _fname_cache

def reinit_caches(ana_path=None, fname_path=None):
    """Reinitialize the default caches to point at the specified
    paths."""
    global ana_cache
    global fname_cache
    if ana_path:
        ana_cache = rccore.Cache(
              repository=ana_path
            , strat=forever , warn_on_empty_repository=True)
    if fname_path:
        fname_cache = rccore.Cache(
              repository=fname_path
            , strat=forever , warn_on_empty_repository=True)


#
# Miscellaneous cache-related stuff
#

def get_request():
    return rreq.storage.current_request()


class Expert(object):
    """
    Direct access to cache storage by those who know what they're doing.

    This class allows the insertion of arbitrary data into the RAVE
    cache framework, keyed using a label (to classify a category of
    cache inserts, corresponding to an operation type), a version
    (corresponding to operation version) and an identifier (analogous
    to parameters used to call an operation).

    Developers of complex RAVE operations may use this class to cache
    data that may be partially reusable by other calls to the
    operation with different arguments. (For example, a request for a
    full day's data can be reused by certain calls looking for an hour
    within that day.) The operation developer is wholly responsible
    for figuring out what"partially reusable" means. Use with caution.
    """
    def __init__(self, label, version, identifier,
                 mime_type="application/octet-stream"):
        dummy = DummyOperation.get_instance(label, version, mime_type)
        norm_kwargs = dummy.normalize(identifier)
        key, meta = dummy._register_or_wait(norm_kwargs)
        self._meta = None
        self._tran = None
        if meta:
            # An active op finished and bequeathed unto us a value
            self._registered = False
            self._meta = meta
        else:
            # No active op running, but now _we're_ the active op.
            # check the cache, and return appropriately.
            self._registered = True
            meta = dummy.cache().get(dummy, norm_kwargs)
            if meta:
                self._meta = meta
            else:
                self._tran = dummy.cache().reserve(dummy, norm_kwargs)
        self._key = key
        self._dummy = dummy
    def __del__(self):
        self.rollback()
    def is_cached(self):
        return bool(self._meta)
    def data_file(self):
        """
        Returns: fname
        fname: str
            If is_cached() is True, the name of the file that
            contains the data.  Otherwise, the name of the file to
            which you should write data.
        """
        if self.is_cached():
            return self._meta.data_file()
        else:
            return self._tran.filename()
    def commit(self, expires=None, writeval=None):
        """
        Finalize the insertion of the item into the cache. If the item is
        already cached, this method does nothing.

        Parameters:
        expires: None | float
            When the cached item should expire
        writeval: None | object
            If writeval is specified, serialize its contents to data_file()
        """
        if not self.is_cached():
            try:
                if writeval is not None:
                    utf8 = codecs.getwriter("utf8")
                    f = utf8(open(self.data_file(), 'wb'))
                    try:
                        try:
                            rjson.unparse({'rave_return_value': writeval}, f)
                        except Exception, e:
                            raise rexcept.DecoratorError(
                                "Couldn't serialize value: %s" % str(e)
                            )
                    finally:
                        f.close()
                self._meta = self._dummy.cache().put(self._tran, expires)
            finally:
                self._dummy.active_ops.unregister(self._key)
                self._registered = False
    def rollback(self):
        """
        Clean up an aborted cache insertion. If the item is already cached
        or rollback() has already been called, this method does nothing.
        """
        if self._registered:
            self._dummy.active_ops.unregister(self._key)
            self._registered = False
    def value(self):
        """
        Get the underlying serialized value from the cache.

        This method assumes the value has been serialized (i.e.,
        it's the result of calling an @op or calling commit() with
        writeval != None). To read non-serialized data, get the
        file name using data_file() and read the file directly.
        """
        if not self.is_cached():
            raise rexcept.DecoratorError(
                "Can't call Expert.value() on an uncached value"
            )
        f = open(self.data_file(), 'rb')
        try:
            try:
                return rjson.parse(f)['rave_return_value']
            except KeyError:
                raise rexcept.DecoratorError(
                    "Invalid data format for serialized cache value"
                )
            except Exception, e:
                raise rexcept.DecoratorError(
                    "Couldn't deserialize value: %s" % str(e)
                )
        finally:
            f.close()
        
        
                


#
# Other helper classes
#



class ActiveList(object):
    """
    Mapping of running instances of a given operation. Keys are args used to
    call the operation, values are Event objects.
    """
    def __init__(self):
        self.ops = {}
        self.lock = threading.Lock()
    def unregister(self, key):
        # This is probably a little more locking than needs to happen,
        # but I'm paranoid....
        self.lock.acquire()
        try:
            try:
                evt = self.ops.pop(key)
            except KeyError:
                get_log().warn(
                    "Tried to unregister '%s', but it's already gone", str(key))
        finally:
            self.lock.release()
        evt.set()
    def register(self, key):
        """
        Register an invocation in the active list, unless a matching
        invocation is already running.
        Returns: (evt, already_running)
            evt (threading.Event)
                Event associated with this operation invocation
            already_running (bool)
                True if the event was already in the active list (and the
                invocation is already running somewhere else)
        """
        self.lock.acquire()
        try:
            if self.ops.has_key(key):
                return (self.ops[key], True)
            else:
                evt = threading.Event()
                self.ops[key] = evt
                return (evt, False)
        finally:
            self.lock.release()
           

#
# Operation classes
#

class RaveDecorator(object):
    """Base class for all RAVE decorators that do not simply return
    the decorated function."""
    # The prefix used to denote options passed to the decorator
    opt_prefix = "rave_"

    @classmethod
    def _extract_options(cls, kwargs):
        options = {}
        nonoptions = {}
        for k, v in kwargs.items():
            if cls._is_option(k, v):
                opt_name = cls._clean_option(k)
                options[opt_name] = v
            else:
                nonoptions[k] = v
        return options, nonoptions

    @classmethod
    def _is_option(cls, opt_k, opt_v):
        return opt_k.startswith(cls.opt_prefix) and len(opt_k) > 1

    @classmethod
    def _clean_option(cls, opt_k):
        return opt_k[len(cls.opt_prefix):]

    def __init__(self, wrapped):
        self.wrapped = wrapped
    def __str__(self):
        try:
            # Are we wrapping the core function?
            childname = self.wrapped.func_name
        except AttributeError:
            # No. Use child's __str__
            childname = str(self.wrapped)
        return "<%s of %s>" % (
            self.__class__.__name__, childname
        )
    def core_function(self):
        "The core function being decorated."
        try:
            # Assume wrapped is another Operation
            return self.wrapped.core_function()
        except AttributeError:
            # Wrapped is the actual core function
            return self.wrapped
    def core_name(self):
        "The name of the core function being decorated."
        # Maybe we've already gotten the name
        try:
            return self.wrapped_name
        except AttributeError:
            self.wrapped.name = self.core_function().func_name
            return self.wrapped.name
    def core_filename(self):
        "The file name (if it exists) of the core function being decorated."
        # Maybe we've already gotten the name
        try:
            return self.wrapped_fname
        except AttributeError:
            self.wrapped_fname = inspect.getabsfile(self.core_function())
            return self.wrapped_fname

class Operation(RaveDecorator):
    def rave_exportable(self):
        """All objects that can be exported via RAVE's namespace facility
        should possess this method, which should return True."""
        return False
    def export(self):
        """If rave_exportable is True, this returns a callable which is
        called identically to the normal Operation, and returns a location
        in the filename cache where one can obtain the information."""
        raise NotImplemented()
    def cache(self):
        raise NotImplemented()




class CacheableOperation(Operation):
    """
    An Operation that adds RAVE cachability to properly implemented functions
    or callables. The exact way in which it does this is determined by
    subclasses.
    """
    def __init__(self, wrapped): 
        super(CacheableOperation, self).__init__(wrapped)
        core = self.core_function()
        def get_from_core(attrname, default):
            if hasattr(core, attrname):
                return getattr(core, attrname)
            else:
                return default
        self.mime_type = get_from_core('_rave_mime_type',
                                       'application/octet-stream')
        self.expire_strat = get_from_core(
            '_rave_strat', lambda req: self.cache().default_strategy()(req))
        version = get_from_core('_rave_version', None)
        if version is None:
            try:
                version = os.stat(self.core_filename()).st_mtime
            except OSError:
                if (self.core_filename().endswith('<stdin>') 
                    or self.core_filename().endswith('<ipython console>')):
                    # Function defined on the command-line with no
                    # version: use current time, so any redefinition
                    # invalidates the previously cached versions.
                    version = time.time()
                else:
                    raise
        self.version = version
        self.active_ops = ActiveList()

        # Do typemap last because it depends on a lot of instance methods.
        argspec = self.on_describe()
        tm_raw = get_from_core('_rave_typemap',
                               ProtoTypemap.default(argspec[0]))
        try:
           mapped_types = argmanip.reconcile(
                self.core_name(), argspec,
                tm_raw.args, tm_raw.kwargs, is_typemap=True) 
        except TypeError, e:
            raise rexcept.TypemapTypeError(str(e))
        self.typemap = (mapped_types, tm_raw.varargs)

    def rave_exportable(self):
        return True

    def export(self):
        return self

    def cache(self):
        return fname_cache

    def normalize(self, *args, **kwargs):
        """Normalize arguments into the form the cache expects. Operations
        that must manipulate the cache directly must do this so the cache can
        compare entries effectively. This is an advanced technique; most
        operations should never use this method."""
        options, kwargs = self._extract_options(kwargs)
        return self._parse_input(args, kwargs, options)[2]

    def fqname(self):
        "Returns the fully-qualified name of the wrapped function."
        try:
            return '.'.join((self.wrapped.__module__, self.wrapped.__name__))
        except AttributeError:
            try:
                return self.wrapped.fqname()
            except AttributeError:
                raise rexcept.ParameterError(
                    "%s object must implement __module__ and __name__ \
                     attributes or fqname method"
                )

    def on_describe(self):
        """
        Called to determine what the underlying function's signature looks 
        like. RAVE uses the output of this function to map values back to 
        arguments in the function signature for type mapping and insertion 
        into the cache.  If you always call the underlying function 
        with exactly the arguments that you use to call the Operation, this 
        should be the output of inspect.getargspec on the underlying 
        function. (RAVE knows to strip off the invocation options passed to 
        it -- i.e., anything beginning with "rave_" -- so you needn't worry 
        about those.)

        You will only need to override this function if you manipulate the 
        invocation arguments in on_call_cache or on_call_nocache. (For 
        instance, the MediaOperation prepends the name of the cached file or 
        the value of rave_tofile to the arguments so the underlying function 
        knows where to write its file.) In that case, you may need to use 
        describe to "undo" the changes you make in on_call_[no]cache so that 
        the invocation arguments and the function signature match up. (For 
        instance, the MediaOperation removes the first positional argument 
        in the signature, which takes the argument it inserts later.)

        It's generally desirable to avoid using this function. To do so:

            * Don't add or remove arguments. If you must adjust the 
              arguments, replace existing ones.

            * Add a **kwargs argument to your function, and insert 
              additional arguments by name.
        """
        return inspect.getargspec(self.core_function())

    def on_call_nocache(self, request):
        """
        Called to execute and return the underlying function without caching.
        """
        raise NotImplemented()

    def on_call_cache(self, request, transaction):
        """
        Called to execute the underlying function and write the results into
        a file in the cache.
        """
        raise NotImplemented()

    def on_return_cache_async(self, request, transaction):
        """
        Called to return preliminary results to a non-blocking call to this
        operation. This function should return something that makes sense on a 
        non-blocking context, or raise NotImplemented to indicate that it can't
        be called asynchronously.

        By default, this method returns the value of
        transaction.new_filename()

        Parameters:
        request: Request
            The Request object associated with this invocation
        transaction: rave.cache.CachePutTransaction
            The CachePutTransaction object reserved for the results of this
            invocation.
        """
        return transaction.new_filename(
            relative=request.option('relative_path'))

    def on_return_cache_hit(self, request, meta):
        """
        Called to return the results of a cached invocation. This function
        should return the data in the cache in a way that will make sense
        to the caller.

        By default, this method returns the value of meta.data_file()

        Parameters:
        request: Request
            The Request object associated with this invocation
        meta: rave.cache.CacheMeta
            The CacheMeta object associated with the cached data
        """
        return meta.data_file(
            relative=request.option('relative_path'))

    #
    # Methods that support __call__
    #

    def _register_or_wait(self, norm_kwargs):
        """
        Check the active list to see if an invocation of this operation
        is that matches the args in norm_kwargs is already running. If so, wait
        for the invocation to finish and returns the cached results. If not,
        register this invocation in the active list.
     
        Parameters:
        norm_kwargs
            The dictionary of post-processed arguments returned from normalize()
       
        Returns: (arghash, meta)
        arghash (tuple)
            A digest of norm_kwargs usable as a key in the active list
        meta (rave.cache.CacheMeta | None)
            The results of the already-running invocation, or None if no other
            invocation was running
        """
        log = get_log("operation")(str(threading.currentThread().getName()))
        log.debug("not cached")
        log.debug("kwargs is %s", str(norm_kwargs))
        arghash = rutil.immute_dict(norm_kwargs)
        evt_op_finished, already_running = self.active_ops.register(arghash)       
        if already_running:
            log.debug(
                "Operation is already running. Waiting for it to finish.")
            while not evt_op_finished.isSet():
                evt_op_finished.wait(60)
                log.debug(
                    "Operation still pending. Continuing to wait.")
            log.debug(
                "Operation complete.")
            # Note: If the item expires from the cache before we
            # retrieve it, (e.g., because it used the Never strategy),
            # this will return None, which is okay.
            meta = self.cache().get(self, norm_kwargs)
            return (arghash, meta)
        else:
            log.debug(
                "Operation not running. Position reserved in active list.")
            return (arghash, None)
            

    def _parse_input(self, args, kwargs, options={}):
        if options.has_key('params'):
            get_log().debug("reconciling params from rave_params (%s)",
                            options['params'])
            reconciled = argmanip.reconcile(
                self.core_name(), self.on_describe(), (), options['params'])
        else:
            get_log().debug("reconciling params from direct call")
            reconciled = argmanip.reconcile(
                self.core_name(), self.on_describe(), args, kwargs)
        get_log().debug("reconciled is %s", reconciled)
        # Special case: if there are no mapped arguments (e.g.,
        # "(*args, **kwargs)") and there is no typemap, do no further
        # argument processing.
        if (len(reconciled['mapped']) == 0
            and _is_empty_typemap(self.typemap)):
            return args, kwargs, reconciled
        try:
            args, kwargs, reconciled = argmanip.coerce(self, reconciled)
        except argmanip.UnmappedValueError, e:
            raise rexcept.ConfigurationError(
                "Argument '%s' not present in typemap" % str(e))
        return args, kwargs, reconciled


    def _debug_log_options(self, options):
        if options is None:
            get_log().debug("No options to %s", self.core_name())
        else:
            get_log().debug("Options to %s", self.core_name())
            for k, v in options.items():
                get_log().debug("%s: %s", str(k), str(v))
    

    def _run_and_cache(self, req):
        # If this is a dry run, we have failed to find the data in the
        # cache. Return that failure.
        if req.option('dry_run'):
            return None
        # Check whether this operation is already running (e.g., in
        # another thread). If so, wait for that to finish and return
        # (newly) cached output.
        #
        # TODO: this should be done in a scheduler thread if
        # req.option('scheduler') is not None.
        arghash, meta = self._register_or_wait(req.cache_key())
        if meta:
            return self.on_return_cache_hit(req, meta)
        else:
            # Not running, and we've reserved our place. Execute the
            # underlying function and save the results.
            transaction = self.cache().reserve(self, req.cache_key())
            l = get_log()
            # The next bit needs to be done either inline or
            # asynchronously (if rave_scheduler is set), so we put it
            # in a function and run it as appropriate.
            def finish_running():
                try:
                    l.debug("Calling on_call_cache for %s",
                            self.core_name())
                    try:
                        self.on_call_cache(req, transaction)
                    except Exception, e:
                        transaction.rollback()
                        raise
                    l.debug("Done calling on_call_cache")
                    return self.cache().put(transaction, self.expire_strat(req))
                    l.debug("Done putting %s in cache", self.core_name())
                finally:
                    l.debug("Unregistering from active_ops")
                    self.active_ops.unregister(arghash)

                    l.debug("Should I set on_done?")
                    if req.option('on_done'):
                        l.debug("Why yes, I should set on_done")
                        req.option('on_done').set()
                    else:
                        l.debug("Not setting on_done -- not supplied")
            # if _scheduler, do asynchronously
            if req.option('scheduler'):
                # Get a reference to the global request so we can pass
                # it into the child thread.
                req_state = rreq.storage.export_state()
                def runner():
                    # Set up request in _this_ thread's local storage
                    rreq.storage.import_state(req_state)
                    finish_running()
                req.option('scheduler').schedule(runner)
                req.job_scheduled(True)
                return self.on_return_cache_async(req, transaction)
            else:
                meta = finish_running()
                return self.on_return_cache_hit(req, meta)


    def __call__(self, *args, **kwargs):
        """
        Executes the underlying function and inserts its results in the
        cache, as customized by subclasses.
        """
        # Note that if a user passed in an event via rave_on_done, we
        # MUST set it before we exit this function, unless we have
        # scheduled a job to run, in which case the worker thread will
        # set it when it's finished.
        get_log().debug("%s(%s, %s)", self.core_name(), args, kwargs)
        options, kwargs = self._extract_options(kwargs)
        self._debug_log_options(options)
        on_done = options.get('on_done', None)
        req = None
        try:
            args, kwargs, cache_key = self._parse_input(args, kwargs, options)
            req = rreq.storage.allocate(self, options, args, kwargs, cache_key)
            try:
                # If we're called with _tofile, just execute the
                # underlying function without checking or writing to
                # the cache.
                if req.option('tofile'):
                    return self.on_call_nocache(req)

                # Similarly, if we're told to unconditionally refresh,
                # execute the underlying function without checking the
                # cache, but still write it back into the cache.
                if (req.option('refresh') or
                    req.option('deep_refresh', check_global=True)):
                    return self._run_and_cache(req)

                # The normal case. Return a cached copy if it exists,
                # or generate a new one if it doesn't.
                meta = self.cache().get(self, cache_key)
                if meta:
                    rc = self.on_return_cache_hit(req, meta)
                    if req.option('on_done'):
                        req.option('on_done').set()
                    return rc
                else:
                    # Cache miss
                    rc = self._run_and_cache(req)
                    if req.option('on_done') and not req.option('scheduler'):
                        req.option('on_done').set()
                    return rc
            finally:
                # We're through (unless a job is still scheduled)
                get_log().debug(
                    "Call complete to %s(%s, %s)",
                    self.core_name(), str(args), kwargs
                )
                rreq.storage.free()
                get_log().debug("request unallocated")
        finally:
            if on_done:
                get_log().debug("on_done was passed to this function")
                if req is None or not req.job_scheduled():
                    get_log().debug("Job was not scheduled. Setting on_done.")
                    on_done.set()
                else:
                    get_log().debug("Job was scheduled. " 
                                    "Thread will have to take care of it.")
            else:
                get_log().debug("on_done was NOT passed to this function")



class DummyOperation(CacheableOperation):
    """An operation that does nothing, but provides a stub with which to
    insert items into the cache not associated one-to-one with a real
    Operation."""
    live_ops = {}
    @classmethod
    #def get_instance(cls, label, dummy_version, identifier,
    def get_instance(cls, label, dummy_version,
            mime_type="application/octet-stream"):
        key = (label, dummy_version)
        if not cls.live_ops.has_key(key):
            #op = cls(label, dummy_version, identifier, mime_type)
            op = cls(label, dummy_version, mime_type)
            cls.live_ops[key] = op
        return cls.live_ops[key]
    #def __init__(self, label, dummy_version, identifier,
    def __init__(self, label, dummy_version,
                 mime_type="application/octet-stream"):
        self.label = label
        self.version = dummy_version
        #self.identifier = identifier
        self.mime_type = mime_type
        @version(self.version)
        def dummyfunc(*args, **kwargs):
            pass
        
        self.dummyfunc = dummyfunc
        super(DummyOperation, self).__init__(dummyfunc)
    def core_function(self):
        return self.dummyfunc
    def core_name(self):
        return "dummy-%s" % self.label
    def core_filename(self):
        return "%s/%s" % (self.label, self.version)
    def cache(self):
        return fname_cache
    def __call__(self, *args, **kwargs):
        raise NotImplemented("DummyOperations are not callable")

class ExternalFileOperation(CacheableOperation):
    """
    Enables functions that write to external files to cache their output in
    RAVE.

    The wrapped function must take a string representing a filename as its
    first argument, and write its data into that file.
    """
    def export(self):
        return self
    def cache(self):
        return fname_cache
    def on_describe(self):
        argnames, varargs, varkwargs, defaults = \
            inspect.getargspec(self.core_function())
        if len(argnames):
            # If the first argument has a default value, remove it, too
            if defaults and len(defaults) == len(argnames):
                defaults.pop(0)
            argnames.pop(0)
        return argnames, varargs, varkwargs, defaults
    def on_call_nocache(self, request):
        args = [request.option('tofile')] + list(request.args())
        self.wrapped(*args, **request.kwargs())
        return request.option('tofile')
    def on_call_cache(self, request, transaction):
        args = [transaction.filename()] + list(request.args())
        get_log().debug("args: %s", args)
        get_log().debug("kwargs: %s", request.kwargs())
        self.wrapped(*args, **request.kwargs())
    # Uses default implementations of:
    #     on_return_cache_hit
    #     on_return_cache_async


class _ExportedSerializedOperation(ExternalFileOperation):
    """
    A special ExternalFileOperation that wraps SerializedOperations
    for exporting.
    """
    def write_wrapped_to_file(self, args, kwargs, filename):
        # Write output of wrapped to file and return filename
        output = self.wrapped(*args, **kwargs)
        f = open(filename, 'wb')
        try:
            f.write(output)
        finally:
            f.close()
        return filename
    def core_name(self):
        # Use name of op we're wrapping, mutated so it's unique to the cache
        return "__exported__.%s" % self.wrapped.core_name()
    def core_filename(self):
        # Use filename of op we're wrapping
        return self.wrapped.core_filename()
    def on_describe(self):
        # Describe argspec of op we're wrapping
        return inspect.getargspec(self.wrapped.core_function())
    def on_call_nocache(self, request):
        return self.write_wrapped_to_file(
            request.args(), request.kwargs(), request.option('tofile'))
    def on_call_cache(self, request, transaction):
        return self.write_wrapped_to_file(
            request.args(), request.kwargs(), transaction.filename())
        

class SerializedOperation(CacheableOperation):
    """
    Takes output from a normal python function and serializes it in the
    cache.
    """
    def rave_exportable(self):
        return self.mime_type[:5] == "text/"
    def export(self):
        if self.mime_type[:5] != "text/":
            raise NotImplemented(
                "Can only export operations with text MIME types")
        return _ExportedSerializedOperation(self)
    def cache(self):
        return ana_cache
    def on_call_cache(self, request, transaction):
        rc = {'rave_return_value': self.wrapped(*request.args(),
                                                **request.kwargs())}
        utf8 = codecs.getwriter("utf8")
        f = utf8(open(transaction.filename(), 'wb'))
        try:
            try:
                rjson.unparse(rc, f)
            except Exception, e:
                raise rexcept.DecoratorError(
                    "Couldn't serialize value: %s" % str(e)
                )
        finally:
            f.close()
    def on_call_nocache(self, request):
        return self.wrapped(*request.args, **request.kwargs)
    def on_return_cache_async(self, request, transaction):
        # We can't know what our value is going to be until we run, so...
        raise NotImplemented()
    def on_return_cache_hit(self, request, meta):
        get_log().debug("About to load serialized object from %s",
                        meta.data_file()) 
        f = open(meta.data_file(), 'rb')
        try:
            try:
                return rjson.parse(f)['rave_return_value']
            except KeyError:
                raise rexcept.DecoratorError(
                    "Invalid data format for serialized operator"
                )
            except Exception, e:
                raise rexcept.DecoratorError(
                    "Couldn't deserialize value: %s" % str(e)
                )
        finally:
            f.close()


#
# Decorators
#

#
# Strategy decorators
#

        
# examples:
# @op
# @use_strategy(duration_strategy(30))
# @version(123)
# @mime_type('text/html')
# def foobar(...):
#   ...
#
#
#
# @op_file
# @use_strategy(crossover_strategy(duration_strategy(300), forever,
#                                   3600, 'etime'))
# def foobar(...):
#   ...


def core_function(op_or_fn):
    try:
        return op_or_fn.core_function()
    except AttributeError:
        return op_or_fn

def function_name(op_or_fn):
    try:
        return op_or_fn.core_name()
    except AttributeError:
        return op_or_fn.__name__


def set_rave_modifier(op_or_fn, name, val):
    """
    Puts the RAVE decorator modifer in the right place, regardless of where
    the modifier decorator is placed relative to the core function.
    """
    setattr(core_function(op_or_fn), name, val)

def expires(strat):
    """
    Decorator specifying the expiration strategy to use for a RAVE operation.

    Parameters:
    strat: function
        An expiration strategy function that will be run to determine when
        the data returned by the operation will no longer be considered
        valid.
    """
    if not callable(strat):
        # Assume strat is really an expiration time
        strat = lambda req: strat
    def wrapper(fn):
        set_rave_modifier(fn, '_rave_strat', strat)
        return fn
    return wrapper

# Synonym for use_strategy
use_strategy = expires

def parse_interval_format(interval):
    unit_multipliers = {
        's': 1,
        'm': 60,
        'h': 60*60,
        'd': 24*60*60
    }
    units = 'dhms'
    times = dict((k,None) for k in units)
    atom = r''
    interval_re = re.compile(r'([0-9]+)([%s])' % units)
    interval_mutable = interval
    while interval_mutable:
        m = interval_re.match(interval_mutable)
        if m is None:
            raise rexcept.ConfigurationError("Syntax error in interval '%s'")
        duration, unit = m.groups()
        duration = int(duration)
        if times[unit] is not None:
            raise rexcept.ConfigurationError(
                "Syntax error in interval '%s': "
                "multiple values for unit '%s'" % (interval, unit)
            )
        times[unit] = duration
        interval_mutable = interval_mutable[m.end():]
    return sum([v * unit_multipliers[k] 
                for k, v in times.items() 
                if v is not None])

def duration_strategy(interval):
    """
    Expiration strategy factory. Generates strategies that expire  data after
    a given time interval.

    Parameters:
    interval: numeric | str
        Number of seconds in the future, after which the data should
        no longer be considered valid.

        Optionally, interval can be a string of duration specifiers
        of the form "<number><unit>", where "unit" can be one of
        "h" (hours), "m" (minutes), or "s" (seconds). For example,
        you can specify "5m" for five minutes, "2h" for two hours,
        "2h30m" for two hours and thirty minutes, and so on.
    """
    if isinstance(interval, str):
    	interval = parse_interval_format(interval)
    def expires(req):
    	return time.time() + interval
    return expires


def crossover_strategy(strat_a, strat_b, age, age_arg='etime'):
    """
    Expiration strategy factory. A combinator that returns either strat_a or 
    strat_b depending on the age of the parameter specified by age_arg.

    Parameters:
    strat_a: function
        One of the expiration strategy functions from which to choose.
    strat_b: function
        The other expiration strategy functions from which to choose.
    age: numeric
        The crossover age. If the value in age_arg is more recent
        or equal to this point, strat_a will be used to determine
        how long the data is useful. If not, strat_b will be used.
    age_arg: string
        The name of the argument on the Request object which will
        be used to determine the expiration strategy.
    """
    def expires(req):
        t = rtimes.datetime_obj(req.kwargs()[age_arg])
        n = datetime.utcnow()
        if t + timedelta(seconds=age) >= n:
                return strat_a(req)
        else:
                return strat_b(req)
    return expires

def forever(req):
    """Expiration strategy. Indicates that the data returned will always be
    good."""
    return rccore.NEVER

def nocache(req):
    """Expiration strategy. Indicates that the data returned should never be
    stored in the cache."""
    return rccore.NOW


# MIME type decorator
def mime_type(mt):
    """
    Decorator specifying a MIME type for a RAVE op_file.

    MIME type of the operation's output. This value is only important if 
    this operation is exported; in this case, this is the MIME type that 
    will be returned to clients requesting this operation, and may affect 
    how the operation is written to disk.

    If not specified, the default MIME type is 'text/plain'.
    """
    def wrapper(fn):
        set_rave_modifier(fn, '_rave_mime_type', mt)
        return fn
    return wrapper


# Typemap decorator
class ProtoTypemap(object):
    @classmethod
    def default(cls, argnames):
        args = []
        for name in argnames:
            args.append(any)
        return cls(args=args)
    def __init__(self, args=[], kwargs={}, rave_varargs=[]):
        self.args = args
        self.kwargs = kwargs
        self.varargs = rave_varargs
    def __str__(self):
        return "<ProtoTypemap: args: %s; kwargs: %s; varargs: %s>" % (
            self.args, self.kwargs, self.varargs)

def _is_empty_typemap(t):
    mapped, varargs = t
    return not (mapped['mapped'] or
                mapped['kw'] or
                mapped['positional'])

def typemap(*args, **kwargs):
    """
    Decorator specifying a type mapping for RAVE operations.

    This decorator takes a dictionary of parameters and their data types. If 
    the decorated function is exported to and called from RAVE, this 
    dictionary is used to convert data types from external sources (such as 
    network requests) to the function's expected data types. Any function 
    argument omitted is considered to be of type str. If this parameter is 
    omitted, the types of all arguments are considered to be of type str.  
    (Note that this dictionary is ignored if the function is called by 
    another function, and not by RAVE.)
    """
    def wrapper(fn):
        # All the values supplied to typemap should be callables
        for x in range(len(args)):
            if args[x] is not None and not callable(args[x]):
                raise rexcept.IllegalTypemap(
                    "Argument %d to typemap of'%s' not callable (or None)" % (
                         x, function_name(fn)))
        for k, v in kwargs.items():
            if v is not None and not callable(v):
                raise rexcept.IllegalTypemap(
                    "Argument '%s' to typemap of '%s'not callable (or None)" % (
                        k, function_name(fn)))
                
        varargs = kwargs.pop('rave_varargs', {})
        set_rave_modifier(fn, '_rave_typemap', ProtoTypemap(args, kwargs, varargs))
        return fn
    return wrapper

# Converter functions for typemapping

def list_of(*converters):
    """
    Coerces a sequence of values into a set of types specified by
    *converters. If the number of elements in the source sequence
    is greater than the length of the list of converters, the
    converters list will recycle from the start. So:

        list_of(str)

    specifies a list of strings, while

        list_of(str, int)

    specifies a list of alternating strings and ints. An odd-length
    list -- e.g., [1, 2, 3] -- would not generate an error. (This
    example would convert to ["1", 2, "3"].)
    """
    if not len(converters):
        raise rexcept.IllegalTypemap("list_of: argument cannot be empty")
    def do_convert(src):
        converted = []
        if not len(src):
            return converted
        for i in xrange(len(src)):
            converted.append(
                argmanip.convert(src[i], converters[i % len(converters)])
            )
        return converted
    return do_convert


def one_of(*converters):
    """
    Returns a converter that attempts to coerce a value into one
    of (potentially) several types of values in sequence. Returns
    the first successfully coerced value.

    For example:

        one_of(None, str)

    returns a converter that will attempt to coerce a value into a
    string, or into None. (Note that None precedes str. one_of(str,
    None) would coerce the value None into the string "None" so we
    put None first.
    """
    def do_convert(src):
        for c in converters:
            try:
                return argmanip.convert(src, c)
            except TypeError, e:
                pass # Ignore
        raise TypeError(
            "Could not convert '%s' using any of: %s" % (
                str(src), str(converters)
            )
        )
    return do_convert
 

def any(src):
    """
    Passes value through without any type checking or coercion.
    """
    return src

# Version decorator
def version(v):
    """
    Decorator specifying the version of a RAVE operation.

    If supplied, this number should be incremented every time the
    function changes in a way that would change its output, or else
    cached values from the old function will still be returned until
    they expire.  If no version is supplied, the modification time
    of the file containing the operation will be used as the version,
    so changes anywhere in the file will have the same effect as a
    version rev.

    """
    def wrapper(fn):
        set_rave_modifier(fn, '_rave_version', v)
        return fn
    return wrapper

#
# Op decorators
#

def op(fn):
    """
    "Top-level" RAVE decorator. Indicates that a function is a RAVE 
    operation; its output will be cached, and its parameters will, as 
    necessary, be coerced into the appropriate types.

    Operations decorated with an @op are normal Python functions whose 
    return values happen to be cached. The return values should be numbers, 
    strings, dictionaries, lists or combinations of the above.
    """
    return SerializedOperation(fn)


def op_file(fn):
    """
    "Top-level" RAVE decorator. Indicates that a function is a RAVE file 
    operation; it will write a file which will be cached. The file to which 
    it should write is passed as the first argument to the function. It 
    returns nothing -- or rather, anything it returns is ignored.
    """
    return ExternalFileOperation(fn)


__all__ = [
    'reinit_caches', 'op', 'op_file', 'typemap', 'mime_type',
    'use_strategy', 'expires', 'version', 'forever', 'nocache',
    'duration_strategy', 'crossover_strategy', 'RaveDecorator',
    'get_request', 'set_rave_modifier', #'expert_cache_get_or_reserve',
#    'expert_cache_put', 'expert_cache_abort',
#    'expert_cache_get_or_reserve_value', 'expert_cache_put_value',
    'Expert', 'list_of', 'one_of', 'any'
]
