How to gracefully return exception from a WebApi Message Handler

I have a global ExceptionFilter in my Mvc\WebApi application registered as:

public virtual void RegisterHttpFilters(HttpConfiguration config)
{
    config.Filters.Add(new MyExceptionFilter(_exceptionHandler));
}

where MyExceptionFilter is:

public class MyExceptionFilter : ExceptionFilterAttribute
{
    private readonly IMyExceptionHandler m_exceptionHandler;

    public MyExceptionFilter(IMyExceptionHandler exceptionHandler)
    {
        m_exceptionHandler = exceptionHandler;
    }

    public override void OnException(HttpActionExecutedContext context)
    {
        Exception ex = context.Exception;
        if (ex != null)
        {
            object response = null;
            HttpStatusCode statusCode = m_exceptionHandler != null
                ? m_exceptionHandler.HandleException(ex, out response)
                : HttpStatusCode.InternalServerError;
            context.Response = context.Request.CreateResponse(statusCode, response ?? ex);
        }

        base.OnException(context);
    }
}

This filter returns all exceptions as json-objects and allows some IMyExceptionHandler-implementation to customize the object being returned.

All this works well. Till I have an exception in some of my message handlers:

public class FooMessageHandler : DelegatingHandler
{
    private readonly Func<IBar> _barFactory;
    public FooMessageHandler(Func<IBar> barFactory)
    {
        _barFactory = varFactory;
    }

    protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
    {
        request.Properties["MY_BAR"] = _barFactory();
        return base.SendAsync(request, cancellationToken);
    }
}

As you can see this handler creates some component and put it into the current http request message. When an exception happens in FuncOfIBar then I get the Yellow Screen of Death. My ExceptionFilter isn't called.

I tried to specifically catch the exception in the message handler and return HttpResponseException but it doesn't change anything - still getting YSOD:

public class XApplicationInitializerMessageHandler : DelegatingHandler
{
    private readonly Func<IXCoreApplication> _appFactory;
    private readonly IXExceptionHandler m_exceptionHandler;

    public XApplicationInitializerMessageHandler(Func<IXCoreApplication> appFactory, IXExceptionHandler exceptionHandler)
    {
        ArgumentValidator.EnsureArgumentNotNull(appFactory, "appFactory");
        _appFactory = appFactory;
        m_exceptionHandler = exceptionHandler;
    }

    protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
    {
        try
        {
            request.SetApplication(_appFactory());
        }
        catch (Exception ex)
        {
            object resultObject = null;
            HttpStatusCode statusCode = m_exceptionHandler != null
                ? m_exceptionHandler.HandleException(ex, out resultObject)
                : HttpStatusCode.InternalServerError;
            HttpResponseMessage responseMessage = request.CreateResponse(statusCode, resultObject ?? ex);

            throw new HttpResponseException(responseMessage);
        }
        return base.SendAsync(request, cancellationToken);
    }
}

}

I want behavior of my app to be same regardless where an exception happens: in a ApiController or a message handler.

How to do this?

I know about Application_Error but I'd like to keep HttpApplication customization untouchable.