Optimize Nginx + PHP-FPM for faster response times (for Openx adserving)

I'm currently running Nginx + PHP-FPM for serving ads on OpenX. Currently my response times are horrible, even during times of low load. However, my CPU and Memory resources are fine, so I can't seem to figure out what the bottleneck is.

My current config for nginx and php-fpm is:

worker_processes 20;
worker_rlimit_nofile 50000;

error_log  /var/log/nginx/error.log;
pid        /var/run/nginx.pid;

events {
    worker_connections  15000;
    multi_accept off;
    use epoll;
}

http {
    include       /etc/nginx/mime.types;

    access_log  /var/log/nginx/access.log;

    sendfile        on;
    tcp_nopush     off;

    keepalive_timeout  0;
    #keepalive_timeout  65;
    tcp_nodelay        on;

    gzip  on;
    gzip_disable "MSIE [1-6]\.(?!.*SV1)";
    gzip_comp_level 2;
    gzip_proxied    any;
    gzip_types    text/plain text/html text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript;

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}

server {
    listen   80;
    server_name  localhost;
    access_log  /var/log/nginx/localhost.access.log;

# Default location
    location / {
        root   /var/www;
        index  index.php;
    }

## Parse all .php file in the /var/www directory
    location ~ .php$ {
        fastcgi_pass   localhost:9000;
        fastcgi_index  index.php;
        fastcgi_param  SCRIPT_FILENAME  /var/www$fastcgi_script_name;
        include fastcgi_params;
        fastcgi_param  QUERY_STRING     $query_string;
        fastcgi_param  REQUEST_METHOD   $request_method;
        fastcgi_param  CONTENT_TYPE     $content_type;
        fastcgi_param  CONTENT_LENGTH   $content_length;
        fastcgi_ignore_client_abort     off;
    }

PHP-FPM:
rlimit_files = 50000
max_children = 500

I only included the PHP-FPM paramaters I've changed for PHP-FPM.

Does anyone have any tips on how I can optimize it so I can serve more requests? I'm seeing horrendous response times right now.