How to make a custom activation function in tensorflow

I think this one could serve you. I have only used functions that incorporate tensorflow in that way it is he who manages the backpropagation.

If you use python functions you would have to program both the forward and the backward. But the problem is when you have to save the function's masks of the piecewise function in a "cache" (personally I do not know how it is done and it would be interesting to know).

import numpy as np
import tensorflow as tf

def new_relu(x, k=0.2):
    part_1 = tf.to_float(tf.math.less_equal(0.0, x))
    part_2 = tf.to_float(tf.math.logical_and(tf.math.less_equal(-1.0, x), tf.math.less(x, 0.0)))
    part_3 = tf.to_float(tf.math.less(x, -1.0))
    return part_1*x + part_2*x*k #+ part_3*0

def new_relu_test():

    # create data
    x = tf.random_normal([10])*10000
    y = new_relu(x)
    with tf.Session():
        diff = tf.test.compute_gradient_error(x, [10], y, [10])
        print(diff)

    # use in dense
    x = tf.placeholder(shape=[None, 3], dtype=tf.float32)
    nn = tf.layers.dense(x, 3, activation=new_relu)

EDIT: If you want the second parameter to be a tensor too, you must be the same size as the input.

import numpy as np
import tensorflow as tf

def new_relu(x, k=0.2):
    part_1 = tf.to_float(tf.math.less_equal(0.0, x))
    part_2 = tf.to_float(tf.math.logical_and(tf.math.less_equal(-1.0, x), tf.math.less(x, 0.0)))
    part_3 = tf.to_float(tf.math.less(x, -1.0))
    return part_1*x + part_2*x*k #+ part_3*0

def new_relu_test():
    # create data
    x = tf.random_normal([10])*10000
    y = new_relu(x)
    with tf.Session():
        diff = tf.test.compute_gradient_error(x, [10], y, [10])
        print(diff)

    # use in dense
    x = tf.placeholder(shape=[None, 3], dtype=tf.float32)
    x_b = tf.placeholder(shape=[None], dtype=tf.float32)
    nn_1 = tf.layers.dense(x, 3)
    nn_2 = tf.layers.dense(x, 3)
    nn = tf.layers.dense(nn_2, 1, activation=None)
    new_r = new_relu(x, tf.tile(tf.expand_dims(x_b, -1), [1, 3]))

    with tf.Session() as sess:
        sess.run(tf.initializers.global_variables())
        sess.run(new_r, feed_dict={x: np.random.rand(100, 3), x_b: np.random.rand(100)})

new_relu_test()

EDIT 2:

Using conv2d

import numpy as np
import tensorflow as tf

def new_relu(x, k=0.2):
    part_1 = tf.to_float(tf.math.less_equal(0.0, x))
    part_2 = tf.to_float(tf.math.logical_and(tf.math.less_equal(-1.0, x), tf.math.less(x, 0.0)))
    part_3 = tf.to_float(tf.math.less(x, -1.0))
    return part_1*x + part_2*x*k #+ part_3*0

def new_relu_test():
    # create data
    x = tf.random_normal([10])*10000
    y = new_relu(x)
    with tf.Session():
        diff = tf.test.compute_gradient_error(x, [10], y, [10])
        print(diff)

    # use in dense
    x = tf.placeholder(shape=[None, 28, 28, 3], dtype=tf.float32)

    conv1_weights = tf.get_variable("weight",[3,3,3,32],initializer=tf.truncated_normal_initializer(stddev=0.1))
    conv1_biases = tf.get_variable("bias", [32], initializer=tf.constant_initializer(0.0))
    conv1 = tf.nn.conv2d(x, conv1_weights, strides=[1, 1, 1, 1], padding='SAME')
    relu1 = new_relu(tf.nn.bias_add(conv1, conv1_biases))

    with tf.Session() as sess:
        sess.run(tf.initializers.global_variables())
        sess.run(relu1, feed_dict={x: np.random.rand(100, 28, 28, 3)})
new_relu_test()