Sequence classification binary model LSTM from scratch

Question

I am writing a LSTM sequence classifier from scratch (no use of AI library).

I first tried with a classical RNN which I started from a many to many model for a many to one model, with a forward propagation looking like that:

def rnn_forward(inputs,rnnNet):

    fw_cache = []

    hidden_state = np.zeros((rnnNet.d[0], 1))

    fw_cache = []

    for t in range(len(inputs)):

        hidden_state = cm.tanh( np.dot(rnnNet.p['U'], inputs[t]) + np.dot(rnnNet.p['V'], hidden_state) + rnnNet.p['b_h'] )

        fw_cache.append(hidden_state.copy())

    outputs = cm.softmax( np.dot(rnnNet.p['W'], hidden_state) + rnnNet.p['b_o'],rnn=True)

    return outputs, fw_cache

I could rewrite my parameters dimensions accordingly and this is working as expected.

However, I struggle with doing the same thing on a LSTM network. Below is the forward prop:

def lstm_forward(inputs,lstmNet):

    fw_cache = []
    # lstmNet.d[0] is the hidden_size
    h_prev = np.zeros((lstmNet.d[0], 1))
    C_prev = np.zeros((lstmNet.d[0], 1))

    for x in inputs:

        cache = {'C': C_prev, 'h': h_prev}

        # Concatenate input and hidden state
        cache['z'] = np.row_stack((cache['h'], x))
        # Calculate forget gate
        cache['f'] = cm.sigmoid(np.dot(lstmNet.p['W_f'], cache['z']) + lstmNet.p['b_f'])
        # Calculate input gate
        cache['i'] = cm.sigmoid(np.dot(lstmNet.p['W_i'], cache['z']) + lstmNet.p['b_i'])
        # Calculate candidate
        cache['g'] = cm.tanh(np.dot(lstmNet.p['W_g'], cache['z']) + lstmNet.p['b_g'])
        # Calculate memory state
        C_prev = cache['f'] * cache['C'] + cache['i'] * cache['g']
        # Calculate output gate
        cache['o'] = cm.sigmoid(np.dot(lstmNet.p['W_o'], cache['z']) + lstmNet.p['b_o'])
        # Calculate hidden state
        h_prev = cache['o'] * cm.tanh(cache['C'])
        # Calculate logits
        cache['v'] = np.dot(lstmNet.p['W_v'], h_prev) + lstmNet.p['b_v']
        # Calculate softmax

        fw_cache.append(copy.deepcopy(cache))

    outputs = cm.softmax(cache['v'],rnn=True)

    return outputs, fw_cache

My parameters are:

def init_params(lstmNet):

    hidden_size = lstmNet.d[0]
    vocab_size = lstmNet.d[1]
    z_size = lstmNet.d[2]
    output_size = lstmNet.d[3]

    # Weight matrix (forget gate)
    lstmNet.p['W_f'] = np.random.randn(hidden_size, z_size)

    # Bias for forget gate
    lstmNet.p['b_f'] = np.zeros((hidden_size, 1))

    # Weight matrix (input gate)
    lstmNet.p['W_i'] = np.random.randn(hidden_size, z_size)

    # Bias for input gate
    lstmNet.p['b_i'] = np.zeros((hidden_size, 1))

    # Weight matrix (candidate)
    lstmNet.p['W_g'] = np.random.randn(hidden_size, z_size)

    # Bias for candidate
    lstmNet.p['b_g'] = np.zeros((hidden_size, 1))

    # Weight matrix of the output gate !!! I expect this to change dimensions
    lstmNet.p['W_o'] = np.random.randn(hidden_size, z_size)
    lstmNet.p['b_o'] = np.zeros((hidden_size, 1))

    # Weight matrix relating the hidden-state to the output !!! I expect this to change dimensions
    lstmNet.p['W_v'] = np.random.randn(vocab_size, hidden_size)
    lstmNet.p['b_v'] = np.zeros((vocab_size, 1))

Any help in passing from this LSTM many to many model to a many to one model with output only on the last cell / input would be much appreciated.

Sequence classification binary model LSTM from scratch

0 Answers