Currently, I am trying to implement MLP-based MNIST classification training code, mimicking the operational flow of computing-in-memory (CIM).
To mimic the CIM operation and match the memory-array size, I divided 3 hidden layers and 1 output layer into 13, 8, 8 and 8 layers, respectively.
After dividing each hidden and output layer, I implemented the hardware noise layers between those.
import os
import tensorflow as tf
import numpy as np
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.layers import Input, Lambda
from tensorflow.keras.models import Model
import larq
import sys
sys.path.append("PACKAGE_PATH")
from variation import shrinkPS_func as ShrinkPS
from variation import ADC_4bModel_func as ADC_4bModel
# Import MNIST dataset
(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()
train_images = train_images.reshape((60000, 28, 28))
test_images = test_images.reshape((10000, 28, 28))
# Normalize pixel values to be between -1 and +1, and binarize it to -1 and +1
train_images, test_images = train_images / 127.5 -1, test_images / 127.5 -1
train_images_binary = np.where(train_images > 0, 1, -1)
test_images_binary = np.where(test_images > 0, 1, -1)
del train_images, test_images
# Returns input tensor and split the tensor according to the memory array size
nonflat_input_layer = Input(shape=(28, 28, 1))
input_layer = tf.keras.layers.Flatten()(nonflat_input_layer)
input_layer0, input_layer1, input_layer2 = input_layer[:,0:64], input_layer[:,64:128], input_layer[:,128:192]
input_layer3, input_layer4, input_layer5 = input_layer[:,192:256], input_layer[:,256:320], input_layer[:,320:384]
input_layer6, input_layer7, input_layer8 = input_layer[:,384:448], input_layer[:,448:512], input_layer[:,512:576]
input_layer9, input_layer10, input_layer11 = input_layer[:,576:640], input_layer[:,640:704], input_layer[:,704:768]
input_layer12 = input_layer[:,768:784]
# First hidden layer
hidden1_0 = larq.layers.QuantDense(512,
kernel_quantizer= "ste_sign",
kernel_constraint= "weight_clip",
name = "first_hidden_layer0")(input_layer0)
## ellipsis ##
hidden1_12 = larq.layers.QuantDense(512,
kernel_quantizer= "ste_sign",
kernel_constraint= "weight_clip",
name = "first_hidden_layer12")(input_layer12)
# Shrinking the PS according to the SPICE data
shrink1_0 = Lambda(lambda x: ShrinkPS(x), name="shrink1_0")(hidden1_0)
## ellipsis ##
shrink1_12 = Lambda(lambda x: ShrinkPS(x), name="shrink1_12")(hidden1_12)
# 4-bit SAR ADC conversion with the variation measured by SPICE
ADC1_0 = Lambda(lambda x: ADC_4bModel(x), name="ADC1_0")(shrink1_0)
## ellipsis ##
ADC1_12 = Lambda(lambda x: ADC_4bModel(x), name="ADC1_12")(shrink1_12)
# Digital Sum
ADCsum_1 = layers.Add()([ADC1_0, ADC1_1, ADC1_2, ADC1_3, ADC1_4, ADC1_5, ADC1_6,
ADC1_7, ADC1_8, ADC1_9, ADC1_10, ADC1_11, ADC1_12])
# Binarize the digital sum
pre_binarize1 = layers.Subtract()([ADCsum_1, tf.constant(91, dtype = tf.float32, shape = (1))])
quantize1 = larq.quantizers.SteSign()(pre_binarize1)
# Split
quantize1_0 = quantize1[:,0:64]
## ellipsis ##
quantize1_7 = quantize1[:,448:512]
The similar code is implemented for second and third hidden layers and its output. For the output of the model, the code is implemented as follows.
# Output layer
output_0 = larq.layers.QuantDense(10,
kernel_quantizer= "ste_sign",
kernel_constraint= "weight_clip",
name = "output_layer0")(quantize3_0)
## ellipsis ##
output_7 = larq.layers.QuantDense(10,
kernel_quantizer= "ste_sign",
kernel_constraint= "weight_clip",
name = "output_layer7")(quantize3_7)
# Shrinking the PS according to the SPICE data
shrinkOut_0 = Lambda(lambda x: ShrinkPS(x), name="shrinkOut_0")(output_0)
## ellipsis ##
shrinkOut_7 = Lambda(lambda x: ShrinkPS(x), name="shrinkOut_7")(output_7)
# 4-bit SAR ADC conversion with the variation measured by SPICE
ADCOut_0 = Lambda(lambda x: ADC_4bModel(x), name="ADCOut_0")(shrinkOut_0)
## ellipsis ##
ADCOut_7 = Lambda(lambda x: ADC_4bModel(x), name="ADCOut_7")(shrinkOut_7)
# Digital Sum
ADCsum_Out = layers.Add()([ADCOut_0, ADCOut_1, ADCOut_2, ADCOut_3, ADCOut_4, ADCOut_5, ADCOut_6, ADCOut_7])
# Activation
pre_binarizeOut = layers.Subtract()([ADCsum_Out, tf.constant(56, dtype = tf.float32, shape = (1))])
# Softmax
output = layers.Softmax(name = "output")(pre_binarizeOut)
# Implementing a model based on given functional API codes
model = Model(inputs= nonflat_input_layer, outputs= output)
# Compile & Train the model
model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=0.001),
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False),
metrics = ['accuracy'])
model.fit(train_images_binary, train_labels, batch_size = 300, epochs = 100,
validation_data=(test_images_binary, test_labels))
test_loss, test_acc = model.evaluate(test_images_binary, test_labels)
print(f"Test accuracy {test_acc * 100:.2f} %")
And code below describes the function for ShrinkPS and ADC_4bModel which is implemented via Lambda layer in the code above.
# Shrinking the partial sum according to the variation of bitcell array's Vbl output.
def shrinkPS_func(inputPS):
coeff_a = tf.constant(0.0003883012)
coeff_b = tf.constant(0.7028718)
coeff_c = tf.constant(0.16390099)
outputs = tf.math.multiply(coeff_a, tf.math.square(inputPS))
outputs = tf.math.add(outputs, tf.math.multiply(coeff_b, inputPS))
outputs = tf.math.add(outputs, coeff_c)
return outputs
# Implementing the ADC model which has fitted variation.
@tf.function
def ADC_4bModel_func(in_ADC_PS):
# Loading the data
inputPS = np.loadtxt("~/inputPS.csv", dtype = float, delimiter = ',')
PSmean_ADC = np.loadtxt("~/PSmean_ADC.csv", dtype = float, delimiter = ',')
PS_Vdiff_Mean = np.loadtxt("~/PS_Vdiff_Mean.csv", dtype = float, delimiter = ',')
PS_Vdiff_Std = np.loadtxt("~/PS_Vdiff_Std.csv", dtype = float, delimiter = ',')
P1_PSmean = PSmean_ADC[:, 1]
P2_PSmean = PSmean_ADC[:, 2]
P3_PSmean = PSmean_ADC[:, 3]
P4_PSmean = PSmean_ADC[:, 4]
# Implementing tensor-based ADC model
CompBndry = tf.constant(0.0, dtype = tf.float64)
"From Phase 0 to 1"
Func_P0to1Mean = scipy.interpolate.interp1d(inputPS, PS_Vdiff_Mean[:, 0], bounds_error = False, fill_value = 'extrapolate')
Func_P0to1Std = scipy.interpolate.interp1d(inputPS, PS_Vdiff_Std[:, 0], bounds_error = False, fill_value = 'extrapolate')
"From Phase 1 to 2"
Func_P1to2Mean = scipy.interpolate.interp1d(P1_PSmean, PS_Vdiff_Mean[:, 1], bounds_error = False, fill_value = 'extrapolate')
Func_P1to2Std = scipy.interpolate.interp1d(P1_PSmean, PS_Vdiff_Std[:, 1], bounds_error = False, fill_value = 'extrapolate')
"From Phase 2 to 3"
idxP2sort = np.argsort(P2_PSmean)
P2_SortPS = np.array([P2_PSmean[idxP2sort], PS_Vdiff_Mean[:, 2][idxP2sort], PS_Vdiff_Std[:, 2][idxP2sort]])
idxP2Low = np.where(P2_SortPS[0] < 0)
idxP2High = np.where(P2_SortPS[0] > 0)
Func_P2to3LowMean = scipy.interpolate.UnivariateSpline(P2_SortPS[0][idxP2Low], P2_SortPS[1][idxP2Low])
Func_P2to3HighMean = scipy.interpolate.UnivariateSpline(P2_SortPS[0][idxP2High], P2_SortPS[1][idxP2High])
Func_P2to3LowStd = scipy.interpolate.UnivariateSpline(P2_SortPS[0][idxP2Low], P2_SortPS[2][idxP2Low])
Func_P2to3HighStd = scipy.interpolate.UnivariateSpline(P2_SortPS[0][idxP2High], P2_SortPS[2][idxP2High])
"From Phase 3 to 4"
idxP3sort = np.argsort(P3_PSmean)
P3_SortPS = np.array([P3_PSmean[idxP3sort], PS_Vdiff_Mean[:, 3][idxP3sort], PS_Vdiff_Std[:, 3][idxP3sort]])
idxP3Low = np.where(P3_SortPS[0] < 0)
idxP3High = np.where(P3_SortPS[0] > 0)
Func_P3to4LowMean = scipy.interpolate.UnivariateSpline(P3_SortPS[0][idxP3Low], P3_SortPS[1][idxP3Low])
Func_P3to4HighMean = scipy.interpolate.UnivariateSpline(P3_SortPS[0][idxP3High], P3_SortPS[1][idxP3High])
Func_P3to4LowStd = scipy.interpolate.UnivariateSpline(P3_SortPS[0][idxP3Low], P3_SortPS[2][idxP3Low])
Func_P3to4HighStd = scipy.interpolate.UnivariateSpline(P3_SortPS[0][idxP3High], P3_SortPS[2][idxP3High])
"From Phase 4 to 5"
idxP4sort = np.argsort(P4_PSmean)
P4_SortPS = np.array([P4_PSmean[idxP4sort], PS_Vdiff_Mean[:, 4][idxP4sort], PS_Vdiff_Std[:, 4][idxP4sort]])
idxP4Low = np.where(P4_SortPS[0] < 0)
idxP4High = np.where(P4_SortPS[0] > 0)
Func_P4to5LowMean = scipy.interpolate.UnivariateSpline(P4_SortPS[0][idxP4Low], P4_SortPS[1][idxP4Low])
Func_P4to5HighMean = scipy.interpolate.UnivariateSpline(P4_SortPS[0][idxP4High], P4_SortPS[1][idxP4High])
Func_P4to5LowStd = scipy.interpolate.UnivariateSpline(P4_SortPS[0][idxP4Low], P4_SortPS[2][idxP4Low])
Func_P4to5HighStd = scipy.interpolate.UnivariateSpline(P4_SortPS[0][idxP4High], P4_SortPS[2][idxP4High])
"[Transition #1] Phase 0 -> 1"
trns0to1_var = tf.random.normal(shape = tf.shape(in_ADC_PS),
mean = tf.py_function(Func_P0to1Mean, [in_ADC_PS], tf.float64),
stddev = tf.py_function(Func_P0to1Std, [in_ADC_PS], tf.float64),
dtype = tf.float64)
P1VarOut_ADC_PS = tf.cast(tf.math.scalar_mul(-1.0, trns0to1_var), tf.float64)
"[Transition #2] Phase 1 -> 2"
trns1to2_var = tf.random.normal(shape = tf.shape(P1VarOut_ADC_PS),
mean = tf.py_function(Func_P1to2Mean, [P1VarOut_ADC_PS], tf.float64),
stddev = tf.py_function(Func_P1to2Std, [P1VarOut_ADC_PS], tf.float64),
dtype = tf.float64)
P2VarOut_ADC_PS = tf.cast(tf.math.add(P1VarOut_ADC_PS, trns1to2_var), tf.float64)
SARSET_trns1to2 = tf.math.sign(tf.math.subtract(CompBndry, P2VarOut_ADC_PS))
"[Transition #3] Phase 2 -> 3"
SARSET_trns2to3 = tf.math.sign(tf.math.subtract(CompBndry, P2VarOut_ADC_PS))
if tf.math.less(P2VarOut_ADC_PS, CompBndry) is True:
phase2to3Mean = tf.py_function(Func_P2to3LowMean, [P2VarOut_ADC_PS], tf.float64)
phase3ErrStd = tf.py_function(Func_P2to3LowStd, [P2VarOut_ADC_PS], tf.float64)
else:
phase2to3Mean = tf.py_function(Func_P2to3HighMean, [P2VarOut_ADC_PS], tf.float64)
phase3ErrStd = tf.py_function(Func_P2to3HighStd, [P2VarOut_ADC_PS], tf.float64)
trns2to3_var = tf.math.multiply(SARSET_trns2to3, tf.random.normal(shape = tf.shape(P2VarOut_ADC_PS),
mean = phase2to3Mean,
stddev = phase3ErrStd,
dtype = tf.float64))
P3VarOut_ADC_PS = tf.cast(tf.math.add(P2VarOut_ADC_PS, trns2to3_var), tf.float64)
"[Transition #4] Phase 3 -> 4"
SARSET_trns3to4 = tf.math.sign(tf.math.subtract(CompBndry, P3VarOut_ADC_PS))
if tf.math.less(P3VarOut_ADC_PS, CompBndry) is True:
phase3to4Mean = tf.py_function(Func_P3to4LowMean, [P3VarOut_ADC_PS], tf.float64)
phase4ErrStd = tf.py_function(Func_P3to4LowStd, [P3VarOut_ADC_PS], tf.float64)
else:
phase3to4Mean = tf.py_function(Func_P3to4HighMean, [P3VarOut_ADC_PS], tf.float64)
phase4ErrStd = tf.py_function(Func_P3to4HighStd, [P3VarOut_ADC_PS], tf.float64)
trns3to4_var = tf.math.multiply(SARSET_trns3to4, tf.random.normal(shape = tf.shape(P3VarOut_ADC_PS),
mean = phase3to4Mean,
stddev = phase4ErrStd,
dtype = tf.float64))
P4VarOut_ADC_PS = tf.cast(tf.math.add(P3VarOut_ADC_PS, trns3to4_var), tf.float64)
"[Transition #5] Phase 4 -> 5"
SARSET_trns4to5 = tf.math.sign(tf.math.subtract(CompBndry, P4VarOut_ADC_PS))
if tf.math.less(P4VarOut_ADC_PS, CompBndry) is True:
phase4to5Mean = tf.py_function(Func_P4to5LowMean, [P4VarOut_ADC_PS], tf.float64)
phase5ErrStd = tf.py_function(Func_P4to5LowStd, [P4VarOut_ADC_PS], tf.float64)
else:
phase4to5Mean = tf.py_function(Func_P4to5HighMean, [P4VarOut_ADC_PS], tf.float64)
phase5ErrStd = tf.py_function(Func_P4to5HighStd, [P4VarOut_ADC_PS], tf.float64)
trns4to5_var = tf.math.multiply(SARSET_trns4to5, tf.random.normal(shape = tf.shape(P4VarOut_ADC_PS),
mean = phase4to5Mean,
stddev = phase5ErrStd,
dtype = tf.float64))
P5VarOut_ADC_PS = tf.cast(tf.math.add(P4VarOut_ADC_PS, trns4to5_var), tf.float64)
"SARSET of Phase 5 & bit position setting"
SARSET_phase5 = tf.math.sign(tf.math.subtract(CompBndry, P5VarOut_ADC_PS))
Bit0 = tf.scalar_mul(8, tf.cast(tf.math.less(P2VarOut_ADC_PS, CompBndry), dtype = tf.int32))
Bit1 = tf.scalar_mul(4, tf.cast(tf.math.less(P3VarOut_ADC_PS, CompBndry), dtype = tf.int32))
Bit2 = tf.scalar_mul(2, tf.cast(tf.math.less(P4VarOut_ADC_PS, CompBndry), dtype = tf.int32))
Bit3 = tf.cast(tf.math.less(P5VarOut_ADC_PS, CompBndry), dtype = tf.int32)
ModelOut = tf.math.add(Bit3, tf.math.add(Bit2, tf.math.add(Bit1, Bit0)))
ModelOut = tf.cast(ModelOut, dtype = tf.float32)
return ModelOut
After implementing the neural network algorithm code, the error message showed up as follows.
ValueError: No gradients provided for any variable: ([],).
(in the error message's [], all of the hidden dense layers were indicated)
I know that this kind of error is obtained when I don't define the label data in model.fit() code. Though I defined the label data both for training and evaluation, I still get this error. Are there any different reasons for this error? How can I solve it?