having an array pass problem for multi scale can network

Viewed 40

I am dealing with audio data (.wav) files, I have created two CNN network to pass the data. so the two models at the end will use the function of Concatenate the output of these two models and give out the classification. but I am having problem for training it, it says the model expects to see 2 arrays but it got one array. any one who can help me with this problem I will appreciate, I have been struggling for weeks.

this is my config file

class Config:
def __init__(self, mode='conv', nfilt=26, nfeat=40, nfft=4096, rate=16000):
    self.mode = mode
    self.nfilt = nfilt
    self.nfeat = nfeat
    self.nfft = nfft
    self.rate = rate
    self.step = int(rate / 10)
    self.model_path = os.path.join('models', mode + '.model')
    self.p_path = os.path.join('pickles', mode + '.p')

I build one function for my mfcc

build a function for build

def build_rand_feat():

X = []
y = []
_min, _max = float('inf'), -float('inf')
for _ in tqdm(range(n_sample)):
    rand_class = np.random.choice(class_dist.index, p=prob_dist)
    file = np.random.choice(df[df.label == rand_class].index)
    data, rate = sf.read('audios/' + file)
    label = df.at[file, 'label']
    rand_index = np.random.randint(0, data.shape[0] - config.step)
    sample = data[rand_index:rand_index + config.step]
    X_sample = mfcc(sample, rate, winlen=0.05, winstep=0.02, numcep=config.nfeat, nfilt=config.nfilt, nfft=config.nfft)
    
    _min = min(np.amin(X_sample), _min)
    _max = max(np.amax(X_sample), _max)
    X.append(X_sample)  
    y.append(classes.index(label))
config.min = _min
X = np.array(X)
y = np.array(y)
X = (X - _min) / (_max - _min)
if config.mode == 'conv':
    X = X.reshape(X.shape[0], X.shape[1], X.shape[2], 1)
    print(X.shape)
elif config.mode == 'time':
    X = X.reshape(X.shape[0], X.shape[1], X.shape[2])
y = to_categorical(y, num_classes=10)
config.data = (X, y)


return X, y

and this is my multi scale network

def get_conv_model():

main_model = Sequential()
main_model.add(Conv2D(128, (3, 3), activation='relu', strides=(1, 1), padding='same', input_shape=input_shape))
main_model.add(MaxPool2D((2, 2)))
main_model.add(Conv2D(128, (3, 3), activation='relu', strides=(1, 1), padding='same'))
main_model.add(MaxPool2D((2, 2), padding='same'))
main_model.add(Flatten())

second_model = Sequential()
second_model.add(Conv2D(256, (3, 3), activation='relu', strides=(1, 1), padding='same', input_shape=input_shape))
second_model.add(MaxPool2D((2, 2), padding='same'))
second_model.add(Conv2D(256, (3, 3), activation='relu', strides=(1, 1), padding='same'))
second_model.add(MaxPool2D((2, 2)))
second_model.add(Flatten())

# first model upper
main_model = Sequential()
main_model.add(Conv2D(64, (3, 3), activation='relu', strides=(1, 1), padding='same', input_shape=input_shape))
main_model.add(BatchNormalization())
main_model.add(Dropout(0.3))
main_model.add(Conv2D(128, (3, 3), activation='relu', strides=(1, 1), padding='same'))
main_model.add(BatchNormalization())
main_model.add(Dropout(0.3))
main_model.add(Conv2D(256, (3, 3), activation='relu', strides=(1, 1), padding='same'))
main_model.add(BatchNormalization())
main_model.add(Dropout(0.3))
main_model.add(Flatten())

# second model lower
lower_model1 = Sequential()
lower_model1.add(MaxPool2D(strides=(1, 1), padding='same', input_shape=input_shape))
lower_model1.add(Conv2D(128, (3, 3), activation='relu', strides=(1, 1), padding='same', input_shape=input_shape))
lower_model1.add(BatchNormalization())
lower_model1.add(Dropout(0.3))
lower_model1.add(Conv2D(256, (3, 3), activation='relu', strides=(1, 1), padding='same'))
lower_model1.add(BatchNormalization())
lower_model1.add(Dropout(0.3))
lower_model1.add(Conv2D(512, (3, 3), activation='relu', strides=(1, 1), padding='same'))
lower_model1.add(Flatten())

# merged models
merged_model = Concatenate()([main_model.output, lower_model1.output])
x = Dense(256, activation='relu')(merged_model)
x = Dropout(0.3)(x)
x = Dense(512, activation='relu')(x)
x = Dropout(0.3)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.3)(x)
output = Dense(10, activation='softmax')(x)
final_model = Model(inputs=[main_model.input, lower_model1.input], outputs=[output])
final_model.summary()
final_model.compile(loss="categorical_crossentropy", optimizer=Adam(0.001), metrics=['acc'])
print(K.eval(final_model.optimizer.lr))
#class_weight = compute_class_weight('balanced', np.unique(y_flat), y_flat)
final_model.fit(X,y, epochs=10, batch_size=64, shuffle=True, validation_split=0.3)
return main_model, lower_model1, merged_model

this is the error

0 Answers
Related