I am attempting to apply Bayesian Optimization to discover the best combination of hyperparameters for my image segmentation model. For now, this is only for learning rate (lr) and weight decay (wd) values. Please would someone kindly troubleshoot my error message. Note for legacy reasons, I am using Fastai v1.
> RuntimeError: The size of tensor a (3744) must match the size of tensor b (101376) at non-singleton dimension 1
CODE:
# source: https://github.com/muellerzr/BaysianOptimizationFastAI/blob/master/Baysian_Optimization_in_FastAI.ipynb
from fastai.vision import *
from fastai.vision.interpret import *
from fastai.callbacks.hooks import *
# split image-mask pairs into train and valid sets
get_y_fn = lambda x: path_rtk_gt_masks/f'{x.stem}{x.suffix}' # func: gets image's corresponding mask
src = (SegmentationItemList.from_folder(path_rtk_gt_images)
.split_by_fname_file('../valid.txt')
.label_from_func(get_y_fn, classes=codes))
# create databunch
src_size = get_dimensions(rtk_gt_images[0])
data = (src.transform(get_transforms(), size=src_size, tfm_y=True)
.databunch(bs=8)
.normalize(imagenet_stats))
# custom fit parameters
def fit_with(lr, wd):
''' create and fit learner '''
# create learner
learn = unet_learner(data, models.resnet34, metrics=accuracy, wd=wd) # main purpose of Learner is to train model using Learner.fit. learn is a trainer for model using data to minimize loss_func with optimizer opt_func
# train model at specified learning rate
with progress_disabled_ctx(learn) as learn:
learn.fit_one_cycle(3, max_lr=lr)
# save, print and return model's overall accuracy
acc = float(learn.validate(learn.data.valid_dl)[1])
return acc
# call 'fit_with' func and return accuracy
pbounds = {'lr': (1e-5, 1e-2), 'wd': (4e-4, 0.4)} # specify parameter limits
optimizer = BayesianOptimization(
f = fit_with, # use custom fit function
pbounds=pbounds, # use specified limits
verbose = 2, # 1 prints a maximum only when it is observed, 0 is completely silent
random_state=1)
optimizer.maximize()
print(optimizer.max)
DATABUNCH:
Train: LabelList (561 items)
x: SegmentationItemList
Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352)
y: SegmentationLabelList
ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352)
Path: <...>;
Valid: LabelList (140 items)
x: SegmentationItemList
Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352),Image (3, 288, 352)
y: SegmentationLabelList
ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352),ImageSegment (1, 288, 352)
Path: <...>;
ERROR:
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/bayes_opt/target_space.py in probe(self, params)
190 try:
--> 191 target = self._cache[_hashable(x)]
192 except KeyError:
KeyError: (0.004176049826978714, 0.2882416675794864)
During handling of the above exception, another exception occurred:
RuntimeError Traceback (most recent call last)
13 frames
<ipython-input-74-de2a7cddfda6> in <module>
62 random_state=1)
63
---> 64 optimizer.maximize()
65 print(optimizer.max)
/usr/local/lib/python3.7/dist-packages/bayes_opt/bayesian_optimization.py in maximize(self, init_points, n_iter, acq, kappa, kappa_decay, kappa_decay_delay, xi, **gp_params)
183 iteration += 1
184
--> 185 self.probe(x_probe, lazy=False)
186
187 if self._bounds_transformer:
/usr/local/lib/python3.7/dist-packages/bayes_opt/bayesian_optimization.py in probe(self, params, lazy)
114 self._queue.add(params)
115 else:
--> 116 self._space.probe(params)
117 self.dispatch(Events.OPTIMIZATION_STEP)
118
/usr/local/lib/python3.7/dist-packages/bayes_opt/target_space.py in probe(self, params)
192 except KeyError:
193 params = dict(zip(self._keys, x))
--> 194 target = self.target_func(**params)
195 self.register(x, target)
196 return target
<ipython-input-74-de2a7cddfda6> in fit_with(lr, wd)
47 # train model at specified learning rate
48 with progress_disabled_ctx(learn) as learn:
---> 49 learn.fit_one_cycle(3, max_lr=lr)
50
51 # save, print and return model's overall accuracy
/usr/local/lib/python3.7/dist-packages/fastai/train.py in fit_one_cycle(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)
21 callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,
22 final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))
---> 23 learn.fit(cyc_len, max_lr, wd=wd, callbacks=callbacks)
24
25 def fit_fc(learn:Learner, tot_epochs:int=1, lr:float=defaults.lr, moms:Tuple[float,float]=(0.95,0.85), start_pct:float=0.72,
/usr/local/lib/python3.7/dist-packages/fastai/basic_train.py in fit(self, epochs, lr, wd, callbacks)
198 else: self.opt.lr,self.opt.wd = lr,wd
199 callbacks = [cb(self) for cb in self.callback_fns + listify(defaults.extra_callback_fns)] + listify(callbacks)
--> 200 fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
201
202 def create_opt(self, lr:Floats, wd:Floats=0.)->None:
/usr/local/lib/python3.7/dist-packages/fastai/basic_train.py in fit(epochs, learn, callbacks, metrics)
104 if not cb_handler.skip_validate and not learn.data.empty_val:
105 val_loss = validate(learn.model, learn.data.valid_dl, loss_func=learn.loss_func,
--> 106 cb_handler=cb_handler, pbar=pbar)
107 else: val_loss=None
108 if cb_handler.on_epoch_end(val_loss): break
/usr/local/lib/python3.7/dist-packages/fastai/basic_train.py in validate(model, dl, loss_func, cb_handler, pbar, average, n_batch)
61 if not is_listy(yb): yb = [yb]
62 nums.append(first_el(yb).shape[0])
---> 63 if cb_handler and cb_handler.on_batch_end(val_losses[-1]): break
64 if n_batch and (len(nums)>=n_batch): break
65 nums = np.array(nums, dtype=np.float32)
/usr/local/lib/python3.7/dist-packages/fastai/callback.py in on_batch_end(self, loss)
306 "Handle end of processing one batch with `loss`."
307 self.state_dict['last_loss'] = loss
--> 308 self('batch_end', call_mets = not self.state_dict['train'])
309 if self.state_dict['train']:
310 self.state_dict['iteration'] += 1
/usr/local/lib/python3.7/dist-packages/fastai/callback.py in __call__(self, cb_name, call_mets, **kwargs)
248 "Call through to all of the `CallbakHandler` functions."
249 if call_mets:
--> 250 for met in self.metrics: self._call_and_update(met, cb_name, **kwargs)
251 for cb in self.callbacks: self._call_and_update(cb, cb_name, **kwargs)
252
/usr/local/lib/python3.7/dist-packages/fastai/callback.py in _call_and_update(self, cb, cb_name, **kwargs)
239 def _call_and_update(self, cb, cb_name, **kwargs)->None:
240 "Call `cb_name` on `cb` and update the inner state."
--> 241 new = ifnone(getattr(cb, f'on_{cb_name}')(**self.state_dict, **kwargs), dict())
242 for k,v in new.items():
243 if k not in self.state_dict:
/usr/local/lib/python3.7/dist-packages/fastai/callback.py in on_batch_end(self, last_output, last_target, **kwargs)
342 if not is_listy(last_target): last_target=[last_target]
343 self.count += first_el(last_target).size(0)
--> 344 val = self.func(last_output, *last_target)
345 if self.world:
346 val = val.clone()
/usr/local/lib/python3.7/dist-packages/fastai/metrics.py in accuracy(input, targs)
28 input = input.argmax(dim=-1).view(n,-1)
29 targs = targs.view(n,-1)
---> 30 return (input==targs).float().mean()
31
32 def accuracy_thresh(y_pred:Tensor, y_true:Tensor, thresh:float=0.5, sigmoid:bool=True)->Rank0Tensor:
RuntimeError: The size of tensor a (3744) must match the size of tensor b (101376) at non-singleton dimension 1