I am trying to train a model by using tensorflow Object Detection API. Using tensorflow-gpu=1.15.0,and pretrained model mask_rcnn_inception_v2_coco. when i run train.py,an error occured,it shows below
(0) Invalid argument: assertion failed: [Condition x == y did not hold element-wise:] [x (Loss/BoxClassifierLoss/assert_equal_5/x:0) = ] [0] [y (Loss/BoxClassifierLoss/assert_equal_5/y:0) = ] [1]
[[node Loss/BoxClassifierLoss/assert_equal_5/Assert/Assert (defined at D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\framework\ops.py:1748) ]]
[[Loss/RPNLoss/BalancedPositiveNegativeSampler/Where/_963]]
(1) Invalid argument: assertion failed: [Condition x == y did not hold element-wise:] [x (Loss/BoxClassifierLoss/assert_equal_5/x:0) = ] [0] [y (Loss/BoxClassifierLoss/assert_equal_5/y:0) = ] [1]
[[node Loss/BoxClassifierLoss/assert_equal_5/Assert/Assert (defined at D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\framework\ops.py:1748) ]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'Loss/BoxClassifierLoss/assert_equal_5/Assert/Assert':
File "D:/proj/models/research/build/lib/object_detection/legacy/train.py", line 186, in <module>
tf.app.run()
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\platform\app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "C:\Users\fxl\AppData\Roaming\Python\Python36\site-packages\absl\app.py", line 303, in run
_run_main(main, args)
File "C:\Users\fxl\AppData\Roaming\Python\Python36\site-packages\absl\app.py", line 251, in _run_main
sys.exit(main(argv))
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\util\deprecation.py", line 324, in new_func
return func(*args, **kwargs)
File "D:/proj/models/research/build/lib/object_detection/legacy/train.py", line 182, in main
graph_hook_fn=graph_rewriter_fn)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\object_detection-0.1-py3.6.egg\object_detection\legacy\trainer.py", line 290, in train
clones = model_deploy.create_clones(deploy_config, model_fn, [input_queue])
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\object_detection-0.1-py3.6.egg\deployment\model_deploy.py", line 192, in create_clones
outputs = model_fn(*args, **kwargs)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\object_detection-0.1-py3.6.egg\object_detection\legacy\trainer.py", line 205, in _create_losses
losses_dict = detection_model.loss(prediction_dict, true_image_shapes)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\object_detection-0.1-py3.6.egg\object_detection\meta_architectures\faster_rcnn_meta_arch.py", line 2285, in loss
fields.DetectionResultFields.num_detections)))
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\object_detection-0.1-py3.6.egg\object_detection\meta_architectures\faster_rcnn_meta_arch.py", line 2571, in _loss_box_classifier
gt_weights_batch=groundtruth_weights_list)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\object_detection-0.1-py3.6.egg\object_detection\core\target_assigner.py", line 515, in batch_assign
gt_weights)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\object_detection-0.1-py3.6.egg\object_detection\core\target_assigner.py", line 184, in assign
groundtruth_boxes.get())[:1])
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\object_detection-0.1-py3.6.egg\object_detection\utils\shape_utils.py", line 324, in assert_shape_equal
return tf.assert_equal(shape_a, shape_b)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\ops\check_ops.py", line 658, in assert_equal
data, summarize, message, name)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\ops\check_ops.py", line 371, in _binary_assert
return control_flow_ops.Assert(condition, data, summarize=summarize)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\util\tf_should_use.py", line 198, in wrapped
return _add_should_use_warning(fn(*args, **kwargs))
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\ops\control_flow_ops.py", line 165, in Assert
return gen_logging_ops._assert(condition, data, summarize, name="Assert")
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\ops\gen_logging_ops.py", line 74, in _assert
name=name)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\framework\op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\framework\ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\framework\ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\framework\ops.py", line 1748, in __init__
self._traceback = tf_stack.extract_stack()
Traceback (most recent call last):
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\client\session.py", line 1365, in _do_call
return fn(*args)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\client\session.py", line 1350, in _run_fn
target_list, run_metadata)
File "D:\SoftWare\Anaconda\envs\object_detection_api_by_tf\lib\site-packages\tensorflow_core\python\client\session.py", line 1443, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: assertion failed: [Condition x == y did not hold element-wise:] [x (Loss/BoxClassifierLoss/assert_equal_5/x:0) = ] [0] [y (Loss/BoxClassifierLoss/assert_equal_5/y:0) = ] [1]
[[{{node Loss/BoxClassifierLoss/assert_equal_5/Assert/Assert}}]]
[[Loss/RPNLoss/BalancedPositiveNegativeSampler/Where/_963]]
(1) Invalid argument: assertion failed: [Condition x == y did not hold element-wise:] [x (Loss/BoxClassifierLoss/assert_equal_5/x:0) = ] [0] [y (Loss/BoxClassifierLoss/assert_equal_5/y:0) = ] [1]
[[{{node Loss/BoxClassifierLoss/assert_equal_5/Assert/Assert}}]]
Here is my config file
# Mask R-CNN with Inception V2
# Configured for MSCOCO Dataset.
# Users should configure the fine_tune_checkpoint field in the train config as
# well as the label_map_path and input_path fields in the train_input_reader and
# eval_input_reader. Search for "PATH_TO_BE_CONFIGURED" to find the fields that
# should be configured.
model {
faster_rcnn {
num_classes: 1
image_resizer {
keep_aspect_ratio_resizer {
min_dimension: 800
max_dimension: 1365
}
}
number_of_stages: 3
feature_extractor {
type: 'faster_rcnn_inception_v2'
first_stage_features_stride: 16
}
first_stage_anchor_generator {
grid_anchor_generator {
scales: [0.25, 0.5, 1.0, 2.0]
aspect_ratios: [0.5, 1.0, 2.0]
height_stride: 16
width_stride: 16
}
}
first_stage_box_predictor_conv_hyperparams {
op: CONV
regularizer {
l2_regularizer {
weight: 0.0
}
}
initializer {
truncated_normal_initializer {
stddev: 0.01
}
}
}
first_stage_nms_score_threshold: 0.0
first_stage_nms_iou_threshold: 0.7
first_stage_max_proposals: 300
first_stage_localization_loss_weight: 2.0
first_stage_objectness_loss_weight: 1.0
initial_crop_size: 14
maxpool_kernel_size: 2
maxpool_stride: 2
second_stage_box_predictor {
mask_rcnn_box_predictor {
use_dropout: false
dropout_keep_probability: 1.0
predict_instance_masks: true
mask_height: 15
mask_width: 15
mask_prediction_conv_depth: 0
mask_prediction_num_conv_layers: 2
fc_hyperparams {
op: FC
regularizer {
l2_regularizer {
weight: 0.0
}
}
initializer {
variance_scaling_initializer {
factor: 1.0
uniform: true
mode: FAN_AVG
}
}
}
conv_hyperparams {
op: CONV
regularizer {
l2_regularizer {
weight: 0.0
}
}
initializer {
truncated_normal_initializer {
stddev: 0.01
}
}
}
}
}
second_stage_post_processing {
batch_non_max_suppression {
score_threshold: 0.0
iou_threshold: 0.6
max_detections_per_class: 100
max_total_detections: 300
}
score_converter: SOFTMAX
}
second_stage_localization_loss_weight: 2.0
second_stage_classification_loss_weight: 1.0
second_stage_mask_prediction_loss_weight: 4.0
}
}
train_config: {
batch_size: 1
optimizer {
momentum_optimizer: {
learning_rate: {
manual_step_learning_rate {
initial_learning_rate: 0.0002
schedule {
step: 900000
learning_rate: .00002
}
schedule {
step: 1200000
learning_rate: .000002
}
}
}
momentum_optimizer_value: 0.9
}
use_moving_average: false
}
gradient_clipping_by_norm: 10.0
fine_tune_checkpoint: "D:/proj/models/research/build/lib/object_detection/mask_rcnn_inception_v2_coco_2018_01_28/model.ckpt"
from_detection_checkpoint: true
# Note: The below line limits the training process to 200K steps, which we
# empirically found to be sufficient enough to train the pets dataset. This
# effectively bypasses the learning rate schedule (the learning rate will
# never decay). Remove the below line to train indefinitely.
num_steps: 200000
data_augmentation_options {
random_horizontal_flip {
}
}
}
train_input_reader: {
tf_record_input_reader {
input_path: "D:/proj/models/research/build/lib/object_detection/training/train.record"
}
label_map_path: "D:/proj/models/research/build/lib/object_detection/training/PVD.pbtxt"
load_instance_masks: true
mask_type: PNG_MASKS
}
eval_config: {
num_examples: 30
# Note: The below line limits the evaluation process to 10 evaluations.
# Remove the below line to evaluate indefinitely.
max_evals: 10
}
eval_input_reader: {
tf_record_input_reader {
input_path: "D:/proj/models/research/build/lib/object_detection/training/val.record"
}
label_map_path: "D:/proj/models/research/build/lib/object_detection/training/PVD.pbtxt"
load_instance_masks: true
mask_type: PNG_MASKS
shuffle: false
num_readers: 1
}
From utils/create_pascal_tf_record.py,I got the .record dataset by transforming pascal voc dataset.Is there anything wrong with them,please tell me and how to fix that,thanks