PySide: QListWIdget item disappearing when I remove and add item sometimes

I have a QListWidget with custom widgets. I am seeing an issue where if I do the following:

1. add two items
2. Remove the second item
3. Add another item

then the first item's widget contents disappears until I either resize the window, or add a third item.

import sys
from PySide.QtGui import *
from PySide.QtCore import *

class StagingWidget(QGroupBox):
    def __init__(self,parent=None):
        #QWidget.__init__(self,parent)
        super(StagingWidget,self).__init__()
        self.itemWidgets = list()
        self.count = 1  
        self.createUi()

    def createUi(self):
        self.widget_layout=QVBoxLayout(self)

        self.list_widget=QListWidget()

        self.setFixedWidth(450)
        self.setFixedHeight(600)
        self.list_widget.setStyleSheet("QListWidget::item:selected{background:lightblue}")
        self.widget_layout.addWidget(self.list_widget)
        self.buttonHLayout = QHBoxLayout()
        self.add = QPushButton("Add Item")
        self.add.clicked.connect(self.addListItem)
        self.buttonHLayout.addWidget(self.add)
        self.widget_layout.addLayout(self.buttonHLayout)

    def addListItem(self):
        itemN = QListWidgetItem()
        widget = QWidget()

        Button1 = QPushButton(str(self.count))
        Button2 =  QPushButton("Remove")
        Button2.clicked.connect(lambda item=itemN:self.removeJob(item))
        widgetLayout = QHBoxLayout()
        widgetLayout.addWidget(Button1)
        widgetLayout.addWidget(Button2)
        widget.setLayout(widgetLayout)
        itemN.setSizeHint(widget.sizeHint())    
        self.itemWidgets.append(widget)
        self.list_widget.addItem(itemN)
        self.list_widget.setItemWidget(itemN, self.itemWidgets[-1])
        self.count = self.count + 1

    def removeJob(self,item):
        print("Removing Job")
        row = self.list_widget.indexFromItem(item).row()
        self.list_widget.takeItem(row)


if __name__ == '__main__':
    app = QApplication(sys.argv)
    widget = StagingWidget()
    widget.show()
    sys.exit(app.exec_())  

Example: Add 1 item:

Add another item:

Remove item 2:

Add another item(You can see widget of item 1 disappear):

Add another item(Widget of item 1 reappears):

This happens every single time I do the above steps.