How to exclude rangebreak data together with rangeselector?

I have set rangebreak to exclude trading hours. But when i choose a button that looks at the past 1 day (which includes the hours excluded in the rangebreak), the plot suddenly shows the non-trading hours. However when i choose past 2 days or more, the non-trading hours suddenly disappears.

import yfinance as yf
import plotly.graph_objs as go
from plotly.subplots import make_subplots
data = yf.Ticker('SPY').history(period='5d', interval='1m')

fig = make_subplots(rows=1, cols=1,)

## Scatter
_ = fig.append_trace(
    go.Scatter(
        x=data.index,
        y=data['Close'],
        fill='tozeroy',
        line=dict(color='rgb(0,0,255)'),
    ),
    row=1, col=1,
)

## X-Axes
_ = fig.update_xaxes(
    rangeslider_visible=True,
    rangeselector=dict(
        buttons=[
            dict(count=6.5, label="Intraday(6.5hrs)", step="hour", stepmode="todate"),
            dict(count=1, label="1d", step="day", stepmode="backward"),
            dict(count=2, label="2d", step="day", stepmode="backward"),
            dict(count=3, label="3d", step="day", stepmode="backward"),
            dict(count=4, label="4d", step="day", stepmode="backward"),
            dict(count=5, label="5d", step="day", stepmode="backward"),
            dict(step="all")
        ],
        font={"color": "white"},
        bgcolor = 'black',
    ),
    rangebreaks=[
        dict(bounds=["sat", "mon"]),  # hide weekends, eg. hide sat to before mon
        dict(bounds=[16, 9.5], pattern="hour"),  # hide hours outside of 9.30am-4pm 
    ],
    row=1, col=1,
)
## Y-Axes
minVal = min(data['Close'])
maxVal = max(data['Close'])
_ = fig.update_yaxes(
    fixedrange=False,
    range=[minVal, maxVal],
    row=1, col=1,
    )

fig.show()

The main plot that excludes the trading hours

After clicking 1d (past 1 day)

3d (3 days)