How can I give a scrollablie listview gesture priority over a sliding panel in Flutter?

I have a sliding panel (sliding_up_panel 1.0.0 - https://pub.dev/packages/sliding_up_panel#-example-tab-) that allows me to slide up a panel from the bottom of the screen and obviously back down to it's hidden state. In this panel, I have added a scrollable separated list view. The panel movement itself works fine, as I can show and hide the panel as intended. The problem I have is with the list. When I scroll down the list with a vertical up gesture, the list scrolls perfectly. However, when I scroll back up the list with a vertical down gesture, the panel moves to its hidden view along with the list scrolling. I would have liked to show you a video here, but I don't have the ability, but here are a couple of screenshots of the two different states.

I have used gestureRecognizer in the GoogleMap part of my app, but, of course, ListView doesn't have this, so how can I scroll the list both ways without the panel going back to its hidden state?

Hidden State

Shown State

Panel Code

Widget _floatingCollapsed(BuildContext context, _fontSize) {
    return Container(
      padding: EdgeInsets.all(0),
      decoration: BoxDecoration(
          color: Theme.UniColour.primary[900],
          borderRadius: BorderRadius.only(
              topLeft: Radius.circular(14.0), topRight: Radius.circular(14.0)),
          boxShadow: [
            BoxShadow(
              blurRadius: 7.0,
              color: Colors.black87,
            ),
          ]),
      margin: EdgeInsets.fromLTRB(28.0, 42.0, 28.0, 0.0),
      child: Center(
        child: Text(
          "Slide up to select a building",
          style: TextStyle(color: Colors.white, fontSize: _fontSize),
        ),
      ),
    );
  }

  Widget _floatingPanel(sc, _fontSize) {
    return Container(
      decoration: BoxDecoration(
          color: Colors.white,
          borderRadius: BorderRadius.all(
            Radius.circular(14.0),
          ),
          boxShadow: [
            BoxShadow(
              blurRadius: 7.0,
              color: Colors.black87,
            ),
          ]),
      margin: EdgeInsets.fromLTRB(28.0, 42.0, 28.0, 35.0),
      child: Center(
        child: Container(
          margin: EdgeInsets.fromLTRB(0.0, 0.0, 0.0, 0.0),
          child: Column(children: [
            //Expanded(
            ////  flex: 4,
                          ClipRRect(
                borderRadius: BorderRadius.circular(14.0),
                child: Image(
                  image:
                      AssetImage('assets/imgs/generic/BuildingsPanelHeader.jpg'),
                  width: MediaQuery.of(context).size.width,
                  fit: BoxFit.contain,
                ),
              ),
           // ),
            Expanded(
              flex: 1,
              child: _showBuildingList(_fontSize),
            ),
          ]),
        ),
      ),
    );
  }

Building List Code

_showBuildingList(_fontSize) {
  return LayoutBuilder(
      builder: (BuildContext context, BoxConstraints constraints) {
    return Container(
      //color: Colors.transparent,
      //width: MediaQuery.of(context).size.width * 0.85,
      //height: (MediaQuery.of(context).size.height * 0.85),
      padding: EdgeInsets.symmetric(horizontal: 20.0),
      child: FutureBuilder(
        future: getBuildingsfromXML(context),
        builder: (context, results) {
          if (results.hasData) {
            List<Building> buildings = results.data;
            return ListView.separated(
                shrinkWrap: true,
                physics: BouncingScrollPhysics(),
                separatorBuilder: (context, index) =>
                    Divider(color: Theme.UniColour.primary[700]),
                itemCount: buildings.length,
                itemBuilder: (context, index) {
                  return ListTile(
                    contentPadding: EdgeInsets.zero,
                    leading: Icon(Icons.business,
                        color: Theme.UniColour.primary[500]),
                    title: Text(
                      buildings[index].name,
                      style: TextStyle(
                          fontSize: _fontSize, fontWeight: FontWeight.normal),
                    ),
                  );
                });
          } else {
            return Center(
              child: CircularProgressIndicator(
                valueColor:
                    AlwaysStoppedAnimation(Theme.UniColour.secondary[900]),
                backgroundColor: Theme.UniColour.primary[900],
              ),
            );
          }
        },
      ),
    );
  });
}

Future<List<Building>> getBuildingsfromXML(BuildContext context) async {
  String xmlString = await DefaultAssetBundle.of(context)
      .loadString('assets/campus/buildings.kml');

  var raw = xml.parse(xmlString);
  var elements = raw.findAllElements("Placemark");

  return elements.map((element) {
    return Building(element.findElements("name").first.text);
  }).toList();
}

thanks