Issue with scroll boundaries with InteractiveViewer on Flutter

I'm having an issue with an InteractiveViewer in which the scroll limits are not working as expected: user can scroll past the limit of the content, and the content inside the InteractiveViewer can get out of sight. I believe this is an issue related to the boundaryMargin property, which is adding that extra scrollable margin after the end of the rows/cols. The thing is that if I put no boundaryMargin at all, as soon as I start doing pinch to zoom, zooming gets stuck and I can't resize the content anymore.

Code:

class TableGrid extends StatefulWidget {
  final ReferenceEntity roomRef;
  const TableGrid({
    required this.roomRef,
    Key? key,
  }) : super(key: key);

  @override
  State<TableGrid> createState() => _TableGridState();
}

class _TableGridState extends State<TableGrid> with TickerProviderStateMixin {
  late TransformationController transformationController;
  late AnimationController animationController;
  late Animation<Matrix4> animation;

  @override
  void initState() {
    super.initState();
    transformationController = TransformationController();
    animationController = AnimationController(
      duration: const Duration(milliseconds: 200),
      vsync: this,
    );
  }

  @override
  void dispose() {
    super.dispose();
    transformationController.dispose();
  }

  @override
  Widget build(BuildContext context) {
    final gridData = context.watch(GetRef.view.tables.tableListComputed.roomGrid(widget.roomRef));

    return StateListener(
      watchable: GetRef.view.tables.tableListStateRef.select((state) => state.triggerReset),
      onStateChanged: _handleReset,
      child: LayoutBuilder(builder: (context, constraints) {
        final maxSize = min(
          constraints.maxHeight * 0.25,
          constraints.maxWidth * 0.25,
        );
        final widgetSize = Size(constraints.maxWidth, constraints.maxHeight);
        final tgiSize = min(
          maxSize,
          min(widgetSize.width / gridData.item1.columns, widgetSize.height / gridData.item1.rows),
        );
        final maxScale = maxSize / tgiSize;
        return InteractiveViewer(
          transformationController: transformationController,
          maxScale: maxScale < 1 ? 1 : maxScale,
          minScale: 1,
          boundaryMargin: EdgeInsets.only(right: widgetSize.width, bottom: widgetSize.height),
          constrained: false,
          child: GestureDetector(
            behavior: HitTestBehavior.translucent,
            onTap: () => context.use(GetRef.view.tables.tableListLogic).setSelectedTable(null),
            child: Column(
              children: List.generate(
                gridData.item1.rows,
                (row) => Row(
                  children: List.generate(
                    gridData.item1.columns,
                    (column) => SizedBox(
                      width: tgiSize,
                      height: tgiSize,
                      child: TableGridItem(table: gridData.item2['$row, $column']),
                    ),
                  ),
                ),
              ),
            ),
          ),
        );
      }),
    );
  }
  }
}