How to avoid child rebuilding if parent updates

I have a strange requirement. I saw this this question on SO but i can't make it work for my case.

I have an animated container which represent my screen. On pressing the ADD icon. I'm transforming the screen like this (The column in right side image is below the homescreen) . But inside that AnimatedContainer there is a LIST(as child).

Every time I do transfromation. The list is re building itself. Is there any way i can avoid it. ?

You can imagine that the homescreen is pinned to wall with two nails in the left and right top. As I press FAB. The left top nail is pulled out and the screen hangs on right nail support. and when i again press FAB, the left top is again pinned with nail.

This is the widget I'm using

https://pub.dev/packages/matrix4_transform

Here is minimal code to see the rebuilding

import 'package:flutter/material.dart';
import 'package:matrix4_transform/matrix4_transform.dart';

void main() => runApp(MyApp());

class MyApp extends StatelessWidget {
  static const String _title = 'Flutter Code Sample';

  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      title: _title,
      home: MyStatefulWidget(),
    );
  }
}

class MyStatefulWidget extends StatefulWidget {
  MyStatefulWidget({Key? key}) : super(key: key);

  @override
  _MyStatefulWidgetState createState() => _MyStatefulWidgetState();
}

_MyStatefulWidgetState? home;

class _MyStatefulWidgetState extends State<MyStatefulWidget> {
  DrawerManager drawerManager = DrawerManager();

  callSetState() {
    setState(() {});
  }

  @override
  Widget build(BuildContext context) {
    print('Rebuild');
    home = this;
    return AnimatedContainer(
      transform: Matrix4Transform()
          .translate(x: drawerManager.xOffSet, y: drawerManager.yOffSet)
          .rotate(drawerManager.angle)
          .matrix4,
      duration: Duration(milliseconds: 500),
      child: Scaffold(
        body: MyList(drawerManager),
      ),
    );
  }
}

class MyList extends StatelessWidget {
  final Data myData = Data();
  final DrawerManager drawerManager;

  MyList(this.drawerManager);

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      body: ListView.builder(
        physics: const BouncingScrollPhysics(),
        itemCount: myData.data.length+1,
        itemBuilder: (context, index) {
          print('Building list' + index.toString());
          if(index == 4){
            return GestureDetector(
              child: IconButton(
                  icon: Icon(Icons.add),
                  onPressed: () {
                    drawerManager.callback(drawerManager.isOpened);
                  }),
            );
          }
          else{return ListTile(
            title: Text(myData.data[index]),
          );}
        },
      ),
    );
  }
}
class Data {
  List<String> data = ['Hello1', 'Hello2', 'Hello3', 'Hello4'];
}
class DrawerManager {
  double xOffSet = 0;
  double yOffSet = 0;
  double angle = 0;
  bool isOpened = false;

  void callback(bool isOpen) {
    print('Tapped');
    if (isOpen) {
      xOffSet = 0;
      yOffSet = 0;
      angle = 0;
      isOpened = false;
    } else {
      xOffSet = 150;
      yOffSet = 80;
      angle = -0.2;
      isOpened = true;
    }
    callSetState();
  }
  void callSetState() {
    home!.callSetState();
  }
}

You can see that When you press that + icon. Screen transforms and the lists are rebuilding.