Flutter) Problem with offset after zooming

I use the Transform.scale() widget and the CustomPaint() widget for zoom and draw functions.

I'm going to zoom in or out and do custom painting However, if I change the scale to Zoom In or Zoom Out and draw, the line is output elsewhere.

The following is the offset difference between the default zoom and the 2x zoom when I draw point is taken in the same position.

Offset(595,417) // zoom x1.0
Offset(595,471) // zoom x2.0

And this difference gets bigger as move away from the middle of the screen. The image below is Zoom In with scale = 2. And in that state, when I drawing(the box of images), it is printed on a different position I dragged.

How can I keep the offset value constant even after changing the scale?

The following is part of my code

draw screen widget

      // zoom and translate screen
      return Transform.translate(
        offset: offset + sessionOffset, // moving
        child: Transform.scale(
          scale: scale, 
          child: Stack(
            children: [
              // allLines
              buildAllPaths(allLines: allLines),

              // new drawing line (onUpdate)
              buildCurrentPath(
                allLines: allLines,
                currentLine: currentLine,
              ),
            ],
          ),
        ),
      );

GestureDetector code

  Widget buildCurrentPath({
    required List<DrawingModel> allLines,
    DrawingModel? currentLine,
  }) {
    return GestureDetector(
      onScaleStart: (details) {
    RenderBox box = context.findRenderObject() as RenderBox;
    // zoom scale and session offset (moving offset)
    double scale = context.read<DrawProvider>().scale;
    Offset offset = context.read<DrawProvider>().offset;
    Offset sessionOffset = context.read<DrawProvider>().sessionOffset;
    Offset translateOffset = offset + sessionOffset;

    if (_isZoom) {
      context.read<DrawProvider>().onScaleStart(details);
    } else if (_isUseTool) {
      if (tool == Tool.pen) {
        Offset point = box.globalToLocal(details.focalPoint);
        // calculation moving offset
        point = Offset(
          (point.dx - translateOffset.dx), 
          (point.dy - translateOffset.dy),
        );

       // TODO add scale calculation
      }
    }
  },

  onScaleUpdate: (details) {
    RenderBox box = context.findRenderObject() as RenderBox;
    // zoom scale and session offset (moving offset)
    double scale = context.read<DrawProvider>().scale;
    Offset offset = context.read<DrawProvider>().offset;
    Offset sessionOffset = context.read<DrawProvider>().sessionOffset;
    Offset translateOffset = offset + sessionOffset;

    if (_isZoom) {
      context.read<DrawProvider>().onScaleUpdate(details);
    } else if (_isUseTool) {
      if (tool == Tool.pen) {
        Offset point = box.globalToLocal(details.focalPoint); 
        point = Offset(
          (point.dx - translateOffset.dx), 
          (point.dy - translateOffset.dy),
        );
      }
    }
  },

  onScaleEnd: (details) {
    if (_isZoom) {
      context.read<DrawProvider>().onScaleEnd(details);
    } 
  },

  child: RepaintBoundary(
    child: Container(
      width: 1200,
      height: 1200,
      decoration: BoxDecoration(
        color: Colors.transparent,
      ),
      child: StreamBuilder<Shape>(
          stream: currentLineStreamController.stream,
          builder: (context, snapshot) {
            // add null check
            // (shape != null
            if (snapshot.hasData) {
              return CustomPaint(
                painter: Sketcher(shapes),
              );
            }

            // Not yet Draw
            return Container();
          }),
    ),
  )

Zoom Functions

class DrawProvider extends ChangeNotifier {
  Offset _offset = Offset.zero;
  Offset _initialFocalPoint = Offset.zero;
  Offset _sessionOffset = Offset.zero;
  double _scale = 1.0;
  double _initialScale = 1.0;

  void onScaleStart(ScaleStartDetails details) {
  _initialFocalPoint = details.focalPoint;
  _initialScale = _scale;
  notifyListeners();
 }

  void onScaleUpdate(ScaleUpdateDetails details) {
    _sessionOffset = details.focalPoint - _initialFocalPoint;
    _scale = _initialScale * details.scale;
    notifyListeners();
  }

  void onScaleEnd(ScaleEndDetails details) {
    _offset += _sessionOffset; 
    _sessionOffset = Offset.zero;
    notifyListeners();
  }
}