C# memory access optimization that offset large array (Why is this code slow)

I'm working on image processing in C#. I have a problem getting the performance of offset large memory access in C#. The speed is significantly different from the same size of zero offset memory. In the case of C ++, the difference was not as great as in C#.

Can you please tell me why my code has this problem ? Also, are there any solutions ?

Source

using System;
using System.Runtime.InteropServices;
using System.Diagnostics;
using System.Numerics;

namespace Test
{
    class Program
    {
        unsafe static void Main(string[] args)
        {
            var width = 8000;
            var height = 8000;

            // var data = new Vector4[height * width]; <- similar problem occur
            var data = (Vector4*)Marshal.AllocHGlobal(height * width * sizeof(Vector4));
            var data2 = (Vector4*)Marshal.AllocHGlobal(height * width * sizeof(Vector4));

            // MATRIX
            float m11 = .7297023F, m12 = 0, m13 = 0, m14 = 0, m21 = 0, m22 = .6109577F,
                m23 = 0, m24 = 0, m31 = 0, m33 = .597218F, m32 = 0, m34 = 0, m41 = 0, m42 = 0,
                m43 = 0, m44 = 1F, m51 = .105F, m52 = .145F, m53 = .155F, m54 = 0;

            var sw = new Stopwatch();
            sw.Start();

            for (int y = 0; y < height; ++y)
            {
                var offset = width * y;
                for (int x = 0; x < width; ++x)
                {
                    // Slow ( 600ms )
                    ref var sData = ref data[offset + x];
                    ref var dData = ref data2[offset + x];

                    // Fast ( 200ms )
                    // ref var sData = ref data[x];
                    // ref var dData = ref data2[x];

                    float b = sData.X;
                    float g = sData.Y;
                    float r = sData.Z;
                    float a = sData.W;

                    dData.X = (b * m11) + (g * m21) + (r * m31) + (a * m41) + m51;
                    dData.Y = (b * m12) + (g * m22) + (r * m32) + (a * m42) + m52;
                    dData.Z = (b * m13) + (g * m23) + (r * m33) + (a * m43) + m53;
                    dData.W = (b * m14) + (g * m24) + (r * m34) + (a * m44) + m54;
                }
            }

            sw.Stop();
            Console.WriteLine(sw.ElapsedMilliseconds);

            Marshal.FreeHGlobal((IntPtr)data);
            Marshal.FreeHGlobal((IntPtr)data2);
        }
    }
}

When using managed array pointer

var array1 = new Vector4[width * height];
var array2 = new Vector4[width * height];
fixed (Vector4* data = &array1[0])
fixed (Vector4* data2 = &array2[0])
    for (int y = 0; y < height; ++y)
    {
        for (int x = 0; x < width; ++x)
        {
            // Slow ( 600ms )
            ref var sData = ref data[width * y + x];
            ref var dData = ref data2[width * y + x];

Offsetting the pointer in the outer loop

（A little improvement）

for (int y = 0; y < height; ++y)
{
    var offsetData1 = data + width * y;
    var offsetData2 = data2 + width * y;
    for (int x = 0; x < width; ++x)
    {
        // Slow ( 470ms )
        ref var sData = ref offsetData1[x];
        ref var dData = ref offsetData2[x];

C++ ver

#include <iostream>
#include <chrono>

struct Vector4 {
    float X = 0;
    float Y = 0;
    float Z = 0;
    float W = 0;
};

int main()
{
    long width = 8000;
    long height = 8000;

    auto buffer = new Vector4[width * height];
    auto buffer2 = new Vector4[width * height];

    // MATRIX
    float m11 = .7297023F, m12 = 0, m13 = 0, m14 = 0, m21 = 0, m22 = .6109577F,
        m23 = 0, m24 = 0, m31 = 0, m33 = .597218F, m32 = 0, m34 = 0, m41 = 0, m42 = 0,
        m43 = 0, m44 = 1, m51 = .105F, m52 = .145F, m53 = .155F, m54 = 0;

    std::chrono::system_clock::time_point  start, end;
    start = std::chrono::system_clock::now();

    for (int y = 0; y < height; ++y)
    {
        int offset = width * y;
        for (int x = 0; x < width; ++x)
        {
            Vector4& sData = buffer[offset + x];
            Vector4& dData = buffer2[offset + x];

            float b = sData.X;
            float g = sData.Y;
            float r = sData.Z;
            float a = sData.W;
            dData.X = (b * m11) + (g * m21) + (r * m31) + (a * m41) + m51;
            dData.Y = (b * m12) + (g * m22) + (r * m32) + (a * m42) + m52;
            dData.Z = (b * m13) + (g * m23) + (r * m33) + (a * m43) + m53;
            dData.W = (b * m14) + (g * m24) + (r * m34) + (a * m44) + m54;
        }
    }

    end = std::chrono::system_clock::now();
    double elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
    std::cout << elapsed << "\n";

    delete[] buffer;
    delete[] buffer2;
}

Benchmarks

Other info

IL in SharpLab