IgnoreLayerCollision2D not working with a foreach loop

I'm working on a game where the player has to ignore collision with a square if the square and the player have the same sprite color. The squares act as gates that allow the player to pass through only when the color of the gate and player are similar.

I'm using a for each loop to check if the player has the gate color then ignore collision. But this only works for my player's initial color gate and when the player changes color to access a different colored gate the 'ignore collision' doesn't work.

Here is my code:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Rendering.Universal;

public class PlayerColor : MonoBehaviour
{
    private Color currentColor;
    private Light2D olight;
    private SpriteRenderer sprite;

    public Color[] colorArray = {
        new Color(0.07843138f, 0.7490196f, 0.1803922f, 1.0f), //Green 14BF2E 
        new Color(0.09019608f, 0.5019608f, 0.9098039f, 1.0f), //Blue 1780E8 
        new Color(0.6313726f, 0.2705882f, 0.9019608f, 1.0f), //Purple A145E6 
        new Color(1f, 0.6392157f, 0.09019608f, 1.0f)  //Orange FFA317
    };

    // Start is called before the first frame update
    void Start()
    {
        olight = GetComponent<Light2D>();
        sprite = GetComponent<SpriteRenderer>();

        ChangeColor(sprite.color);
        
    }


    public Color ChangeColor(Color _color)
    {
        Color oldCol = currentColor;
        currentColor = _color;
        olight.color = currentColor;
        sprite.color = currentColor;

        int index = 6;
        foreach(Color color in colorArray)
        {
            if(color == sprite.color)
            {
                Physics2D.IgnoreLayerCollision(3, index);
            }
            else
            {
                Physics2D.IgnoreLayerCollision(3, index, false);
            }
            index++;
        }
        return oldCol;
    }
}