List<T>.RemoveAll does not work on List objects

I have the below implementation.

members.RemoveAll doesn't seem to be doing the job by the way. I wonder why and how to fix this. thanks!

public class Member
{
    public string member { get; set; }
}

public class SpecialMember
{
    public string Upn { get; set; }
}

public class RemoveFromList
{
    public static List<Member> Remove()
    {
        var specialMembers = new List<SpecialMember>();
        specialMembers.Add(new SpecialMember { Upn = "a@b.com" });

        var members = new List<Member>();
        members.Add(new Member { member = "a@b.com" });
        members.Add(new Member { member = "c@d.com" });

        var toRemove = members.Select(f => f.member)
                              .Intersect(specialMembers.Select(s => s.Upn))
                              .Select(m => new Member
                              {
                                  member = m
                              }).ToList();


        members.RemoveAll(m =>
        {
            var b = toRemove.Contains(m);
            return b;
        });

        return members;
    }
}