Typescript discriminant union with nested property with a long list of possibilities

I read at a few places that discriminant unions from a nested property are not something feasible at this moment, or at least not without guard functions and I wanted to know if there's a way not to use guard functions at all to determine a type.

The goal is to try to use Enums for narrowing because multiple CustomFieldType can have the same type of value, and it would be simpler to register cases every time by using a switch statement rather than if-elseif to narrow down.

Here is a typescript playground that tries the different possibilities.

enum CustomFieldType {
    INT = 'int',
    STRING = 'string',
    MULTISELECT = 'multiselect',
    SELECT = 'select'
    // Possibly 20 others
}

interface CustomFieldValueBase<T extends CustomFieldType> {
    customField: {
        type: T;
    }
    value: any;
}

interface CustomFieldValueInt extends CustomFieldValueBase<CustomFieldType.INT> {
    value: number;
}
interface CustomFieldValueString extends CustomFieldValueBase<CustomFieldType.STRING> {
    value: string;
}
interface CustomFieldValueMultiSelect extends CustomFieldValueBase<CustomFieldType.MULTISELECT> {
    value: number[];
}
interface CustomFieldValueSelect extends CustomFieldValueBase<CustomFieldType.SELECT> {
    value: number[];
}

type CustomFieldValue =
  | CustomFieldValueInt
  | CustomFieldValueString
  | CustomFieldValueMultiSelect
  | CustomFieldValueSelect


// Solution wanted
// Won't narrow
function guessType(cfv: CustomFieldValue) {
    switch (cfv.customField.type) {
        case CustomFieldType.INT:
            return cfv.value // should be narrowed to number
        case CustomFieldType.STRING:
            return cfv.value // should be narrowed to string
        case CustomFieldType.MULTISELECT:
        case CustomFieldType.SELECT:
            return cfv.value // should be narrowed to number[]
        default: throw new Error()
    }
}