Compile-time decision of call parameters based on function prototype

You can do this in standard C using _Generic:

//  Define sample functions.
int foo(int a, int b)        { return a+b;   }
int bar(int a, int b, int c) { return a+b+c; }

//  Define names for the old and new types.
typedef int (*Type0)(int, int);
typedef int (*Type1)(int, int, int );

/*  Given an identifier (f) for a function and the arguments (a and b) we
    want to pass to it, the MyFunctio macro below uses _Generic to test which
    type the function is and call it with appropriate arguments.

    However, the unchosen items in the _Generic would contain function calls
    with mismatched arguments.  Even though these are never evaluated, they
    violate translation-time contraints for function calls.  One solution would
    be to cast the function (automatically converted to a function pointer) to
    the type being used in the call expression.  However, GCC sees through this
    and complains the function is called through a non-compatible type, even
    though it never actually is.  To solve this, the Sanitize macro is used.

    The Sanitize macro is given a function type and a function.  It uses a
    _Generic to select either the function (when it matches the type) or a null
    pointer of the type (when the function does not match the type).  (And GCC,
    although unhappy with a call to a function with wrong parameters, is happy
    with a call to a null pointer.)
*/
#define Sanitize(Type, f)   \
    _Generic((f), Type: (f), default: (Type) 0)
#define MyFunction(f, a, b)                                      \
    _Generic(f,                                                  \
        Type0:  Sanitize(Type0, (f)) ((a), (b)),   \
        Type1:  Sanitize(Type1, (f)) ((a), (b), 0) \
    )


#include <stdio.h>


int main(void)
{
    printf("%d\n", MyFunction(foo, 3, 4));
    printf("%d\n", MyFunction(bar, 3, 4));
}

The above use a parameterized function name (f) to allow us to demonstrate this with both functions (foo and bar) defined. For the situation in the question, there is only one function definition and with only one name, so we can simplify the macro. We can also use the function name for the macro name. (It is not recursively substituted during preprocessing both because C does not recursively substitute macros and because the name does not appear in the replacement text with a left parenthesis following it.) This would look like:

#define Sanitize(Type, f)   \
    _Generic((f), Type: (f), default: (Type) 0)
#define foo(a, b)                                  \
    _Generic(foo,                                  \
        Type0:  Sanitize(Type0, foo) ((a), (b)),   \
        Type1:  Sanitize(Type1, foo) ((a), (b), 0) \
    )
…
    printf("%d\n", foo(3, 4));

(The definition of Sanitize used here was provided by Artyer in this answer.)

The standard and best approach is writing a wrapper with the #ifdefs testing inside it:

int my_function(int param)
{
#if LIBRARY_VERSION > 5
    return lib_my_function(param, 2, 3);
#elif LIBRARY_VERSION > 4
    return lib_my_function(param, 2);
#else
    return lib_my_function(param);
#endif
}

Which then you call like:

my_function(1);

No need for macro magic, type checking works as usual, future API changes in the library are localized, IDEs and coworkers do not get confused, etc.

Side-note: if you really want to go with your proposed solution, then you can keep the type-checking by creating the proper pointer for each alternative:

if (__builtin_types_compatible_p(typeof(f), int(int, int)))
    ret = ((int(*)(int, int)) f)(3, 4);
else if (__builtin_types_compatible_p(typeof(f), int(int)))
    ret = ((int(*)(int)) f)(1);
else
    printf("no matching call\n");

But really, there is no advantage for that since you have anyway #ifdef for defining the proper f. At that point, you might as well define it as shown above and avoid all that, plus no need for GNU extensions.

Assuming this:

int f (int a) {         // OLD API
    return a + 5;
}

int f (int a, int b) {  // NEW API
    return a + b;
}

Then you can write a macro that checks the number of parameters passed, then call the actual function or a wrapper based on that:

static int f_wrapper (int a)
{
  return f(a, 0);
}

#define p(...) _Generic( &(int[]){__VA_ARGS__},       \
                 int(*)[2]: f,                        \
                 int(*)[1]: f_wrapper) (__VA_ARGS__)

Here the (int[]){ __VA_ARGS__} creates a dummy int[] array in the form of a compound literal. It will get 1 or 2 items based on the caller code. Type safety during storage to this dummy array will be the same as during function call - parameters/expressions convertible to int are accepted, others will yield compiler errors.

The &(int[]) ... takes the address of this dummy compound literal to generate an array pointer type - we can't have array types inside _Generic (because an array passed to _Generic will decay to a pointer) but we can have distinct array pointer types. So depending on if we ended up with an int[2] or int[1], the corresponding _Generic clause will get picked (or if no matches, compiler error).

The main advantage of this code is that no modifications are needed in the caller code, they can use p() with either 1 or 2 parameters.

Full example:

#include <stdio.h>

/* 
int f (int a) {         // OLD API
    return a + 5;
}
*/

int f (int a, int b) {  // NEW API
    return a + b;
}


static int f_wrapper (int a)
{
  return f(a, 0);
}

#define p(...) _Generic( &(int[]){__VA_ARGS__},       \
                 int(*)[2]: f,                        \
                 int(*)[1]: f_wrapper) (__VA_ARGS__)

int main (void) 
{
  int ret = 0;
  ret = p(3, 4);
  printf("ret: %d\n", ret);
  ret = p(1);
  printf("ret: %d\n", ret);
  return 0;
}

Output:

ret: 7
ret: 1

gcc x86 -O3

f:
        lea     eax, [rdi+rsi]
        ret
.LC0:
        .string "ret: %d\n"
main:
        sub     rsp, 8
        mov     esi, 7
        mov     edi, OFFSET FLAT:.LC0
        xor     eax, eax
        call    printf
        mov     esi, 1
        mov     edi, OFFSET FLAT:.LC0
        xor     eax, eax
        call    printf
        xor     eax, eax
        add     rsp, 8
        ret

As we can see from the disassembly, everything was handled in compile time and inlined - no temporary arrays etc are actually created.