How do you use malloc of struct inside a function?

There are two values you have to pass back from add() to main(), num and the malloc()ed array itself. As you can return only one value, there are two positilities:

a) return num and have a test ** parameter to pass back the array

int add( struct test **ar )
{
   int num = 10;

   *ar = malloc( num * sizeof **ar );
   // initialize everything, you have to replace ar[i] by (*ar)[i]
   return num;
}

call it

struct test *ar;
int num = add( &ar );

b) return the array and have a int * parameter to pass back num

struct test *add( int *num )
{
   *num = 10;
   
   struct test *ar = malloc( *num * sizeof *ar );
   // initialize everything as you do now
   return ar;
}

call it

 int num;
 struct test *ar = add( &num );

As @Bodo mentioned, either way, you have to call free( ar ); in main() when you don't need ar anymore. You could call it directly or think about having a cleanup function like free_test( struct test *ar, int num ); that does the job. Such a function is especially useful if you have more malloc()s to allocate memory for the single struct elements (eg. if they contained char * elements to store strings).

• use the argument(s) to pass argument(s)
• use the return value to return stuff to the caller

#include <stdio.h>
#include <stdlib.h>

struct test
{
  int i;
  double data;
} ;

struct test *test_create(unsigned ntest)
{
  struct test *pp;
  unsigned idx;

  pp = malloc(sizeof *pp * ntest);
  for (idx = 0; idx < ntest; idx++)
  {
    pp[idx].i = idx;
    pp[idx].data = idx * idx;
  }

return pp;
}

int main(void)
{
  struct test *ptr;
  ptr = test_create(10);

  for (int i = 0; i < 10; i++)
  {
    printf("%d %f\n", ptr[i].i, ptr[i].data);
  }
  return 0;
}

Update: if you want to return more than a single item, you could put the items together in a struct:

#include <stdio.h>
#include <stdlib.h>

struct dope {
        unsigned size;
        struct {
                int i;
                double data;
                } *array;
        } ;

struct dope *dope_create(void)
{
  struct dope *pp;
  unsigned omg =10;

  pp = malloc(sizeof *pp );
  pp->array = malloc(sizeof *pp->array * omg);
  pp->size = omg;

  for (omg = 0; omg < pp->size; omg++)
  {
    pp->array[omg].i = omg;
    pp->array[omg].data = omg * omg;
  }

return pp;
}

int main(void)
{
  struct dope *ptr;
  ptr = dope_create();

  for (int iii = 0; iii < ptr->size; iii++)
  {
    printf("%d %f\n", ptr->array[iii].i, ptr->array[iii].data);
  }
  return 0;
}

You have already good answers as alternatives. Anyway I will show you code for 2 common ways of writing this.

As I see in your code, the factory function will determine the actual number of structs test to allocate. I will in the example:

• generate a random number of structs, using rand() since it makes no difference here
• fill them in like (1,1.01), (2,2.02) ... just to have a known value for testing
• show the structs' contents on screen
• free() them at exit
• assume that can also be 0 structures created

1: use a NULL-terminated sequence of pointers

As it is used with success in all C strings :) we can use it here. As in strings (C strings), you need to search for the terminator in order to get the array size (Of course you can set the first pointer apart for the size, Pascal-like). It can or can not be of importance to know before-hand the # of structs.

Test**      add_vector()
{
    // returns a null terminated array of pointers
    // to 'Test', pointing to actual instances of
    // 'Test', allocated and numbered with 'i' starting
    //  at 1 and 'data' starting at 1.01
    int num = rand() % 10;  // 0 to 9 Test
    fprintf(stderr,
        "add_vector(): creating %d structs\n", num);
    // at least one pointer, the terminating one
    Test** ar = (Test**)malloc((1 + num) * sizeof(Test*));
    int    ix = 0;
    for (ix = 0; ix < num; ix += 1)
    {
        ar[ix] = (Test*)malloc(sizeof(Test));
        // sets up ar[i] to a known value
        ar[ix]->i    = 1 + ix;
        ar[ix]->data = 1 + ix + (1 + ix) / 100.;
    };              // for()
    ar[ix] = NULL;  // the terminator one, as in strings
    return ar;
}

To use it you just call

    Test** vector = add_vector();

as you see in the example below. A single pointer is returned. No need for arguments. If the number of structs is zero a single pointer is returned, like it would be with an empty string.

The # of structs is defined inside the function, and all structs instances are allocated and numbered before returning to caller.

2: return a pointer to a struct

typedef struct
{
    int    i;
    double data;

}   Test;

typedef struct
{
    unsigned size;
    Test*    test;

} V_Test;

V_Test*     add_struct();      

The allocation function returns a pointer to a V_Test struct, that contains an array of Test strucs and a size elements, an in main() for every C program

    int main(in argc, char** argv)

Here is the code:

V_Test*     add_struct()
{
    // returns a vector of structs inside V_Test,
    // with known 'size', like in main( int,char**) 
    // The structs are initialized with 'i' starting
    //  at 1 and 'data' starting at 1.01
    unsigned num = rand() % 10;  // 0 to 9 Test
    fprintf(stderr, "add_struct(): creating %d structs\n", num);
    V_Test* v  = (V_Test*) malloc(sizeof(V_Test));
    v->size   = num;
    if (num == 0)
    {
        v->test = NULL;
        return v;
    };
    v->test  = (Test*)malloc(num * sizeof(Test));
    for (unsigned ix = 0; ix < num; ix += 1)
    {
        v->test[ix].i = 1 + ix;
        v->test[ix].data = 1 + ix + (1 + ix) / 100.;
    };              // for()
    return v;
}

To used it you just call

    V_Test* vector = add_struct();

And here also there are no arguments. A new V_Test is allocated and returned.

Also here the # of structs is defined inside the function, and all structs instances are allocated and numbered before returning to caller.

In the code you will see a way of use both functions, create the structs, fill them in, display the contents and release the allocated memory.

Example output

add_vector(): creating 9 structs
#  1: [1, 1.01]
#  2: [2, 2.02]
#  3: [3, 3.03]
#  4: [4, 4.04]
#  5: [5, 5.05]
#  6: [6, 6.06]
#  7: [7, 7.07]
#  8: [8, 8.08]
#  9: [9, 9.09]
9 records were created
9 records were free()'d
add_struct(): creating 4 structs
#  1: [1, 1.01]
#  2: [2, 2.02]
#  3: [3, 3.03]
#  4: [4, 4.04]
4 records were created
4 records were free()'d

C Code

I compiled just once under MSVC.

#include <stdio.h>
#include <stdlib.h>

typedef struct
{
    int    i;
    double data;

}   Test;

typedef struct
{
    unsigned size;
    Test*    test;

} V_Test;

V_Test*     add_struct();      
Test**      add_vector();

void as_vector_of_pointers();
void as_struct_of_struct();

int main(void)
{
    srand(210728);
    as_vector_of_pointers();
    as_struct_of_struct();
    return 0;
}


Test**      add_vector()
{
    // returns a null terminated array of pointers
    // to test
    int num = rand() % 10;  // 0 to 9 Test
    fprintf(stderr,
        "add_vector(): creating %d structs\n", num);
    // at least one pointer, the terminating one
    Test** ar = (Test**)malloc((1 + num) * sizeof(Test*));
    int    ix = 0;
    for (ix = 0; ix < num; ix += 1)
    {
        ar[ix] = (Test*)malloc(sizeof(Test));
        // sets up ar[i] to a known value
        ar[ix]->i    = 1 + ix;
        ar[ix]->data = 1 + ix + (1 + ix) / 100.;
    };              // for()
    ar[ix] = NULL;  // the terminator one, as in strings
    return ar;
}


V_Test*     add_struct()
{
    // returns a vector of structs inside V_Test,
    // with known 'size', like in 
    // main( int,char**) 
    unsigned num = rand() % 10;  // 0 to 9 Test
    fprintf(stderr, "add_struct(): creating %d structs\n", num);
    V_Test* v  = (V_Test*) malloc(sizeof(V_Test));
    v->size   = num;
    if (num == 0)
    {
        v->test = NULL;
        return v;
    };
    v->test  = (Test*)malloc(num * sizeof(Test));
    for (unsigned ix = 0; ix < num; ix += 1)
    {
        v->test[ix].i = 1 + ix;
        v->test[ix].data = 1 + ix + (1 + ix) / 100.;
    };              // for()
    return v;
}


void as_struct_of_struct()
{
    V_Test* vector = add_struct();
    if (vector->size == 0)
    {
        printf("No records were created!\n");
        free(vector);
        return;
    }
    for ( unsigned count = 0; count < vector->size; count+=1)
    {
        printf("#%3d: [%d, %.2f]\n",
            1 + count,
            vector->test[count].i,
            vector->test[count].data
        );
    };  // for()
    printf("%d records were created\n", vector->size);

    // to free() vector:
    free(vector->test);
    printf("%d records were free()'d\n", vector->size);
    free(vector);
    return;
};


void as_vector_of_pointers()
{
    Test** vector = add_vector();
    // now test the vector to count the number of
    // records generated in the funcion
    if (vector[0] == NULL)
    {
        printf("No records were created!\n");
        free(vector);
        return;
    }
    unsigned count = 0;
    while (vector[count] != NULL)
    {
        printf("#%3d: [%d, %.2f]\n", 1 + count, (vector[count])->i,
               (vector[count])->data);
        count += 1;
    };
    printf("%d records were created\n", count);

    // to free() vector, same way:
    for (unsigned i = 0; i < count; i += 1) free(vector[i]);
    free(vector);
    printf("%d records were free()'d\n", count);
    return;
}

SO vigilants: I always cast malloc() pointers, as I reminder to myself and others reading the code. No implicit conversions. No need to pointing that.