const-ness as template argument

I have two structs:

  // ----- non-const -----
  struct arg_adapter
  {
      EArgType type;  // fmtA, fmtB, ...

      union
      {
        TypeA * valueA;
        TypeB * valueB;
        // ... more types
      }

      arg_adapter(TypeA & value) : type(fmtA), valueA(&value) {}
      arg_adapter(TypeB & value) : type(fmtB), valueB(&value) {}
      // ...
  }

  // ----- const version -----
  struct const_arg_adapter
  {
      EArgType type;  // fmtA, fmtB, ...

      union
      {
        TypeA const * valueA;
        TypeB const * valueB;
        // ... more types
      }

      arg_adapter(TypeA const & value) : type(fmtA), valueA(&value) {}
      arg_adapter(TypeB const & value) : type(fmtB), valueB(&value) {}
      // ...
  }

They are supposed to be used in methods such as:

  Convert(const_arg_adapter from, arg_adapter to)

There are multiple TypeX' (about 5, may become more), most of them primitive. This is to avoid maintaining different prototypes.

Now my question ;-)

Is there a way to make the const-ness a template parameter? My goal is to maintain only one struct, i.e.

template <Qualifier CONSTNESS>
struct arg_adapter_t
{
   ...
   CONSTNESS TypeA * valueA;
   ...
}