How to copy an OleVariant array efficiently to my own structure?

Question

I am trying to copy an OleVariant array to my own structure. I received the OleVariant from an external COM call.

The size is 1000 x 500 elements (I don't know if its the same as this Structure Definition: array of array of OleVariant).

Currently, I am trying to achieve something like:

result := Copy(Source, Amount)

But the OleVariant structure is in my way.

If I use a "classical" loop, it works, but it is slow (very slow).

aResult is currently defined as TData = array of array of string;

procedure CopyResult(aResultCount: Integer; var aResult: TData; aSource: Variant);
var
  i, j: Integer;
  bVariantConversion: boolean;
begin    
  SetLength(aResult, aResultCount, VarArrayHighBound(aSource[0], 1));
  bVariantConversion := NullStrictConvert; // settings to manage how string conversion for Variant is handled.
  NullStrictConvert := False;
  try
    for i := VarArrayLowBound(aSource, 1) to VarArrayHighBound(aSource, 1) do
    begin
      for j := VarArrayLowBound(aSource[i], 1) to pred(VarArrayHighBound(aSource[i], 1)) do
      begin
        //nearly every execution pause is somewhere in this String Conversion or Array Function.
        aResult[i][j] := aSource[i][j]; //implicit conversion to string ... 
      end;
    end;
  finally
    NullStrictConvert := bVariantConversion;
  end;
end;

As @Remy Lebau mentioned the bounds Check for the Vararray[x][y] access is the routine my source burns its time. I am trying to eliminate this kind of acces by going directly to the OleVariantArray Elements.

Aftermath... trying to Determine my Structure i think i found the Root.

  tmyVarType := VarType(aSource); //8204 => Array(VT_ARRAY = 0x2000 = 8192) + variant(VT_VARIANT = 0x000C = 12) 
  tmyVarType := VarType(aSource[0]); //8204
  tmyVarType := VarType(aSource[0][0]); //3 VT_I4 = 0x0003 = 3 is integer and this is correctly changin for the fields.

So i try to acess the Source without the build in functions to avoid the bounds check.

user_65863 · Answer

The biggest bottleneck in this code is the bounds checking performed by the [] operator on each Variant array, and potentially on your aResult array, too. Since you are already handling the bounds in each loop, there is no need to verify the bounds inside of the loops as well.

So, if performance is an issue for you, then you can use VarArrayLock() to access the underlying Variant elements in each array, using pointer arithmetic to move between them, eliminating those redundant bounds checks.

You should also reduce the redundant calls to VarArray(Low|High)Bound(aSource[i], 1) on each iteration of the outer array, since you claim the inner arrays all have the same length. So you can calculate that up front before entering the loops.

Try something like this:

type
  TStrArr = array of string;
  PStrArr = ^TStrArr;
  TData = array of TStrArr;

procedure CopyResult(aResultCount: Integer; var aResult: TData; aSource: Variant);
var
  i, j,
  OuterLBound, OuterHBound, OuterCount,
  InnerLBound, InnerHBound, InnerCount: Integer;
  pOuterVarArr, pInnerVarArr: PVariant;
  pOuterDynArr: PStrArr;
  pInnerDynArr: PString;
  bVariantConversion: boolean;
begin    
  aResult := nil;

  Assert(VarIsType(aSource, varArray or varVariant));
  Assert(VarArrayDimCount(aSource) = 1);

  OuterLBound := VarArrayLowBound(aSource, 1);
  OuterHBound := VarArrayHighBound(aSource, 1);
  OuterCount := {aResultCount} OuterHBound - OuterLBound + 1;

  if OuterCount < 1 then Exit;

  Assert(VarIsType(aSource[0], varArray or varVariant));
  Assert(VarArrayDimCount(aSource[0]) = 1);

  InnerLBound := VarArrayLowBound(aSource[0], 1);
  InnerHBound := VarArrayHighBound(aSource[0], 1);
  InnerCount := InnerHBound - InnerLBound + 1;

  SetLength(aResult, {aResultCount} OuterCount, InnerCount);

  bVariantConversion := NullStrictConvert; // settings to manage how string conversion for Variant is handled.
  NullStrictConvert := False;
  try
    pOuterDynArr := PStrArr(aResult);
    pOuterVarArr := PVariant(VarArrayLock(aSource));
    try
      for i := OuterLBound to OuterHBound do
      begin
        pInnerDynArr := PString(pOuterDynArr^);
        pInnerVarArr := PVariant(VarArrayLock(pOuterVarArr^));
        try
          //System.Variants.DynArrayFromVariant(pOuterDynArr^, pInnerVarArr^, TypeInfo(String));

          for j := InnerLBound to InnerHBound do
          begin
            pInnerDynArr^ := pInnerVarArr^; //implicit conversion to string ... 
            Inc(pInnerDynArr);
            Inc(pInnerVarArr);
          end;
        finally
          VarArrayUnlock(pOuterVarArr^);
        end;
        Inc(pOuterDynArr);
        Inc(pOuterVarArr);
      end;
    finally
      VarArrayUnlock(aSource);
    end;
  finally
    NullStrictConvert := bVariantConversion;
  end;
end;

On the other hand, if there is ever a chance that the inner arrays have different lengths, then you can try this adjustment instead:

type
  TStrArr = array of string;
  PStrArr = ^TStrArr;
  TData = array of TStrArr;

procedure CopyResult(aResultCount: Integer; var aResult: TData; aSource: Variant);
var
  i, j,
  OuterLBound, OuterHBound, OuterCount,
  InnerLBound, InnerHBound, InnerCount: Integer;
  pOuterVarArr, pInnerVarArr: PVariant;
  pOuterDynArr: PStrArr;
  pInnerDynArry: PString;
  bVariantConversion: boolean;
begin    
  aResult := nil;

  Assert(VarIsType(aSource, varArray or varVariant);
  Assert(VarArrayDimCount(aSource) = 1);

  OuterLBound := VarArrayLowBound(aSource, 1);
  OuterHBound := VarArrayHighBound(aSource, 1);
  OuterCount := {aResultCount} OuterHBound - OuterLBound + 1;

  if OuterCount < 1 then Exit;

  SetLength(aResult, {aResultCount} OuterCount);

  bVariantConversion := NullStrictConvert; // settings to manage how string conversion for Variant is handled.
  NullStrictConvert := False;
  try
    pOuterDynArr := PStrArr(aResult);
    pOuterVarArr := PVariant(VarArrayLock(aSource));
    try
      for i := OuterLBound to OuterHBound do
      begin
        pInnerVarArr := PVariant(VarArrayLock(pOuterVarArr^));
        try
          //System.Variants.DynArrayFromVariant(pOuterDynArr^, pInnerVarArr^, TypeInfo(String));

          Assert(VarIsType(pInnerVarArr^, varArray or varVariant);
          Assert(VarArrayDimCount(pInnerVarArr^) = 1);

          InnerLBound := VarArrayLowBound(pInnerVarArr^, 1);
          InnerHBound := VarArrayHighBound(pInnerVarArr^, 1);
          InnerCount := InnerHBound - InnerLBound + 1;

          SetLength(pOuterDynArr^, InnerCount);
          pInnerDynArr := PString(pOuterDynArr^);

          for j := InnerLBound to InnerHBound do
          begin
            pInnerDynArr^ := pInnerVarArr^; //implicit conversion to string ... 
            Inc(pInnerDynArr);
            Inc(pInnerVarArr);
          end;
        finally
          VarArrayUnlock(pOuterVarArr^);
        end;
        Inc(pOuterDynArr);
        Inc(pOuterVarArr);
      end;
    finally
      VarArrayUnlock(aSource);
    end;
  finally
    NullStrictConvert := bVariantConversion;
  end;
end;

Edit: I Only tested the Source version for all Entrys the same length but it works my own partial [] free Version used ~5 Million cycles with Tstopwatch ElapsedTicks and this one only took around ~2 Millon (more like 1.6) Thanks

How to copy an OleVariant array efficiently to my own structure?

1 Answers