Read byte slice into struct with variable-size members

Nevo 0
Viewed 36

I'm trying to convert some Python code that relies on the AWS Encryption SDK to Golang. AWS does not provide a Golang version of this SDK. My needs are well constrained so I only need to convert some very specific functions.

One of the data structures defined in the SDK (translated to Go by me) is:

type encryptionSDK_Header_v1 struct {
    Version     byte
    Type        byte
    AlgorithmID [2]byte
    MessageID   [16]byte
    AADLength   [2]byte //Really a little-endian uint16
    AAD         []byte
    KeyCount    [2]byte //Again, a little-endian uint16
    DataKeys    []byte
    ContentType byte
    Reserved    [4]byte
    IVLength    byte    //Maybe a uint8?
    FrameLength [4]byte //A little-endian uint32
    HeaderAuth  []byte
}

I start with a Base64 encoded string which I can easily enough convert to a byte slice that needs to be parsed into this struct.

I had some moderate success reading the fixed-length fields but I'm at a loss as to how to read in the variable length fields. Can anyone help?

The encoding/binary package can help me convert the various fields to uint8/uint16/uint32 so I'm not too terribly concerned about getting help with that.

A description of the struct format can be found at https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/message-format.htm.

go
1 Answers

I can't help with reading HeaderAuth since it depends on the algorithm.

For other fields I wouldn't keep their lenghts and count in the final structure since they are encoded in the actual slice lenght.

I'd make a type to read variable fields from a reader:

type ByteField []byte

func (bf *ByteField) ReadFrom(r io.Reader) error {
    var length uint16
    err := binary.Read(r, binary.LittleEndian, &length)
    if err != nil {
        return err
    }
    *bf = make([]byte, length)
    _, err = io.ReadFull(r, *bf)
    if err != nil {
        return err
    }
    return nil
}

How to use it. Let's consider encrypted keys:

type EncryptedDataKey struct {
    KeyProviderID   ByteField
    KeyProviderInfo ByteField
    EncryptedKey    ByteField
}

func (k *EncryptedDataKey) ReadFrom(r io.Reader) error {
    if err := k.KeyProviderID.ReadFrom(r); err != nil {
        return err
    }
    if err := k.KeyProviderInfo.ReadFrom(r); err != nil {
        return err
    }
    if err := k.EncryptedKey.ReadFrom(r); err != nil {
        return err
    }

    return nil
}

With ByteField it is easy to decode AAD

type AAD map[string][]byte

func (a *AAD) ReadFrom(r io.Reader) error {
    var key ByteField
    var value ByteField
    var err error

    for {
        if err = key.ReadFrom(r); err != nil {
            if err == io.EOF {
                return nil
            } else {
                return err
            }
        }
        if err = value.ReadFrom(r); err != nil {
            return err
        }
        (*a)[string(key)] = []byte(value)
    }
}

Method AAD.ReadFrom reads until the end of buffer, that's why here I check that EOF happened while reading a key. I treat it as no more k-v pairs in the input stream.

Now we are equipped to read the whole header:

type EncryptionSDK_Header_v1 struct {
    Version     byte
    Type        byte
    AlgorithmID uint16
    MessageID   [16]byte
    AAD         AAD
    DataKeys    []EncryptedDataKey
    ContentType byte
    Reserved    [4]byte
    IVLength    byte   //Maybe a uint8?
    FrameLength uint32 //A little-endian uint32
    HeaderAuth  []byte
}

func (h *EncryptionSDK_Header_v1) ReadFrom(r io.Reader) error {
    var err error
    if err = binary.Read(r, binary.LittleEndian, &h.Version); err != nil {
        return err
    }
    if err = binary.Read(r, binary.LittleEndian, &h.Type); err != nil {
        return err
    }
    if err = binary.Read(r, binary.LittleEndian, &h.AlgorithmID); err != nil {
        return err
    }
    if _, err = io.ReadFull(r, h.MessageID[:]); err != nil {
        return err
    }
    var aadLen uint16
    if err = binary.Read(r, binary.LittleEndian, &aadLen); err != nil {
        return err
    }

    h.AAD = make(AAD)
    if aadLen > 0 {
        aadBuf := make([]byte, aadLen)

        if _, err = io.ReadFull(r, aadBuf); err != nil {
            return err
        }
        aadReader := bytes.NewReader(aadBuf)
        if err = h.AAD.ReadFrom(aadReader); err != nil {
            return err
        }
    }

    var keyCount uint16
    if err = binary.Read(r, binary.LittleEndian, keyCount); err != nil {
        return err
    }
    h.DataKeys = make([]EncryptedDataKey, keyCount)

    for i := uint16(0); i < keyCount; i++ {
        if err = h.DataKeys[i].ReadFrom(r); err != nil {
            return err
        }
    }

    if err = binary.Read(r, binary.LittleEndian, &h.ContentType); err != nil {
        return err
    }

    if _, err = io.ReadFull(r, h.Reserved[:]); err != nil {
        return err
    }

    if err = binary.Read(r, binary.LittleEndian, &h.IVLength); err != nil {
        return err
    }
    if err = binary.Read(r, binary.LittleEndian, &h.FrameLength); err != nil {
        return err
    }

    // Stopped at Header Authentication
    return nil
}
Pak Uula0
Related