DRF data not validating nexted object while processed with file data (multipart/form-data)

Viewed 129
===========JS==========
const data = {
    "id": "d33e6dca-9152-4ded-96e3-51b2f24423d8",
    "logo": null,
    "name": "Company 1",
    "account": {
        "id": "d33e6dca-9152-4ded-96e3-51b2f24423d9",
        "name": "Account Name",
        "company": "d33e6dca-9152-4ded-96e3-51b2f24423d8"
    },
    "modules": [
        {
            "id": "d33e6dca-9152-4ded-96e3-51b2f24423d1",
            "name": "Module 1"
        },
        {
            "id": "d33e6dca-9152-4ded-96e3-51b2f24423d2",
            "name": "Module 2"
        },
    ],
    "addresses": [
        {
            "id": "d33e6dca-9152-4ded-96e3-51b2f24423d4",
            "name": "Address 1",
            "company": "d33e6dca-9152-4ded-96e3-51b2f24423d8"
        },
        {
            "id": "d33e6dca-9152-4ded-96e3-51b2f24423d5",
            "name": "Address 2",
            "company": "d33e6dca-9152-4ded-96e3-51b2f24423d8"
        },
    ],
}

let formData = new FormData();
Object.keys(data).map(key=>{
      formData.append(key, typeof data[key] === "object" && key!=="logo"? JSON.stringify(data[key]):data[key])
    })

const config = {
      headers: {'Content-Type': 'multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW'}
    };

//axios.put(url, data).then(res => {console.log(".....Response!!!  UPDATE")}).catch(err => {console.log(err.response.data)})
axios.put(url, formData, config).then(res => {console.log(".....Response!!!  UPDATE")}).catch(err => {console.log(err.response.data)})

===========Models==========
class Address(BaseModel):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(_('name'), max_length=254, unique=True)
    company = models.ForeignKey("Company", on_delete=models.CASCADE, related_name="our_addresses")

class Module(BaseModel):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=32)

class Account(CompanyBaseModel):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(max_length=32)
    company = models.OneToOneField("Company", on_delete=models.CASCADE, related_name="account")

class Company(BaseModel):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
    name = models.CharField(_('name'), max_length=254, unique=True)
    logo = models.ImageField(_('Company Logo'), upload_to=logo_path, storage=PublicMediaStorage(),
                             blank=True, null=True, default=None)
    addresses = models.ManyToManyField(Address, related_name="companies")
    modules = models.ManyToManyField(Module)

===========Serializers==========
class AddressFormSerializer(serializers.ModelSerializer):
    id = serializers.UUIDField(read_only=False, required=False, allow_null=True)

    class Meta:
        model = Address
        fields = '__all__'
        read_only_fields = ['company', ]

class ModuleFormSerializer(serializers.ModelSerializer):
    id = serializers.UUIDField(read_only=False)     # required ONLY for M2M Update cos validation will fail without it 

    class Meta:
        model = Module
        fields = "__all__"

class AccountSerializer(serializers.ModelSerializer):

    class Meta:
        model = Account
        fields = "__all__"
        read_only_fields = ['id', 'company']

class CompanyFormSerializer(serializers.ModelSerializer):
    logo = serializers.ImageField(max_length=None, use_url=True, allow_null=True)
    account = AccountSerializer()
    modules = ModuleFormSerializer(many=True)
    addresses = AddressFormSerializer(many=True)

    class Meta:
        model = Company
        fields = "__all__"

    def create(self, validated_data):
        with transaction.atomic():
            request = self.context['request']
            account_data = validated_data.pop('account')
            modules_data = validated_data.pop('modules')
            address_data = validated_data.pop('addresses')
            company = Company.objects.create(**validated_data)

            Billing.objects.create(company=company, **account_data)

            addresses = [Address.objects.create(company=company, **a) for a in address_data]
            company.addresses.add(*addresses)
            modules = Module.objects.filter(id__in=[m.get('id') for m in modules_data])
            company.modules.add(*modules)

            return company

    def update(self, instance, validated_data):
        with transaction.atomic():
            #... lets me not bore you with the logic here
            return instance
===========ModelViewSet==========
class CompanyViewSet(viewsets.ModelViewSet):
    permission_classes = (IsAuthenticated, )

    def create(self, request, *args, **kwargs):
        serializer = self.get_serializer(data=self.request.data, context={'request': request})
        serializer.is_valid(raise_exception=True)
        serializer.save()
        headers = self.get_success_headers(serializer.data)
        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)

    def update(self, request, *args, **kwargs):
            partial = kwargs.pop('partial', False)
            instance = self.get_object()
            log.warning(type(request.data))

            #json.loads(request.data['account']) is serialized as String if 'multipart/form-data' 

            # request.data._mutable = True
            # request.data['account'] = json.loads(request.data['account'])
            # request.data['addresses'] = json.loads(request.data['addresses'])
            # request.data['modules'] = json.loads(request.data['modules'])
            # request.data._mutable = False
            
            serializer = self.get_serializer(instance, data=request.data, partial=partial, context={'request': request})
            serializer.is_valid(raise_exception=True)     # It fails here....
            company = serializer.save()

            serializer = CompanyFormSerializer(company)

            if getattr(instance, '_prefetched_objects_cache', None):
                # If 'prefetch_related' has been applied to a queryset, we need to
                # forcibly invalidate the prefetch cache on the instance.
                instance._prefetched_objects_cache = {}

            return Response(serializer.data, status=status.HTTP_201_CREATED)

    def get_serializer_class(self):
        if self.action in ['create', 'update', 'retrieve', 'partial_update']:
            return CompanyFormSerializer
        else:
            return CompanySerializer
===========Settings Files==========
REST_FRAMEWORK = {
    ......,
    'DEFAULT_PARSER_CLASSES': (
        'rest_framework.parsers.JSONParser',
        'rest_framework.parsers.FormParser',
        'rest_framework.parsers.MultiPartParser',
    ),
}

If we focus on update (though it is same for both POST & PUT), I expected serializer.is_valid(raise_exception=True) to validate but it does not validate instead it keep complaining that account, modules, addresses is required

Note that everything works well when I POST OR PUT with axios.put(url, data) so the issue starts when I attempt to use multipart/form-data passing FormData instead of pure JSON

1 Answers

A serializer just needs a dictionary of data. Doesn't have to be request.data. So you can just build a good dict yourself from what the parser made of your request:

data = {
    'account': json.loads(self.request.data.get('account'))
    # etc..
}
data['logo'] = request.FILES.get('logo')
serializer = self.get_serializer(instance, data=data, ...)

If anything, it should fail at a different spot now.

The Long Answer

So I played around with this and below is the view and test I used to verify the method is working. The only failure is that I don't provide a valid image, but the rest of the fields are accepted.

I hooked it up to /main/api/companies/ as you can see from the test case.

# view.py
import json

from rest_framework import generics, status
from rest_framework.response import Response

from . import serializers, models


class CompanyUpdate(generics.RetrieveUpdateAPIView):
    serializer_class = serializers.CompanyFormSerializer
    queryset = models.Company.objects.all()
    json_fields = ("account", "modules", "addresses")
    normal_fields = ("id", "name", "logo")

    def update(self, request, *args, **kwargs):
        partial = kwargs.pop("partial", False)
        instance = self.get_object()
        data = dict(
            ((key, json.loads(self.request.data.get(key))) for key in self.json_fields)
        )
        data.update(((key, request.data.get(key)) for key in self.normal_fields))
        serializer: serializers.CompanyFormSerializer = self.get_serializer(
            instance=instance, data=data, context={"request": request}
        )
        serializer.is_valid(raise_exception=True)
        return Response(status=status.HTTP_200_OK)  # NOT_REACHED

Test case:

from io import BytesIO
import json
import typing as t
import uuid

from django.test import TestCase
from rest_framework.test import APIClient
from rest_framework import status

from . import models



class UploadTest(TestCase):
    client_class = APIClient
    base_url = "/main/api/companies/"

    @classmethod
    def setUpTestData(cls):
        company = models.Company.objects.create(name="honest mike",)
        account = models.Account.objects.create(name="account name", company=company)
        gardening = models.Module.objects.create(name="gardening")
        kitchen = models.Module.objects.create(name="kitchen")
        first_address = models.Address.objects.create(name="address 1", company=company)
        second_address = models.Address.objects.create(
            name="address 2", company=company
        )
        img = BytesIO(b"\x89PNG\rbogus")
        img.name = "fake.png"
        cls.original = {
            "id": str(company.id),
            "logo": img,
            "name": company.name.title(),
            "account": {
                "id": str(account.id),
                "name": account.name.title(),
                "company": str(company.id),
            },
            "modules": [
                {"id": str(gardening.id), "name": gardening.name.title()},
                {"id": str(kitchen.id), "name": kitchen.name.title()},
            ],
            "addresses": [
                {
                    "id": str(first_address.id),
                    "name": "Main Street 1",
                    "company": str(company.id),
                },
                {
                    "id": str(second_address.id),
                    "name": "Church Avenue 2",
                    "company": company.id,
                },
            ],
        }
        cls.company = company
        cls.account = account
        cls.gardening = gardening
        cls.kitchen = kitchen
        cls.first_address = first_address
        cls.second_address = second_address
        cls.json_fields = ("account", "modules", "addresses")

    def test_url(self):
        url = f"{self.base_url}{str(self.company.id)}/"
        response = self.client.get(url)
        self.assertEqual(str(self.company.id), response.data.get("id"))
        data = self.jsonify()
        response = self.client.put(url, data=data, format="multipart")
        print(response.data)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertTrue('modules' not in response.data)
        self.assertEqual(len(response.data.keys()), 1)
        self.assertTrue('logo' in response.data)

    def jsonify(self):
        class Encoder(json.JSONEncoder):
            def default(self, o: t.Any) -> t.Any:
                if isinstance(o, uuid.UUID):
                    return str(o)
                return super().default(o)

        data = self.original
        for field in self.json_fields:
            data[field] = json.dumps(self.original[field], cls=Encoder)
        return data
Related