Pytorch based Resnet18 achieves low accuracy on CIFAR100

I'm training a resnet18 on CIFAR100 dataset. After about 50 iterations the validation accuracy converged at about 34%. While the training accuracy reached almost 100%.

I doubt it's kinda overfitting, so i applied data augmentation like RandomHorizontalFlip and RandomRotation, which made the validation converge at about 40%.

I also tried decaying learning rate [0.1, 0.03, 0.01, 0.003, 0.001], decaying after each 50 iterations. Decaying learning rate seems not enhancing the performance.

Heard that Resnet on CIFAR100 may get 70%~80% accuracy. What trick else could I apply? Or is there anything wrong in my implementation? The same code on CIFAR10 can achieve about 80% accuracy.

My whole training and evaluation code is here below:

import torch
from torch import nn
from torch import optim
from torch.utils.data import DataLoader
from torchvision.models import resnet18
from torchvision.transforms import Compose, ToTensor, RandomHorizontalFlip, RandomRotation, Normalize
from torchvision.datasets import CIFAR10, CIFAR100
import os
from datetime import datetime
import matplotlib.pyplot as plt


def draw_loss_curve(histories, legends, save_dir):
    os.makedirs(save_dir, exist_ok=True)
    for key in histories[0][0].keys():
        if key != "epoch":
            plt.figure()
            plt.title(key)
            for history in histories:
                x = [h["epoch"] for h in history]
                y = [h[key] for h in history]
                # plt.ylim(ymin=0, ymax=3.0)
                plt.plot(x, y)
            plt.legend(legends)
            plt.savefig(os.path.join(save_dir, key + ".png"))


def cal_acc(out, label):
    batch_size = label.shape[0]
    pred = torch.argmax(out, dim=1)
    num_true = torch.nonzero(pred == label).shape[0]
    acc = num_true / batch_size
    return torch.tensor(acc)


class LrManager(optim.lr_scheduler.LambdaLR):
    def __init__(self, optimizer, lrs):
        def f(epoch):
            rate = 1
            for k in sorted(lrs.keys()):
                if epoch >= k:
                    rate = lrs[k]
                else:
                    break
            return rate
        super(LrManager, self).__init__(optimizer, f)


def main(cifar=100, epochs=250, batches_show=100):
    if torch.cuda.is_available():
        device = "cuda"
    else:
        device = "cpu"
        print("warning: CUDA is not available, using CPU instead")
    dataset_cls = CIFAR10 if cifar == 10 else CIFAR100
    dataset_train = dataset_cls(root=f"data/{dataset_cls.__name__}/", download=True, train=True,
                                transform=Compose([RandomHorizontalFlip(), RandomRotation(15), ToTensor(), Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]))
    dataset_val = dataset_cls(root=f"data/{dataset_cls.__name__}/", download=True, train=False,
                              transform=Compose([ToTensor(), Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))]))
    loader_train = DataLoader(dataset_train, batch_size=128, shuffle=True)
    loader_val = DataLoader(dataset_val, batch_size=128, shuffle=True)
    model = resnet18(pretrained=False, num_classes=cifar).to(device)
    optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9, weight_decay=1e-5)
    lr_scheduler = LrManager(optimizer, {0: 1.0, 50: 0.3, 100: 0.1, 150: 0.03, 200: 0.01})
    criterion = nn.CrossEntropyLoss()

    history = []
    model.train()
    for epoch in range(epochs):
        print("-------------------  TRAINING  -------------------")
        loss_train = 0.0
        running_loss = 0.0
        acc_train = 0.0
        running_acc = 0.0
        for batch, data in enumerate(loader_train, 1):
            img, label = data[0].to(device), data[1].to(device)
            optimizer.zero_grad()
            pred = model(img)
            loss = criterion(pred, label)
            loss.backward()
            optimizer.step()

            running_loss += loss.item()
            loss_train += loss.item()
            acc = cal_acc(pred, label)
            running_acc += acc.item()
            acc_train += acc.item()

            if batch % batches_show == 0:
                print(f"epoch: {epoch}, batch: {batch}, loss: {running_loss/batches_show:.4f}, acc: {running_acc/batches_show:.4f}")
                running_loss = 0.0
                running_acc = 0.0
        loss_train = loss_train / batch
        acc_train = acc_train / batch
        lr_scheduler.step()

        print("------------------- EVALUATING -------------------")
        with torch.no_grad():
            running_acc = 0.0
            for batch, data in enumerate(loader_val, 1):
                img, label = data[0].to(device), data[1].to(device)
                pred = model(img)
                acc = cal_acc(pred, label)
                running_acc += acc.item()
            acc_val = running_acc / batch
            print(f"epoch: {epoch}, acc_val: {acc_val:.4f}")

        history.append({"epoch": epoch, "loss_train": loss_train, "acc_train": acc_train, "acc_val": acc_val})
    draw_loss_curve([history], legends=[f"resnet18-CIFAR{cifar}"], save_dir=f"history/resnet18-CIFAR{cifar}[{datetime.now()}]")


if __name__ == '__main__':
    main()