fill argument nullifies position argument in geom_label()

When I use a fill argument in geom_label, my position argument is nullified. This does not happen with geom_text(). How can I use fill and position arguments with geom_label()?

library(dplyr) library(ggplot2)

ds_mt <- 
  mtcars %>% 
  group_by(cyl, am) %>% 
  summarise(mpg = mean(mpg)) %>% 
  ungroup() %>% 
  mutate_at(vars(cyl, am), funs(as.factor)) 

ds_mt %>% 
  ggplot(aes(x = cyl, fill = as.factor(am), y = mpg, label = am)) + 
  geom_col(position = position_dodge()) + 
  geom_label(position = position_dodge(width = .9))

# Adding fill argument to geom_label() nullifies the position argument
ds_mt %>% 
  ggplot(aes(x = cyl, fill = as.factor(am), y = mpg, label = am)) + 
  geom_col(position = position_dodge()) + 
  geom_label(fill = "white", position = position_dodge(width = .9))

ds_mt %>% 
  ggplot(aes(x = cyl, fill = as.factor(am), y = mpg, label = am)) + 
  geom_col(position = position_dodge()) + 
  geom_label(position = position_dodge(width = .9), fill = "white")

# no problems with geom_text
ds_mt %>% 
  ggplot(aes(x = cyl, fill = as.factor(am), y = mpg, label = am)) + 
  geom_col(position = position_dodge()) + 
  geom_text(position = position_dodge(width = .9), color = "blue")