You could achieve this by setting the breaks for the fill scale:
df <- data.frame(
group = c("A", "B", "C", "D", "E", "F", "G"),
sample.size = c(2L, 3L, 1L, 25L, 23L, 20L, 3L)
)
library(ggplot2)
library(dplyr)
top_group <- df %>% top_n(3, sample.size) %>% pull(group)
ggplot(df, aes(group, sample.size, fill = group)) +
geom_col() +
scale_fill_discrete(breaks = top_group)

EDIT In case of scale_fill_manual one option would be to name your list of colors. This has the additional benefit that you could assign colors to names or Categories without bothering about the order in which you pass the colors to the values argument of the scale:
# Example color palette
colourslist <- scales::hue_pal()(length(unique(df$group)))
# Name your list of colors
names(colourslist) <- unique(df$group)
ggplot(df, aes("1", sample.size, fill = group)) +
geom_col(width = 1, color="darkgrey") +
scale_fill_manual(values = colourslist, breaks = top_group) +
coord_polar("y", start = 0)
