This could be achieved as follows:
- Arrange your df by cat in descending order so that A and B come last
- Map the condition
cat %in% c("A", "B") on linetype
- Set the linetypes for TRUE and FALSE using
scale_linetype_manual
- get rid of the linetype legend using
guides
library(reshape2)
library(ggplot2)
df = data.frame(cat = LETTERS[1:6], VAR1 = runif(6), VAR2 = runif(6), VAR3 = runif(6), VAR4 = runif(6))
df_melted = melt(df, id.vars = 'cat')
df_melted <- arrange(df_melted, desc(cat))
#> Error in arrange(df_melted, desc(cat)): could not find function "arrange"
ggplot(df_melted, aes(x = variable, y = value)) +
geom_line(aes(color = cat, group = cat, linetype = cat %in% c("A", "B"))) +
scale_linetype_manual(values = c("TRUE" = "solid", "FALSE" = "dotted")) +
guides(linetype = FALSE)

EDIT An alternative approach to achieve your result may look like so. Here I also use the same pattern to adjust the size. What I like about this approach is that we don't need a condition and we only have one legend:
library(reshape2)
library(ggplot2)
library(dplyr)
df = data.frame(cat = LETTERS[1:6], VAR1 = runif(6), VAR2 = runif(6), VAR3 = runif(6), VAR4 = runif(6))
df_melted = melt(df, id.vars = 'cat')
df_melted <- arrange(df_melted, desc(cat))
ggplot(df_melted, aes(x = variable, y = value, group = cat)) +
geom_line(aes(color = cat, linetype = cat, size = cat)) +
scale_linetype_manual(values = c(A = "solid", B = "solid", rep("dotted", 4))) +
scale_size_manual(values = c(A = 1, B = 1, rep(.5, 4)))
