Vertical RecyclerView inside horizontal RecyclerView lag when scrolling

I have a horizontal RecyclerView. Each child contains a TextView and a vertical RecyclerView. The vertical RecyclerView's children contains only a TextView.

Vertical scroll is very smooth, but the horizontal scroll lag a lot. I have tried to use swapAdapter in onBindViewHolder instead of setAdapter, but this didn't solve the problem. I have also tried to change the data set and call notifyDataSetChanged(), but this didn't help either.

Edit

I have just found out that the scrolling is actually smooth in landscape mode.

Horizontal RecyclerView.Adapter

private SparseArray<GroupArrayAdapter> adapters;

private ArrayList<Category> categories;

private Activity context;

public CategoryArrayAdapter(Context context, ArrayList<Category> categories) {
    this.categories = categories;
    this.context = (Activity)context;

    adapters = new SparseArray<GroupArrayAdapter>();
    for (int i = 0; i < categories.size(); i++) {
        adapters.append(i, new GroupArrayAdapter(context, categories.get(i).getGroups()));
    }
}

public class ViewHolder extends RecyclerView.ViewHolder {
    public TextView name;
    public RecyclerView list;

    public ViewHolder(View view) {
        super(view);
        name = (TextView) view.findViewById(R.id.lblCategoryName);
        list = (RecyclerView) view.findViewById(R.id.lstGroups);
        list.setLayoutManager(new LinearLayoutManager(context, LinearLayoutManager.VERTICAL, false));
        list.addItemDecoration(new GroupItemDecoration((int)context.getResources().getDimension(R.dimen.gap)));
        //list.setAdapter(new GroupArrayAdapter(context, new ArrayList<Group>()));
    }
}

@Override
public int getItemCount() {
    return categories.size();
}

@Override
public void onBindViewHolder(ViewHolder viewHolder, int position) {
    Category category = categories.get(position);
    viewHolder.name.setText(category.getName());
    viewHolder.list.swapAdapter(adapters.get(position), false);
}

@Override
public ViewHolder onCreateViewHolder(ViewGroup parent, int viewType) {
    View view = LayoutInflater.from(parent.getContext()).inflate(R.layout.item_category, parent, false);
    ViewHolder viewHolder = new ViewHolder(view);
    return viewHolder;
}

Vertical RecyclerView.Adapter

private ArrayList<Group> groups;

public GroupArrayAdapter(Context context, ArrayList<Group> groups) {
    this.groups = groups;
}

public class ViewHolder extends RecyclerView.ViewHolder {
    public TextView name;

    public ViewHolder(View view) {
        super(view);
        name = (TextView) view.findViewById(R.id.lblGroupName);
    }
}

@Override
public int getItemCount() {
    return groups.size();
}

@Override
public void onBindViewHolder(ViewHolder viewHolder, int position) {
    Group group = groups.get(position);
    viewHolder.name.setText(group.getName());
}

@Override
public ViewHolder onCreateViewHolder(ViewGroup parent, int viewType) {
    View view = null;
    view = LayoutInflater.from(parent.getContext()).inflate(R.layout.item_group_text, parent, false);
    ViewHolder viewHolder = new ViewHolder(view);
    return viewHolder;
}