Recycler view horizontal scroll not working in Navigation Drawer header

I am trying to implements recyclerView on header of navigation bar, where I can show list of users, But issue is recyclerView is not scrolling there. when I scroll from right to left it scrolls, but when I am trying to scroll from left to right Navigation drawer get closed (coming from left to right).

below is my nav_header.xml

<?xml version="1.0" encoding="utf-8"?>
<androidx.cardview.widget.CardView xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:layout_width="match_parent"
    android:layout_height="@dimen/nav_header_height"
    android:elevation="8dp"

    android:paddingLeft="@dimen/activity_horizontal_margin"
    android:paddingTop="@dimen/activity_vertical_margin"
    android:paddingRight="@dimen/activity_horizontal_margin"
    android:paddingBottom="@dimen/activity_vertical_margin"
    app:cardCornerRadius="10dp"
    app:cardElevation="15dp">

    <LinearLayout
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:gravity="bottom"
        android:orientation="vertical"
        android:theme="@style/ThemeOverlay.AppCompat.Dark">

        <LinearLayout
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            android:gravity="center"
            android:orientation="vertical">

            <TextView
                android:id="@+id/txt_empty"
                android:layout_width="wrap_content"
                android:layout_height="wrap_content"
                android:layout_gravity="center"
                android:layout_margin="8dp"
                android:ellipsize="marquee"
                android:visibility="gone"
                android:marqueeRepeatLimit="marquee_forever"
                android:singleLine="true"
                android:text="@string/nav_header_title"
                android:textAppearance="@style/TextAppearance.AppCompat.Body1"
                android:textColor="@color/colorButton"
                android:textSize="@dimen/text_size_midnormal"
                android:textStyle="bold" />



            <androidx.recyclerview.widget.RecyclerView
                android:id="@+id/rv_users"
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                android:layout_centerHorizontal="true"
                android:layout_marginTop="30dp" />


        </LinearLayout>

    </LinearLayout>
</androidx.cardview.widget.CardView>