Unable to paginate with selenium-scrapy, only extracting data for first page

I am scraping a website for most recent customer rating, with several pages.

The problem is that I am able to interact with the "sortby" option and select "most recent" using Selenium, and scrape data for first page using Scrapy. However, I am unable to extract the data for other pages, the Selenium Web driver somehow does not render the next page. My intension is to automate data scraping.

I am a newb to web scraping. A snippet of the code is attached here (Some information is removed due to confidentiality)

import scrapy
import selenium.webdriver as webdriver
from selenium.webdriver.edge.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait,Select
import time
from selenium.webdriver.support import expected_conditions as EC
from scrapy import Selector
from selenium.webdriver.edge.options import Options
from scrapy.utils.project import get_project_settings


class ABC(scrapy.Spider):
    #"........."
   

    def start_requests(self):
        #"  ......  "

            yield scrapy.Request(url)

    def parse(self, response):
         settings =get_project_settings()
         driver_path = settings.get('EDGE_DRIVER_PATH')
         options = Options()
         options.add_argument("headless")
            
         ser=Service(driver_path)
         driver = webdriver.Edge(service=ser,options = options)

         driver.get(response.url)
         WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID,"sort-order-dropdown")))
         element_dropdown=driver.find_element(By.ID,"sort-order-dropdown")
         select=Select(element_dropdown)
         select.select_by_value("recent")
         time.sleep(5)

            for review in response.css('[data-hook="review"]':
               res={
                   "rating": review.css('[class="a-icon-alt"]::text').get(),
                 }
               yield res 
                     
            
         next_page =response.xpath('//a[text()="Next page"]/@href').get()
         if next_page:
             yield scrapy.Request(response.urljoin(next_page))
       
         driver.quit()