Optimalizing code for extracting information from many html files

I am trying to extract specific information from a directory of html files that I earlier extracted using the request library of python. Extraction of the htmls was already slow since I built in a random wait timer but now that I want to iterate over each retrieved html file, it seems like my script is not very well optimized. This is a problem since I want to iterate over 42000 html files, each with > 8000 lines. This would take a lot of time probably.

Since I have never run into these problems that were this demanding for my computer, I do not know where to start learning to optimize my code. My question to you, should I approach this problem differently, possibly in a more time efficient way?? Your suggestions would be very much appreciated.

Here is the code I am using, I changed some sensitive information:

#empty lists of features of houses
link = []
name_list = []
agent_list = []
description_list = []
features_list = []

#here link_list is a list that I previously retrieved and holds all the links to the original html files extracted in a previous step.
for i in range(1,len(link_list)):
    html = open_html('C:\\Users\\Documents\\file_p{}.html'.format(i))
    soup = BeautifulSoup(html, 'html.parser')
    link.append(link_list[i])
    name_list.append(soup.select_one('.object-header__title').text)
    agent_list.append(soup.select_one('.object-contact-agent-link').text)
    description_list.append(soup.select_one('.object-description-body').text)
    features_list.append(soup.select_one('.object-features').text)
    

d = {'Link_P': link, 'Name_P': name_list, 'Features_P': features_list, 'Description_P': description_list, 'Agent_P': agent_list}
df = pd.DataFrame(data=d)
df

Update: Thanks to the help here I managed to make my code more efficient. I ended up trying out multiprocessing among other things and did a lot of testing on speed using the time module in Python. I learned a lot from this about optimizing my code.

import numpy as np
import requests
from bs4 import BeautifulSoup
from time import sleep
import time
from tqdm import tqdm_notebook
import json
from random import randint
import csv
import concurrent.futures
import re
from itertools import product
from multiprocessing import Pool

"""Retrieve individual pages html source code and save this, 
then open individual page and retrieve specific data. 
This data is finally saved in a dedicated csv """         
def extract_indiv_page_html(page, start_num, end_num):
    sleep(randint(1,3))
    url = "https://www.xxx.xx{}".format(page)
    headers = {'authority': 'www.xxx.xx', 'pragma': 'no-cache', 'cache-control': 'no-cache', 'upgrade-insecure-requests': '1', 'user-agent': xxx, 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,/;q=0.8,application/signed-exchange;v=b3;q=0.9', 'sec-fetch-site': 'same-origin', 'sec-fetch-mode': 'navigate', 'sec-fetch-user': '?1', 'sec-fetch-dest': 'document', 'referer': 'https://www.xxx.xx/xxxx/', 'accept-language': 'en;q=0.9,en-GB;q=0.8,en-US;q=0.7'}
    r = requests.get(url, headers = headers )
    name = re.findall('.+?/',page)
    save_html(r.content, 'path\\file_{}_{}.html'.format(name[1].replace('/', ''), name[2].replace('/', '')))
    html = open_html('path\\file_{}_{}.html'.format(name[1].replace('/', ''), name[2].replace('/', '')))
    soup = BeautifulSoup(html, 'html.parser')
    link = url
    try:
        Name_P = soup.select_one('.object-header__title').text
    except AttributeError:
        Name_P = ''
    try:
        Agent_P = soup.select_one('.object-contact-agent-link').text
    except AttributeError:
        Agent_P = ''
    try:
        Description_P = soup.select_one('.object-description-body').text
    except AttributeError:
        Description_P = ''
    try:
        Features_P = soup.select_one('.object-features').text
    except AttributeError:
        Features_P = ''
    with open('path\\P_details_df_date_{}_{}.csv'.format(start_num, end_num), 'a', newline='') as file:
            writer = csv.writer(file)
            writer.writerow([link, Name_P, Agent_P, Description_P, Features_P, start_num, end_num])


start = time.perf_counter()
pages = 3015

#creates a list of all the links that we could extract from the original listing htmls
link_list = []
if __name__ == "__main__":
    with concurrent.futures.ProcessPoolExecutor() as executor:
          indiv_p_link = executor.map(extract_links, [i for i in range(1, pages)])
          [link_list.extend(i) for i in indiv_p_link]

"""segments the list in slices of 1% of its total. 
This is done so that we split the saved data over more csv files. 
This will decrease working time if we are trying to extract data from many html links."""
start_indices = [(len(link_list)//100*i) for i in range(0, 100, 1)]
end_indices = [(len(link_list)//100*i) for i in range(1, 100, 1)]
end_indices.append(len(link_list))

#creating a tuple list, this is needed to pass into the starmap method of our pools
tuple_list = []
for indice_segment in range(0, 100):
    for number in range(start_indices[indice_segment], end_indices[indice_segment]):
        extract_indiv_page_tuple = (link_list[number], start_indices[indice_segment], end_indices[indice_segment])
        tuple_list.append(extract_indiv_page_tuple)
        
 
if __name__ == "__main__":
    p = Pool(4)
    p.starmap(extract_indiv_page_html, tuple_list)
    p.terminate()
    p.join()




finish = time.perf_counter()       
print('Finished in {} seconds'.format(round(finish-start, 2)))