Open a XML File with Selenium in the browser without downloading it

user_9640175 0
Viewed 324

Currently I am working on a program that opens a website with selenium and then clicks on a link. The problem is that the links redirects to an xml file that I only can download. Is there a way to prevent this and just open the xml source code in selenium? Here is my code:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
from time import sleep

# Initialize webdriver
driver_options = Options()
driver_options.headless = True
driver = webdriver.Chrome(options=driver_options, executable_path="bin/chromedriver")


# Get Index page
url = "https://www.ems-ph.org/journals/all_issues.php?issn=1660-8933"
driver.get(url)
sleep(10)

for td in driver.find_elements(By.CLASS_NAME, 'remark'):
    td.find_elements(By.LINK_TEXT, 'Metadata')[0].click()
    sleep(10)
selenium-webdriver selenium python
1 Answers

A workaround to the problem highlighted by @cruisepandey is to send requests to each link and save the returned XML source. You can run a Javascript snippet via driver.execute_async_script that sends an XMLHttpRequest to each of the meta XML links and returns the result, along with the volume and issue data from the main webpage:

from selenium import webdriver
d = webdriver.Chrome('/path/to/chromedriver')
d.get('https://www.ems-ph.org/journals/all_issues.php?issn=1660-8933')
xml = d.execute_async_script('''
  async function get_xml_source(link){
     var r = await fetch(link);
     return await r.text();
  }
  async function get_remarks(){
     var volume = null
     var results = []
     for (var i of document.querySelectorAll('table tr')){
        if (i.querySelector('td.basket-header') != null){
           volume = i.querySelector('td.basket-header').textContent;
        }
        if (i.querySelector('td.remark > a') != null){
           results.push({
              volume:volume,
              issue:Array.from(i.querySelectorAll('td[width="10%"]')).map(x => x.textContent).join(' '),
              xml:await get_xml_source(i.querySelector('td.remark > a').href)
           })
        }
    }
    return results;
 }
 var done = arguments[0];
 get_remarks().then(function(x){
    done(x)
 })
''')

Now, xml stores the volume, issue (not available if the metadata is for a volume listing), and the XML as a string:

print([{**a, 'xml':a['xml'][:100]} for a in xml])

Output (results with XML truncated to the first 100 characters):

[{'issue': '', 'volume': 'Volume 17 (2020)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 1539-2115)', 'volume': 'Volume 17 (2020)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2/3 (pp. 797-1528)', 'volume': 'Volume 17 (2020)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 6-783)', 'volume': 'Volume 17 (2020)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 16 (2019)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2739-3606)', 'volume': 'Volume 16 (2019)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1855-2726)', 'volume': 'Volume 16 (2019)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 911-1839)', 'volume': 'Volume 16 (2019)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-895)', 'volume': 'Volume 16 (2019)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 15 (2018)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2715-3374)', 'volume': 'Volume 15 (2018)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1857-2701)', 'volume': 'Volume 15 (2018)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 805-1844)', 'volume': 'Volume 15 (2018)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-792)', 'volume': 'Volume 15 (2018)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 14 (2017)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2781-3560)', 'volume': 'Volume 14 (2017)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1987-2767)', 'volume': 'Volume 14 (2017)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 1067-1971)', 'volume': 'Volume 14 (2017)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-1051)', 'volume': 'Volume 14 (2017)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 13 (2016)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2705-3372)', 'volume': 'Volume 13 (2016)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1821-2691)', 'volume': 'Volume 13 (2016)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 963-1807)', 'volume': 'Volume 13 (2016)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-948)', 'volume': 'Volume 13 (2016)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 12 (2015)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2605-3331)', 'volume': 'Volume 12 (2015)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1759-2592)', 'volume': 'Volume 12 (2015)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 849-1746)', 'volume': 'Volume 12 (2015)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-836)', 'volume': 'Volume 12 (2015)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 11 (2014)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2481-3254)', 'volume': 'Volume 11 (2014)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1695-2468)', 'volume': 'Volume 11 (2014)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 957-1681)', 'volume': 'Volume 11 (2014)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-941)', 'volume': 'Volume 11 (2014)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 10 (2013)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2837-3485)', 'volume': 'Volume 10 (2013)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1929-2823)', 'volume': 'Volume 10 (2013)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 963-1913)', 'volume': 'Volume 10 (2013)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-950)', 'volume': 'Volume 10 (2013)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 9 (2012)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2879-3633)', 'volume': 'Volume 9 (2012)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1961-2866)', 'volume': 'Volume 9 (2012)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 1035-1948)', 'volume': 'Volume 9 (2012)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-1019)', 'volume': 'Volume 9 (2012)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 8 (2011)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2745-3286)', 'volume': 'Volume 8 (2011)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1857-2733)', 'volume': 'Volume 8 (2011)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 993-1843)', 'volume': 'Volume 8 (2011)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-978)', 'volume': 'Volume 8 (2011)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 7 (2010)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2573-3348)', 'volume': 'Volume 7 (2010)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1703-2560)', 'volume': 'Volume 7 (2010)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 955-1690)', 'volume': 'Volume 7 (2010)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-939)', 'volume': 'Volume 7 (2010)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 6 (2009)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2483-3100)', 'volume': 'Volume 6 (2009)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1787-2470)', 'volume': 'Volume 6 (2009)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 983-1774)', 'volume': 'Volume 6 (2009)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-970)', 'volume': 'Volume 6 (2009)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 5 (2008)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2419-3271)', 'volume': 'Volume 5 (2008)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1603-2406)', 'volume': 'Volume 5 (2008)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 759-1588)', 'volume': 'Volume 5 (2008)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-746)', 'volume': 'Volume 5 (2008)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 4 (2007)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2543-3420)', 'volume': 'Volume 4 (2007)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1663-2530)', 'volume': 'Volume 4 (2007)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 833-1650)', 'volume': 'Volume 4 (2007)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-820)', 'volume': 'Volume 4 (2007)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 3 (2006)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2579-30056)', 'volume': 'Volume 3 (2006)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1743-2566)', 'volume': 'Volume 3 (2006)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 895-1730)', 'volume': 'Volume 3 (2006)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-882)', 'volume': 'Volume 3 (2006)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 2 (2005)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2447-3246)', 'volume': 'Volume 2 (2005)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1601-2434)', 'volume': 'Volume 2 (2005)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 827-1588)', 'volume': 'Volume 2 (2005)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-813)', 'volume': 'Volume 2 (2005)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': '', 'volume': 'Volume 1 (2004)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 4 (pp. 2229-3036)', 'volume': 'Volume 1 (2004)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 3 (pp. 1461-2216)', 'volume': 'Volume 1 (2004)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 2 (pp. 785-1448)', 'volume': 'Volume 1 (2004)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}, {'issue': 'Issue 1 (pp. 5-772)', 'volume': 'Volume 1 (2004)', 'xml': '<?xml version="1.0" encoding="utf-8"?>\n<ems_journal_articles>\n  <head>\n    <item>journal articles me'}]
user_73267380
Related