Why is my R webscraper slower than webmonitor apps?

Viewed 34

Every day around 12:30 the Swiss government publishes corona data on a website. I run a scraper that is supposed to notify a slack-channel as soon as the numbers are published. However, every day my scraper is a few minutes slower than the webmonitor apps my colleagues use. I don't know how to make it even faster. At this point it runs on a repeat loop that scrapes the website every 0-2 seconds and I start it around 11:00 with a Linux crontab. Any suggestions?

#Packages

if (!require('lubridate')) {install.packages('lubridate')};library('lubridate')
if (!require('magrittr')) {install.packages('magrittr')};library('magrittr')
if (!require('rvest')) {install.packages('rvest')};library('rvest')
if (!require('slackr')) {install.packages('slackr')};library('slackr')
if (!require('XML')) {install.packages('XML')};library('XML')


#Basics
linux <- "/home/pi/R/projects"
setwd(linux)
slackr_setup(config_file = ".coronapush")
y <- 1

#scraping
bag_alt <- xml2::read_html('https://www.bag.admin.ch/bag/de/home/krankheiten/ausbrueche-epidemien-pandemien/aktuelle-ausbrueche-epidemien/novel-cov/situation-schweiz-und-international.html')
anzahl_faelle_alt <- bag_alt %>% 
  html_nodes('#2030838475+ .mod-table tr:nth-child(1) td:nth-child(2)') %>% 
  html_text()
anzahl_faelle_alt <- gsub("[[:space:]]", "", anzahl_faelle_alt) %>%
  as.integer()
anzahl_tote_alt <- bag_alt %>%
  html_nodes('#2030838475+ .mod-table tr:nth-child(3) td:nth-child(2)') %>% 
  html_text()
anzahl_tote_alt <- gsub("[[:space:]]", "", anzahl_tote_alt) %>%
  as.integer()
anzahl_tests_alt <- bag_alt %>%
  html_nodes('tr:nth-child(4) td:nth-child(2)')%>%
  html_text()
anzahl_tests_alt <- gsub("[[:space:]]", "", anzahl_tests_alt) %>%
  as.integer()

#prepare loop
tests_gemacht <- F
faelle_gemacht <- F
tote_gemacht <- F
y <- 1

#Loop

repeat {
  bag_neu <- xml2::read_html('https://www.bag.admin.ch/bag/de/home/krankheiten/ausbrueche-epidemien-pandemien/aktuelle-ausbrueche-epidemien/novel-cov/situation-schweiz-und-international.html')  
    anzahl_faelle_neu <- bag_neu %>% 
      html_nodes('#2030838475+ .mod-table tr:nth-child(1) td:nth-child(2)') %>% 
      html_text() 
    anzahl_faelle_neu <- gsub("[[:space:]]", "", anzahl_faelle_neu) %>%
      as.integer()
    
    anzahl_tote_neu <- bag_neu %>%
      html_nodes('#2030838475+ .mod-table tr:nth-child(3) td:nth-child(2)') %>% 
      html_text() 
    anzahl_tote_neu <- gsub("[[:space:]]", "", anzahl_tote_neu) %>%
      as.integer()

    anzahl_tests_neu <- bag_neu %>%
      html_nodes('tr:nth-child(4) td:nth-child(2)')%>%
      html_text()
    anzahl_tests_neu <- gsub("[[:space:]]", "", anzahl_tests_neu) %>%
      as.integer()
    
    if(anzahl_faelle_alt != anzahl_faelle_neu) {
      ansteckungen <- paste0("Neue BAG-Zahlen: ", anzahl_faelle_neu, " Ansteckungen,")
      slackr_bot(ansteckungen)
      faelle_gemacht <- T
      }
    if(anzahl_tests_alt != anzahl_tests_neu) { 
      anzahl_tests_neu_formatiert <- format(anzahl_tests_neu, big.mark = "'")
      tests <- paste0(anzahl_tests_neu_formatiert, " Tests")
        slackr_bot(tests)
        tests_gemacht <- T
        }
   
    if(tests_gemacht == T && faelle_gemacht == T) {
      positivitaet <- anzahl_faelle_neu/anzahl_tests_neu*100
      positivitaet<-  round(positivitaet,1) %>%
        as.character()
      positivitaet <- paste0(positivitaet, "% Positivität")
      slackr_bot(positivitaet)
    }
    if(anzahl_tote_alt != anzahl_tote_neu) { 
      tote <- paste0(anzahl_tote_neu, " Tote")
      slackr_bot(tote)
      tote_gemacht <- T
    }
    
    if(tests_gemacht == T && faelle_gemacht == T && tote_gemacht == F){
      tote <- paste0(anzahl_tote_neu, " Tote")
      slackr_bot(tote)
      tote_gemacht <- T
    }
y <-y+1    
if (faelle_gemacht == T && tote_gemacht == T && tests_gemacht == T){
  quit()
}
if (y == 15000){
  quit()
}
x <- runif(1,0,2)
Sys.sleep(x)
print(y)
}
0 Answers
Related