Every day around 12:30 the Swiss government publishes corona data on a website. I run a scraper that is supposed to notify a slack-channel as soon as the numbers are published. However, every day my scraper is a few minutes slower than the webmonitor apps my colleagues use. I don't know how to make it even faster. At this point it runs on a repeat loop that scrapes the website every 0-2 seconds and I start it around 11:00 with a Linux crontab. Any suggestions?
#Packages
if (!require('lubridate')) {install.packages('lubridate')};library('lubridate')
if (!require('magrittr')) {install.packages('magrittr')};library('magrittr')
if (!require('rvest')) {install.packages('rvest')};library('rvest')
if (!require('slackr')) {install.packages('slackr')};library('slackr')
if (!require('XML')) {install.packages('XML')};library('XML')
#Basics
linux <- "/home/pi/R/projects"
setwd(linux)
slackr_setup(config_file = ".coronapush")
y <- 1
#scraping
bag_alt <- xml2::read_html('https://www.bag.admin.ch/bag/de/home/krankheiten/ausbrueche-epidemien-pandemien/aktuelle-ausbrueche-epidemien/novel-cov/situation-schweiz-und-international.html')
anzahl_faelle_alt <- bag_alt %>%
html_nodes('#2030838475+ .mod-table tr:nth-child(1) td:nth-child(2)') %>%
html_text()
anzahl_faelle_alt <- gsub("[[:space:]]", "", anzahl_faelle_alt) %>%
as.integer()
anzahl_tote_alt <- bag_alt %>%
html_nodes('#2030838475+ .mod-table tr:nth-child(3) td:nth-child(2)') %>%
html_text()
anzahl_tote_alt <- gsub("[[:space:]]", "", anzahl_tote_alt) %>%
as.integer()
anzahl_tests_alt <- bag_alt %>%
html_nodes('tr:nth-child(4) td:nth-child(2)')%>%
html_text()
anzahl_tests_alt <- gsub("[[:space:]]", "", anzahl_tests_alt) %>%
as.integer()
#prepare loop
tests_gemacht <- F
faelle_gemacht <- F
tote_gemacht <- F
y <- 1
#Loop
repeat {
bag_neu <- xml2::read_html('https://www.bag.admin.ch/bag/de/home/krankheiten/ausbrueche-epidemien-pandemien/aktuelle-ausbrueche-epidemien/novel-cov/situation-schweiz-und-international.html')
anzahl_faelle_neu <- bag_neu %>%
html_nodes('#2030838475+ .mod-table tr:nth-child(1) td:nth-child(2)') %>%
html_text()
anzahl_faelle_neu <- gsub("[[:space:]]", "", anzahl_faelle_neu) %>%
as.integer()
anzahl_tote_neu <- bag_neu %>%
html_nodes('#2030838475+ .mod-table tr:nth-child(3) td:nth-child(2)') %>%
html_text()
anzahl_tote_neu <- gsub("[[:space:]]", "", anzahl_tote_neu) %>%
as.integer()
anzahl_tests_neu <- bag_neu %>%
html_nodes('tr:nth-child(4) td:nth-child(2)')%>%
html_text()
anzahl_tests_neu <- gsub("[[:space:]]", "", anzahl_tests_neu) %>%
as.integer()
if(anzahl_faelle_alt != anzahl_faelle_neu) {
ansteckungen <- paste0("Neue BAG-Zahlen: ", anzahl_faelle_neu, " Ansteckungen,")
slackr_bot(ansteckungen)
faelle_gemacht <- T
}
if(anzahl_tests_alt != anzahl_tests_neu) {
anzahl_tests_neu_formatiert <- format(anzahl_tests_neu, big.mark = "'")
tests <- paste0(anzahl_tests_neu_formatiert, " Tests")
slackr_bot(tests)
tests_gemacht <- T
}
if(tests_gemacht == T && faelle_gemacht == T) {
positivitaet <- anzahl_faelle_neu/anzahl_tests_neu*100
positivitaet<- round(positivitaet,1) %>%
as.character()
positivitaet <- paste0(positivitaet, "% Positivität")
slackr_bot(positivitaet)
}
if(anzahl_tote_alt != anzahl_tote_neu) {
tote <- paste0(anzahl_tote_neu, " Tote")
slackr_bot(tote)
tote_gemacht <- T
}
if(tests_gemacht == T && faelle_gemacht == T && tote_gemacht == F){
tote <- paste0(anzahl_tote_neu, " Tote")
slackr_bot(tote)
tote_gemacht <- T
}
y <-y+1
if (faelle_gemacht == T && tote_gemacht == T && tests_gemacht == T){
quit()
}
if (y == 15000){
quit()
}
x <- runif(1,0,2)
Sys.sleep(x)
print(y)
}