Download specific files from url in r

Im sure this can be optimised but give this Base R solution a try (note this is not tested due to the size of files):

# Store the parent url of files we want to download:
# base_url => character scalar
base_url <- "https://www.star.nesdis.noaa.gov/pub/corp/scsb/wguo/data/Blended_VH_4km/geo_TIFF/"

# Read in the html: html_string => character vector
html_string <- readLines(
  base_url
)

# Store the range of years to search for, 
# flattened into a regex pattern: 
# rng => character scalar
rng <- paste0(
  seq(
    1981, 
    2011,
    by = 1
  ),
  collapse = "|"
)

# Create a vector of urls requiring download: 
# input_urls => character vector
input_urls <- Filter(
  function(y){
    grepl(
      rng, 
      y
    )
  },
  unlist(
    lapply(
      strsplit(
        html_string, 
        "href\\="
      ),
      function(x){
        ir <- paste0(
          base_url, 
          unlist(
            strsplit(
              gsub(
              '^"', 
              '', 
              gsub(
                "(\\w+VCI\\.tif).*", 
                "\\1", 
                noquote(x)
              )
            ),
          "\\<\\/td\\>\\<td\\>\\<a"
            )
          )
        )[2]
      }
    )
  )
)

# Store the desired output folder here: 
# dir_path => character scalar
dir_path <- paste0(
  getwd(),
  "/geo_tifs"
)

# Function to create a directory if it doesnt already
# exist: r_mkdir => function() 
r_mkdir <- function(dir_path){
  if(dir.exists(dir_path)){
    invisible()
  }else{
    dir.create(dir_path)
  }
}

# Create the directory if it doesnt already exist: 
# directory => stdout(file system)
r_mkdir(dir_path)

# Generate a character vector of output file paths: 
# output_file_paths => character vector
output_file_paths <- vapply(
  strsplit(tmp, "geo_TIFF"),
  function(x){
    paste0(dir_path, x[2])
  },
  character(1)
)

# Download the files to the output paths: 
# .tif files => stdout(file path)
lapply(
  seq_along(output_file_paths),
  function(i){
    download.file(
      input_urls[i],
      output_file_paths[i],
      quiet = TRUE,
      cacheOK = FALSE
    )
  }
)

The following uses wget and it works at least with the first 2 files, I have tested the downloads of a (very) small subset of the wanted files.

suppressPackageStartupMessages({
  library(httr)
  library(rvest)
  library(dplyr)
  library(stringr)
})

# big files need greater timeout values,
# since I'm using wget this is probably
# unnecessary
old_timeout <- options(timeout = 300)
getOption("timeout")

year_start <- 1981
year_end <- 2011
download_dir <- "~/Temp/"
wget_cmd_line <- c("-P", download_dir, "")

link <- "https://www.star.nesdis.noaa.gov/pub/corp/scsb/wguo/data/Blended_VH_4km/geo_TIFF/"
page <- read_html(link)

files_urls <- page %>%
  html_elements("a") %>%
  html_attr("href")

wanted_urls <- files_urls %>%
  str_extract(pattern = "^.*\\.VCI\\.tif$") %>%
  na.omit() %>%
  data.frame(filename = .) %>% 
  mutate(year = str_extract(filename, "\\d{7}"),
         year = str_extract(year, "^\\d{4}"),
         year = as.integer(year)) %>%
  filter(year >= year_start & year <= year_end)

wanted_urls %>%
  #
  # to test the code I only download 2 files;
  # comment out this instruction to download all of them
  head(n = 2) %>%
  #
  pull(filename) %>%
  lapply(\(x) {
    wget_cmd <- wget_cmd_line
    wget_cmd[3] <- paste0(link, x)
    system2("wget", args = wget_cmd, stdout = TRUE, stderr = TRUE)
  })

# put saved value back
options(old_timeout)

I would like to download only the files that contain the name VCI.tif and only the years between 1981 - 2011.

This sound like task for Recursive Retrieval feature of GNU wget, firstly consider more limited in size example: to download *VCI.tif files from https://www.star.nesdis.noaa.gov/pub/corp/scsb/wguo/data/Blended_VH_4km/geo_TIFF/ for year 2021 into current working directory, this can be done as follows

wget --no-clobber --no-directories --recursive --level=1 --accept "*P2021*VCI.tif" https://www.star.nesdis.noaa.gov/pub/corp/scsb/wguo/data/Blended_VH_4km/geo_TIFF/

meaning of used options: do not download file which was already downloaded, do not create subdirectories (save to current working directory), do recursive download, do not move more than 1 time (i.e. use only links given in page) and get only files with names like *P2021*VCI.tif.

I do not know way to describe 1981-2011 with single pattern so I suggest to use 3 following subsequently

*P19[89][0123456789]*VCI.tif
*P200[01234567890]*VCI.tif
*P201[01]*VCI.tif