I am scraping a website and some urls are missing. As a result, my code can't get all the data I'm trying to save.
Here is my code :
informations = []
0.upto(94) do |n|
begin
html_file = open(departement[n]).read
html_doc = Nokogiri::HTML(html_file)
unless n == 19
html_doc.search('a').each do |element|
if n < 9
x = "http://www.annuaire-des-mairies.com/0#{n + 1}/" + element.text.strip
informations << x
elsif n > 9 && n < 19
y = "http://www.annuaire-des-mairies.com/#{n + 1}/" + element.text.strip
informations << y
elsif n > 19
z= "http://www.annuaire-des-mairies.com/#{n + 2}/" + element.text.strip
informations << z
csv_options = { col_sep: ',', force_quotes: true, quote_char: '"' }
filepath = 'mairiesscrapées.csv'
informations.each do |information|
next unless information.include? ".html"
html_file = open(URI.parse(URI.escape(information))).read
html_doc = Nokogiri::HTML(html_file)
g = html_doc.search("td")
CSV.open(filepath, 'ab', csv_options) do |csv|
csv << [g[0].to_s, g[3].to_s, g[7].to_s, g[13].to_s]
puts csv
rescue OpenURI::HTTPError => ex
puts "Handle missing link here"
end
end
end
end
end
end
end
I've tried to insert a begin rescue method but my console keeps answering me with :
1: from /Users/pierrelaburthe/.rbenv/versions/2.6.3/lib/ruby/2.6.0/open-uri.rb:756:in `buffer_open'
/Users/pierrelaburthe/.rbenv/versions/2.6.3/lib/ruby/2.6.0/open-uri.rb:378:in `open_http': 404 Not Found (OpenURI::HTTPError)