I am working on a Firebase Function website scraper to grab basic Open Graph meta tags from any website.
My function works perfectly fine when running locally as an emulator, but once I deploy my function up to the cloud and call it from my app it gets blocked by all major sites.
It seems once the scraper is run from a server it is detected as a robot, and stops working altogether. Any ideas why this may be happening?
Scraper code:
import * as functions from "firebase-functions";
import fetch, { Headers } from "node-fetch";
const cheerio = require("cheerio");
const headers = new Headers({
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36",
Accept:
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate",
Connection: "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Cache-Control": "max-age=0",
});
export const getUrlData = functions.https.onCall(
async (data, context) => {
try {
const response = await fetch(data.url, { headers });
const html = await response.text();
const $ = cheerio.load(html);
const title = $('meta[property = "og:title"]').attr("content") || $("title").html();
const image = $('meta[property = "og:image"]').attr("content");
return {
title,
image
};
} catch (e) {
throw Error(e)
}
}
);