I am new to Node JS, using node-fetch and cheerio packages. I am trying to scrape data from different websites, so I am testing by passing many different URLs, and selectors. However, in the following code, no matter what selector, or URL I pass as input, .text() returns an empty string, and .html() returns null.
const cheerio= require('cheerio');
const fetch = require('node-fetch');
fetch('https://www.npmjs.com/package/node-fetch/')
.then((res)=>{
if(res.ok){
let $=cheerio.load(res);
console.log(res);
let siteData = $('#readme > p:nth-child(8)');
console.log(siteData.text());
console.log(siteData.html());
return res.text();
}else{
throw new Error(res.statusText);
}
})
.then(body => console.log(body))
.catch(error => console.log(error))
I have even written the output of res.text() to a file and compared it with the source HTML of the website. It is almost the same. The value of res is printed as follows:
Response {
size: 0,
timeout: 0,
prev: null,
next: null,
root: {
type: 'root',
name: 'root',
namespace: 'http://www.w3.org/1999/xhtml',
attribs: [Object: null prototype] {},
'x-attribsNamespace': [Object: null prototype] {},
'x-attribsPrefix': [Object: null prototype] {},
children: [ [Circular] ],
parent: null,
prev: null,
next: null
},
parent: null,
[Symbol(Body internals)]: {
body: Gunzip {
_writeState: [Uint32Array],
_readableState: [ReadableState],
readable: true,
_events: [Object: null prototype],
_eventsCount: 6,
_maxListeners: undefined,
_writableState: [WritableState],
writable: true,
allowHalfOpen: true,
_transformState: [Object],
_hadError: false,
bytesWritten: 0,
_handle: [Zlib],
_outBuffer: <Buffer 80 00 f4 9f 03 02 00 00 f0 80 f2 9f 03 02 00 00 20 46 00 00 00 00 00 00 d8 73 dd 9f 03 02 00 00 0f 00 00 00 7f ae f8 39 01 5d dd 9f 03 02 00 00 d0 68 ... 16334 more bytes>,
_outOffset: 0,
_chunkSize: 16384,
_defaultFlushFlag: 2,
_finishFlushFlag: 2,
_defaultFullFlushFlag: 3,
_info: undefined,
_level: -1,
_strategy: 0,
[Symbol(kCapture)]: false
},
disturbed: false,
error: null
},
[Symbol(Response internals)]: {
url: 'https://www.npmjs.com/package/node-fetch',
status: 200,
statusText: 'OK',
headers: Headers { [Symbol(map)]: [Object: null prototype] },
counter: 1
}
}
Even the object for siteData is printed as follows:
initialize {
options: {
withDomLvl1: true,
normalizeWhitespace: false,
xml: false,
decodeEntities: true
},
_root: initialize {
'0': {
type: 'root',
name: 'root',
namespace: 'http://www.w3.org/1999/xhtml',
attribs: [Object: null prototype] {},
'x-attribsNamespace': [Object: null prototype] {},
'x-attribsPrefix': [Object: null prototype] {},
children: [Array],
parent: null,
prev: null,
next: null
},
options: {
withDomLvl1: true,
normalizeWhitespace: false,
xml: false,
decodeEntities: true
},
length: 1,
_root: [Circular]
},
length: 0,
prevObject: initialize {
'0': {
type: 'root',
name: 'root',
namespace: 'http://www.w3.org/1999/xhtml',
attribs: [Object: null prototype] {},
'x-attribsNamespace': [Object: null prototype] {},
'x-attribsPrefix': [Object: null prototype] {},
children: [Array],
parent: null,
prev: null,
next: null
},
options: {
withDomLvl1: true,
normalizeWhitespace: false,
xml: false,
decodeEntities: true
},
length: 1,
_root: [Circular]
}
}
Still, siteData.text() is an empty string, and siteData.html() is null. Please advise on what is wrong with this code, I have been going through many stack overflow pages, and reading cheerio documentation, but I still haven't found the answer.
Thanking you in advance.