I am using this Quora scraper by Bright Data and getting different results every time. For example, for this page I got these results on the first run:
- number of native answers:1
- number of related answers: 31
- 1st_answer_author_name: Joseph Rowena
On the second run, I get "null" for all those fields. This also happens on other pages.
This is the navigation code:
country('us') //proxy USA where load site below in navigate()
navigate(input.url, {wait_until: 'domcontentloaded', timeout: 90e3}); // load site
try{wait('.q-text.puppeteer_test_question_title')} catch{ throw 'Something went wrong'} // wait title of page
scroll_to('bottom') // scroll page to the end of page (need for load fully page with all info)
wait(10e3) //wait 10 seconds
try{
wait(".qu-alignItems--center.qu-py--small.qu-justifyContent--space-between .PopoverMenu___StyledInlineFlex-sc-11ubalt-0:first-child")
//wait buttons with info
$(".qu-alignItems--center.qu-py--small.qu-justifyContent--space-between .PopoverMenu___StyledInlineFlex-sc-11ubalt-0:first-child").click()
//click on button
if(el_exists('.q-box.puppeteer_test_popover_menu > div:nth-child(2)')){
click('.q-box.puppeteer_test_popover_menu > div:nth-child(2)')
wait(10e3)
$(".qu-alignItems--center.qu-py--small.qu-justifyContent--space-between .PopoverMenu___StyledInlineFlex-sc-11ubalt-0:first-child").click()
}
} catch {}
wait(10e3)
let data = parse(); //parse data from Parser code below
if(data.is_error || (data.number_of_related_answers && data.number_of_native_answers == 0 && data["1st_answer_author_name"]))
throw 'Not load all authors'
else{
delete data.is_error
collect(data); // return and crawl data from Parser code
}And this is the parser code:
let number_of_native_answers = +$(".qu-alignItems--center.qu-py--small.qu-justifyContent--space-between div.qu-px--medium.qu-py--small:contains('Answer')").first().text().trim()?.replace(/\D+/g, '') || null
let authors = []
if (number_of_native_answers)
authors = $('div[class*="dom_annotate_question_answer"] .qu-alignItems--flex-start .q-inlineFlex.qu-alignItems--center.qu-wordBreak--break-word > span > span').toArray().map(e=>$(e).text().trim()).slice(0, 3)
console.log(authors)
let is_error = 0;
if ((authors.length <= 2 && number_of_native_answers > 2) || (authors.length && number_of_native_answers && authors.length > number_of_native_answers))
is_error = 1
// got all authors from page in array
return {
question_title: $('.q-text.puppeteer_test_question_title').text().trim(),
number_of_related_answers: +$(".qu-alignItems--center.qu-py--small.qu-justifyContent--space-between div.qu-px--medium.qu-py--small:contains('All related')").first().text().trim()?.replace(/\D+/g, '') || null,
number_of_native_answers,
"1st_answer_author_name": authors[0],
"2nd_answer_author_name": authors[1],
"3rd_answer_author_name": authors[2],
is_error
};Is the problem in my parser code, or perhaps it's a proxy issue and Quora is just blocking me in some unknown way? from what I understand the job runs through a residential proxy IP so it should work fine.