I am trying to scrap linkedin profiles and download as json, using chrome extension. as everything is work fine contact information needed to be load asynchronously as, i have to simulate a click and wait contact info to load in DOM. callback form onMessage is triggered before DOM is loaded properly. result in incomplete details. Any help is highly appreciated.
content.js
'use strict';
function getElementByXpath(path) {
return document.evaluate(path, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
}
let profileArray = [];
let arrayEducation = [];
let arrayExp = [];
let arrayLanguage = [];
const profile_data = {
}
let profile_data1 ="";
chrome.runtime.onMessage.addListener(gotMessage);
function gotMessage(message,sender,sendResponse){
if(message === "extension_go"){
let userName = getElementByXpath("/html/body/div[6]/div[3]/div/div/div/div/div[3]/div/div/main/div/section/div[2]/div[2]/div[1]/div[1]/h1");
if(userName === null || userName === undefined){
Object.assign(profile_data,{name:""});
}else{
Object.assign(profile_data,{name:userName.textContent})
}
let profilePic = getElementByXpath("/html/body/div[6]/div[3]/div/div/div/div/div[3]/div/div/main/div/section/div[2]/div[1]/div[1]/div/button/img");
let profilePic_src = profilePic.src;
if(profilePic_src === null || profilePic_src === undefined){
Object.assign(profile_data,{profile_pic:""});
}else{
Object.assign(profile_data,{profile_pic:profilePic_src});
}
let location = getElementByXpath("/html/body/div[6]/div[3]/div/div/div/div/div[3]/div/div/main/div/section/div[2]/div[2]/div[2]/span[1]");
if(location === null || location === undefined){
Object.assign(profile_data,{location:""});
}else{
Object.assign(profile_data,{location:location.textContent.replace( /[\r\n]+/gm, "" ).trim()});
}
let about = getElementByXpath("/html/body/div[6]/div[3]/div/div/div/div/div[3]/div/div/main/div/div/div[3]/section/div");
if(about === null || about === undefined){
Object.assign(profile_data,{about:""});
}else{
Object.assign(profile_data,{about:about.textContent.replace( /[\r\n]+/gm, "" ).trim()});
}
let designation = getElementByXpath("/html/body/div[6]/div[3]/div/div/div/div/div[3]/div/div/main/div/section/div[2]/div[2]/div[1]/div[2]");
if(designation === null || designation === undefined){
Object.assign(profile_data,{designation:""});
}else{
Object.assign(profile_data,{designation:designation.textContent.replace( /[\r\n]+/gm, "" ).trim()});
}
let company = getElementByXpath("/html/body/div[6]/div[3]/div/div/div/div/div[3]/div/div/main/div/section/div[2]/div[2]/ul/li[1]/a/h2/div");
if(company === null || company === undefined){
Object.assign(profile_data,{company:""});
}else{
Object.assign(profile_data,{company:company.textContent.replace( /[\r\n]+/gm, "" ).trim()});
}
let experience = document.getElementsByClassName("pv-entity__position-group-pager pv-profile-section__list-item ember-view")
for(let i=0;i<experience.length;i++){
let experienceStr=experience[i].textContent;
let ExperienceList = experienceStr.replace( /[\r\n]+/gm, "" );
let experienceList = ExperienceList.trim();
arrayExp.push(experienceList);
let ExperienceInfo = Object.assign({}, arrayExp);
Object.assign(profile_data,{ExperienceInfo:ExperienceInfo});
}
let education = document.getElementsByClassName("pv-profile-section__list-item pv-education-entity pv-profile-section__card-item ember-view")
for(let i=0;i<education.length;i++){
let allEducation = education[i].textContent;
let EducationList = allEducation.replace( /[\r\n]+/gm, "" );
let educationList = EducationList.trim();
arrayEducation.push(educationList);
let educationInfo = Object.assign({}, arrayEducation);
Object.assign(profile_data,{educationInfo:educationInfo});
}
let languages = document.getElementsByClassName("pv-accomplishments-block__summary-list-item")
for(let i=0;i<languages.length;i++){
let allLanguages = languages[i].textContent;
let LanguagesList = allLanguages.replace( /[\r\n]+/gm, "" );
let languagesList = LanguagesList.trim();
console.log(languagesList)
arrayLanguage.push(languagesList);
let languageInfo = Object.assign({}, arrayLanguage);
Object.assign(profile_data,{languageInfo:languageInfo});
}
document.getElementsByClassName("ember-view link-without-visited-state cursor-pointer text-heading-small inline-block break-words")[0].click()
function waitForElm(selector) {
return new Promise(resolve => {
if (getElementByXpath(selector)) {
return resolve(getElementByXpath(selector));
}
const observer = new MutationObserver(mutations => {
if (getElementByXpath(selector)) {
resolve(getElementByXpath(selector));
observer.disconnect();
}
});
observer.observe(document.body, {
childList: true,
subtree: true
});
});
}
waitForElm('/html/body/div[3]/div/div/div[3]/section/div').then((elm) => {
var lis = elm.getElementsByTagName("section");
for(let i=1;i<lis.length+1;i++){
// console.log(lis[i].getElementsByTagName("h3").textContent);
let h31 = getElementByXpath("/html/body/div[3]/div/div/div[3]/section/div/div/div/section["+i+"]/h3").textContent
let h32="";
if(h31 !== null || h31 !== undefined){
if(h31.includes("Profile")||h31.includes("Email")||h31.includes("Web")){
h32 = getElementByXpath("/html/body/div[3]/div/div/div[3]/section/div/div/div/section["+i+"]/div/a").textContent
}else if(h31.includes("Phone")){
h32 = getElementByXpath("/html/body/div[3]/div/div/div[3]/section/div/div/div/section["+i+"]/ul/li/span[1]").textContent
}else{
h32 = getElementByXpath("/html/body/div[3]/div/div/div[3]/section/div/div/div/section["+i+"]/div/span").textContent
}
}
let profileDe = h32.replace( /[\r\n]+/gm, "" );
let profileDetails = profileDe.trim();
profileArray.push(profileDetails);
let contactInfo = Object.assign({}, profileArray);
Object.assign(profile_data,{contactInfo:contactInfo});
// console.log(lis[i].getElementsByTagName("div").textContent)
}
});
sendResponse(profile_data)
}
// sendResponse(profile_data)
}
background.js
chrome.browserAction.onClicked.addListener(buttonClicked);
function buttonClicked(tab) {
let params = {
active:true,
currentWindow:true
}
chrome.tabs.query(params,function(tab){
var CurrTab = tab[0];
chrome.tabs.sendMessage(CurrTab.id,"extension_go",(data)=>{
console.log(data);
let data1 = JSON.stringify(data);
var blob = new Blob([data1], {type: "application/json"});
var url = URL.createObjectURL(blob);
chrome.downloads.download({
url: url
})
});
})
manifest.json
{
"manifest_version":2,
"name": "Web_Scraper",
"version" : "1.0",
"description" : "LinkedIn_WebScraper",
"content_scripts":[
{
"matches":[
"<all_urls>"
],
"js":["content.js"]
}
],
"permissions": [
"downloads"
],
"browser_action":{
"default_icon":"icon.png"
},
"background":{
"scripts":["background.js"]
}
}