Веб-скрейпинг органических результатов Google Jobs с помощью Nodejs

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

puppeteer.use(StealthPlugin());

const searchString = "javascript developer"; // what we want to search
const encodedString = encodeURI(searchString); // what we want to search for in URI encoding

const requestParams = {
  q: encodedString, // our encoded search string
  hl: "en", // parameter defines the language to use for the Google search
  uule: "w+CAIQICIKY2FsaWZvcm5pYQ", // encoded location
};

const domain = `https://www.google.com`;

async function scrollPage(page, scrollContainer) {
  let lastHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
  while (true) {
    await page.evaluate(`document.querySelector("${scrollContainer}").scrollTo(0, document.querySelector("${scrollContainer}").scrollHeight)`);
    await page.waitForTimeout(2000);
    let newHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
    if (newHeight === lastHeight) {
      break;
    }
    lastHeight = newHeight;
  }
}

async function fillInfoFromPage(page) {
  return await page.evaluate(async () => {
    return Array.from(document.querySelectorAll(".iFjolb")).map((el) => ({
      title: el.querySelector(".BjJfJf").textContent.trim(),
      companyName: el.querySelector(".vNEEBe").textContent.trim(),
      location: el.querySelectorAll(".Qk80Jf")[0].textContent.trim(),
      via: el.querySelectorAll(".Qk80Jf")[1].textContent.trim(),
      thumbnail: el.querySelector(".pJ3Uqf img")?.getAttribute("src"),
      extensions: Array.from(el.querySelectorAll(".oNwCmf .I2Cbhb .LL4CDc")).map((el) => el.textContent.trim()),
    }));
  });
}

async function getJobsInfo() {
  const browser = await puppeteer.launch({
    headless: false,
    args: ["--no-sandbox", "--disable-setuid-sandbox"],
  });

  const page = await browser.newPage();

  const URL = `${domain}/search?ibp=htl;jobs&hl=${requestParams.hl}&q=${requestParams.q}&uule=${requestParams.uule}`;

  await page.setDefaultNavigationTimeout(60000);
  await page.goto(URL);
  await page.waitForSelector(".iFjolb");
  await page.waitForTimeout(1000);

  await scrollPage(page, ".zxU94d");

  const jobs = await fillInfoFromPage(page);

  await browser.close();

  return jobs;
}

getJobsInfo().then((result) => console.dir(result, { depth: null }));

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

puppeteer.use(StealthPlugin());

const searchString = "javascript developer"; // what we want to search
const encodedString = encodeURI(searchString); // what we want to search for in URI encoding

const requestParams = {
  q: encodedString, // our encoded search string
  hl: "en", // parameter defines the language to use for the Google search
  uule: "w+CAIQICIKY2FsaWZvcm5pYQ", // encoded location
};

const domain = `https://www.google.com`;

async function scrollPage(page, scrollContainer) {
  ...
}

let lastHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
while (true) {
  await page.evaluate(`document.querySelector("${scrollContainer}").scrollTo(0, document.querySelector("${scrollContainer}").scrollHeight)`);
  await page.waitForTimeout(2000);
  let newHeight = await page.evaluate(`document.querySelector("${scrollContainer}").scrollHeight`);
  if (newHeight === lastHeight) {
    break;
  }
  lastHeight = newHeight;
}

async function fillInfoFromPage(page) {
  ...
}

return await page.evaluate(async () => {
    return Array.from(document.querySelectorAll(".iFjolb")).map((el) => ({

  title: el.querySelector(".BjJfJf").textContent.trim(),
  companyName: el.querySelector(".vNEEBe").textContent.trim(),
  location: el.querySelectorAll(".Qk80Jf")[0].textContent.trim(),
  via: el.querySelectorAll(".Qk80Jf")[1].textContent.trim(),
  thumbnail: el.querySelector(".pJ3Uqf img")?.getAttribute("src"),
  extensions: Array.from(el.querySelectorAll(".oNwCmf .I2Cbhb .LL4CDc")).map((el) => el.textContent.trim()),

async function getJobsInfo() {
  ...
}

const browser = await puppeteer.launch({
  headless: false,
  args: ["--no-sandbox", "--disable-setuid-sandbox"],
});

const page = await browser.newPage();

const URL = `${domain}/search?ibp=htl;jobs&hl=${requestParams.hl}&q=${requestParams.q}&uule=${requestParams.uule}`;

await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);
await page.waitForSelector(".iFjolb");

await scrollPage(page, ".zxU94d");

const jobs = await fillInfoFromPage(page);

await browser.close();

return jobs;

$ node YOUR_FILE_NAME # YOUR_FILE_NAME is the name of your .js file

[
   {
      "title":"Python Developer Python-JavaScript and vue.js",
      "companyName":"Dice",
      "location":"San Francisco, CA",
      "via":"via LinkedIn",
      "thumbnail":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQKlgydP7sElaJC9qPrtNHwBhyTMHYgii1RPWsy&s=0",
      "extensions":[
         "5 days ago",
         "Contractor"
      ]
   },
   {
      "title":"Remote Senior JavaScript Developer",
      "companyName":"Jobot",
      "location":"Las Vegas, NV",
      "via":"via Central Illinois Proud Jobs",
      "extensions":[
         "4 days ago",
         "Full-time",
         "No degree mentioned"
      ]
   },
   ... and other results
]

npm i google-search-results-nodejs

const SerpApi = require("google-search-results-nodejs");
const search = new SerpApi.GoogleSearch(process.env.API_KEY);

const searchString = "javascript developer"; // what we want to search

const params = {
  engine: "google_jobs", // search engine
  q: searchString, // search query
  hl: "en", // Parameter defines the language to use for the Google search
  uule: "w+CAIQICIKY2FsaWZvcm5pYQ", // encoded location
};

const getJson = () => {
  return new Promise((resolve) => {
    search.json(params, resolve);
  });
};

const getResults = async () => {
  const organicResults = [];
  while (true) {
    const json = await getJson();
    if (json.search_information?.jobs_results_state === "Fully empty") break;
    organicResults.push(...json.jobs_results);
    params.start ? (params.start += 10) : (params.start = 10);
  }
  return organicResults;
};

getResults().then((result) => console.dir(result, { depth: null }));

const SerpApi = require("google-search-results-nodejs");
const search = new SerpApi.GoogleSearch(API_KEY);

const searchString = "javascript developer"; // what we want to search

const params = {
  engine: "google_jobs", // search engine
  q: searchString, // search query
  hl: "en", // Parameter defines the language to use for the Google search
  uule: "w+CAIQICIKY2FsaWZvcm5pYQ", // encoded location
};

const getJson = () => {
  return new Promise((resolve) => {
    search.json(params, resolve);
  });
};

const getResults = async () => {
  ...
};

const organicResults = [];

while (true) {
  const json = await getJson();
  if (json.search_information?.jobs_results_state === "Fully empty") break;
  organicResults.push(...json.jobs_results);
  params.start ? (params.start += 10) : (params.start = 10);
}
return organicResults;

getResults().then((result) => console.dir(result, { depth: null }));

[
  {
    "title": "Python Developer Python-JavaScript and vue.js",
    "company_name": "Dice",
    "location": "San Francisco, CA",
    "via": "via LinkedIn",
    "description": "Dice is the leading career destination for tech experts at every stage of their careers. Our client, Mitchell Martin, Inc., is seeking the following. Apply via Dice today!\n\nPython Developer Python-JavaScript and vue.js...\n\nPosition Type: Contract\n\nJob responsibilities:\n\nAs a member of the Company Bioinformatics team, you will work closely with other Bioinformatics developers and laboratory staff to provide technical leadership, and develop & deploy workflows for our laboratory LIMS that enable automated high throughput workflows for our DNA sequencing laboratories.\n• Develop and deploy software that manages the operational activities in our specialty genetics laboratories\n• Ensure availability, performance, and scalability of workflows\n• Work closely with product owners, software engineers and R&D scientists to gather and implement requirements\n• Build and maintain code that interacts with a 3rd party vendor application\n• Guide and mentors other engineers and project team members\n\nRequired Skills and Qualifications 5+ years of experience Python, JavaScript and vue.js\n• Proficient in Python, JavaScript and Vue.js Experience in using version control tools, e.g., Gitlab\n• B.S. in Bioengineering, Computer Science, MS/PhD preferred\n• 3+ years of experience working in a regulated industrial life sciences environment or equivalent\n• 5+ years of experience Python, JavaScript and vue.js\n• Experienced in using version control tools, e.g., Gitlab\n• Familiar with working in a Linux environment\n• Familiar with writing unit tests\n• Familiarity with typical laboratory workflows and robotic automation used by DNA sequencing laboratories is a plus\n• Knowledge of L7 ESP LIMS is a plus\n• Demonstrated ability to work with vendor APIs (or file-based communication) for integration and development\n• Experience to develop APIs in MuleSoft is a plus\n• Experience supporting and maintaining applications that interact with 3rd party Vendor software\n• Demonstrated ability to work in a team and communicate effectively with laboratory personal and R&D scientists\n• Proficient in Python, JavaScript and Vue.js Experience in using version control tools, e.g., Gitlab\n• provided by Dice",
    "thumbnail": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTchgwk0qIvqPnMlAcqO5451PRYsMDccWFDcD5pGeE&s",
    "extensions": ["5 days ago", "Contractor"],
    "detected_extensions": {
      "posted_at": "5 days ago",
      "schedule_type": "Contractor"
    },
    "job_id": "eyJqb2JfdGl0bGUiOiJQeXRob24gRGV2ZWxvcGVyIFB5dGhvbi1KYXZhU2NyaXB0IGFuZCB2dWUuanMiLCJodGlkb2NpZCI6InVvWXpSMGhPWjZvQUFBQUFBQUFBQUE9PSIsInV1bGUiOiJ3K0NBSVFJQ0lLWTJGc2FXWnZjbTVwWVEiLCJobCI6ImVuIiwiZmMiOiJFcUlDQ3VJQlFVRjBWbXhpUVV0V1kyZFRiWGszZWxwcmNHWnBjVmswTVhCUk9EQkVUems1VkVocWJFWmtWRXBFWHpNNVIzSjFVMkZCZVZoU1FVNTFSVmhDTUhZd2NGZFpRVTVvTTFGWWVtUk5WbnBmZDFOTWJUazBVblJqV21OcVlXb3RVMUpFU0VSck5GWnNWV0l6TjA1NE5XMWhiMnQyUmpWd1UxODViR042YVV0QmJsUTVTalJ2YzFWaFMwSlVNM2xHUWpFdGNGcEllVkpzUWpWeVRGQlRSbDl2Y1VsMlh6TlNkaTFIZFZCWU9WVm1SaTFNV0hkMlpTMDJjVGRqWWxaaU16Rk9jakl0YVZvMVJISnhla2hXWkZkT1dGOVdjRkpGZVRCNlkzUlNSMVF6VHpadVFSSVhNelYzVlZrMVlsWkVjbGN4Y1hSelVHdFBkVTF0UVVrYUlrRkVWWGxGUjJSdE5FUlVNaTFxUkdWbmRHbHBObWhZY1VOcmNYQXdOSGhhVmxFIiwiZmN2IjoiMyIsImZjX2lkIjoiZmNfMSIsImFwcGx5X2xpbmsiOnsidGl0bGUiOiIubkZnMmVie2ZvbnQtd2VpZ2h0OjUwMH0uQmk2RGRje2ZvbnQtd2VpZ2h0OjUwMH1BcHBseSBvbiBMaW5rZWRJbiIsImxpbmsiOiJodHRwczovL3d3dy5saW5rZWRpbi5jb20vam9icy92aWV3L3B5dGhvbi1kZXZlbG9wZXItcHl0aG9uLWphdmFzY3JpcHQtYW5kLXZ1ZS1qcy1hdC1kaWNlLTMyNDU2NzQxMTU/dXRtX2NhbXBhaWduPWdvb2dsZV9qb2JzX2FwcGx5XHUwMDI2dXRtX3NvdXJjZT1nb29nbGVfam9ic19hcHBseVx1MDAyNnV0bV9tZWRpdW09b3JnYW5pYyJ9fQ=="
  },
  {
    "title": "Staff JavaScript Developer - 50% REMOTE",
    "company_name": "Jobot",
    "location": "Los Angeles, CA",
    "via": "via KTLA Jobs",
    "description": "Growing technology company in Cambridge, MA looking for a sharp Senior JavaScript Developer to join their growing team!\n\nThis Jobot Job is hosted by Roxy Kupfert...\n\nAre you a fit? Easy Apply now by clicking the Apply button and sending us your resume.\n\nSalary $120,000 - $220,000 per year\n\nA Bit About Us\n\nLocated in Cambridge, MA we are a rapidly growing company in the internet technology space. We are looking for a sharp Senior JavaScript Developer to join our team and hit the ground running!\n\nWhy join us?\n\nWe offer a comprehensive compensation package including but not limited to\n• A highly competitive base salary ranging from $120K-$220K + EQUITY + BONUSES!\n• Full benefits (Medical, Dental, Vision)\n• 401K with match\n• Great work/life balance - ability to work partially remote / partially in the office\n• Opportunity to work alongside other brilliant engineers\n• Flexible work schedule\n• Catered lunches\n• Paid gym membership\n• Foosball and Ping Pong tables\nJob Details\n• Integrating user components on server-side JavaScript\n• Building performant applications with high availability and low latency\n• Ensuring security, accessibility, and privacy concerns are handled\n• Writing maintainable code with extensive test coverage, including load tests\nMUST HAVE, experience with\n• Modern JavaScript\n• React and/or Redux\n• Developing well-structured, performant web applications with component-based architectures\nNICE TO HAVE, experience with\n• Security and data concerns such as privacy, data integrity, etc.\n• Node.js\n• Containerization / cloud environments\n• REST, JSON, API design and micro-services\n• Common UX patterns, accessibility, and cross-browser, cross-device implementations.\n• Understanding of algorithms, data structures and design patterns\n• CI/CD pipelines\nIf this sounds like you, please apply through the link or email your resume directly to roxy.kupfert@!\n\nInterested in hearing more? Easy Apply now by clicking the Apply button",
    "extensions": ["4 days ago", "Full-time", "No degree mentioned", "Health insurance", "Dental insurance"],
    "detected_extensions": {
      "posted_at": "4 days ago",
      "schedule_type": "Full-time"
    },
    "job_id": "eyJqb2JfdGl0bGUiOiJTdGFmZiBKYXZhU2NyaXB0IERldmVsb3BlciAtIDUwJSBSRU1PVEUiLCJodGlkb2NpZCI6InZsVmN0d2s5RklFQUFBQUFBQUFBQUE9PSIsInV1bGUiOiJ3K0NBSVFJQ0lLWTJGc2FXWnZjbTVwWVEiLCJobCI6ImVuIiwiZmMiOiJFb3dDQ3N3QlFVRjBWbXhpUTJoYVYwMUhiWHB6Y1hwYVNrZDRTRzVUZVdaUkxVNUpRWFZvUTJGV01XZFNRMVZ1V0cxcVJETjZjVGMwZURsMVMyaEhZM2x3ZFVSaVRXZDVjREJGVmt4TU9GQklhR050ZFVzNFFtODFTWEJJVDNwcVJGRndkSE5aVGkxVmRuZzVaRU5UU0RaWVJsaEpZVXB4Tm5WTWJURllUbTF2Wm1WMmFGQkxURjlZVTFCeVNISkRUVFZ1TjA1UE9FeHliWFZ2Ym1acmR6TlplblpzVWpJd2NXZExaVzVhY2xrMFVrSlVheTAzZFY5T1ZFcGhSMDh4WjFkSmFWWmtUMkZGYlVaNFVVVkpZblZCRWhjek5YZFZXVFZpVmtSeVZ6RnhkSE5RYTA5MVRXMUJTUm9pUVVSVmVVVkhabDlNZFZSb2RrZ3dRek56Y0ZaSFkxQTFiek5sZW13eE9IUldVUSIsImZjdiI6IjMiLCJmY19pZCI6ImZjXzMiLCJhcHBseV9saW5rIjp7InRpdGxlIjoiQXBwbHkgb24gS1RMQSBKb2JzIiwibGluayI6Imh0dHBzOi8vam9icy5rdGxhLmNvbS9qb2JzL3N0YWZmLWphdmFzY3JpcHQtZGV2ZWxvcGVyLTUwLXJlbW90ZS1sb3MtYW5nZWxlcy1jYWxpZm9ybmlhLzY5OTIyMTQ2My0yLz91dG1fY2FtcGFpZ249Z29vZ2xlX2pvYnNfYXBwbHlcdTAwMjZ1dG1fc291cmNlPWdvb2dsZV9qb2JzX2FwcGx5XHUwMDI2dXRtX21lZGl1bT1vcmdhbmljIn19"
  }
]