How to aggregate permitted job listings with a web extraction API

Developer guide · reviewed September 2026

Job data is spread across many source formats. Company career pages, licensed feeds, and job boards use different HTML structures, filters, and pagination. A small aggregation pipeline can normalize permitted public sources into one searchable feed.

Here's how to build one with Node.js and ExtractAPI. The API can help with public pages whose content needs browser rendering; your application still owns source permission, normalization, storage, scheduling, and quality checks.

Architecture Overview

The system has three components:

1. Source list: URLs for job listing pages across multiple boards
2. Extraction pipeline: Fetch each URL, extract structured job data, normalize fields
3. Storage + search: SQLite database with full-text search

Step 1: Define Sources

Start with sources you are authorized to read. Company career pages or licensed feeds are often easier to govern than general job boards. The URLs below are placeholders for synthetic examples; replace them only after checking each source's terms and robots guidance:

// sources.json
[
  { "source": "company-a", "url": "https://jobs.example.com/search?q=engineer", "type": "career-page" },
  { "source": "company-b", "url": "https://careers.example.org/open-roles", "type": "career-page" }
]

Step 2: Extract Job Data

Use ExtractAPI with CSS selectors to pull structured job fields from each page. The key is normalizing the output — different sites use different HTML structures, but we want a uniform schema:

const EXTRACTAPI_KEY = process.env.EXTRACTAPI_KEY;
if (!EXTRACTAPI_KEY || /^(YOUR_API_KEY|ext_your)/i.test(EXTRACTAPI_KEY)) {
  throw new Error('Set EXTRACTAPI_KEY before making a request');
}

async function extractJobs(source) {
  const res = await fetch('https://extractapi.app/v1/extract', {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${EXTRACTAPI_KEY}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      url: source.url,
      waitFor: 'a[href*="job"], a[href*="career"], .job-listing, .career-listing',
      javascript: true,
      extractLinks: true,
      extractImages: false
    })
  });

  const data = await res.json();
  if (data.status !== 'ok') {
    console.error(`Failed to extract ${source.source}: ${data.error}`);
    return [];
  }

  // Parse links and text to find job listings
  const allLinks = data.data.links || [];
  const headings = data.data.headings || [];

  // Strategy: find links that look like job listings
  const jobLinks = allLinks.filter(link => {
    const url = (link.href || '').toLowerCase();
    const text = (link.text || '').toLowerCase();
    return url.includes('/job') || url.includes('/career') || url.includes('/position') ||
           text.includes('engineer') || text.includes('developer') || text.includes('senior') ||
           text.includes('staff') || text.includes('lead') || text.includes('manager');
  });

  // Build normalized job objects
  const jobs = [];
  const seen = new Set();

  for (const link of jobLinks) {
    // Deduplicate by URL
    const fullUrl = link.href.startsWith('http') ? link.href : new URL(link.href, source.url).href;
    if (seen.has(fullUrl)) continue;
    seen.add(fullUrl);

    // Extract location from nearby text or URL
    const locationMatch = (link.text || '').match(/(?:in|—|-)\s*([A-Z][a-z]+(?:\s*,\s*[A-Z]{2})?)/);
    const location = locationMatch ? locationMatch[1] : null;

    jobs.push({
      title: link.text || 'Untitled Position',
      url: fullUrl,
      source: source.source,
      location: location,
      discoveredAt: new Date().toISOString()
    });
  }

  return jobs;
}

Step 3: Store and Search

SQLite with FTS5 gives you full-text search without running a separate database:

const Database = require('better-sqlite3');
const db = new Database('jobs.db');

// Enable WAL mode for concurrent reads
db.pragma('journal_mode = WAL');

// Create schema with full-text search
db.exec(`
  CREATE TABLE IF NOT EXISTS jobs (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    title TEXT NOT NULL,
    url TEXT UNIQUE NOT NULL,
    source TEXT NOT NULL,
    location TEXT,
    first_seen TEXT NOT NULL,
    last_seen TEXT NOT NULL
  );

  CREATE VIRTUAL TABLE IF NOT EXISTS jobs_fts USING fts5(
    title, source, location, content='jobs', content_rowid='id'
  );

  CREATE TRIGGER IF NOT EXISTS jobs_ai AFTER INSERT ON jobs BEGIN
    INSERT INTO jobs_fts(rowid, title, source, location)
    VALUES (new.id, new.title, new.source, new.location);
  END;
`);

function insertJobs(jobs) {
  const insert = db.prepare(`
    INSERT OR IGNORE INTO jobs (title, url, source, location, first_seen, last_seen)
    VALUES (@title, @url, @source, @location, @discoveredAt, @discoveredAt)
  `);

  const update = db.prepare(`
    UPDATE jobs SET last_seen = @discoveredAt WHERE url = @url
  `);

  const transaction = db.transaction((jobs) => {
    let inserted = 0;
    for (const job of jobs) {
      const result = insert.run(job);
      if (result.changes > 0) inserted++;
      else update.run(job);
    }
    return inserted;
  });

  return transaction(jobs);
}

// Search function
function searchJobs(query) {
  return db.prepare(`
    SELECT j.* FROM jobs j
    JOIN jobs_fts f ON j.id = f.rowid
    WHERE jobs_fts MATCH ?
    ORDER BY j.last_seen DESC
  `).all(query);
}

Step 4: Put It All Together

// aggregate.js
const SOURCES = require('./sources.json');

async function main() {
  console.log(`Starting job aggregation: ${new Date().toISOString()}`);
  let totalNew = 0;

  for (const source of SOURCES) {
    try {
      const jobs = await extractJobs(source);
      const newCount = insertJobs(jobs);
      totalNew += newCount;
      console.log(`  ${source.source}: ${jobs.length} found, ${newCount} new`);
    } catch (err) {
      console.error(`  ${source.source}: ERROR — ${err.message}`);
    }
    // Rate limit: be respectful to source sites
    await new Promise(r => setTimeout(r, 5000));
  }

  console.log(`Done. ${totalNew} new jobs added.`);

  // Print top 5 newest
  const recent = db.prepare('SELECT * FROM jobs ORDER BY first_seen DESC LIMIT 5').all();
  console.log('\nRecent listings:');
  recent.forEach(j => console.log(`  ${j.title} @ ${j.source} — ${j.location}`));
}

main().catch(console.error);

Going Further

Once you have the pipeline running, add:

• Notifications: Email or Slack alerts for jobs matching specific keywords
• Deduplication: Same job title across multiple boards — fuzzy match on title + company name
• Deep extraction: Follow each job link to extract full description, salary range, and requirements
• Dashboard: A simple web UI to browse and filter aggregated listings

Related: JavaScript website to JSON · Browser vs HTTP requests · API docs