How to Aggregate Job Listings From Multiple Boards

Published July 24, 2026 · 6 min read

Job hunting across 10 different boards is exhausting. LinkedIn, Indeed, Glassdoor, company career pages, niche boards — each with its own UI, filters, and notification system. A job aggregator solves this by pulling everything into one searchable feed.

Here's how to build one with Node.js and ExtractAPI — no headless browser infrastructure required.

Architecture Overview

The system has three components:

1. Source list: URLs for job listing pages across multiple boards
2. Extraction pipeline: Fetch each URL, extract structured job data, normalize fields
3. Storage + search: SQLite database with full-text search

Step 1: Define Sources

Start with company career pages — they're the least protected and often have the freshest listings:

// sources.json
[
  { "source": "stripe", "url": "https://stripe.com/jobs/search?q=engineer", "type": "career-page" },
  { "source": "vercel", "url": "https://vercel.com/careers", "type": "career-page" },
  { "source": "linear", "url": "https://linear.app/careers", "type": "career-page" },
  { "source": "indeed", "url": "https://www.indeed.com/jobs?q=software+engineer&l=remote&fromage=3", "type": "job-board" },
  { "source": "linkedin", "url": "https://www.linkedin.com/jobs/search?keywords=software+engineer&location=remote&f_TPR=r86400", "type": "job-board" }
]

Step 2: Extract Job Data

Use ExtractAPI with CSS selectors to pull structured job fields from each page. The key is normalizing the output — different sites use different HTML structures, but we want a uniform schema:

const EXTRACT_API_KEY = process.env.EXTRACT_API_KEY;

async function extractJobs(source) {
  const res = await fetch('https://extractapi.app/v1/extract', {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${EXTRACT_API_KEY}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      url: source.url,
      waitFor: 'a[href*="job"], a[href*="career"], .job-listing, .career-listing',
      javascript: true,
      extractLinks: true,
      extractImages: false
    })
  });

  const data = await res.json();
  if (data.status !== 'ok') {
    console.error(`Failed to extract ${source.source}: ${data.error}`);
    return [];
  }

  // Parse links and text to find job listings
  const allLinks = data.data.links || [];
  const headings = data.data.headings || [];

  // Strategy: find links that look like job listings
  const jobLinks = allLinks.filter(link => {
    const url = (link.href || '').toLowerCase();
    const text = (link.text || '').toLowerCase();
    return url.includes('/job') || url.includes('/career') || url.includes('/position') ||
           text.includes('engineer') || text.includes('developer') || text.includes('senior') ||
           text.includes('staff') || text.includes('lead') || text.includes('manager');
  });

  // Build normalized job objects
  const jobs = [];
  const seen = new Set();

  for (const link of jobLinks) {
    // Deduplicate by URL
    const fullUrl = link.href.startsWith('http') ? link.href : new URL(link.href, source.url).href;
    if (seen.has(fullUrl)) continue;
    seen.add(fullUrl);

    // Extract location from nearby text or URL
    const locationMatch = (link.text || '').match(/(?:in|—|-)\s*([A-Z][a-z]+(?:\s*,\s*[A-Z]{2})?)/);
    const location = locationMatch ? locationMatch[1] : 'Remote';

    jobs.push({
      title: link.text || 'Untitled Position',
      url: fullUrl,
      source: source.source,
      location: location,
      discoveredAt: new Date().toISOString()
    });
  }

  return jobs;
}

Step 3: Store and Search

SQLite with FTS5 gives you full-text search without running a separate database:

const Database = require('better-sqlite3');
const db = new Database('jobs.db');

// Enable WAL mode for concurrent reads
db.pragma('journal_mode = WAL');

// Create schema with full-text search
db.exec(`
  CREATE TABLE IF NOT EXISTS jobs (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    title TEXT NOT NULL,
    url TEXT UNIQUE NOT NULL,
    source TEXT NOT NULL,
    location TEXT,
    first_seen TEXT NOT NULL,
    last_seen TEXT NOT NULL
  );

  CREATE VIRTUAL TABLE IF NOT EXISTS jobs_fts USING fts5(
    title, source, location, content='jobs', content_rowid='id'
  );

  CREATE TRIGGER IF NOT EXISTS jobs_ai AFTER INSERT ON jobs BEGIN
    INSERT INTO jobs_fts(rowid, title, source, location)
    VALUES (new.id, new.title, new.source, new.location);
  END;
`);

function insertJobs(jobs) {
  const insert = db.prepare(`
    INSERT OR IGNORE INTO jobs (title, url, source, location, first_seen, last_seen)
    VALUES (@title, @url, @source, @location, @discoveredAt, @discoveredAt)
  `);

  const update = db.prepare(`
    UPDATE jobs SET last_seen = @discoveredAt WHERE url = @url
  `);

  const transaction = db.transaction((jobs) => {
    let inserted = 0;
    for (const job of jobs) {
      const result = insert.run(job);
      if (result.changes > 0) inserted++;
      else update.run(job);
    }
    return inserted;
  });

  return transaction(jobs);
}

// Search function
function searchJobs(query) {
  return db.prepare(`
    SELECT j.* FROM jobs j
    JOIN jobs_fts f ON j.id = f.rowid
    WHERE jobs_fts MATCH ?
    ORDER BY j.last_seen DESC
  `).all(query);
}

Step 4: Put It All Together

// aggregate.js
const SOURCES = require('./sources.json');

async function main() {
  console.log(`Starting job aggregation: ${new Date().toISOString()}`);
  let totalNew = 0;

  for (const source of SOURCES) {
    try {
      const jobs = await extractJobs(source);
      const newCount = insertJobs(jobs);
      totalNew += newCount;
      console.log(`  ${source.source}: ${jobs.length} found, ${newCount} new`);
    } catch (err) {
      console.error(`  ${source.source}: ERROR — ${err.message}`);
    }
    // Rate limit: be respectful to source sites
    await new Promise(r => setTimeout(r, 5000));
  }

  console.log(`Done. ${totalNew} new jobs added.`);

  // Print top 5 newest
  const recent = db.prepare('SELECT * FROM jobs ORDER BY first_seen DESC LIMIT 5').all();
  console.log('\nRecent listings:');
  recent.forEach(j => console.log(`  ${j.title} @ ${j.source} — ${j.location}`));
}

main().catch(console.error);

Going Further

Once you have the pipeline running, add:

• Notifications: Email or Slack alerts for jobs matching specific keywords
• Deduplication: Same job title across multiple boards — fuzzy match on title + company name
• Deep extraction: Follow each job link to extract full description, salary range, and requirements
• Dashboard: A simple web UI to browse and filter aggregated listings