Crawl & Scrape

// Crawl multiple pages with pagination
const result = await agentbase.runAgent({
  message: `Visit example.com/blog and extract:
    - All article titles
    - Author names
    - Publication dates
    - Follow pagination to get all articles from pages 1-5`,
  mode: "base"
});

// Extract data in specific format
const result = await agentbase.runAgent({
  message: `Scrape example.com/directory and return data as JSON:
    {
      "listings": [
        {
          "name": "...",
          "address": "...",
          "phone": "...",
          "rating": "..."
        }
      ]
    }`,
  mode: "base"
});

const data = JSON.parse(result.content);

async function monitorCompetitors(competitors: string[]) {
  const results = await Promise.all(
    competitors.map(async (url) => {
      return await agentbase.runAgent({
        message: `Visit ${url} and extract:
          - All product names and prices
          - Key features
          - Any promotional banners
          - Last updated date`,
        mode: "base"
      });
    })
  );

  return results.map((r, i) => ({
    competitor: competitors[i],
    data: r.content
  }));
}

// Usage
const intel = await monitorCompetitors([
  'https://competitor1.com/deploy/pricing',
  'https://competitor2.com/products',
  'https://competitor3.com/features'
]);

async function extractLeads(directoryUrl: string) {
  const result = await agentbase.runAgent({
    message: `Scrape ${directoryUrl} and extract all business listings:
      - Business name
      - Industry/category
      - Contact email
      - Phone number
      - Website URL

      Follow pagination to get all listings. Return as CSV format.`,
    mode: "base"
  });

  // Save to file
  fs.writeFileSync('leads.csv', result.content);
  return result.content;
}

async function aggregateNews(topics: string[]) {
  const articles = [];

  for (const topic of topics) {
    const result = await agentbase.runAgent({
      message: `Search for recent articles about "${topic}" and extract:
        - Article title
        - Source/publication
        - Publication date
        - Summary
        - URL

        Get the 10 most recent articles.`,
      mode: "base"
    });

    articles.push({
      topic,
      articles: JSON.parse(result.content)
    });
  }

  return articles;
}

// Usage
const news = await aggregateNews([
  'AI developments',
  'Cloud computing trends',
  'Cybersecurity news'
]);

interface PriceAlert {
  product: string;
  targetPrice: number;
  urls: string[];
}

async function monitorPrices(alerts: PriceAlert[]) {
  for (const alert of alerts) {
    for (const url of alert.urls) {
      const result = await agentbase.runAgent({
        message: `Visit ${url} and extract the current price for ${alert.product}`,
        mode: "flash"
      });

      // Parse price from response
      const priceMatch = result.content.match(/\$?([\d,]+\.?\d*)/);
      if (priceMatch) {
        const currentPrice = parseFloat(priceMatch[1].replace(',', ''));

        if (currentPrice <= alert.targetPrice) {
          await sendAlert({
            product: alert.product,
            currentPrice,
            targetPrice: alert.targetPrice,
            url
          });
        }
      }
    }
  }
}

async function aggregateJobs(searchTerms: string[]) {
  const jobBoards = [
    'https://jobs.example.com',
    'https://careers.example.org',
    'https://opportunities.example.net'
  ];

  const allJobs = [];

  for (const board of jobBoards) {
    for (const term of searchTerms) {
      const result = await agentbase.runAgent({
        message: `Search for "${term}" on ${board} and extract:
          - Job title
          - Company name
          - Location
          - Salary (if available)
          - Posted date
          - Job URL

          Return as JSON array. Get the first 20 results.`,
        mode: "base"
      });

      allJobs.push(...JSON.parse(result.content));
    }
  }

  return allJobs;
}

async function collectReviews(productUrl: string) {
  const result = await agentbase.runAgent({
    message: `Visit ${productUrl} and extract all customer reviews:
      - Reviewer name
      - Rating (stars)
      - Review text
      - Date
      - Verified purchase (if shown)

      Follow pagination to get all reviews. Return as JSON.`,
    mode: "base"
  });

  const reviews = JSON.parse(result.content);

  // Analyze sentiment
  const analysis = await agentbase.runAgent({
    message: `Analyze these reviews and provide:
      - Average rating
      - Common positive themes
      - Common negative themes
      - Overall sentiment

      Reviews: ${JSON.stringify(reviews.slice(0, 50))}`,
    mode: "base"
  });

  return {
    reviews,
    analysis: analysis.content
  };
}

async function scrapeInParallel(urls: string[], concurrency: number = 5) {
  const results = [];

  for (let i = 0; i < urls.length; i += concurrency) {
    const batch = urls.slice(i, i + concurrency);

    const batchResults = await Promise.all(
      batch.map(url =>
        agentbase.runAgent({
          message: `Extract data from ${url}`,
          mode: "base"
        })
      )
    );

    results.push(...batchResults);

    // Rate limit between batches
    if (i + concurrency < urls.length) {
      await new Promise(resolve => setTimeout(resolve, 2000));
    }
  }

  return results;
}

async function crawlSitemap(startUrl: string, maxDepth: number = 3) {
  const visited = new Set();
  const results = [];

  async function crawlPage(url: string, depth: number) {
    if (depth > maxDepth || visited.has(url)) return;

    visited.add(url);

    const result = await agentbase.runAgent({
      message: `Visit ${url} and:
        1. Extract main content
        2. Find all internal links
        3. Return both as JSON`,
      mode: "base"
    });

    results.push({
      url,
      depth,
      content: result.content
    });

    // Crawl linked pages
    const data = JSON.parse(result.content);
    for (const link of data.links || []) {
      await crawlPage(link, depth + 1);
    }
  }

  await crawlPage(startUrl, 0);
  return results;
}