code icon Code

Fetch Gmail Messages

Fetch message details with parallel requests for speed

Source Code

import fs from "fs";
import path from "path";

const [idsInput, format = "metadata", outputPath = "session/messages.json"] = process.argv.slice(2);

if (!idsInput) {
  console.error("Usage: gmail.core.fetch <ids_file_or_list> [format] [outputPath]");
  process.exit(1);
}

// Parse input - either file path or comma-separated IDs
let messageIds = [];
if (idsInput.endsWith(".json") && fs.existsSync(idsInput)) {
  const data = JSON.parse(fs.readFileSync(idsInput, "utf-8"));
  messageIds = data.ids || data;
} else {
  messageIds = idsInput.split(",").map(id => id.trim()).filter(Boolean);
}

if (messageIds.length === 0) {
  console.error("No message IDs provided");
  process.exit(1);
}

const CONCURRENCY = 25;

/**
 * Get header value from message
 */
function getHeader(msg, name) {
  const header = msg.payload?.headers?.find(
    h => h.name.toLowerCase() === name.toLowerCase()
  );
  return header ? header.value : "";
}

/**
 * Extract email from header
 */
function extractEmail(header) {
  if (!header) return "unknown";
  const match = header.match(/<([^>]+)>/);
  return match ? match[1].toLowerCase() : header.toLowerCase().trim();
}

/**
 * Extract name from header
 */
function extractName(header) {
  if (!header) return "Unknown";
  const match = header.match(/^([^<]+)</);
  return match ? match[1].trim().replace(/"/g, "") : header.split("@")[0];
}

/**
 * Extract plain text body from payload
 */
function extractBodyText(payload) {
  if (!payload) return "";

  if (payload.body?.data) {
    try {
      return Buffer.from(payload.body.data, "base64").toString("utf-8");
    } catch {
      return "";
    }
  }

  if (payload.parts) {
    for (const part of payload.parts) {
      if (part.mimeType === "text/plain" && part.body?.data) {
        try {
          return Buffer.from(part.body.data, "base64").toString("utf-8");
        } catch {
          continue;
        }
      }
      if (part.parts) {
        for (const nested of part.parts) {
          if (nested.mimeType === "text/plain" && nested.body?.data) {
            try {
              return Buffer.from(nested.body.data, "base64").toString("utf-8");
            } catch {
              continue;
            }
          }
        }
      }
    }
  }

  return "";
}

/**
 * Fetch messages with parallel requests
 */
async function fetchMessages(messageIds, format) {
  const results = [];

  console.log(`Fetching ${messageIds.length} messages (format=${format})...`);

  for (let i = 0; i < messageIds.length; i += CONCURRENCY) {
    const batch = messageIds.slice(i, i + CONCURRENCY);
    const fetched = await Promise.all(
      batch.map(async id => {
        let url = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${id}?format=${format}`;
        if (format === "metadata") {
          url += "&metadataHeaders=Subject&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Date&metadataHeaders=Cc";
        }
        const res = await fetch(url, {
          headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" },
        });
        if (!res.ok) return null;
        try {
          return await res.json();
        } catch {
          return null;
        }
      })
    );
    results.push(...fetched.filter(Boolean));

    if ((i + CONCURRENCY) % 100 === 0 || i + CONCURRENCY >= messageIds.length) {
      console.log(`  Fetched ${Math.min(i + CONCURRENCY, messageIds.length)}/${messageIds.length}...`);
    }
  }

  return results;
}

try {
  const rawMessages = await fetchMessages(messageIds, format);

  // Transform to standardized format
  const messages = rawMessages.map(msg => {
    const base = {
      id: msg.id,
      threadId: msg.threadId,
      subject: getHeader(msg, "Subject"),
      from: getHeader(msg, "From"),
      fromEmail: extractEmail(getHeader(msg, "From")),
      fromName: extractName(getHeader(msg, "From")),
      to: getHeader(msg, "To"),
      cc: getHeader(msg, "Cc"),
      date: getHeader(msg, "Date"),
      snippet: msg.snippet,
      labelIds: msg.labelIds || [],
      sizeEstimate: msg.sizeEstimate,
    };

    // Include body if format=full
    if (format === "full") {
      base.body = extractBodyText(msg.payload);
    }

    return base;
  });

  // Compute summary stats
  const senderCounts = {};
  for (const msg of messages) {
    senderCounts[msg.fromEmail] = (senderCounts[msg.fromEmail] || 0) + 1;
  }
  const topSenders = Object.entries(senderCounts)
    .sort((a, b) => b[1] - a[1])
    .slice(0, 5)
    .map(([email, count]) => ({ email, count }));

  // Ensure output directory exists
  const dir = path.dirname(outputPath);
  if (dir && dir !== ".") fs.mkdirSync(dir, { recursive: true });

  const output = {
    fetchedAt: new Date().toISOString(),
    format,
    count: messages.length,
    messages,
  };

  fs.writeFileSync(outputPath, JSON.stringify(output, null, 2));

  console.log(`\n✓ Fetched ${messages.length} messages`);
  console.log(`  Written to: ${outputPath}`);
  console.log(`  Top senders:`);
  for (const s of topSenders) {
    console.log(`    - ${s.email}: ${s.count} messages`);
  }

  console.log(JSON.stringify({ success: true, outputPath, count: messages.length, topSenders }));
} catch (error) {
  console.error("Error:", error.message);
  throw error;
}