Fetch Gmail Messages
Fetch message details with parallel requests for speed
Source Code
import fs from "fs";
import path from "path";
const [idsInput, format = "metadata", outputPath = "session/messages.json"] = process.argv.slice(2);
if (!idsInput) {
console.error("Usage: gmail.core.fetch <ids_file_or_list> [format] [outputPath]");
process.exit(1);
}
// Parse input - either file path or comma-separated IDs
let messageIds = [];
if (idsInput.endsWith(".json") && fs.existsSync(idsInput)) {
const data = JSON.parse(fs.readFileSync(idsInput, "utf-8"));
messageIds = data.ids || data;
} else {
messageIds = idsInput.split(",").map(id => id.trim()).filter(Boolean);
}
if (messageIds.length === 0) {
console.error("No message IDs provided");
process.exit(1);
}
const CONCURRENCY = 25;
/**
* Get header value from message
*/
function getHeader(msg, name) {
const header = msg.payload?.headers?.find(
h => h.name.toLowerCase() === name.toLowerCase()
);
return header ? header.value : "";
}
/**
* Extract email from header
*/
function extractEmail(header) {
if (!header) return "unknown";
const match = header.match(/<([^>]+)>/);
return match ? match[1].toLowerCase() : header.toLowerCase().trim();
}
/**
* Extract name from header
*/
function extractName(header) {
if (!header) return "Unknown";
const match = header.match(/^([^<]+)</);
return match ? match[1].trim().replace(/"/g, "") : header.split("@")[0];
}
/**
* Extract plain text body from payload
*/
function extractBodyText(payload) {
if (!payload) return "";
if (payload.body?.data) {
try {
return Buffer.from(payload.body.data, "base64").toString("utf-8");
} catch {
return "";
}
}
if (payload.parts) {
for (const part of payload.parts) {
if (part.mimeType === "text/plain" && part.body?.data) {
try {
return Buffer.from(part.body.data, "base64").toString("utf-8");
} catch {
continue;
}
}
if (part.parts) {
for (const nested of part.parts) {
if (nested.mimeType === "text/plain" && nested.body?.data) {
try {
return Buffer.from(nested.body.data, "base64").toString("utf-8");
} catch {
continue;
}
}
}
}
}
}
return "";
}
/**
* Fetch messages with parallel requests
*/
async function fetchMessages(messageIds, format) {
const results = [];
console.log(`Fetching ${messageIds.length} messages (format=${format})...`);
for (let i = 0; i < messageIds.length; i += CONCURRENCY) {
const batch = messageIds.slice(i, i + CONCURRENCY);
const fetched = await Promise.all(
batch.map(async id => {
let url = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${id}?format=${format}`;
if (format === "metadata") {
url += "&metadataHeaders=Subject&metadataHeaders=From&metadataHeaders=To&metadataHeaders=Date&metadataHeaders=Cc";
}
const res = await fetch(url, {
headers: { Authorization: "Bearer PLACEHOLDER_TOKEN" },
});
if (!res.ok) return null;
try {
return await res.json();
} catch {
return null;
}
})
);
results.push(...fetched.filter(Boolean));
if ((i + CONCURRENCY) % 100 === 0 || i + CONCURRENCY >= messageIds.length) {
console.log(` Fetched ${Math.min(i + CONCURRENCY, messageIds.length)}/${messageIds.length}...`);
}
}
return results;
}
try {
const rawMessages = await fetchMessages(messageIds, format);
// Transform to standardized format
const messages = rawMessages.map(msg => {
const base = {
id: msg.id,
threadId: msg.threadId,
subject: getHeader(msg, "Subject"),
from: getHeader(msg, "From"),
fromEmail: extractEmail(getHeader(msg, "From")),
fromName: extractName(getHeader(msg, "From")),
to: getHeader(msg, "To"),
cc: getHeader(msg, "Cc"),
date: getHeader(msg, "Date"),
snippet: msg.snippet,
labelIds: msg.labelIds || [],
sizeEstimate: msg.sizeEstimate,
};
// Include body if format=full
if (format === "full") {
base.body = extractBodyText(msg.payload);
}
return base;
});
// Compute summary stats
const senderCounts = {};
for (const msg of messages) {
senderCounts[msg.fromEmail] = (senderCounts[msg.fromEmail] || 0) + 1;
}
const topSenders = Object.entries(senderCounts)
.sort((a, b) => b[1] - a[1])
.slice(0, 5)
.map(([email, count]) => ({ email, count }));
// Ensure output directory exists
const dir = path.dirname(outputPath);
if (dir && dir !== ".") fs.mkdirSync(dir, { recursive: true });
const output = {
fetchedAt: new Date().toISOString(),
format,
count: messages.length,
messages,
};
fs.writeFileSync(outputPath, JSON.stringify(output, null, 2));
console.log(`\n✓ Fetched ${messages.length} messages`);
console.log(` Written to: ${outputPath}`);
console.log(` Top senders:`);
for (const s of topSenders) {
console.log(` - ${s.email}: ${s.count} messages`);
}
console.log(JSON.stringify({ success: true, outputPath, count: messages.length, topSenders }));
} catch (error) {
console.error("Error:", error.message);
throw error;
}