export const metadata = { id: "code:social.twitter.fetch_tweets_multiple", name: "Fetch Recent Tweets (Multiple Handles)", description: "Fetch up to 100 recent tweets for multiple Twitter handles in parallel using widget scraping, plus basic profile metadata from SocialData.", packages: [], args: [ { name: "handles", type: "string", description: "Comma-separated Twitter handles (with or without @)", position: 0, default: "", }, ], }; const [handlesArg] = process.argv.slice(2); const rawHandles = (handlesArg || "").trim(); const maxResults = 100; if (!rawHandles) { console.error("Twitter handles are required (comma-separated)"); throw new Error("Missing required parameter: handles"); } // Parse and normalize handles const handles = rawHandles .split(",") .map((h) => h.trim().replace(/^@/, "")) .filter((h) => h.length > 0); if (handles.length === 0) { console.error("No valid Twitter handles provided"); throw new Error("No valid handles after parsing"); } if (handles.length > 10) { console.error("Too many handles. Maximum is 10 to avoid rate limiting."); throw new Error("Maximum 10 handles allowed"); } console.log( `Fetching up to ${maxResults} tweets for ${ handles.length } handle(s): @${handles.join(", @")}` ); async function fetchSocialDataProfile(screenName) { const apiKey = process.env.SOCIAL_DATA_API_KEY; if (!apiKey) { console.warn( "SOCIAL_DATA_API_KEY is not set; skipping SocialData profile fetch." ); return null; } const url = `https://api.socialdata.tools/twitter/user/${encodeURIComponent( screenName )}`; const res = await fetch(url, { method: "GET", headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json", }, }); const text = await res.text(); if (!res.ok) { console.error( `SocialData profile request failed with status ${res.status} for @${screenName}` ); console.error(text.slice(0, 500)); return null; } let json; try { json = JSON.parse(text); } catch (err) { console.error( `Failed to parse SocialData profile JSON for @${screenName}:`, err.message ); return null; } return { id: json.id_str || String(json.id || ""), name: json.name || "", screenName: json.screen_name || screenName, description: json.description || "", location: json.location || "", profileImageUrl: json.profile_image_url_https || "", profileBannerUrl: json.profile_banner_url || "", followersCount: typeof json.followers_count === "number" ? json.followers_count : 0, friendsCount: typeof json.friends_count === "number" ? json.friends_count : 0, statusesCount: typeof json.statuses_count === "number" ? json.statuses_count : 0, createdAt: json.created_at || "", }; } async function fetchTimelineProfile(screenName) { const url = `https://syndication.twitter.com/srv/timeline-profile/screen-name/${screenName}`; const params = new URLSearchParams({ dnt: "false", embedId: "twitter-widget-1", frame: "false", hideHeader: "false", lang: "en", showReplies: "false", }); const headers = { accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", "accept-language": "en-US,en;q=0.9", cookie: `guest_id=v1%3A168987090345885498; kdt=p0eqA5087j6MNpREwxYJcudjCegtrzc9b8J7H2iI; auth_token=e4fa2e1fb11107b1648add7bbd58d3cd4e4b2d5e; ct0=f39f4820e35d45a0b64651eb46acae1694bfeb221af2d1b650b74a81bebd95673869b2504b75d005809bebe0a71e1863c87338e5b534a02c8ce4d10b1730f501127333aadc34cf9c6dece1d74c91b00f; twid=u%3D945756809356300294; dnt=1; btc_opt_in=Y; twtr_pixel_opt_in=Y; *twitter*sess=BAh7CiIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo%250ASGFzaHsABjoKQHVzZWR7ADoPY3JlYXRlZF9hdGwrCAIPOL2LAToMY3NyZl9p%250AZCIlODJlZjUzMTE0ZmZkZWZlNzVhMzM0NTM3M2FhNWZhNmI6B2lkIiU3OGMy%250ANDlmN2Y5NWRjYmVkZTliMTYyZmI2YWVjYjgzZToVaW5pdGlhdGVkX2luX2Fw%250AcCIGMQ%253D%253D--e4fb722064815b66eb4cb5098a47fc74eef01367; fm="WW91IHdpbGwgbm8gbG9uZ2VyIHJlY2VpdmUgZW1haWxzIGxpa2UgdGhpcy4=--61a3e5587a505c23e2499300d1d7f92ff6d971e0"; guest_id_marketing=v1%3A168987090345885498; guest_id_ads=v1%3A168987090345885498; personalization_id="v1_qWZOJ07EYJQV7qtkcyHuQg=="; *ga=GA1.2.891791384.1722577601; *gid=GA1.2.262146840.1723035771; *gat=1`, "user-agent": "Mozilla/5.0 (Skills Bot; +https://skills.dom) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36", }; const response = await fetch(`${url}?${params.toString()}`, { headers }); if (!response.ok) { const body = await response.text(); console.error(`Widget HTTP error for @${screenName}: ${response.status}`); console.error(body.slice(0, 300)); throw new Error( `Failed to fetch widget timeline for @${screenName}: ${response.status}` ); } return await response.text(); } function extractWidgetTweetsFromHtml(html, limit) { const m = html.match( /]*id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i ); if (!m) { throw new Error("__NEXT_DATA__ script not found in widget HTML"); } let data; try { data = JSON.parse(m[1]); } catch (e) { console.error("Failed to parse __NEXT_DATA__ JSON"); throw e; } const entries = data?.props?.pageProps?.timeline?.entries || data?.props?.pageProps?.timeline?.instructions?.[0]?.entries || []; const tweets = entries .filter( (entry) => entry && entry.type === "tweet" && entry.content && entry.content.tweet ) .map((entry) => entry.content.tweet); if (!tweets.length) { throw new Error("No tweet entries found in widget data"); } tweets.sort( (a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime() ); return tweets.slice(0, limit); } function normalizeWidgetTweet(tweet) { return { id: tweet.id_str || tweet.id || "", text: tweet.full_text || tweet.text || "", createdAt: tweet.created_at || "", likeCount: tweet.favorite_count || 0, retweetCount: tweet.retweet_count || 0, replyCount: tweet.reply_count || 0, quoteCount: tweet.quote_count || 0, }; } // Fetch tweets for a single handle async function fetchTweetsForHandle(handle) { console.log(`\n[${handle}] Starting fetch...`); try { // Fetch profile first const profile = await fetchSocialDataProfile(handle).catch((err) => { console.error( `[${handle}] Error fetching SocialData profile:`, err.message ); return null; }); // Fetch tweets via widget const html = await fetchTimelineProfile(handle); const widgetTweets = extractWidgetTweetsFromHtml(html, maxResults); if (!widgetTweets || widgetTweets.length === 0) { console.log(`[${handle}] No tweets found`); return { handle, success: true, profile, tweets: [], }; } const normalized = widgetTweets.map(normalizeWidgetTweet); console.log( `[${handle}] ✓ Fetched ${normalized.length} tweet(s)${ profile ? " with profile metadata" : "" }` ); return { handle, success: true, profile, tweets: normalized, }; } catch (error) { console.error(`[${handle}] ✗ Error:`, error.message); return { handle, success: false, error: error.message, profile: null, tweets: [], }; } } // Fetch all handles in parallel try { console.log("Starting parallel Twitter fetch for multiple handles..."); const results = await Promise.all( handles.map((handle) => fetchTweetsForHandle(handle)) ); // Summarize results const successful = results.filter((r) => r.success); const failed = results.filter((r) => !r.success); console.log("\n=== FETCH SUMMARY ==="); console.log(`Total handles: ${handles.length}`); console.log(`Successful: ${successful.length}`); console.log(`Failed: ${failed.length}`); if (failed.length > 0) { console.log("\nFailed handles:"); failed.forEach((f) => console.log(` - @${f.handle}: ${f.error}`)); } // Output complete results as JSON console.log("\n=== COMPLETE RESULTS ==="); console.log(JSON.stringify({ results }, null, 2)); } catch (error) { console.error("Fatal error during parallel fetch:", error.message); throw error; }