api/twitter: use newer graphql endpoint, refactor (#1436)

it's only present on github, which is cool cuz we get to keep cobalt itself ad-free

README.md

@@ -29,6 +29,20 @@ cobalt is a media downloader that doesn't piss you off. it's friendly, efficient
 
 paste the link, get the file, move on. that simple, just how it should be.
 
+### sponsors
+<div align="center" markdown="1">
+    <sup>special thanks to Warp for sponsoring the development of cobalt</sup>
+    <br>
+    <a href="https://go.warp.dev/cobalt">
+        <img alt="Warp banner" width="400" src="https://raw.githubusercontent.com/warpdotdev/brand-assets/be7d584f98e62b1579fd2e9338d4c7318a732f1b/Github/Sponsor/Warp-Github-LG-03.png">
+    </a>
+
+### [Warp, built for coding with multiple AI agents](https://go.warp.dev/cobalt)
+</div>
+
+#### RoyaleHosting
+cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), and a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
+
 ### cobalt monorepo
 this monorepo includes source code for api, frontend, and related packages:
 - [api tree & readme](/api/)
@@ -53,9 +67,6 @@ same content can be downloaded via dev tools of any modern web browser.
 ### contributing
 if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
 
-### thank you
-cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
-
 ### licenses
 for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
 unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).

api/src/processing/services/twitter.js

@@ -3,12 +3,12 @@ import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
 
-const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId';
+const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail';
 const tokenURL = 'https://api.x.com/1.1/guest/activate.json';
 
-const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false});
+const tweetFeatures = JSON.stringify({"rweb_video_screen_enabled":false,"payments_enabled":false,"rweb_xchat_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false});
 
-const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false});
+const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false});
 
 const commonHeaders = {
     "user-agent": genericUserAgent,
@@ -100,10 +100,14 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
 
     graphqlTweetURL.searchParams.set('variables',
         JSON.stringify({
-            tweetId,
+            focalTweetId: tweetId,
-            withCommunity: false,
+            with_rux_injections: false,
-            includePromotedContent: false,
+            rankingMode: "Relevance",
-            withVoice: false
+            includePromotedContent: true,
+            withCommunity: true,
+            withQuickPromoteEligibilityTweetFields: true,
+            withBirdwatchNotes: true,
+            withVoice: true
         })
     );
     graphqlTweetURL.searchParams.set('features', tweetFeatures);
@@ -129,24 +133,48 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     return result
 }
 
-const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => {
+const parseCard = (cardOuter) => {
-    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+    const card = JSON.parse(
+      (cardOuter?.legacy?.binding_values[0].value
+       || cardOuter?.binding_values?.unified_card)?.string_value,
+    );
+
+    if (!["video_website", "image_website"].includes(card?.type)
+        || !card?.media_entities
+        || card?.component_objects?.media_1?.type !== "media") {
+      return;
+    }
+
+    const mediaId = card.component_objects?.media_1?.data?.id;
+    return [card.media_entities[mediaId]];
+};
+
+const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) => {
+    const addInsn = thread?.data?.threaded_conversation_with_injections_v2?.instructions?.find(
+        insn => insn.type === 'TimelineAddEntries'
+    );
+
+    const tweetResult = addInsn?.entries?.find(
+        entry => entry.entryId === `tweet-${id}`
+    )?.content?.itemContent?.tweet_results?.result;
+
+    let tweetTypename = tweetResult?.__typename;
 
     if (!tweetTypename) {
         return { error: "fetch.empty" }
     }
 
-    if (tweetTypename === "TweetUnavailable") {
+    if (tweetTypename === "TweetUnavailable" || tweetTypename === "TweetTombstone") {
-        const reason = tweet?.data?.tweetResult?.result?.reason;
+        const reason = tweetResult?.result?.reason;
-        switch(reason) {
+        if (reason === 'Protected') {
-            case "Protected":
+            return { error: "content.post.private" };
-                return { error: "content.post.private" };
+        } else if (reason === "NsfwLoggedOut" || tweetResult?.tombstone?.text?.text?.startsWith('Age-restricted')) {
-            case "NsfwLoggedOut":
+            if (!cookie) {
-                if (cookie) {
+                return { error: "content.post.age" };
-                    tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+            }
-                    tweet = await tweet.json();
+
-                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+            const tweet = await requestTweet(dispatcher, id, guestToken, cookie).then(t => t.json());
-                } else return { error: "content.post.age" };
+            return extractGraphqlMedia(tweet, dispatcher, id, guestToken);
         }
     }
 
@@ -154,8 +182,7 @@ const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) =>
         return { error: "content.post.unavailable" }
     }
 
-    let tweetResult = tweet.data.tweetResult.result,
+    let baseTweet = tweetResult.legacy,
-        baseTweet = tweetResult.legacy,
         repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
 
     if (tweetTypename === "TweetWithVisibilityResults") {
@@ -164,81 +191,51 @@ const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) =>
     }
 
     if (tweetResult.card?.legacy?.binding_values?.length) {
-        const card = JSON.parse(tweetResult.card.legacy.binding_values[0].value?.string_value);
+        return parseCard(tweetResult.card);
-
-        if (!["video_website", "image_website"].includes(card?.type) ||
-            !card?.media_entities ||
-            card?.component_objects?.media_1?.type !== "media") {
-            return;
-        }
-
-        const mediaId = card.component_objects?.media_1?.data?.id;
-        return [card.media_entities[mediaId]];
     }
 
     return (repostedTweet?.media || baseTweet?.extended_entities?.media);
 }
 
-const testResponse = (result) => {
-    const contentLength = result.headers.get("content-length");
-
-    if (!contentLength || contentLength === '0') {
-        return false;
-    }
-
-    if (!result.headers.get("content-type").startsWith("application/json")) {
-        return false;
-    }
-
-    return true;
-}
-
 export default async function({ id, index, toGif, dispatcher, alwaysProxy, subtitleLang }) {
     const cookie = await getCookie('twitter');
 
-    let syndication = false;
-
     let guestToken = await getGuestToken(dispatcher);
     if (!guestToken) return { error: "fetch.fail" };
 
-    // for now we assume that graphql api will come back after some time,
-    // so we try it first
-
     let tweet = await requestTweet(dispatcher, id, guestToken);
 
-    // get new token & retry if old one expired
+    if ([403, 404, 429].includes(tweet.status)) {
-    if ([403, 429].includes(tweet.status)) {
+        // get new token & retry if old one expired
-        guestToken = await getGuestToken(dispatcher, true);
+        if ([403, 429].includes(tweet.status)) {
-        if (cookie) {
+            guestToken = await getGuestToken(dispatcher, true);
-            tweet = await requestTweet(dispatcher, id, guestToken, cookie);
-        } else {
-            tweet = await requestTweet(dispatcher, id, guestToken);
         }
+        tweet = await requestTweet(dispatcher, id, guestToken, cookie);
     }
 
-    const testGraphql = testResponse(tweet);
+    let media;
+    try {
+        tweet = await tweet.json();
+        media = await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
+    } catch {}
 
     // if graphql requests fail, then resort to tweet embed api
-    if (!testGraphql) {
+    if (!media || 'error' in media) {
-        syndication = true;
+        try {
-        tweet = await requestSyndication(dispatcher, id);
+            tweet = await requestSyndication(dispatcher, id);
+            tweet = await tweet.json();
 
-        const testSyndication = testResponse(tweet);
+            if (tweet?.card) {
+                media = parseCard(tweet.card);
+            }
+        } catch {}
 
-        // if even syndication request failed, then cry out loud
+        media = tweet?.mediaDetails ?? media;
-        if (!testSyndication) {
-            return { error: "fetch.fail" };
-        }
     }
 
-    tweet = await tweet.json();
+    if (!media || 'error' in media) {
-
+        return { error: media?.error || "fetch.empty" };
-    let media =
+    }
-        syndication
-            ? tweet.mediaDetails
-            : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
-
-    if (!media) return { error: "fetch.empty" };
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {