README.md

@@ -29,20 +29,6 @@ cobalt is a media downloader that doesn't piss you off. it's friendly, efficient
 
 paste the link, get the file, move on. that simple, just how it should be.
 
-### sponsors
-<div align="center" markdown="1">
-    <sup>special thanks to Warp for sponsoring the development of cobalt</sup>
-    <br>
-    <a href="https://go.warp.dev/cobalt">
-        <img alt="Warp banner" width="400" src="https://raw.githubusercontent.com/warpdotdev/brand-assets/be7d584f98e62b1579fd2e9338d4c7318a732f1b/Github/Sponsor/Warp-Github-LG-03.png">
-    </a>
-
-### [Warp, built for coding with multiple AI agents](https://go.warp.dev/cobalt)
-</div>
-
-#### RoyaleHosting
-cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt), and a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
-
 ### cobalt monorepo
 this monorepo includes source code for api, frontend, and related packages:
 - [api tree & readme](/api/)
@@ -67,6 +53,9 @@ same content can be downloaded via dev tools of any modern web browser.
 ### contributing
 if you're considering contributing to cobalt, first of all, thank you! check the [contribution guidelines here](/CONTRIBUTING.md) before getting started, they'll help you do your best right away.
 
+### thank you
+cobalt is sponsored by [royalehosting.net](https://royalehosting.net/?partner=cobalt). a part of our infrastructure is hosted on their network. we really appreciate their kindness and support!
+
 ### licenses
 for relevant licensing information, see the [api](api/README.md) and [web](web/README.md) READMEs.
 unless specified otherwise, the remainder of this repository is licensed under [AGPL-3.0](LICENSE).

api/src/processing/services/twitter.js

@@ -3,12 +3,12 @@ import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
 
-const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail';
+const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId';
 const tokenURL = 'https://api.x.com/1.1/guest/activate.json';
 
-const tweetFeatures = JSON.stringify({"rweb_video_screen_enabled":false,"payments_enabled":false,"rweb_xchat_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false});
+const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false});
 
-const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false});
+const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false});
 
 const commonHeaders = {
     "user-agent": genericUserAgent,
@@ -100,14 +100,10 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
 
     graphqlTweetURL.searchParams.set('variables',
         JSON.stringify({
-            focalTweetId: tweetId,
+            tweetId,
-            with_rux_injections: false,
+            withCommunity: false,
-            rankingMode: "Relevance",
+            includePromotedContent: false,
-            includePromotedContent: true,
+            withVoice: false
-            withCommunity: true,
-            withQuickPromoteEligibilityTweetFields: true,
-            withBirdwatchNotes: true,
-            withVoice: true
         })
     );
     graphqlTweetURL.searchParams.set('features', tweetFeatures);
@@ -133,48 +129,24 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     return result
 }
 
-const parseCard = (cardOuter) => {
+const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => {
-    const card = JSON.parse(
+    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-      (cardOuter?.legacy?.binding_values[0].value
-       || cardOuter?.binding_values?.unified_card)?.string_value,
-    );
-
-    if (!["video_website", "image_website"].includes(card?.type)
-        || !card?.media_entities
-        || card?.component_objects?.media_1?.type !== "media") {
-      return;
-    }
-
-    const mediaId = card.component_objects?.media_1?.data?.id;
-    return [card.media_entities[mediaId]];
-};
-
-const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) => {
-    const addInsn = thread?.data?.threaded_conversation_with_injections_v2?.instructions?.find(
-        insn => insn.type === 'TimelineAddEntries'
-    );
-
-    const tweetResult = addInsn?.entries?.find(
-        entry => entry.entryId === `tweet-${id}`
-    )?.content?.itemContent?.tweet_results?.result;
-
-    let tweetTypename = tweetResult?.__typename;
 
     if (!tweetTypename) {
         return { error: "fetch.empty" }
     }
 
-    if (tweetTypename === "TweetUnavailable" || tweetTypename === "TweetTombstone") {
+    if (tweetTypename === "TweetUnavailable") {
-        const reason = tweetResult?.result?.reason;
+        const reason = tweet?.data?.tweetResult?.result?.reason;
-        if (reason === 'Protected') {
+        switch(reason) {
-            return { error: "content.post.private" };
+            case "Protected":
-        } else if (reason === "NsfwLoggedOut" || tweetResult?.tombstone?.text?.text?.startsWith('Age-restricted')) {
+                return { error: "content.post.private" };
-            if (!cookie) {
+            case "NsfwLoggedOut":
-                return { error: "content.post.age" };
+                if (cookie) {
-            }
+                    tweet = await requestTweet(dispatcher, id, guestToken, cookie);
-
+                    tweet = await tweet.json();
-            const tweet = await requestTweet(dispatcher, id, guestToken, cookie).then(t => t.json());
+                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-            return extractGraphqlMedia(tweet, dispatcher, id, guestToken);
+                } else return { error: "content.post.age" };
         }
     }
 
@@ -182,7 +154,8 @@ const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) =
         return { error: "content.post.unavailable" }
     }
 
-    let baseTweet = tweetResult.legacy,
+    let tweetResult = tweet.data.tweetResult.result,
+        baseTweet = tweetResult.legacy,
         repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
 
     if (tweetTypename === "TweetWithVisibilityResults") {
@@ -191,51 +164,81 @@ const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) =
     }
 
     if (tweetResult.card?.legacy?.binding_values?.length) {
-        return parseCard(tweetResult.card);
+        const card = JSON.parse(tweetResult.card.legacy.binding_values[0].value?.string_value);
+
+        if (!["video_website", "image_website"].includes(card?.type) ||
+            !card?.media_entities ||
+            card?.component_objects?.media_1?.type !== "media") {
+            return;
+        }
+
+        const mediaId = card.component_objects?.media_1?.data?.id;
+        return [card.media_entities[mediaId]];
     }
 
     return (repostedTweet?.media || baseTweet?.extended_entities?.media);
 }
 
+const testResponse = (result) => {
+    const contentLength = result.headers.get("content-length");
+
+    if (!contentLength || contentLength === '0') {
+        return false;
+    }
+
+    if (!result.headers.get("content-type").startsWith("application/json")) {
+        return false;
+    }
+
+    return true;
+}
+
 export default async function({ id, index, toGif, dispatcher, alwaysProxy, subtitleLang }) {
     const cookie = await getCookie('twitter');
 
+    let syndication = false;
+
     let guestToken = await getGuestToken(dispatcher);
     if (!guestToken) return { error: "fetch.fail" };
 
+    // for now we assume that graphql api will come back after some time,
+    // so we try it first
+
     let tweet = await requestTweet(dispatcher, id, guestToken);
 
-    if ([403, 404, 429].includes(tweet.status)) {
+    // get new token & retry if old one expired
-        // get new token & retry if old one expired
+    if ([403, 429].includes(tweet.status)) {
-        if ([403, 429].includes(tweet.status)) {
+        guestToken = await getGuestToken(dispatcher, true);
-            guestToken = await getGuestToken(dispatcher, true);
+        if (cookie) {
+            tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+        } else {
+            tweet = await requestTweet(dispatcher, id, guestToken);
         }
-        tweet = await requestTweet(dispatcher, id, guestToken, cookie);
     }
 
-    let media;
+    const testGraphql = testResponse(tweet);
-    try {
-        tweet = await tweet.json();
-        media = await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
-    } catch {}
 
     // if graphql requests fail, then resort to tweet embed api
-    if (!media || 'error' in media) {
+    if (!testGraphql) {
-        try {
+        syndication = true;
-            tweet = await requestSyndication(dispatcher, id);
+        tweet = await requestSyndication(dispatcher, id);
-            tweet = await tweet.json();
 
-            if (tweet?.card) {
+        const testSyndication = testResponse(tweet);
-                media = parseCard(tweet.card);
-            }
-        } catch {}
 
-        media = tweet?.mediaDetails ?? media;
+        // if even syndication request failed, then cry out loud
+        if (!testSyndication) {
+            return { error: "fetch.fail" };
+        }
     }
 
-    if (!media || 'error' in media) {
+    tweet = await tweet.json();
-        return { error: media?.error || "fetch.empty" };
+
-    }
+    let media =
+        syndication
+            ? tweet.mediaDetails
+            : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
+
+    if (!media) return { error: "fetch.empty" };
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {