Video: Use server generated storyboards (#6557)

package-lock.json

@@ -20,7 +20,6 @@
         "emoji-data-ios": "git+https://github.com/korenskoy/emoji-data-ios#443f1c9",
         "idb-keyval": "^6.2.2",
         "lowlight": "^3.3.0",
-        "mp4box": "^0.5.4",
         "music-metadata": "^11.10.0",
         "opus-recorder": "github:Ajaxy/opus-recorder#116830a",
         "os-browserify": "^0.3.0",
@@ -181,6 +180,7 @@
       "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@babel/code-frame": "^7.27.1",
         "@babel/generator": "^7.28.5",
@@ -2100,6 +2100,7 @@
       "integrity": "sha512-eohl3hKTiVyD1ilYdw9T0OiB4hnjef89e3dMYKz+mVKDzj+5IteTseASUsOB+EU9Tf6VNTCjDePcP6wkDGmLKQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@keyv/serialize": "^1.1.1"
       }
@@ -2218,6 +2219,7 @@
         }
       ],
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=18"
       },
@@ -2241,6 +2243,7 @@
         }
       ],
       "license": "MIT",
+      "peer": true,
       "engines": {
         "node": ">=18"
       }
@@ -3977,8 +3980,7 @@
       "resolved": "https://registry.npmjs.org/@module-federation/error-codes/-/error-codes-0.21.4.tgz",
       "integrity": "sha512-ClpL5MereWNXh+EgDjz7w4RrC1JlisQTvXDa1gLxpviHafzNDfdViVmuhi9xXVuj+EYo8KU70Y999KHhk9424Q==",
       "dev": true,
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/@module-federation/runtime": {
       "version": "0.21.4",
@@ -3986,7 +3988,6 @@
       "integrity": "sha512-wgvGqryurVEvkicufJmTG0ZehynCeNLklv8kIk5BLIsWYSddZAE+xe4xov1kgH5fIJQAoQNkRauFFjVNlHoAkA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@module-federation/error-codes": "0.21.4",
         "@module-federation/runtime-core": "0.21.4",
@@ -3999,7 +4000,6 @@
       "integrity": "sha512-SGpmoOLGNxZofpTOk6Lxb2ewaoz5wMi93AFYuuJB04HTVcngEK+baNeUZ2D/xewrqNIJoMY6f5maUjVfIIBPUA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@module-federation/error-codes": "0.21.4",
         "@module-federation/sdk": "0.21.4"
@@ -4011,7 +4011,6 @@
       "integrity": "sha512-RzFKaL0DIjSmkn76KZRfzfB6dD07cvID84950jlNQgdyoQFUGkqD80L6rIpVCJTY/R7LzR3aQjHnoqmq4JPo3w==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@module-federation/runtime": "0.21.4",
         "@module-federation/webpack-bundler-runtime": "0.21.4"
@@ -4022,8 +4021,7 @@
       "resolved": "https://registry.npmjs.org/@module-federation/sdk/-/sdk-0.21.4.tgz",
       "integrity": "sha512-tzvhOh/oAfX++6zCDDxuvioHY4Jurf8vcfoCbKFxusjmyKr32GPbwFDazUP+OPhYCc3dvaa9oWU6X/qpUBLfJw==",
       "dev": true,
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/@module-federation/webpack-bundler-runtime": {
       "version": "0.21.4",
@@ -4031,7 +4029,6 @@
       "integrity": "sha512-dusmR3uPnQh9u9ChQo3M+GLOuGFthfvnh7WitF/a1eoeTfRmXqnMFsXtZCUK+f/uXf+64874Zj/bhAgbBcVHZA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@module-federation/runtime": "0.21.4",
         "@module-federation/sdk": "0.21.4"
@@ -4073,7 +4070,6 @@
       "dev": true,
       "license": "MIT",
       "optional": true,
-      "peer": true,
       "dependencies": {
         "@emnapi/core": "^1.5.0",
         "@emnapi/runtime": "^1.5.0",
@@ -4524,7 +4520,6 @@
       "integrity": "sha512-liRgxMjHWDL225c41pH4ZcFtPN48LM0+St3iylwavF5JFSqBv86R/Cv5+M+WLrhcihCQsxDwBofipyosJIFmmA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "optionalDependencies": {
         "@rspack/binding-darwin-arm64": "1.6.3",
         "@rspack/binding-darwin-x64": "1.6.3",
@@ -4550,8 +4545,7 @@
       "optional": true,
       "os": [
         "darwin"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/binding-darwin-x64": {
       "version": "1.6.3",
@@ -4565,8 +4559,7 @@
       "optional": true,
       "os": [
         "darwin"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/binding-linux-arm64-gnu": {
       "version": "1.6.3",
@@ -4580,8 +4573,7 @@
       "optional": true,
       "os": [
         "linux"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/binding-linux-arm64-musl": {
       "version": "1.6.3",
@@ -4595,8 +4587,7 @@
       "optional": true,
       "os": [
         "linux"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/binding-linux-x64-gnu": {
       "version": "1.6.3",
@@ -4610,8 +4601,7 @@
       "optional": true,
       "os": [
         "linux"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/binding-linux-x64-musl": {
       "version": "1.6.3",
@@ -4625,8 +4615,7 @@
       "optional": true,
       "os": [
         "linux"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/binding-wasm32-wasi": {
       "version": "1.6.3",
@@ -4638,7 +4627,6 @@
       "dev": true,
       "license": "MIT",
       "optional": true,
-      "peer": true,
       "dependencies": {
         "@napi-rs/wasm-runtime": "1.0.7"
       }
@@ -4655,8 +4643,7 @@
       "optional": true,
       "os": [
         "win32"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/binding-win32-ia32-msvc": {
       "version": "1.6.3",
@@ -4670,8 +4657,7 @@
       "optional": true,
       "os": [
         "win32"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/binding-win32-x64-msvc": {
       "version": "1.6.3",
@@ -4685,8 +4671,7 @@
       "optional": true,
       "os": [
         "win32"
-      ],
+      ]
-      "peer": true
     },
     "node_modules/@rspack/core": {
       "version": "1.6.3",
@@ -4717,8 +4702,7 @@
       "resolved": "https://registry.npmjs.org/@rspack/lite-tapable/-/lite-tapable-1.1.0.tgz",
       "integrity": "sha512-E2B0JhYFmVAwdDiG14+DW0Di4Ze4Jg10Pc4/lILUrd5DRCaklduz2OvJ5HYQ6G+hd+WTzqQb3QnDNfK4yvAFYw==",
       "dev": true,
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/@rtsao/scc": {
       "version": "1.1.0",
@@ -5855,6 +5839,7 @@
       "integrity": "sha512-keKxkZMqnDicuvFoJbzrhbtdLSPhj/rZThDlKWCDbgXmUg0rEUFtRssDXKYmtXluZlIqiC5VqkCgRwzuyLHKHw==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "csstype": "^3.0.2"
       }
@@ -6008,6 +5993,7 @@
       "integrity": "sha512-R48VhmTJqplNyDxCyqqVkFSZIx1qX6PzwqgcXn1olLrzxcSBDlOsbtcnQuQhNtnNiJ4Xe5gREI1foajYaYU2Vg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@eslint-community/regexpp": "^4.10.0",
         "@typescript-eslint/scope-manager": "8.46.4",
@@ -6038,6 +6024,7 @@
       "integrity": "sha512-tK3GPFWbirvNgsNKto+UmB/cRtn6TZfyw0D6IKrW55n6Vbs7KJoZtI//kpTKzE/DUmmnAFD8/Ca46s7Obs92/w==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@typescript-eslint/scope-manager": "8.46.4",
         "@typescript-eslint/types": "8.46.4",
@@ -6726,7 +6713,6 @@
       "integrity": "sha512-u8d0pJ5YFgneF/GuvEiDA61Tf1VDomHHYMjv/wc9XzYj7nopltpG96nXN5dJRstxZhcNpV1g+nT6CydO7pHbjA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18.12.0"
       },
@@ -6741,7 +6727,6 @@
       "integrity": "sha512-coEmDzc2u/ffMvuW9aCjoRzNSPDl/XLuhPdlFRpT9tZHmJ/039az33CE7uH+8s0uL1j5ZNtfdv0HkfaKRBGJsQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18.12.0"
       },
@@ -6860,6 +6845,7 @@
       "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -7903,6 +7889,7 @@
         }
       ],
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "baseline-browser-mapping": "^2.8.25",
         "caniuse-lite": "^1.0.30001754",
@@ -9665,7 +9652,6 @@
       "integrity": "sha512-+zUomDcLXsVkQ37vUqWBvQwLaLlj8eZPSi61llaEFAVBY5mhcXdaSw1pSJVl4yTYD5g/gEfpNl28YYk4IPvrrg==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "bin": {
         "envinfo": "dist/cli.js"
       },
@@ -9965,6 +9951,7 @@
       "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@eslint-community/eslint-utils": "^4.8.0",
         "@eslint-community/regexpp": "^4.12.1",
@@ -10973,7 +10960,6 @@
       "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==",
       "dev": true,
       "license": "BSD-3-Clause",
-      "peer": true,
       "bin": {
         "flat": "cli.js"
       }
@@ -12393,7 +12379,6 @@
       "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=10.13.0"
       }
@@ -13193,6 +13178,7 @@
       "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@jest/core": "30.2.0",
         "@jest/types": "30.2.0",
@@ -14010,6 +13996,7 @@
       "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "bin": {
         "jiti": "lib/jiti-cli.mjs"
       }
@@ -14053,6 +14040,7 @@
       "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "cssstyle": "^4.2.1",
         "data-urls": "^5.0.0",
@@ -15329,12 +15317,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/mp4box": {
-      "version": "0.5.4",
-      "resolved": "https://registry.npmjs.org/mp4box/-/mp4box-0.5.4.tgz",
-      "integrity": "sha512-GcCH0fySxBurJtvr0dfhz0IxHZjc1RP+F+I8xw+LIwkU1a+7HJx8NCDiww1I5u4Hz6g4eR1JlGADEGJ9r4lSfA==",
-      "license": "BSD-3-Clause"
-    },
     "node_modules/ms": {
       "version": "2.1.3",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -16404,6 +16386,7 @@
         }
       ],
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "nanoid": "^3.3.11",
         "picocolors": "^1.1.1",
@@ -16658,6 +16641,7 @@
       "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "cssesc": "^3.0.0",
         "util-deprecate": "^1.0.2"
@@ -17031,7 +17015,6 @@
       "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "resolve": "^1.20.0"
       },
@@ -17583,6 +17566,7 @@
       "integrity": "sha512-Dqh7SiYcaFtdv5Wvku6QgS5IGPm281L+ZtVD1U2FJa7Q0EFRlq8Z3sjYtz6gYObsYThUOz9ArwFqPZx+1azILQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "chokidar": "^4.0.0",
         "immutable": "^5.0.2",
@@ -17685,6 +17669,7 @@
       "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.3",
         "fast-uri": "^3.0.1",
@@ -18885,6 +18870,7 @@
         }
       ],
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@csstools/css-parser-algorithms": "^3.0.5",
         "@csstools/css-tokenizer": "^3.0.4",
@@ -19928,7 +19914,8 @@
       "version": "2.8.1",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
       "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
-      "license": "0BSD"
+      "license": "0BSD",
+      "peer": true
     },
     "node_modules/tsx": {
       "version": "4.20.6",
@@ -19936,6 +19923,7 @@
       "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "esbuild": "~0.25.0",
         "get-tsconfig": "^4.7.5"
@@ -20171,6 +20159,7 @@
       "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
       "dev": true,
       "license": "Apache-2.0",
+      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -20566,6 +20555,7 @@
       "integrity": "sha512-7h/weGm9d/ywQ6qzJ+Xy+r9n/3qgp/thalBbpOi5i223dPXKi04IBtqPN9nTd+jBc7QKfvDbaBnFipYp4sJAUQ==",
       "dev": true,
       "license": "MIT",
+      "peer": true,
       "dependencies": {
         "@types/eslint-scope": "^3.7.7",
         "@types/estree": "^1.0.8",
@@ -20615,7 +20605,6 @@
       "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "@discoveryjs/json-ext": "^0.6.1",
         "@webpack-cli/configtest": "^3.0.1",
@@ -20659,7 +20648,6 @@
       "integrity": "sha512-4B4OijXeVNOPZlYA2oEwWOTkzyltLao+xbotHQeqN++Rv27Y6s818+n2Qkp8q+Fxhn0t/5lA5X1Mxktud8eayQ==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=14.17.0"
       }
@@ -20670,7 +20658,6 @@
       "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "engines": {
         "node": ">=18"
       }
@@ -20888,7 +20875,6 @@
       "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==",
       "dev": true,
       "license": "MIT",
-      "peer": true,
       "dependencies": {
         "clone-deep": "^4.0.1",
         "flat": "^5.0.2",
@@ -21171,8 +21157,7 @@
       "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.1.tgz",
       "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==",
       "dev": true,
-      "license": "MIT",
+      "license": "MIT"
-      "peer": true
     },
     "node_modules/word-wrap": {
       "version": "1.2.5",

package.json

@@ -136,7 +136,6 @@
     "emoji-data-ios": "git+https://github.com/korenskoy/emoji-data-ios#443f1c9",
     "idb-keyval": "^6.2.2",
     "lowlight": "^3.3.0",
-    "mp4box": "^0.5.4",
     "music-metadata": "^11.10.0",
     "opus-recorder": "github:Ajaxy/opus-recorder#116830a",
     "os-browserify": "^0.3.0",

src/api/gramjs/apiBuilders/messageContent.ts

@@ -29,10 +29,17 @@ import type {
   ApiWebPageStoryData,
   BoughtPaidMedia,
   MediaContent,
+  StoryboardInfo,
 } from '../../types';
 import type { UniversalMessage } from './messages';
 
-import { SUPPORTED_PHOTO_CONTENT_TYPES, SUPPORTED_VIDEO_CONTENT_TYPES, VIDEO_WEBM_TYPE } from '../../../config';
+import {
+  STORYBOARD_MAP_MIME,
+  STORYBOARD_MIME,
+  SUPPORTED_PHOTO_CONTENT_TYPES,
+  SUPPORTED_VIDEO_CONTENT_TYPES,
+  VIDEO_WEBM_TYPE,
+} from '../../../config';
 import { addTimestampEntities } from '../../../util/dates/timestamp';
 import { generateWaveform } from '../../../util/generateWaveform';
 import { pick } from '../../../util/iteratees';
@@ -143,8 +150,7 @@ export function buildMessageMediaContent(
   if (photo) return { photo };
 
   const video = buildVideo(media);
-  const altVideos = buildAltVideos(media);
+  if (video) return { video };
-  if (video) return { video, altVideos };
 
   const audio = buildAudio(media);
   if (audio) return { audio };
@@ -208,7 +214,7 @@ function buildPhoto(media: GramJs.TypeMessageMedia): ApiPhoto | undefined {
   return buildApiPhoto(media.photo, media.spoiler);
 }
 
-export function buildVideoFromDocument(document: GramJs.Document, params?: {
+export function buildVideoFromDocument(document: GramJs.Document, altDocuments?: GramJs.TypeDocument[], params?: {
   isSpoiler?: boolean;
   timestamp?: number;
 }): ApiVideo | undefined {
@@ -216,6 +222,8 @@ export function buildVideoFromDocument(document: GramJs.Document, params?: {
     return undefined;
   }
 
+  const altVideos = altDocuments && buildAltVideosFromDocuments(altDocuments);
+
   const { isSpoiler, timestamp } = params || {};
 
   const {
@@ -249,6 +257,7 @@ export function buildVideoFromDocument(document: GramJs.Document, params?: {
   } = videoAttr;
 
   const waveform = isRound ? generateWaveform(duration) : undefined;
+  const storyboardInfo = altDocuments && buildStoryboardInfoFromDocuments(altDocuments);
 
   return {
     mediaType: 'video',
@@ -268,7 +277,9 @@ export function buildVideoFromDocument(document: GramJs.Document, params?: {
     hasVideoPreview,
     previewPhotoSizes,
     waveform,
-    ...(nosound && { noSound: true }),
+    noSound: nosound,
+    altVideos,
+    storyboardInfo,
   };
 }
 
@@ -315,17 +326,19 @@ function buildVideo(media: GramJs.TypeMessageMedia): ApiVideo | undefined {
     return undefined;
   }
 
-  return buildVideoFromDocument(media.document, { isSpoiler: media.spoiler, timestamp: media.videoTimestamp });
+  return buildVideoFromDocument(
+    media.document,
+    media.altDocuments,
+    { isSpoiler: media.spoiler, timestamp: media.videoTimestamp },
+  );
 }
 
-function buildAltVideos(media: GramJs.TypeMessageMedia): ApiVideo[] | undefined {
+function buildAltVideosFromDocuments(altDocuments: GramJs.TypeDocument[], params?: {
-  if (!(media instanceof GramJs.MessageMediaDocument) || !media.altDocuments) {
+  isSpoiler?: boolean;
-    return undefined;
+}): ApiVideo[] | undefined {
-  }
+  const altVideos = altDocuments.filter((d): d is GramJs.Document => (
-
-  const altVideos = media.altDocuments.filter((d): d is GramJs.Document => (
     d instanceof GramJs.Document && d.mimeType.startsWith('video')
-  )).map((alt) => buildVideoFromDocument(alt, { isSpoiler: media.spoiler }))
+  )).map((alt) => buildVideoFromDocument(alt, undefined, params))
     .filter(Boolean);
   if (!altVideos.length) {
     return undefined;
@@ -334,6 +347,34 @@ function buildAltVideos(media: GramJs.TypeMessageMedia): ApiVideo[] | undefined
   return altVideos;
 }
 
+function buildStoryboardInfoFromDocuments(documents: GramJs.TypeDocument[]): StoryboardInfo | undefined {
+  const storyboardMtpFile = documents.find((d): d is GramJs.Document => (
+    d instanceof GramJs.Document && d.mimeType === STORYBOARD_MIME
+  ));
+  const storyboardMapMtpFile = documents.find((d): d is GramJs.Document => (
+    d instanceof GramJs.Document && d.mimeType === STORYBOARD_MAP_MIME
+  ));
+
+  const storyboardFile = storyboardMtpFile && buildApiDocument(storyboardMtpFile);
+  const storyboardMapFile = storyboardMapMtpFile && buildApiDocument(storyboardMapMtpFile);
+
+  const sizeAttribute = storyboardMapMtpFile?.attributes.find((a): a is GramJs.DocumentAttributeImageSize => (
+    a instanceof GramJs.DocumentAttributeImageSize
+  ));
+
+  const frameSize = sizeAttribute && { width: sizeAttribute.w, height: sizeAttribute.h };
+
+  if (!storyboardFile || !storyboardMapFile || !frameSize) {
+    return undefined;
+  }
+
+  return {
+    storyboardFile,
+    storyboardMapFile,
+    frameSize,
+  };
+}
+
 function buildAudio(media: GramJs.TypeMessageMedia): ApiAudio | undefined {
   if (
     !(media instanceof GramJs.MessageMediaDocument)

src/api/gramjs/helpers/localDb.ts

@@ -42,6 +42,13 @@ export function addMediaToLocalDb(media: GramJs.TypeMessageMedia, context?: Medi
   if (media instanceof GramJs.MessageMediaDocument && media.document) {
     const document = addMessageRepairInfo(media.document, context);
     addDocumentToLocalDb(document);
+
+    if (media.altDocuments) {
+      for (const altDocument of media.altDocuments) {
+        const doc = addMessageRepairInfo(altDocument, context);
+        addDocumentToLocalDb(doc);
+      }
+    }
   }
 
   if (media instanceof GramJs.MessageMediaGame) {

src/api/types/messages.ts

@@ -99,6 +99,12 @@ type ApiStickerSetInfoMissing = {
 
 export type ApiStickerSetInfo = ApiStickerSetInfoShortName | ApiStickerSetInfoId | ApiStickerSetInfoMissing;
 
+export interface StoryboardInfo {
+  storyboardFile: ApiDocument;
+  storyboardMapFile: ApiDocument;
+  frameSize: ApiDimensions;
+}
+
 export interface ApiVideo {
   mediaType: 'video';
   id: string;
@@ -120,6 +126,8 @@ export interface ApiVideo {
   noSound?: boolean;
   waveform?: number[];
   timestamp?: number;
+  altVideos?: ApiVideo[];
+  storyboardInfo?: StoryboardInfo;
 }
 
 export interface ApiAudio {
@@ -584,7 +592,6 @@ export type MediaContent = {
   text?: ApiFormattedTextWithEmojiOnlyCount;
   photo?: ApiPhoto;
   video?: ApiVideo;
-  altVideos?: ApiVideo[];
   document?: ApiDocument;
   sticker?: ApiSticker;
   contact?: ApiContact;

src/components/mediaViewer/MediaViewerContent.tsx

@@ -13,7 +13,6 @@ import {
   selectIsMessageProtected, selectTabState,
 } from '../../global/selectors';
 import { selectMessageTimestampableDuration } from '../../global/selectors/media';
-import { ARE_WEBCODECS_SUPPORTED } from '../../util/browser/globalEnvironment';
 import { IS_TOUCH_ENV } from '../../util/browser/windowEnvironment';
 import buildClassName from '../../util/buildClassName';
 import stopEvent from '../../util/stopEvent';
@@ -99,7 +98,6 @@ const MediaViewerContent = ({
     bestData,
     dimensions,
     isGif,
-    isLocal,
     isVideoAvatar,
     mediaSize,
     loadProgress,
@@ -159,7 +157,6 @@ const MediaViewerContent = ({
             fileSize={mediaSize!}
             isMediaViewerOpen={isOpen && isActive}
             isProtected={isProtected}
-            isPreviewDisabled={!ARE_WEBCODECS_SUPPORTED || isLocal}
             noPlay={!isActive}
             onClose={onClose}
             isMuted
@@ -207,6 +204,7 @@ const MediaViewerContent = ({
         <VideoPlayer
           key={media.id}
           url={bestData}
+          storyboardInfo={'storyboardInfo' in media ? media.storyboardInfo : undefined}
           isGif={isGif}
           posterData={bestImageData}
           posterSize={posterSize}
@@ -214,7 +212,6 @@ const MediaViewerContent = ({
           fileSize={mediaSize!}
           isMediaViewerOpen={isOpen && isActive}
           noPlay={!isActive}
-          isPreviewDisabled={!ARE_WEBCODECS_SUPPORTED || isLocal}
           onClose={onClose}
           isMuted={isMuted}
           isHidden={isHidden}

src/components/mediaViewer/SeekLine.module.scss

@@ -27,10 +27,10 @@
   background: #000;
 }
 
-.previewCanvas {
+.previewContainer {
-  display: block;
   width: 100%;
   height: 100%;
+  background-repeat: no-repeat;
 }
 
 body:global(.is-touch-env) .preview {

src/components/mediaViewer/SeekLine.tsx

@@ -1,24 +1,29 @@
-import type { FC } from '../../lib/teact/teact';
+import { setExtraStyles } from '@teact/teact-dom';
 import {
   memo, useEffect, useLayoutEffect,
-  useMemo, useRef, useSignal, useState,
+  useRef, useSignal, useState,
 } from '../../lib/teact/teact';
 
-import type { ApiDimensions } from '../../api/types';
 import type { BufferedRange } from '../../hooks/useBuffering';
+import { ApiMediaFormat, type StoryboardInfo } from '../../api/types';
 
-import { createVideoPreviews, getPreviewDimensions, renderVideoPreview } from '../../lib/video-preview/VideoPreview';
+import { DEBUG } from '../../config';
+import { requestMutation } from '../../lib/fasterdom/fasterdom';
+import { getDocumentMediaHash } from '../../global/helpers';
 import { animateNumber } from '../../util/animation';
 import { IS_TOUCH_ENV } from '../../util/browser/windowEnvironment';
 import buildClassName from '../../util/buildClassName';
+import buildStyle from '../../util/buildStyle';
 import { captureEvents } from '../../util/captureEvents';
 import { formatMediaDuration } from '../../util/dates/dateFormat';
 import getPointerPosition from '../../util/events/getPointerPosition';
 import { clamp, round } from '../../util/math';
+import StoryboardParser from '../../util/media/StoryboardParser';
 
 import { useThrottledSignal } from '../../hooks/useAsyncResolvers';
 import useCurrentTimeSignal from '../../hooks/useCurrentTimeSignal';
 import useLastCallback from '../../hooks/useLastCallback';
+import useMedia from '../../hooks/useMedia';
 import useVideoWaitingSignal from './hooks/useVideoWaitingSignal';
 
 import ShowTransition from '../ui/ShowTransition';
@@ -26,15 +31,13 @@ import ShowTransition from '../ui/ShowTransition';
 import styles from './SeekLine.module.scss';
 
 type OwnProps = {
-  url?: string;
+  storyboardInfo?: StoryboardInfo;
   duration: number;
   bufferedRanges: BufferedRange[];
   playbackRate: number;
   isActive?: boolean;
   isPlaying?: boolean;
-  isPreviewDisabled?: boolean;
   isReady: boolean;
-  posterSize?: ApiDimensions;
   onSeek: (position: number) => void;
   onSeekStart: () => void;
 };
@@ -42,19 +45,17 @@ type OwnProps = {
 const LOCK_TIMEOUT = 250;
 let cancelAnimation: ReturnType<typeof animateNumber> | undefined;
 
-const SeekLine: FC<OwnProps> = ({
+const SeekLine = ({
+  storyboardInfo,
   duration,
   bufferedRanges,
   isReady,
-  posterSize,
   playbackRate,
-  url,
   isActive,
   isPlaying,
-  isPreviewDisabled,
   onSeek,
   onSeekStart,
-}) => {
+}: OwnProps) => {
   const seekerRef = useRef<HTMLDivElement>();
   const [getCurrentTimeSignal] = useCurrentTimeSignal();
   const [getIsWaiting] = useVideoWaitingSignal();
@@ -65,30 +66,48 @@ const SeekLine: FC<OwnProps> = ({
   const isLockedRef = useRef<boolean>(false);
   const [isPreviewVisible, setPreviewVisible] = useState(false);
   const [isSeeking, setIsSeeking] = useState(false);
-  const previewCanvasRef = useRef<HTMLCanvasElement>();
+  const previewContainerRef = useRef<HTMLDivElement>();
   const previewRef = useRef<HTMLDivElement>();
   const progressRef = useRef<HTMLDivElement>();
   const previewTimeRef = useRef<HTMLDivElement>();
+  const storyboardParser = useRef<StoryboardParser>();
 
-  const previewSize = useMemo(() => {
+  const storyboardHash = storyboardInfo && getDocumentMediaHash(storyboardInfo.storyboardFile, 'full');
-    return getPreviewDimensions(posterSize?.width || 0, posterSize?.height || 0);
+  const storyboardMapHash = storyboardInfo && getDocumentMediaHash(storyboardInfo.storyboardMapFile, 'full');
-  }, [posterSize]);
 
-  const setPreview = useLastCallback((time: number) => {
+  const storyboardUrl = useMedia(storyboardHash, !isReady);
-    time = Math.floor(time);
+  const storyboardMapData = useMedia(storyboardMapHash, !isReady, ApiMediaFormat.Text);
-    setPreviewTime(time);
-    renderVideoPreview(time);
-  });
-
-  useEffect(() => {
-    if (isPreviewDisabled || !url || !isReady) return undefined;
-    return createVideoPreviews(url, previewCanvasRef.current!);
-  }, [url, isReady, isPreviewDisabled]);
 
   useEffect(() => {
     setPreviewVisible(false);
   }, [isActive]);
 
+  useEffect(() => {
+    if (!storyboardMapData) return;
+    try {
+      storyboardParser.current = new StoryboardParser(storyboardMapData);
+    } catch (error) {
+      if (DEBUG) {
+        // eslint-disable-next-line no-console
+        console.error(`Error parsing storyboard map data`, error, storyboardMapData);
+      }
+    }
+  }, [storyboardMapData]);
+
+  const setPreview = useLastCallback((time: number) => {
+    const previewContainer = previewContainerRef.current;
+    if (!storyboardParser.current || !previewContainer) return;
+    const frame = storyboardParser.current.getNearestPreview(time);
+
+    setPreviewTime(Math.floor(frame.time));
+
+    requestMutation(() => {
+      setExtraStyles(previewContainer, {
+        backgroundPosition: `${-frame.left}px ${-frame.top}px`,
+      });
+    });
+  });
+
   useEffect(() => {
     if (cancelAnimation) cancelAnimation();
     cancelAnimation = undefined;
@@ -154,7 +173,7 @@ const SeekLine: FC<OwnProps> = ({
     const getPreviewProps = (e: MouseEvent | TouchEvent) => {
       const pageX = getPointerPosition(e).x;
       const t = clamp(duration * ((pageX - seekerSize.left) / seekerSize.width), 0, duration);
-      if (isPreviewDisabled) return [t, 0];
+      if (!storyboardInfo) return [t, 0];
       if (!seekerSize.width) seekerSize = seeker.getBoundingClientRect();
       const preview = previewRef.current!;
       const o = clamp(
@@ -204,7 +223,7 @@ const SeekLine: FC<OwnProps> = ({
       onDrag: handleSeek,
     });
 
-    if (IS_TOUCH_ENV || isPreviewDisabled) {
+    if (IS_TOUCH_ENV) {
       return cleanup;
     }
 
@@ -230,29 +249,27 @@ const SeekLine: FC<OwnProps> = ({
       seeker.removeEventListener('mouseleave', handleSeekMouseLeave);
     };
   }, [
-    duration,
+    duration, setPreview, isActive, onSeek, onSeekStart, setPreviewOffset, setSelectedTime, setIsSeeking,
-    setPreview,
+    isPreviewVisible, playbackRate, storyboardInfo,
-    isActive,
-    onSeek,
-    onSeekStart,
-    setPreviewOffset,
-    setSelectedTime,
-    setIsSeeking,
-    isPreviewDisabled,
-    playbackRate,
   ]);
 
   return (
     <div className={styles.container} ref={seekerRef}>
-      {!isPreviewDisabled && (
+      {storyboardInfo && (
         <ShowTransition
           isOpen
           isHidden={!isPreviewVisible}
           className={styles.preview}
-          style={`width: ${previewSize.width}px; height: ${previewSize.height}px`}
+          style={`width: ${storyboardInfo.frameSize.width}px; height: ${storyboardInfo.frameSize.height}px`}
           ref={previewRef}
         >
-          <canvas className={styles.previewCanvas} ref={previewCanvasRef} />
+          <div
+            ref={previewContainerRef}
+            style={buildStyle(
+              `background-image: url(${storyboardUrl});`,
+            )}
+            className={styles.previewContainer}
+          />
           <div className={styles.previewTime}>
             <span className={styles.previewTimeText} ref={previewTimeRef} />
           </div>

src/components/mediaViewer/VideoPlayer.tsx

@@ -5,7 +5,7 @@ import {
 } from '../../lib/teact/teact';
 import { getActions } from '../../global';
 
-import type { ApiDimensions } from '../../api/types';
+import type { ApiDimensions, StoryboardInfo } from '../../api/types';
 
 import { IS_IOS, IS_TOUCH_ENV, IS_YA_BROWSER } from '../../util/browser/windowEnvironment';
 import getPointerPosition from '../../util/events/getPointerPosition';
@@ -33,12 +33,12 @@ import './VideoPlayer.scss';
 
 type OwnProps = {
   url?: string;
+  storyboardInfo?: StoryboardInfo;
   isGif?: boolean;
   posterData?: string;
   posterSize?: ApiDimensions;
   loadProgress?: number;
   fileSize: number;
-  isPreviewDisabled?: boolean;
   isMediaViewerOpen?: boolean;
   noPlay?: boolean;
   volume: number;
@@ -61,6 +61,7 @@ const REWIND_STEP = 5; // Seconds
 
 const VideoPlayer: FC<OwnProps> = ({
   url,
+  storyboardInfo,
   isGif,
   posterData,
   posterSize,
@@ -75,7 +76,6 @@ const VideoPlayer: FC<OwnProps> = ({
   shouldCloseOnClick,
   isProtected,
   isClickDisabled,
-  isPreviewDisabled,
   isSponsoredMessage,
   timestamp,
   handleSponsoredClick,
@@ -378,7 +378,7 @@ const VideoPlayer: FC<OwnProps> = ({
       )}
       {!isGif && !isSponsoredMessage && !isUnsupported && (
         <VideoPlayerControls
-          url={url}
+          storyboardInfo={storyboardInfo}
           isPlaying={isPlaying}
           bufferedRanges={bufferedRanges}
           bufferedProgress={bufferedProgress}
@@ -386,11 +386,9 @@ const VideoPlayer: FC<OwnProps> = ({
           isFullscreenSupported={Boolean(setFullscreen)}
           isPictureInPictureSupported={isPictureInPictureSupported}
           isFullscreen={isFullscreen}
-          isPreviewDisabled={isPreviewDisabled}
           fileSize={fileSize}
           duration={duration}
           isReady={isReady}
-          posterSize={posterSize}
           isForceMobileVersion={isForceMobileVersion}
           onSeek={handleSeek}
           onChangeFullscreen={handleFullscreenChange}

src/components/mediaViewer/VideoPlayerControls.scss

@@ -58,13 +58,12 @@
   }
 
   .volume-slider {
-
     --volume-slider-width: 4rem;
     --slider-color: #fff;
     --color-borders: rgba(255, 255, 255, 0.5);
 
     width: 0;
-    margin-bottom: 0;
+    margin: 0;
     margin-left: -0.75rem;
     padding: 0.5rem 0.5rem 0.5rem 0.5rem;
     /* stylelint-disable-next-line plugin/no-low-performance-animation-properties */

src/components/mediaViewer/VideoPlayerControls.tsx

@@ -7,7 +7,7 @@ import {
   useSignal,
 } from '../../lib/teact/teact';
 
-import type { ApiDimensions } from '../../api/types';
+import type { StoryboardInfo } from '../../api/types';
 import type { BufferedRange } from '../../hooks/useBuffering';
 import type { IconName } from '../../types/icons';
 
@@ -33,7 +33,7 @@ import SeekLine from './SeekLine';
 import './VideoPlayerControls.scss';
 
 type OwnProps = {
-  url?: string;
+  storyboardInfo?: StoryboardInfo;
   bufferedRanges: BufferedRange[];
   bufferedProgress: number;
   duration: number;
@@ -44,12 +44,10 @@ type OwnProps = {
   isFullscreenSupported: boolean;
   isPictureInPictureSupported: boolean;
   isFullscreen: boolean;
-  isPreviewDisabled?: boolean;
   isBuffered: boolean;
   volume: number;
   isMuted: boolean;
   playbackRate: number;
-  posterSize?: ApiDimensions;
   onChangeFullscreen: (e: React.MouseEvent<HTMLButtonElement, MouseEvent>) => void;
   onPictureInPictureChange?: () => void;
   onVolumeClick: () => void;
@@ -74,7 +72,7 @@ const PLAYBACK_RATES = [
 const HIDE_CONTROLS_TIMEOUT_MS = 3000;
 
 const VideoPlayerControls: FC<OwnProps> = ({
-  url,
+  storyboardInfo,
   bufferedRanges,
   bufferedProgress,
   duration,
@@ -85,16 +83,14 @@ const VideoPlayerControls: FC<OwnProps> = ({
   isFullscreenSupported,
   isFullscreen,
   isBuffered,
-  isPreviewDisabled,
   volume,
   isMuted,
   playbackRate,
-  posterSize,
+  isPictureInPictureSupported,
   onChangeFullscreen,
   onVolumeClick,
   onVolumeChange,
   onPlaybackRateChange,
-  isPictureInPictureSupported,
   onPictureInPictureChange,
   onPlayPause,
   onSeek,
@@ -168,12 +164,10 @@ const VideoPlayerControls: FC<OwnProps> = ({
       onClick={stopEvent}
     >
       <SeekLine
-        url={url}
+        storyboardInfo={storyboardInfo}
         duration={duration}
         isReady={isReady}
         isPlaying={isPlaying}
-        isPreviewDisabled={isPreviewDisabled}
-        posterSize={posterSize}
         bufferedRanges={bufferedRanges}
         playbackRate={playbackRate}
         onSeek={handleSeek}

src/components/mediaViewer/hooks/useMediaProps.ts

@@ -104,8 +104,6 @@ export const useMediaProps = ({
 
   const mediaSize = media && getMediaFileSize(media);
 
-  const isLocal = Boolean(localBlobUrl);
-
   const dimensions = useMemo(() => {
     if (isAvatar) {
       return isVideoAvatar ? VIDEO_AVATAR_FULL_DIMENSIONS : AVATAR_FULL_DIMENSIONS;
@@ -138,7 +136,6 @@ export const useMediaProps = ({
     dimensions,
     contentType,
     isVideoAvatar,
-    isLocal,
     loadProgress,
     mediaSize,
   };

src/components/story/hooks/useStoryPreloader.ts

@@ -101,7 +101,7 @@ function getPreloadMediaHashes(peerId: string, storyId: number) {
     });
     // Thumbnail
     mediaHashes.push({ hash: getStoryMediaHash(story), format: ApiMediaFormat.BlobUrl });
-    if (story.content.altVideos) {
+    if (story.content.video?.altVideos) {
       mediaHashes.push({
         hash: getStoryMediaHash(story, 'full', true)!,
         format: ApiMediaFormat.Progressive,

src/config.ts

@@ -289,6 +289,9 @@ export const CONTENT_TYPES_WITH_PREVIEW = new Set([
   ...SUPPORTED_VIDEO_CONTENT_TYPES,
 ]);
 
+export const STORYBOARD_MIME = 'application/x-tgstoryboard';
+export const STORYBOARD_MAP_MIME = 'application/x-tgstoryboardmap';
+
 // Taken from https://github.com/telegramdesktop/tdesktop/blob/41d9a9fcbd0c809c60ddbd9350791b1436aff7d9/Telegram/SourceFiles/ui/boxes/choose_language_box.cpp#L28
 export const SUPPORTED_TRANSLATION_LANGUAGES = [
   // Official

src/global/helpers/media.ts

@@ -13,17 +13,17 @@ export function getStoryMediaHash(story: ApiStory, size: StorySize): string;
 export function getStoryMediaHash(
   story: ApiStory, size: StorySize = 'preview', isAlt?: boolean,
 ) {
-  const isVideo = Boolean(story.content.video);
+  const video = story.content.video;
-  const isPhoto = Boolean(story.content.photo);
+  const photo = story.content.photo;
 
-  if (isVideo) {
+  if (video) {
-    if (isAlt && !story.content.altVideos) return undefined;
+    if (isAlt && !video.altVideos) return undefined;
-    const media = isAlt ? getPreferredAlt(story.content.altVideos!) : story.content.video!;
+    const media = isAlt ? getPreferredAlt(video.altVideos!) : video;
     return getVideoMediaHash(media, size);
   }
 
-  if (isPhoto) {
+  if (photo) {
-    return getPhotoMediaHash(story.content.photo!, size);
+    return getPhotoMediaHash(photo, size);
   }
 
   return undefined;

src/lib/mediaWorker/index.worker.ts

@@ -1,12 +1,9 @@
 import '../rlottie/rlottie.worker';
-import '../video-preview/video-preview.worker';
 import '../offscreen-canvas/offscreen-canvas.worker';
 
 import type { OffscreenCanvasApi } from '../offscreen-canvas/offscreen-canvas.worker';
 import type { RLottieApi } from '../rlottie/rlottie.worker';
-import type { VideoPreviewApi } from '../video-preview/video-preview.worker';
 
 export type MediaWorkerApi =
   RLottieApi
-  & VideoPreviewApi
   & OffscreenCanvasApi;

src/lib/video-preview/MP4Demuxer.ts

@@ -1,225 +0,0 @@
-import type { MP4ArrayBuffer, MP4Info, MP4VideoTrack } from 'mp4box';
-import MP4Box, { DataStream } from 'mp4box';
-
-import { requestPart } from './requestPart';
-
-const META_PART_SIZE = 128 * 1024;
-const MIN_PART_SIZE = 1024;
-enum Status {
-  loading = 'loading',
-  ready = 'ready',
-  closed = 'closed',
-}
-
-export type MP4DecoderConfig = {
-  codec: string;
-  codedHeight: number;
-  codedWidth: number;
-  description: Uint8Array;
-};
-
-type MP4DemuxerConfig = {
-  stepOffset: number;
-  stepMultiplier: number;
-  isPolyfill: boolean;
-  maxFrames: number;
-  onConfig: (config: any) => void;
-  onChunk: (chunk: any) => void;
-};
-
-export class MP4Demuxer {
-  private readonly url: string;
-
-  private file: MP4Box.MP4File;
-
-  private status = Status.loading;
-
-  private readonly stepOffset: number;
-
-  private readonly stepMultiplier: number;
-
-  private readonly maxFrames: number;
-
-  private readonly isPolyfill: boolean;
-
-  private decodedSamples = new Set<string>();
-
-  private lastSample = 0;
-
-  private readonly onConfig: (config: MP4DecoderConfig) => void;
-
-  private readonly onChunk: (chunk: any) => void;
-
-  constructor(url: string, {
-    onConfig,
-    onChunk,
-    stepOffset,
-    stepMultiplier,
-    isPolyfill,
-    maxFrames,
-  }: MP4DemuxerConfig) {
-    this.url = url;
-    this.stepOffset = stepOffset;
-    this.stepMultiplier = stepMultiplier;
-    this.maxFrames = maxFrames;
-    this.isPolyfill = isPolyfill;
-    this.onConfig = onConfig;
-    this.onChunk = onChunk;
-
-    this.file = MP4Box.createFile();
-    this.file.onError = (e) => {
-      // eslint-disable-next-line no-console
-      console.error(e);
-    };
-    this.file.onReady = this.onReady.bind(this);
-    this.file.onSamples = this.onSamples.bind(this);
-
-    void this.loadMetadata();
-  }
-
-  private async loadMetadata() {
-    let offset: number | undefined = 0;
-    while (offset !== undefined) {
-      try {
-        offset = await this.requestPart(offset, META_PART_SIZE);
-      } catch (e) {
-        // eslint-disable-next-line no-console
-        console.error(e);
-      }
-      if (this.status === Status.ready) break;
-    }
-  }
-
-  private async loadNextFrames(step: number, duration: number, partSize: number) {
-    let tick = step * this.stepOffset;
-    let lastSample = 0;
-    let rap = this.file.seek(tick, true);
-    while (this.status !== Status.closed) {
-      try {
-        await this.requestPart(rap.offset, partSize);
-        if (tick > duration) break;
-        if (this.lastSample > 1 && lastSample < this.lastSample) {
-          tick += step * this.stepMultiplier;
-          lastSample = this.lastSample;
-        }
-        rap = this.file.seek(tick, true);
-      } catch (e) {
-        // eslint-disable-next-line no-console
-        console.error(e);
-      }
-    }
-    this.file.flush();
-  }
-
-  private async requestPart(offset: number, partSize: number, useRap = true) {
-    const reminder = (offset % MIN_PART_SIZE);
-    const start = offset - reminder;
-    const end = start + partSize - 1;
-    let arrayBuffer = await requestPart({ url: this.url, start, end }) as MP4ArrayBuffer;
-    if (!arrayBuffer) {
-      return undefined;
-    }
-    if (reminder) {
-      arrayBuffer = arrayBuffer.slice(reminder) as MP4ArrayBuffer;
-    }
-    arrayBuffer.fileStart = offset;
-    const nextOffset = this.file.appendBuffer(arrayBuffer);
-    if (!useRap) return offset + arrayBuffer.byteLength;
-    return nextOffset;
-  }
-
-  private description(track: MP4VideoTrack) {
-    const t = this.file.getTrackById(track.id);
-    for (const entry of t.mdia.minf.stbl.stsd.entries) {
-      if (entry.avcC || entry.hvcC || entry.av1C) {
-        const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
-        if (entry.avcC) {
-          entry.avcC.write(stream);
-        } else if (entry.hvcC) {
-          entry.hvcC.write(stream);
-        } else if (entry.av1C) {
-          entry.av1C.write(stream);
-        }
-        return new Uint8Array(stream.buffer, 8); // Remove the box header.
-      }
-    }
-    throw new Error('avcC, hvcC ro av1C not found');
-  }
-
-  private onReady(info: MP4Info) {
-    const track = info.videoTracks[0];
-
-    let codec = track.codec;
-    if (codec.startsWith('avc1')) {
-      // Somehow this is the only avc1 codec that works.
-      codec = 'avc1.4d001f';
-    }
-
-    // Generate and emit an appropriate VideoDecoderConfig.
-    this.onConfig({
-      codec,
-      codedHeight: track.video.height,
-      codedWidth: track.video.width,
-      description: this.description(track),
-    });
-
-    const duration = info.duration / info.timescale;
-
-    // If we set a part size too small, the onSamples callback is not called.
-    // If we use polyfill, we need to set a smaller part size to avoid decoding multiple frames.
-    const partSizeDivider = this.isPolyfill ? 24 : 12;
-    const partSize = roundPartSize(track.bitrate / partSizeDivider);
-    const step = calculateStep(duration, this.maxFrames);
-
-    // Start demuxing.
-    this.file.setExtractionOptions(track.id, undefined, { nbSamples: 1 });
-    this.file.start();
-
-    this.status = Status.ready;
-
-    // // Load frames
-    void this.loadNextFrames(step, duration, partSize);
-  }
-
-  private onSamples(trackId: number, ref: any, samples: any) {
-    if (this.status !== Status.ready) return;
-    // Generate and emit an EncodedVideoChunk for each demuxed sample.
-    for (const sample of samples) {
-      const time = sample.cts / sample.timescale;
-      const type = sample.is_sync ? 'key' : 'delta';
-      const id = `${type}${sample.number}`;
-
-      // Skip already decoded samples.
-      if (this.decodedSamples.has(id)) continue;
-
-      // @ts-ignore
-      this.onChunk(new EncodedVideoChunk({
-        type,
-        timestamp: (1e6 * time),
-        duration: (1e6 * sample.duration) / sample.timescale,
-        data: sample.data,
-      }));
-
-      this.decodedSamples.add(id);
-      this.lastSample = parseInt(sample.number, 10);
-
-      if (sample.is_sync) {
-        this.file.releaseUsedSamples(trackId, sample.number);
-      }
-    }
-  }
-
-  close() {
-    this.file.flush();
-    this.file.stop();
-    this.status = Status.closed;
-  }
-}
-
-function roundPartSize(size: number) {
-  return size + MIN_PART_SIZE - (size % MIN_PART_SIZE);
-}
-
-function calculateStep(duration: number, max: number): number {
-  return Math.round((duration + max) / max);
-}

src/lib/video-preview/VideoPreview.ts

@@ -1,134 +0,0 @@
-import { ApiMediaFormat } from '../../api/types';
-
-import { IS_ANDROID, IS_IOS } from '../../util/browser/windowEnvironment';
-import launchMediaWorkers, { MAX_WORKERS } from '../../util/launchMediaWorkers';
-import { callApi } from '../../api/gramjs';
-import { requestMutation } from '../fasterdom/fasterdom';
-
-const IS_MOBILE = IS_ANDROID || IS_IOS;
-const PREVIEW_SIZE_RATIO = (IS_ANDROID || IS_IOS) ? 0.3 : 0.25;
-const MAX_FRAMES = IS_MOBILE ? 40 : 80;
-const PREVIEW_MAX_SIDE = 200;
-
-const connections = launchMediaWorkers();
-
-let videoPreview: VideoPreview | undefined;
-
-export class VideoPreview {
-  frames = new Map<number, ImageBitmap>();
-
-  currentTime = 0;
-
-  canvas: HTMLCanvasElement;
-
-  constructor(url: string, canvas: HTMLCanvasElement) {
-    this.canvas = canvas;
-    connections.forEach(({ connector }, index) => {
-      void connector.request({
-        name: 'video-preview:init',
-        args: [
-          url,
-          MAX_FRAMES,
-          index,
-          MAX_WORKERS,
-          this.onFrame.bind(this),
-        ],
-      });
-    });
-  }
-
-  private onFrame(time: number, frame: ImageBitmap) {
-    this.frames.set(time, frame);
-    if (time === this.currentTime) {
-      this.render(time);
-    }
-  }
-
-  private clearCache() {
-    this.frames.forEach((frame) => {
-      frame.close();
-    });
-    this.frames.clear();
-  }
-
-  render(time: number) {
-    this.currentTime = time;
-    const frame = this.frames.get(time);
-    if (!frame) return false;
-    requestMutation(() => {
-      this.canvas.width = frame.width;
-      this.canvas.height = frame.height;
-      const ctx = this.canvas.getContext('2d')!;
-      ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
-    });
-    return true;
-  }
-
-  destroy() {
-    this.clearCache();
-    connections.forEach(({ connector }) => {
-      void connector.request({
-        name: 'video-preview:destroy',
-        args: [],
-      });
-    });
-  }
-}
-
-export function getPreviewDimensions(width: number, height: number) {
-  width = Math.round(width * PREVIEW_SIZE_RATIO);
-  height = Math.round(height * PREVIEW_SIZE_RATIO);
-  const ratio = width / height;
-  if (width > PREVIEW_MAX_SIDE) {
-    width = PREVIEW_MAX_SIDE;
-    height = Math.round(width / ratio);
-  }
-  if (height > PREVIEW_MAX_SIDE) {
-    height = PREVIEW_MAX_SIDE;
-    width = Math.round(height * ratio);
-  }
-  return { width, height };
-}
-
-connections.forEach(({ worker }) => {
-  worker.addEventListener('message', async (e) => {
-    const { type, messageId, params } = e.data as {
-      type: string;
-      messageId: string;
-      params: { url: string; start: number; end: number };
-    };
-
-    if (type !== 'requestPart') {
-      return;
-    }
-
-    const result = await callApi('downloadMedia', { mediaFormat: ApiMediaFormat.Progressive, ...params });
-    if (!result) {
-      return;
-    }
-
-    const { arrayBuffer } = result;
-
-    worker.postMessage({
-      type: 'partResponse',
-      messageId,
-      result: arrayBuffer,
-    }, [arrayBuffer!]);
-  });
-});
-
-export function createVideoPreviews(url: string, canvas: HTMLCanvasElement) {
-  if (videoPreview) {
-    videoPreview.destroy();
-  }
-  videoPreview = new VideoPreview(url, canvas);
-  return () => {
-    videoPreview?.destroy();
-    videoPreview = undefined;
-  };
-}
-
-export function renderVideoPreview(time: number) {
-  if (!videoPreview) return false;
-  return videoPreview.render(time);
-}

src/lib/video-preview/mp4box.d.ts

@@ -1,93 +0,0 @@
-declare module 'mp4box' {
-
-  export interface MP4MediaTrack {
-    id: number;
-    created: Date;
-    modified: Date;
-    movie_duration: number;
-    layer: number;
-    alternate_group: number;
-    volume: number;
-    track_width: number;
-    track_height: number;
-    timescale: number;
-    duration: number;
-    bitrate: number;
-    codec: string;
-    language: string;
-    nb_samples: number;
-
-  }
-
-  export interface MP4VideoData {
-    width: number;
-    height: number;
-  }
-
-  export interface MP4VideoTrack extends MP4MediaTrack {
-    video: MP4VideoData;
-    [key: string]: any;
-  }
-
-  export interface MP4AudioData {
-    sample_rate: number;
-    channel_count: number;
-    sample_size: number;
-  }
-
-  export interface MP4AudioTrack extends MP4MediaTrack {
-    audio: MP4AudioData;
-    [key: string]: any;
-  }
-
-  export type MP4Track = MP4VideoTrack | MP4AudioTrack;
-
-  export class DataStream {
-    buffer: ArrayBuffer;
-
-    static BIG_ENDIAN: number;
-    constructor(buffer?: ArrayBuffer, offset?: number, endianness?: number);
-  }
-
-  export interface MP4Info {
-    duration: number;
-    timescale: number;
-    fragment_duration: number;
-    isFragmented: boolean;
-    isProgressive: boolean;
-    hasIOD: boolean;
-    brands: string[];
-    created: Date;
-    modified: Date;
-    tracks: MP4Track[];
-    videoTracks: MP4VideoTrack[];
-  }
-
-  export type MP4ArrayBuffer = ArrayBuffer & { fileStart: number };
-
-  export interface MP4File {
-
-    onMoovStart?: () => void;
-    onReady?: (info: MP4Info) => void;
-    onSamples?: (trackId: number, ref: any, samples: any) => void;
-    onError?: (e: string) => void;
-
-    processSamples(last: boolean): void;
-
-    getTrackById(id: number): MP4Track;
-
-    setExtractionOptions(id: number, user?: any, options?: any): void;
-
-    appendBuffer(data: MP4ArrayBuffer): number;
-    start(): void;
-    stop(): void;
-    flush(): void;
-    seek(time: number, useRap?: boolean): { offset: number; time: number };
-    releaseUsedSamples(id: number, sampleNumber: number): void;
-  }
-
-  export function createFile(): MP4File;
-
-  export {};
-
-}

src/lib/video-preview/requestPart.ts

@@ -1,62 +0,0 @@
-import generateUniqueId from '../../util/generateUniqueId';
-import { pause } from '../../util/schedulers';
-
-declare const self: WorkerGlobalScope;
-
-type RequestStates = {
-  resolve: (response: ArrayBuffer) => void;
-  reject: () => void;
-};
-
-type RequestPartParams = { url: string; start: number; end: number };
-
-const PART_TIMEOUT = 30000;
-
-const requestStates = new Map<string, RequestStates>();
-
-export function requestPart(params: RequestPartParams): Promise<ArrayBuffer | undefined> {
-  const messageId = generateUniqueId();
-  const requestState = {} as RequestStates;
-
-  let isResolved = false;
-  const promise = Promise.race([
-    pause(PART_TIMEOUT).then(() => (isResolved ? undefined : Promise.reject(new Error('ERROR_PART_TIMEOUT')))),
-    new Promise<ArrayBuffer>((resolve, reject) => {
-      Object.assign(requestState, { resolve, reject });
-    }),
-  ]);
-
-  requestStates.set(messageId, requestState);
-
-  promise
-    .catch(() => undefined)
-    .finally(() => {
-      requestStates.delete(messageId);
-      isResolved = true;
-    });
-
-  const message = {
-    type: 'requestPart',
-    messageId,
-    params,
-  };
-
-  postMessage(message);
-
-  return promise;
-}
-
-self.addEventListener('message', (e) => {
-  const { type, messageId, result } = (e as any).data as {
-    type: string;
-    messageId: string;
-    result: ArrayBuffer;
-  };
-
-  if (type === 'partResponse') {
-    const requestState = requestStates.get(messageId);
-    if (requestState) {
-      requestState.resolve(result);
-    }
-  }
-});

src/lib/video-preview/video-preview.worker.ts

@@ -1,86 +0,0 @@
-import type { CancellableCallback } from '../../util/PostMessageConnector';
-
-import { createWorkerInterface } from '../../util/createPostMessageInterface';
-
-import { MP4Demuxer } from './MP4Demuxer';
-
-let decoder: any;
-let demuxer: any;
-let onDestroy: VoidFunction | undefined;
-
-function init(
-  url: string,
-  maxFrames: number,
-  workerIndex: number,
-  workersTotal: number,
-  onFrame: CancellableCallback,
-) {
-  const hasWebCodecs = 'VideoDecoder' in globalThis;
-  if (!hasWebCodecs) {
-    // eslint-disable-next-line no-console
-    console.log('[Video Preview] WebCodecs not supported');
-    return new Promise<void>((resolve) => {
-      onDestroy = resolve;
-    });
-  }
-
-  const decodedFrames = new Set<number>();
-
-  // @ts-ignore
-  decoder = new VideoDecoder({
-    async output(frame: any) {
-      const time = frame.timestamp / 1e6;
-      const seconds = Math.floor(time);
-      // Only render whole second frames
-      if (!decodedFrames.has(seconds)) {
-        const bitmap = await createImageBitmap(frame);
-        decodedFrames.add(seconds);
-        onFrame(seconds, bitmap);
-      }
-      frame.close();
-    },
-    error(e: any) {
-      // eslint-disable-next-line no-console
-      console.error('[Video Preview] error', e);
-    },
-  });
-
-  demuxer = new MP4Demuxer(url, {
-    stepOffset: workerIndex,
-    stepMultiplier: workersTotal,
-    isPolyfill: !hasWebCodecs,
-    maxFrames,
-    onConfig(config) {
-      decoder?.configure(config);
-    },
-    onChunk(chunk) {
-      if (decoder?.state !== 'configured') return;
-      decoder?.decode(chunk);
-    },
-  });
-
-  return new Promise<void>((resolve) => {
-    onDestroy = resolve;
-  });
-}
-
-function destroy() {
-  try {
-    decoder?.close();
-    demuxer?.close();
-  } catch {
-    // Ignore
-  }
-  decoder = undefined;
-  demuxer = undefined;
-  onDestroy?.();
-}
-
-const api = {
-  'video-preview:init': init,
-  'video-preview:destroy': destroy,
-};
-
-createWorkerInterface(api, 'media');
-
-export type VideoPreviewApi = typeof api;

src/util/browser/globalEnvironment.ts

@@ -5,7 +5,6 @@ declare const globalThis: ServiceWorkerGlobalScope & WorkerGlobalScope & SharedW
 export const IS_MULTIACCOUNT_SUPPORTED = 'SharedWorker' in globalThis;
 export const IS_INTL_LIST_FORMAT_SUPPORTED = 'ListFormat' in Intl;
 export const IS_BAD_URL_PARSER = new URL('tg://host').host !== 'host';
-export const ARE_WEBCODECS_SUPPORTED = 'VideoDecoder' in globalThis;
 
 export const IS_TAURI = isTauri();
 // @ts-expect-error no types for electron

src/util/media/StoryboardParser.ts

@@ -0,0 +1,63 @@
+interface PreviewFrameInfo {
+  top: number;
+  left: number;
+  width: number;
+  height: number;
+  time: number;
+}
+
+export default class StoryboardParser {
+  private frames: PreviewFrameInfo[];
+
+  /**
+   * Can throw error if the storyboard map data is invalid
+   */
+  constructor(storyboardMapData: string) {
+    const [_file, widthLine, heightLine, ...frameLines] = storyboardMapData.split('\n');
+    const width = Number.parseFloat(widthLine.split('=')[1]);
+    const height = Number.parseFloat(heightLine.split('=')[1]);
+
+    if (Number.isNaN(width) || Number.isNaN(height)) {
+      throw new Error('Invalid storyboard map frame size');
+    }
+
+    this.frames = frameLines.map((frame) => {
+      if (!frame.trim().length) {
+        return undefined;
+      }
+
+      const [timeStr, leftStr, topStr] = frame.split(',');
+      const info = {
+        time: Number.parseFloat(timeStr),
+        left: Number.parseInt(leftStr, 10),
+        top: Number.parseInt(topStr, 10),
+        width,
+        height,
+      };
+
+      if (Number.isNaN(info.time) || Number.isNaN(info.left) || Number.isNaN(info.top)) {
+        throw new Error('Invalid storyboard map data');
+      }
+      return info;
+    }).filter(Boolean);
+
+    if (this.frames.length === 0) {
+      throw new Error('Missing frames in storyboard map data');
+    }
+  }
+
+  getNearestPreview(time: number): PreviewFrameInfo {
+    // Binary search for the nearest frame
+    let left = 0;
+    let right = this.frames.length - 1;
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2);
+      if (this.frames[mid].time <= time) {
+        left = mid + 1;
+      } else {
+        right = mid - 1;
+      }
+    }
+    return this.frames[Math.max(0, left - 1)];
+  }
+}