diff --git a/package-lock.json b/package-lock.json index b637349b0..79cb2f6b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,6 @@ "emoji-data-ios": "git+https://github.com/korenskoy/emoji-data-ios#443f1c9", "idb-keyval": "^6.2.2", "lowlight": "^3.3.0", - "mp4box": "^0.5.4", "music-metadata": "^11.10.0", "opus-recorder": "github:Ajaxy/opus-recorder#116830a", "os-browserify": "^0.3.0", @@ -181,6 +180,7 @@ "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -2100,6 +2100,7 @@ "integrity": "sha512-eohl3hKTiVyD1ilYdw9T0OiB4hnjef89e3dMYKz+mVKDzj+5IteTseASUsOB+EU9Tf6VNTCjDePcP6wkDGmLKQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@keyv/serialize": "^1.1.1" } @@ -2218,6 +2219,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" }, @@ -2241,6 +2243,7 @@ } ], "license": "MIT", + "peer": true, "engines": { "node": ">=18" } @@ -3977,8 +3980,7 @@ "resolved": "https://registry.npmjs.org/@module-federation/error-codes/-/error-codes-0.21.4.tgz", "integrity": "sha512-ClpL5MereWNXh+EgDjz7w4RrC1JlisQTvXDa1gLxpviHafzNDfdViVmuhi9xXVuj+EYo8KU70Y999KHhk9424Q==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@module-federation/runtime": { "version": "0.21.4", @@ -3986,7 +3988,6 @@ "integrity": "sha512-wgvGqryurVEvkicufJmTG0ZehynCeNLklv8kIk5BLIsWYSddZAE+xe4xov1kgH5fIJQAoQNkRauFFjVNlHoAkA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@module-federation/error-codes": "0.21.4", "@module-federation/runtime-core": "0.21.4", @@ -3999,7 +4000,6 @@ "integrity": "sha512-SGpmoOLGNxZofpTOk6Lxb2ewaoz5wMi93AFYuuJB04HTVcngEK+baNeUZ2D/xewrqNIJoMY6f5maUjVfIIBPUA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@module-federation/error-codes": "0.21.4", "@module-federation/sdk": "0.21.4" @@ -4011,7 +4011,6 @@ "integrity": "sha512-RzFKaL0DIjSmkn76KZRfzfB6dD07cvID84950jlNQgdyoQFUGkqD80L6rIpVCJTY/R7LzR3aQjHnoqmq4JPo3w==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@module-federation/runtime": "0.21.4", "@module-federation/webpack-bundler-runtime": "0.21.4" @@ -4022,8 +4021,7 @@ "resolved": "https://registry.npmjs.org/@module-federation/sdk/-/sdk-0.21.4.tgz", "integrity": "sha512-tzvhOh/oAfX++6zCDDxuvioHY4Jurf8vcfoCbKFxusjmyKr32GPbwFDazUP+OPhYCc3dvaa9oWU6X/qpUBLfJw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@module-federation/webpack-bundler-runtime": { "version": "0.21.4", @@ -4031,7 +4029,6 @@ "integrity": "sha512-dusmR3uPnQh9u9ChQo3M+GLOuGFthfvnh7WitF/a1eoeTfRmXqnMFsXtZCUK+f/uXf+64874Zj/bhAgbBcVHZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@module-federation/runtime": "0.21.4", "@module-federation/sdk": "0.21.4" @@ -4073,7 +4070,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "@emnapi/core": "^1.5.0", "@emnapi/runtime": "^1.5.0", @@ -4524,7 +4520,6 @@ "integrity": "sha512-liRgxMjHWDL225c41pH4ZcFtPN48LM0+St3iylwavF5JFSqBv86R/Cv5+M+WLrhcihCQsxDwBofipyosJIFmmA==", "dev": true, "license": "MIT", - "peer": true, "optionalDependencies": { "@rspack/binding-darwin-arm64": "1.6.3", "@rspack/binding-darwin-x64": "1.6.3", @@ -4550,8 +4545,7 @@ "optional": true, "os": [ "darwin" - ], - "peer": true + ] }, "node_modules/@rspack/binding-darwin-x64": { "version": "1.6.3", @@ -4565,8 +4559,7 @@ "optional": true, "os": [ "darwin" - ], - "peer": true + ] }, "node_modules/@rspack/binding-linux-arm64-gnu": { "version": "1.6.3", @@ -4580,8 +4573,7 @@ "optional": true, "os": [ "linux" - ], - "peer": true + ] }, "node_modules/@rspack/binding-linux-arm64-musl": { "version": "1.6.3", @@ -4595,8 +4587,7 @@ "optional": true, "os": [ "linux" - ], - "peer": true + ] }, "node_modules/@rspack/binding-linux-x64-gnu": { "version": "1.6.3", @@ -4610,8 +4601,7 @@ "optional": true, "os": [ "linux" - ], - "peer": true + ] }, "node_modules/@rspack/binding-linux-x64-musl": { "version": "1.6.3", @@ -4625,8 +4615,7 @@ "optional": true, "os": [ "linux" - ], - "peer": true + ] }, "node_modules/@rspack/binding-wasm32-wasi": { "version": "1.6.3", @@ -4638,7 +4627,6 @@ "dev": true, "license": "MIT", "optional": true, - "peer": true, "dependencies": { "@napi-rs/wasm-runtime": "1.0.7" } @@ -4655,8 +4643,7 @@ "optional": true, "os": [ "win32" - ], - "peer": true + ] }, "node_modules/@rspack/binding-win32-ia32-msvc": { "version": "1.6.3", @@ -4670,8 +4657,7 @@ "optional": true, "os": [ "win32" - ], - "peer": true + ] }, "node_modules/@rspack/binding-win32-x64-msvc": { "version": "1.6.3", @@ -4685,8 +4671,7 @@ "optional": true, "os": [ "win32" - ], - "peer": true + ] }, "node_modules/@rspack/core": { "version": "1.6.3", @@ -4717,8 +4702,7 @@ "resolved": "https://registry.npmjs.org/@rspack/lite-tapable/-/lite-tapable-1.1.0.tgz", "integrity": "sha512-E2B0JhYFmVAwdDiG14+DW0Di4Ze4Jg10Pc4/lILUrd5DRCaklduz2OvJ5HYQ6G+hd+WTzqQb3QnDNfK4yvAFYw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@rtsao/scc": { "version": "1.1.0", @@ -5855,6 +5839,7 @@ "integrity": "sha512-keKxkZMqnDicuvFoJbzrhbtdLSPhj/rZThDlKWCDbgXmUg0rEUFtRssDXKYmtXluZlIqiC5VqkCgRwzuyLHKHw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -6008,6 +5993,7 @@ "integrity": "sha512-R48VhmTJqplNyDxCyqqVkFSZIx1qX6PzwqgcXn1olLrzxcSBDlOsbtcnQuQhNtnNiJ4Xe5gREI1foajYaYU2Vg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.46.4", @@ -6038,6 +6024,7 @@ "integrity": "sha512-tK3GPFWbirvNgsNKto+UmB/cRtn6TZfyw0D6IKrW55n6Vbs7KJoZtI//kpTKzE/DUmmnAFD8/Ca46s7Obs92/w==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.46.4", "@typescript-eslint/types": "8.46.4", @@ -6726,7 +6713,6 @@ "integrity": "sha512-u8d0pJ5YFgneF/GuvEiDA61Tf1VDomHHYMjv/wc9XzYj7nopltpG96nXN5dJRstxZhcNpV1g+nT6CydO7pHbjA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=18.12.0" }, @@ -6741,7 +6727,6 @@ "integrity": "sha512-coEmDzc2u/ffMvuW9aCjoRzNSPDl/XLuhPdlFRpT9tZHmJ/039az33CE7uH+8s0uL1j5ZNtfdv0HkfaKRBGJsQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=18.12.0" }, @@ -6860,6 +6845,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -7903,6 +7889,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.25", "caniuse-lite": "^1.0.30001754", @@ -9665,7 +9652,6 @@ "integrity": "sha512-+zUomDcLXsVkQ37vUqWBvQwLaLlj8eZPSi61llaEFAVBY5mhcXdaSw1pSJVl4yTYD5g/gEfpNl28YYk4IPvrrg==", "dev": true, "license": "MIT", - "peer": true, "bin": { "envinfo": "dist/cli.js" }, @@ -9965,6 +9951,7 @@ "integrity": "sha512-BhHmn2yNOFA9H9JmmIVKJmd288g9hrVRDkdoIgRCRuSySRUHH7r/DI6aAXW9T1WwUuY3DFgrcaqB+deURBLR5g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -10973,7 +10960,6 @@ "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "bin": { "flat": "cli.js" } @@ -12393,7 +12379,6 @@ "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10.13.0" } @@ -13193,6 +13178,7 @@ "integrity": "sha512-F26gjC0yWN8uAA5m5Ss8ZQf5nDHWGlN/xWZIh8S5SRbsEKBovwZhxGd6LJlbZYxBgCYOtreSUyb8hpXyGC5O4A==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "30.2.0", "@jest/types": "30.2.0", @@ -14010,6 +13996,7 @@ "integrity": "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==", "dev": true, "license": "MIT", + "peer": true, "bin": { "jiti": "lib/jiti-cli.mjs" } @@ -14053,6 +14040,7 @@ "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "cssstyle": "^4.2.1", "data-urls": "^5.0.0", @@ -15329,12 +15317,6 @@ "node": ">=10" } }, - "node_modules/mp4box": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/mp4box/-/mp4box-0.5.4.tgz", - "integrity": "sha512-GcCH0fySxBurJtvr0dfhz0IxHZjc1RP+F+I8xw+LIwkU1a+7HJx8NCDiww1I5u4Hz6g4eR1JlGADEGJ9r4lSfA==", - "license": "BSD-3-Clause" - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -16404,6 +16386,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -16658,6 +16641,7 @@ "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -17031,7 +17015,6 @@ "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "resolve": "^1.20.0" }, @@ -17583,6 +17566,7 @@ "integrity": "sha512-Dqh7SiYcaFtdv5Wvku6QgS5IGPm281L+ZtVD1U2FJa7Q0EFRlq8Z3sjYtz6gYObsYThUOz9ArwFqPZx+1azILQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "chokidar": "^4.0.0", "immutable": "^5.0.2", @@ -17685,6 +17669,7 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -18885,6 +18870,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@csstools/css-parser-algorithms": "^3.0.5", "@csstools/css-tokenizer": "^3.0.4", @@ -19928,7 +19914,8 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" + "license": "0BSD", + "peer": true }, "node_modules/tsx": { "version": "4.20.6", @@ -19936,6 +19923,7 @@ "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "~0.25.0", "get-tsconfig": "^4.7.5" @@ -20171,6 +20159,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -20566,6 +20555,7 @@ "integrity": "sha512-7h/weGm9d/ywQ6qzJ+Xy+r9n/3qgp/thalBbpOi5i223dPXKi04IBtqPN9nTd+jBc7QKfvDbaBnFipYp4sJAUQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -20615,7 +20605,6 @@ "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", @@ -20659,7 +20648,6 @@ "integrity": "sha512-4B4OijXeVNOPZlYA2oEwWOTkzyltLao+xbotHQeqN++Rv27Y6s818+n2Qkp8q+Fxhn0t/5lA5X1Mxktud8eayQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=14.17.0" } @@ -20670,7 +20658,6 @@ "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -20888,7 +20875,6 @@ "integrity": "sha512-hXXvrjtx2PLYx4qruKl+kyRSLc52V+cCvMxRjmKwoA+CBbbF5GfIBtR6kCvl0fYGqTUPKB+1ktVmTHqMOzgCBg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "clone-deep": "^4.0.1", "flat": "^5.0.2", @@ -21171,8 +21157,7 @@ "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.1.tgz", "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/word-wrap": { "version": "1.2.5", diff --git a/package.json b/package.json index dbffc7d5a..7b2c7b151 100644 --- a/package.json +++ b/package.json @@ -136,7 +136,6 @@ "emoji-data-ios": "git+https://github.com/korenskoy/emoji-data-ios#443f1c9", "idb-keyval": "^6.2.2", "lowlight": "^3.3.0", - "mp4box": "^0.5.4", "music-metadata": "^11.10.0", "opus-recorder": "github:Ajaxy/opus-recorder#116830a", "os-browserify": "^0.3.0", diff --git a/src/api/gramjs/apiBuilders/messageContent.ts b/src/api/gramjs/apiBuilders/messageContent.ts index 1754e040c..80bbe1fe5 100644 --- a/src/api/gramjs/apiBuilders/messageContent.ts +++ b/src/api/gramjs/apiBuilders/messageContent.ts @@ -29,10 +29,17 @@ import type { ApiWebPageStoryData, BoughtPaidMedia, MediaContent, + StoryboardInfo, } from '../../types'; import type { UniversalMessage } from './messages'; -import { SUPPORTED_PHOTO_CONTENT_TYPES, SUPPORTED_VIDEO_CONTENT_TYPES, VIDEO_WEBM_TYPE } from '../../../config'; +import { + STORYBOARD_MAP_MIME, + STORYBOARD_MIME, + SUPPORTED_PHOTO_CONTENT_TYPES, + SUPPORTED_VIDEO_CONTENT_TYPES, + VIDEO_WEBM_TYPE, +} from '../../../config'; import { addTimestampEntities } from '../../../util/dates/timestamp'; import { generateWaveform } from '../../../util/generateWaveform'; import { pick } from '../../../util/iteratees'; @@ -143,8 +150,7 @@ export function buildMessageMediaContent( if (photo) return { photo }; const video = buildVideo(media); - const altVideos = buildAltVideos(media); - if (video) return { video, altVideos }; + if (video) return { video }; const audio = buildAudio(media); if (audio) return { audio }; @@ -208,7 +214,7 @@ function buildPhoto(media: GramJs.TypeMessageMedia): ApiPhoto | undefined { return buildApiPhoto(media.photo, media.spoiler); } -export function buildVideoFromDocument(document: GramJs.Document, params?: { +export function buildVideoFromDocument(document: GramJs.Document, altDocuments?: GramJs.TypeDocument[], params?: { isSpoiler?: boolean; timestamp?: number; }): ApiVideo | undefined { @@ -216,6 +222,8 @@ export function buildVideoFromDocument(document: GramJs.Document, params?: { return undefined; } + const altVideos = altDocuments && buildAltVideosFromDocuments(altDocuments); + const { isSpoiler, timestamp } = params || {}; const { @@ -249,6 +257,7 @@ export function buildVideoFromDocument(document: GramJs.Document, params?: { } = videoAttr; const waveform = isRound ? generateWaveform(duration) : undefined; + const storyboardInfo = altDocuments && buildStoryboardInfoFromDocuments(altDocuments); return { mediaType: 'video', @@ -268,7 +277,9 @@ export function buildVideoFromDocument(document: GramJs.Document, params?: { hasVideoPreview, previewPhotoSizes, waveform, - ...(nosound && { noSound: true }), + noSound: nosound, + altVideos, + storyboardInfo, }; } @@ -315,17 +326,19 @@ function buildVideo(media: GramJs.TypeMessageMedia): ApiVideo | undefined { return undefined; } - return buildVideoFromDocument(media.document, { isSpoiler: media.spoiler, timestamp: media.videoTimestamp }); + return buildVideoFromDocument( + media.document, + media.altDocuments, + { isSpoiler: media.spoiler, timestamp: media.videoTimestamp }, + ); } -function buildAltVideos(media: GramJs.TypeMessageMedia): ApiVideo[] | undefined { - if (!(media instanceof GramJs.MessageMediaDocument) || !media.altDocuments) { - return undefined; - } - - const altVideos = media.altDocuments.filter((d): d is GramJs.Document => ( +function buildAltVideosFromDocuments(altDocuments: GramJs.TypeDocument[], params?: { + isSpoiler?: boolean; +}): ApiVideo[] | undefined { + const altVideos = altDocuments.filter((d): d is GramJs.Document => ( d instanceof GramJs.Document && d.mimeType.startsWith('video') - )).map((alt) => buildVideoFromDocument(alt, { isSpoiler: media.spoiler })) + )).map((alt) => buildVideoFromDocument(alt, undefined, params)) .filter(Boolean); if (!altVideos.length) { return undefined; @@ -334,6 +347,34 @@ function buildAltVideos(media: GramJs.TypeMessageMedia): ApiVideo[] | undefined return altVideos; } +function buildStoryboardInfoFromDocuments(documents: GramJs.TypeDocument[]): StoryboardInfo | undefined { + const storyboardMtpFile = documents.find((d): d is GramJs.Document => ( + d instanceof GramJs.Document && d.mimeType === STORYBOARD_MIME + )); + const storyboardMapMtpFile = documents.find((d): d is GramJs.Document => ( + d instanceof GramJs.Document && d.mimeType === STORYBOARD_MAP_MIME + )); + + const storyboardFile = storyboardMtpFile && buildApiDocument(storyboardMtpFile); + const storyboardMapFile = storyboardMapMtpFile && buildApiDocument(storyboardMapMtpFile); + + const sizeAttribute = storyboardMapMtpFile?.attributes.find((a): a is GramJs.DocumentAttributeImageSize => ( + a instanceof GramJs.DocumentAttributeImageSize + )); + + const frameSize = sizeAttribute && { width: sizeAttribute.w, height: sizeAttribute.h }; + + if (!storyboardFile || !storyboardMapFile || !frameSize) { + return undefined; + } + + return { + storyboardFile, + storyboardMapFile, + frameSize, + }; +} + function buildAudio(media: GramJs.TypeMessageMedia): ApiAudio | undefined { if ( !(media instanceof GramJs.MessageMediaDocument) diff --git a/src/api/gramjs/helpers/localDb.ts b/src/api/gramjs/helpers/localDb.ts index 1dea37f94..853c64d5a 100644 --- a/src/api/gramjs/helpers/localDb.ts +++ b/src/api/gramjs/helpers/localDb.ts @@ -42,6 +42,13 @@ export function addMediaToLocalDb(media: GramJs.TypeMessageMedia, context?: Medi if (media instanceof GramJs.MessageMediaDocument && media.document) { const document = addMessageRepairInfo(media.document, context); addDocumentToLocalDb(document); + + if (media.altDocuments) { + for (const altDocument of media.altDocuments) { + const doc = addMessageRepairInfo(altDocument, context); + addDocumentToLocalDb(doc); + } + } } if (media instanceof GramJs.MessageMediaGame) { diff --git a/src/api/types/messages.ts b/src/api/types/messages.ts index be7c1bb8a..c963533b6 100644 --- a/src/api/types/messages.ts +++ b/src/api/types/messages.ts @@ -99,6 +99,12 @@ type ApiStickerSetInfoMissing = { export type ApiStickerSetInfo = ApiStickerSetInfoShortName | ApiStickerSetInfoId | ApiStickerSetInfoMissing; +export interface StoryboardInfo { + storyboardFile: ApiDocument; + storyboardMapFile: ApiDocument; + frameSize: ApiDimensions; +} + export interface ApiVideo { mediaType: 'video'; id: string; @@ -120,6 +126,8 @@ export interface ApiVideo { noSound?: boolean; waveform?: number[]; timestamp?: number; + altVideos?: ApiVideo[]; + storyboardInfo?: StoryboardInfo; } export interface ApiAudio { @@ -584,7 +592,6 @@ export type MediaContent = { text?: ApiFormattedTextWithEmojiOnlyCount; photo?: ApiPhoto; video?: ApiVideo; - altVideos?: ApiVideo[]; document?: ApiDocument; sticker?: ApiSticker; contact?: ApiContact; diff --git a/src/components/mediaViewer/MediaViewerContent.tsx b/src/components/mediaViewer/MediaViewerContent.tsx index 70d5bc6c4..b6e2b88bf 100644 --- a/src/components/mediaViewer/MediaViewerContent.tsx +++ b/src/components/mediaViewer/MediaViewerContent.tsx @@ -13,7 +13,6 @@ import { selectIsMessageProtected, selectTabState, } from '../../global/selectors'; import { selectMessageTimestampableDuration } from '../../global/selectors/media'; -import { ARE_WEBCODECS_SUPPORTED } from '../../util/browser/globalEnvironment'; import { IS_TOUCH_ENV } from '../../util/browser/windowEnvironment'; import buildClassName from '../../util/buildClassName'; import stopEvent from '../../util/stopEvent'; @@ -99,7 +98,6 @@ const MediaViewerContent = ({ bestData, dimensions, isGif, - isLocal, isVideoAvatar, mediaSize, loadProgress, @@ -159,7 +157,6 @@ const MediaViewerContent = ({ fileSize={mediaSize!} isMediaViewerOpen={isOpen && isActive} isProtected={isProtected} - isPreviewDisabled={!ARE_WEBCODECS_SUPPORTED || isLocal} noPlay={!isActive} onClose={onClose} isMuted @@ -207,6 +204,7 @@ const MediaViewerContent = ({ void; onSeekStart: () => void; }; @@ -42,19 +45,17 @@ type OwnProps = { const LOCK_TIMEOUT = 250; let cancelAnimation: ReturnType | undefined; -const SeekLine: FC = ({ +const SeekLine = ({ + storyboardInfo, duration, bufferedRanges, isReady, - posterSize, playbackRate, - url, isActive, isPlaying, - isPreviewDisabled, onSeek, onSeekStart, -}) => { +}: OwnProps) => { const seekerRef = useRef(); const [getCurrentTimeSignal] = useCurrentTimeSignal(); const [getIsWaiting] = useVideoWaitingSignal(); @@ -65,30 +66,48 @@ const SeekLine: FC = ({ const isLockedRef = useRef(false); const [isPreviewVisible, setPreviewVisible] = useState(false); const [isSeeking, setIsSeeking] = useState(false); - const previewCanvasRef = useRef(); + const previewContainerRef = useRef(); const previewRef = useRef(); const progressRef = useRef(); const previewTimeRef = useRef(); + const storyboardParser = useRef(); - const previewSize = useMemo(() => { - return getPreviewDimensions(posterSize?.width || 0, posterSize?.height || 0); - }, [posterSize]); + const storyboardHash = storyboardInfo && getDocumentMediaHash(storyboardInfo.storyboardFile, 'full'); + const storyboardMapHash = storyboardInfo && getDocumentMediaHash(storyboardInfo.storyboardMapFile, 'full'); - const setPreview = useLastCallback((time: number) => { - time = Math.floor(time); - setPreviewTime(time); - renderVideoPreview(time); - }); - - useEffect(() => { - if (isPreviewDisabled || !url || !isReady) return undefined; - return createVideoPreviews(url, previewCanvasRef.current!); - }, [url, isReady, isPreviewDisabled]); + const storyboardUrl = useMedia(storyboardHash, !isReady); + const storyboardMapData = useMedia(storyboardMapHash, !isReady, ApiMediaFormat.Text); useEffect(() => { setPreviewVisible(false); }, [isActive]); + useEffect(() => { + if (!storyboardMapData) return; + try { + storyboardParser.current = new StoryboardParser(storyboardMapData); + } catch (error) { + if (DEBUG) { + // eslint-disable-next-line no-console + console.error(`Error parsing storyboard map data`, error, storyboardMapData); + } + } + }, [storyboardMapData]); + + const setPreview = useLastCallback((time: number) => { + const previewContainer = previewContainerRef.current; + if (!storyboardParser.current || !previewContainer) return; + const frame = storyboardParser.current.getNearestPreview(time); + + setPreviewTime(Math.floor(frame.time)); + + requestMutation(() => { + setExtraStyles(previewContainer, { + backgroundPosition: `${-frame.left}px ${-frame.top}px`, + }); + }); + }); + useEffect(() => { if (cancelAnimation) cancelAnimation(); cancelAnimation = undefined; @@ -154,7 +173,7 @@ const SeekLine: FC = ({ const getPreviewProps = (e: MouseEvent | TouchEvent) => { const pageX = getPointerPosition(e).x; const t = clamp(duration * ((pageX - seekerSize.left) / seekerSize.width), 0, duration); - if (isPreviewDisabled) return [t, 0]; + if (!storyboardInfo) return [t, 0]; if (!seekerSize.width) seekerSize = seeker.getBoundingClientRect(); const preview = previewRef.current!; const o = clamp( @@ -204,7 +223,7 @@ const SeekLine: FC = ({ onDrag: handleSeek, }); - if (IS_TOUCH_ENV || isPreviewDisabled) { + if (IS_TOUCH_ENV) { return cleanup; } @@ -230,29 +249,27 @@ const SeekLine: FC = ({ seeker.removeEventListener('mouseleave', handleSeekMouseLeave); }; }, [ - duration, - setPreview, - isActive, - onSeek, - onSeekStart, - setPreviewOffset, - setSelectedTime, - setIsSeeking, - isPreviewDisabled, - playbackRate, + duration, setPreview, isActive, onSeek, onSeekStart, setPreviewOffset, setSelectedTime, setIsSeeking, + isPreviewVisible, playbackRate, storyboardInfo, ]); return (
- {!isPreviewDisabled && ( + {storyboardInfo && ( - +
diff --git a/src/components/mediaViewer/VideoPlayer.tsx b/src/components/mediaViewer/VideoPlayer.tsx index 2a17085b0..644887b4b 100644 --- a/src/components/mediaViewer/VideoPlayer.tsx +++ b/src/components/mediaViewer/VideoPlayer.tsx @@ -5,7 +5,7 @@ import { } from '../../lib/teact/teact'; import { getActions } from '../../global'; -import type { ApiDimensions } from '../../api/types'; +import type { ApiDimensions, StoryboardInfo } from '../../api/types'; import { IS_IOS, IS_TOUCH_ENV, IS_YA_BROWSER } from '../../util/browser/windowEnvironment'; import getPointerPosition from '../../util/events/getPointerPosition'; @@ -33,12 +33,12 @@ import './VideoPlayer.scss'; type OwnProps = { url?: string; + storyboardInfo?: StoryboardInfo; isGif?: boolean; posterData?: string; posterSize?: ApiDimensions; loadProgress?: number; fileSize: number; - isPreviewDisabled?: boolean; isMediaViewerOpen?: boolean; noPlay?: boolean; volume: number; @@ -61,6 +61,7 @@ const REWIND_STEP = 5; // Seconds const VideoPlayer: FC = ({ url, + storyboardInfo, isGif, posterData, posterSize, @@ -75,7 +76,6 @@ const VideoPlayer: FC = ({ shouldCloseOnClick, isProtected, isClickDisabled, - isPreviewDisabled, isSponsoredMessage, timestamp, handleSponsoredClick, @@ -378,7 +378,7 @@ const VideoPlayer: FC = ({ )} {!isGif && !isSponsoredMessage && !isUnsupported && ( = ({ isFullscreenSupported={Boolean(setFullscreen)} isPictureInPictureSupported={isPictureInPictureSupported} isFullscreen={isFullscreen} - isPreviewDisabled={isPreviewDisabled} fileSize={fileSize} duration={duration} isReady={isReady} - posterSize={posterSize} isForceMobileVersion={isForceMobileVersion} onSeek={handleSeek} onChangeFullscreen={handleFullscreenChange} diff --git a/src/components/mediaViewer/VideoPlayerControls.scss b/src/components/mediaViewer/VideoPlayerControls.scss index 84af2d5da..fe157ca89 100644 --- a/src/components/mediaViewer/VideoPlayerControls.scss +++ b/src/components/mediaViewer/VideoPlayerControls.scss @@ -58,13 +58,12 @@ } .volume-slider { - --volume-slider-width: 4rem; --slider-color: #fff; --color-borders: rgba(255, 255, 255, 0.5); width: 0; - margin-bottom: 0; + margin: 0; margin-left: -0.75rem; padding: 0.5rem 0.5rem 0.5rem 0.5rem; /* stylelint-disable-next-line plugin/no-low-performance-animation-properties */ diff --git a/src/components/mediaViewer/VideoPlayerControls.tsx b/src/components/mediaViewer/VideoPlayerControls.tsx index 7f8cf2271..5d12e727d 100644 --- a/src/components/mediaViewer/VideoPlayerControls.tsx +++ b/src/components/mediaViewer/VideoPlayerControls.tsx @@ -7,7 +7,7 @@ import { useSignal, } from '../../lib/teact/teact'; -import type { ApiDimensions } from '../../api/types'; +import type { StoryboardInfo } from '../../api/types'; import type { BufferedRange } from '../../hooks/useBuffering'; import type { IconName } from '../../types/icons'; @@ -33,7 +33,7 @@ import SeekLine from './SeekLine'; import './VideoPlayerControls.scss'; type OwnProps = { - url?: string; + storyboardInfo?: StoryboardInfo; bufferedRanges: BufferedRange[]; bufferedProgress: number; duration: number; @@ -44,12 +44,10 @@ type OwnProps = { isFullscreenSupported: boolean; isPictureInPictureSupported: boolean; isFullscreen: boolean; - isPreviewDisabled?: boolean; isBuffered: boolean; volume: number; isMuted: boolean; playbackRate: number; - posterSize?: ApiDimensions; onChangeFullscreen: (e: React.MouseEvent) => void; onPictureInPictureChange?: () => void; onVolumeClick: () => void; @@ -74,7 +72,7 @@ const PLAYBACK_RATES = [ const HIDE_CONTROLS_TIMEOUT_MS = 3000; const VideoPlayerControls: FC = ({ - url, + storyboardInfo, bufferedRanges, bufferedProgress, duration, @@ -85,16 +83,14 @@ const VideoPlayerControls: FC = ({ isFullscreenSupported, isFullscreen, isBuffered, - isPreviewDisabled, volume, isMuted, playbackRate, - posterSize, + isPictureInPictureSupported, onChangeFullscreen, onVolumeClick, onVolumeChange, onPlaybackRateChange, - isPictureInPictureSupported, onPictureInPictureChange, onPlayPause, onSeek, @@ -168,12 +164,10 @@ const VideoPlayerControls: FC = ({ onClick={stopEvent} > { if (isAvatar) { return isVideoAvatar ? VIDEO_AVATAR_FULL_DIMENSIONS : AVATAR_FULL_DIMENSIONS; @@ -138,7 +136,6 @@ export const useMediaProps = ({ dimensions, contentType, isVideoAvatar, - isLocal, loadProgress, mediaSize, }; diff --git a/src/components/story/hooks/useStoryPreloader.ts b/src/components/story/hooks/useStoryPreloader.ts index b6773208c..23ecc1e4f 100644 --- a/src/components/story/hooks/useStoryPreloader.ts +++ b/src/components/story/hooks/useStoryPreloader.ts @@ -101,7 +101,7 @@ function getPreloadMediaHashes(peerId: string, storyId: number) { }); // Thumbnail mediaHashes.push({ hash: getStoryMediaHash(story), format: ApiMediaFormat.BlobUrl }); - if (story.content.altVideos) { + if (story.content.video?.altVideos) { mediaHashes.push({ hash: getStoryMediaHash(story, 'full', true)!, format: ApiMediaFormat.Progressive, diff --git a/src/config.ts b/src/config.ts index fdb1378b5..a234f5245 100644 --- a/src/config.ts +++ b/src/config.ts @@ -289,6 +289,9 @@ export const CONTENT_TYPES_WITH_PREVIEW = new Set([ ...SUPPORTED_VIDEO_CONTENT_TYPES, ]); +export const STORYBOARD_MIME = 'application/x-tgstoryboard'; +export const STORYBOARD_MAP_MIME = 'application/x-tgstoryboardmap'; + // Taken from https://github.com/telegramdesktop/tdesktop/blob/41d9a9fcbd0c809c60ddbd9350791b1436aff7d9/Telegram/SourceFiles/ui/boxes/choose_language_box.cpp#L28 export const SUPPORTED_TRANSLATION_LANGUAGES = [ // Official diff --git a/src/global/helpers/media.ts b/src/global/helpers/media.ts index 59fc8dcdf..4676c7452 100644 --- a/src/global/helpers/media.ts +++ b/src/global/helpers/media.ts @@ -13,17 +13,17 @@ export function getStoryMediaHash(story: ApiStory, size: StorySize): string; export function getStoryMediaHash( story: ApiStory, size: StorySize = 'preview', isAlt?: boolean, ) { - const isVideo = Boolean(story.content.video); - const isPhoto = Boolean(story.content.photo); + const video = story.content.video; + const photo = story.content.photo; - if (isVideo) { - if (isAlt && !story.content.altVideos) return undefined; - const media = isAlt ? getPreferredAlt(story.content.altVideos!) : story.content.video!; + if (video) { + if (isAlt && !video.altVideos) return undefined; + const media = isAlt ? getPreferredAlt(video.altVideos!) : video; return getVideoMediaHash(media, size); } - if (isPhoto) { - return getPhotoMediaHash(story.content.photo!, size); + if (photo) { + return getPhotoMediaHash(photo, size); } return undefined; diff --git a/src/lib/mediaWorker/index.worker.ts b/src/lib/mediaWorker/index.worker.ts index 38ef94d4f..ee4906875 100644 --- a/src/lib/mediaWorker/index.worker.ts +++ b/src/lib/mediaWorker/index.worker.ts @@ -1,12 +1,9 @@ import '../rlottie/rlottie.worker'; -import '../video-preview/video-preview.worker'; import '../offscreen-canvas/offscreen-canvas.worker'; import type { OffscreenCanvasApi } from '../offscreen-canvas/offscreen-canvas.worker'; import type { RLottieApi } from '../rlottie/rlottie.worker'; -import type { VideoPreviewApi } from '../video-preview/video-preview.worker'; export type MediaWorkerApi = RLottieApi - & VideoPreviewApi & OffscreenCanvasApi; diff --git a/src/lib/video-preview/MP4Demuxer.ts b/src/lib/video-preview/MP4Demuxer.ts deleted file mode 100644 index e53821a97..000000000 --- a/src/lib/video-preview/MP4Demuxer.ts +++ /dev/null @@ -1,225 +0,0 @@ -import type { MP4ArrayBuffer, MP4Info, MP4VideoTrack } from 'mp4box'; -import MP4Box, { DataStream } from 'mp4box'; - -import { requestPart } from './requestPart'; - -const META_PART_SIZE = 128 * 1024; -const MIN_PART_SIZE = 1024; -enum Status { - loading = 'loading', - ready = 'ready', - closed = 'closed', -} - -export type MP4DecoderConfig = { - codec: string; - codedHeight: number; - codedWidth: number; - description: Uint8Array; -}; - -type MP4DemuxerConfig = { - stepOffset: number; - stepMultiplier: number; - isPolyfill: boolean; - maxFrames: number; - onConfig: (config: any) => void; - onChunk: (chunk: any) => void; -}; - -export class MP4Demuxer { - private readonly url: string; - - private file: MP4Box.MP4File; - - private status = Status.loading; - - private readonly stepOffset: number; - - private readonly stepMultiplier: number; - - private readonly maxFrames: number; - - private readonly isPolyfill: boolean; - - private decodedSamples = new Set(); - - private lastSample = 0; - - private readonly onConfig: (config: MP4DecoderConfig) => void; - - private readonly onChunk: (chunk: any) => void; - - constructor(url: string, { - onConfig, - onChunk, - stepOffset, - stepMultiplier, - isPolyfill, - maxFrames, - }: MP4DemuxerConfig) { - this.url = url; - this.stepOffset = stepOffset; - this.stepMultiplier = stepMultiplier; - this.maxFrames = maxFrames; - this.isPolyfill = isPolyfill; - this.onConfig = onConfig; - this.onChunk = onChunk; - - this.file = MP4Box.createFile(); - this.file.onError = (e) => { - // eslint-disable-next-line no-console - console.error(e); - }; - this.file.onReady = this.onReady.bind(this); - this.file.onSamples = this.onSamples.bind(this); - - void this.loadMetadata(); - } - - private async loadMetadata() { - let offset: number | undefined = 0; - while (offset !== undefined) { - try { - offset = await this.requestPart(offset, META_PART_SIZE); - } catch (e) { - // eslint-disable-next-line no-console - console.error(e); - } - if (this.status === Status.ready) break; - } - } - - private async loadNextFrames(step: number, duration: number, partSize: number) { - let tick = step * this.stepOffset; - let lastSample = 0; - let rap = this.file.seek(tick, true); - while (this.status !== Status.closed) { - try { - await this.requestPart(rap.offset, partSize); - if (tick > duration) break; - if (this.lastSample > 1 && lastSample < this.lastSample) { - tick += step * this.stepMultiplier; - lastSample = this.lastSample; - } - rap = this.file.seek(tick, true); - } catch (e) { - // eslint-disable-next-line no-console - console.error(e); - } - } - this.file.flush(); - } - - private async requestPart(offset: number, partSize: number, useRap = true) { - const reminder = (offset % MIN_PART_SIZE); - const start = offset - reminder; - const end = start + partSize - 1; - let arrayBuffer = await requestPart({ url: this.url, start, end }) as MP4ArrayBuffer; - if (!arrayBuffer) { - return undefined; - } - if (reminder) { - arrayBuffer = arrayBuffer.slice(reminder) as MP4ArrayBuffer; - } - arrayBuffer.fileStart = offset; - const nextOffset = this.file.appendBuffer(arrayBuffer); - if (!useRap) return offset + arrayBuffer.byteLength; - return nextOffset; - } - - private description(track: MP4VideoTrack) { - const t = this.file.getTrackById(track.id); - for (const entry of t.mdia.minf.stbl.stsd.entries) { - if (entry.avcC || entry.hvcC || entry.av1C) { - const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN); - if (entry.avcC) { - entry.avcC.write(stream); - } else if (entry.hvcC) { - entry.hvcC.write(stream); - } else if (entry.av1C) { - entry.av1C.write(stream); - } - return new Uint8Array(stream.buffer, 8); // Remove the box header. - } - } - throw new Error('avcC, hvcC ro av1C not found'); - } - - private onReady(info: MP4Info) { - const track = info.videoTracks[0]; - - let codec = track.codec; - if (codec.startsWith('avc1')) { - // Somehow this is the only avc1 codec that works. - codec = 'avc1.4d001f'; - } - - // Generate and emit an appropriate VideoDecoderConfig. - this.onConfig({ - codec, - codedHeight: track.video.height, - codedWidth: track.video.width, - description: this.description(track), - }); - - const duration = info.duration / info.timescale; - - // If we set a part size too small, the onSamples callback is not called. - // If we use polyfill, we need to set a smaller part size to avoid decoding multiple frames. - const partSizeDivider = this.isPolyfill ? 24 : 12; - const partSize = roundPartSize(track.bitrate / partSizeDivider); - const step = calculateStep(duration, this.maxFrames); - - // Start demuxing. - this.file.setExtractionOptions(track.id, undefined, { nbSamples: 1 }); - this.file.start(); - - this.status = Status.ready; - - // // Load frames - void this.loadNextFrames(step, duration, partSize); - } - - private onSamples(trackId: number, ref: any, samples: any) { - if (this.status !== Status.ready) return; - // Generate and emit an EncodedVideoChunk for each demuxed sample. - for (const sample of samples) { - const time = sample.cts / sample.timescale; - const type = sample.is_sync ? 'key' : 'delta'; - const id = `${type}${sample.number}`; - - // Skip already decoded samples. - if (this.decodedSamples.has(id)) continue; - - // @ts-ignore - this.onChunk(new EncodedVideoChunk({ - type, - timestamp: (1e6 * time), - duration: (1e6 * sample.duration) / sample.timescale, - data: sample.data, - })); - - this.decodedSamples.add(id); - this.lastSample = parseInt(sample.number, 10); - - if (sample.is_sync) { - this.file.releaseUsedSamples(trackId, sample.number); - } - } - } - - close() { - this.file.flush(); - this.file.stop(); - this.status = Status.closed; - } -} - -function roundPartSize(size: number) { - return size + MIN_PART_SIZE - (size % MIN_PART_SIZE); -} - -function calculateStep(duration: number, max: number): number { - return Math.round((duration + max) / max); -} diff --git a/src/lib/video-preview/VideoPreview.ts b/src/lib/video-preview/VideoPreview.ts deleted file mode 100644 index 2ec464663..000000000 --- a/src/lib/video-preview/VideoPreview.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { ApiMediaFormat } from '../../api/types'; - -import { IS_ANDROID, IS_IOS } from '../../util/browser/windowEnvironment'; -import launchMediaWorkers, { MAX_WORKERS } from '../../util/launchMediaWorkers'; -import { callApi } from '../../api/gramjs'; -import { requestMutation } from '../fasterdom/fasterdom'; - -const IS_MOBILE = IS_ANDROID || IS_IOS; -const PREVIEW_SIZE_RATIO = (IS_ANDROID || IS_IOS) ? 0.3 : 0.25; -const MAX_FRAMES = IS_MOBILE ? 40 : 80; -const PREVIEW_MAX_SIDE = 200; - -const connections = launchMediaWorkers(); - -let videoPreview: VideoPreview | undefined; - -export class VideoPreview { - frames = new Map(); - - currentTime = 0; - - canvas: HTMLCanvasElement; - - constructor(url: string, canvas: HTMLCanvasElement) { - this.canvas = canvas; - connections.forEach(({ connector }, index) => { - void connector.request({ - name: 'video-preview:init', - args: [ - url, - MAX_FRAMES, - index, - MAX_WORKERS, - this.onFrame.bind(this), - ], - }); - }); - } - - private onFrame(time: number, frame: ImageBitmap) { - this.frames.set(time, frame); - if (time === this.currentTime) { - this.render(time); - } - } - - private clearCache() { - this.frames.forEach((frame) => { - frame.close(); - }); - this.frames.clear(); - } - - render(time: number) { - this.currentTime = time; - const frame = this.frames.get(time); - if (!frame) return false; - requestMutation(() => { - this.canvas.width = frame.width; - this.canvas.height = frame.height; - const ctx = this.canvas.getContext('2d')!; - ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height); - }); - return true; - } - - destroy() { - this.clearCache(); - connections.forEach(({ connector }) => { - void connector.request({ - name: 'video-preview:destroy', - args: [], - }); - }); - } -} - -export function getPreviewDimensions(width: number, height: number) { - width = Math.round(width * PREVIEW_SIZE_RATIO); - height = Math.round(height * PREVIEW_SIZE_RATIO); - const ratio = width / height; - if (width > PREVIEW_MAX_SIDE) { - width = PREVIEW_MAX_SIDE; - height = Math.round(width / ratio); - } - if (height > PREVIEW_MAX_SIDE) { - height = PREVIEW_MAX_SIDE; - width = Math.round(height * ratio); - } - return { width, height }; -} - -connections.forEach(({ worker }) => { - worker.addEventListener('message', async (e) => { - const { type, messageId, params } = e.data as { - type: string; - messageId: string; - params: { url: string; start: number; end: number }; - }; - - if (type !== 'requestPart') { - return; - } - - const result = await callApi('downloadMedia', { mediaFormat: ApiMediaFormat.Progressive, ...params }); - if (!result) { - return; - } - - const { arrayBuffer } = result; - - worker.postMessage({ - type: 'partResponse', - messageId, - result: arrayBuffer, - }, [arrayBuffer!]); - }); -}); - -export function createVideoPreviews(url: string, canvas: HTMLCanvasElement) { - if (videoPreview) { - videoPreview.destroy(); - } - videoPreview = new VideoPreview(url, canvas); - return () => { - videoPreview?.destroy(); - videoPreview = undefined; - }; -} - -export function renderVideoPreview(time: number) { - if (!videoPreview) return false; - return videoPreview.render(time); -} diff --git a/src/lib/video-preview/mp4box.d.ts b/src/lib/video-preview/mp4box.d.ts deleted file mode 100644 index 23903c64b..000000000 --- a/src/lib/video-preview/mp4box.d.ts +++ /dev/null @@ -1,93 +0,0 @@ -declare module 'mp4box' { - - export interface MP4MediaTrack { - id: number; - created: Date; - modified: Date; - movie_duration: number; - layer: number; - alternate_group: number; - volume: number; - track_width: number; - track_height: number; - timescale: number; - duration: number; - bitrate: number; - codec: string; - language: string; - nb_samples: number; - - } - - export interface MP4VideoData { - width: number; - height: number; - } - - export interface MP4VideoTrack extends MP4MediaTrack { - video: MP4VideoData; - [key: string]: any; - } - - export interface MP4AudioData { - sample_rate: number; - channel_count: number; - sample_size: number; - } - - export interface MP4AudioTrack extends MP4MediaTrack { - audio: MP4AudioData; - [key: string]: any; - } - - export type MP4Track = MP4VideoTrack | MP4AudioTrack; - - export class DataStream { - buffer: ArrayBuffer; - - static BIG_ENDIAN: number; - constructor(buffer?: ArrayBuffer, offset?: number, endianness?: number); - } - - export interface MP4Info { - duration: number; - timescale: number; - fragment_duration: number; - isFragmented: boolean; - isProgressive: boolean; - hasIOD: boolean; - brands: string[]; - created: Date; - modified: Date; - tracks: MP4Track[]; - videoTracks: MP4VideoTrack[]; - } - - export type MP4ArrayBuffer = ArrayBuffer & { fileStart: number }; - - export interface MP4File { - - onMoovStart?: () => void; - onReady?: (info: MP4Info) => void; - onSamples?: (trackId: number, ref: any, samples: any) => void; - onError?: (e: string) => void; - - processSamples(last: boolean): void; - - getTrackById(id: number): MP4Track; - - setExtractionOptions(id: number, user?: any, options?: any): void; - - appendBuffer(data: MP4ArrayBuffer): number; - start(): void; - stop(): void; - flush(): void; - seek(time: number, useRap?: boolean): { offset: number; time: number }; - releaseUsedSamples(id: number, sampleNumber: number): void; - } - - export function createFile(): MP4File; - - export {}; - -} diff --git a/src/lib/video-preview/requestPart.ts b/src/lib/video-preview/requestPart.ts deleted file mode 100644 index 0e39f93c4..000000000 --- a/src/lib/video-preview/requestPart.ts +++ /dev/null @@ -1,62 +0,0 @@ -import generateUniqueId from '../../util/generateUniqueId'; -import { pause } from '../../util/schedulers'; - -declare const self: WorkerGlobalScope; - -type RequestStates = { - resolve: (response: ArrayBuffer) => void; - reject: () => void; -}; - -type RequestPartParams = { url: string; start: number; end: number }; - -const PART_TIMEOUT = 30000; - -const requestStates = new Map(); - -export function requestPart(params: RequestPartParams): Promise { - const messageId = generateUniqueId(); - const requestState = {} as RequestStates; - - let isResolved = false; - const promise = Promise.race([ - pause(PART_TIMEOUT).then(() => (isResolved ? undefined : Promise.reject(new Error('ERROR_PART_TIMEOUT')))), - new Promise((resolve, reject) => { - Object.assign(requestState, { resolve, reject }); - }), - ]); - - requestStates.set(messageId, requestState); - - promise - .catch(() => undefined) - .finally(() => { - requestStates.delete(messageId); - isResolved = true; - }); - - const message = { - type: 'requestPart', - messageId, - params, - }; - - postMessage(message); - - return promise; -} - -self.addEventListener('message', (e) => { - const { type, messageId, result } = (e as any).data as { - type: string; - messageId: string; - result: ArrayBuffer; - }; - - if (type === 'partResponse') { - const requestState = requestStates.get(messageId); - if (requestState) { - requestState.resolve(result); - } - } -}); diff --git a/src/lib/video-preview/video-preview.worker.ts b/src/lib/video-preview/video-preview.worker.ts deleted file mode 100644 index 6d6cfb34b..000000000 --- a/src/lib/video-preview/video-preview.worker.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { CancellableCallback } from '../../util/PostMessageConnector'; - -import { createWorkerInterface } from '../../util/createPostMessageInterface'; - -import { MP4Demuxer } from './MP4Demuxer'; - -let decoder: any; -let demuxer: any; -let onDestroy: VoidFunction | undefined; - -function init( - url: string, - maxFrames: number, - workerIndex: number, - workersTotal: number, - onFrame: CancellableCallback, -) { - const hasWebCodecs = 'VideoDecoder' in globalThis; - if (!hasWebCodecs) { - // eslint-disable-next-line no-console - console.log('[Video Preview] WebCodecs not supported'); - return new Promise((resolve) => { - onDestroy = resolve; - }); - } - - const decodedFrames = new Set(); - - // @ts-ignore - decoder = new VideoDecoder({ - async output(frame: any) { - const time = frame.timestamp / 1e6; - const seconds = Math.floor(time); - // Only render whole second frames - if (!decodedFrames.has(seconds)) { - const bitmap = await createImageBitmap(frame); - decodedFrames.add(seconds); - onFrame(seconds, bitmap); - } - frame.close(); - }, - error(e: any) { - // eslint-disable-next-line no-console - console.error('[Video Preview] error', e); - }, - }); - - demuxer = new MP4Demuxer(url, { - stepOffset: workerIndex, - stepMultiplier: workersTotal, - isPolyfill: !hasWebCodecs, - maxFrames, - onConfig(config) { - decoder?.configure(config); - }, - onChunk(chunk) { - if (decoder?.state !== 'configured') return; - decoder?.decode(chunk); - }, - }); - - return new Promise((resolve) => { - onDestroy = resolve; - }); -} - -function destroy() { - try { - decoder?.close(); - demuxer?.close(); - } catch { - // Ignore - } - decoder = undefined; - demuxer = undefined; - onDestroy?.(); -} - -const api = { - 'video-preview:init': init, - 'video-preview:destroy': destroy, -}; - -createWorkerInterface(api, 'media'); - -export type VideoPreviewApi = typeof api; diff --git a/src/util/browser/globalEnvironment.ts b/src/util/browser/globalEnvironment.ts index fa9cb3e60..82c7751ba 100644 --- a/src/util/browser/globalEnvironment.ts +++ b/src/util/browser/globalEnvironment.ts @@ -5,7 +5,6 @@ declare const globalThis: ServiceWorkerGlobalScope & WorkerGlobalScope & SharedW export const IS_MULTIACCOUNT_SUPPORTED = 'SharedWorker' in globalThis; export const IS_INTL_LIST_FORMAT_SUPPORTED = 'ListFormat' in Intl; export const IS_BAD_URL_PARSER = new URL('tg://host').host !== 'host'; -export const ARE_WEBCODECS_SUPPORTED = 'VideoDecoder' in globalThis; export const IS_TAURI = isTauri(); // @ts-expect-error no types for electron diff --git a/src/util/media/StoryboardParser.ts b/src/util/media/StoryboardParser.ts new file mode 100644 index 000000000..794c20894 --- /dev/null +++ b/src/util/media/StoryboardParser.ts @@ -0,0 +1,63 @@ +interface PreviewFrameInfo { + top: number; + left: number; + width: number; + height: number; + time: number; +} + +export default class StoryboardParser { + private frames: PreviewFrameInfo[]; + + /** + * Can throw error if the storyboard map data is invalid + */ + constructor(storyboardMapData: string) { + const [_file, widthLine, heightLine, ...frameLines] = storyboardMapData.split('\n'); + const width = Number.parseFloat(widthLine.split('=')[1]); + const height = Number.parseFloat(heightLine.split('=')[1]); + + if (Number.isNaN(width) || Number.isNaN(height)) { + throw new Error('Invalid storyboard map frame size'); + } + + this.frames = frameLines.map((frame) => { + if (!frame.trim().length) { + return undefined; + } + + const [timeStr, leftStr, topStr] = frame.split(','); + const info = { + time: Number.parseFloat(timeStr), + left: Number.parseInt(leftStr, 10), + top: Number.parseInt(topStr, 10), + width, + height, + }; + + if (Number.isNaN(info.time) || Number.isNaN(info.left) || Number.isNaN(info.top)) { + throw new Error('Invalid storyboard map data'); + } + return info; + }).filter(Boolean); + + if (this.frames.length === 0) { + throw new Error('Missing frames in storyboard map data'); + } + } + + getNearestPreview(time: number): PreviewFrameInfo { + // Binary search for the nearest frame + let left = 0; + let right = this.frames.length - 1; + while (left <= right) { + const mid = Math.floor((left + right) / 2); + if (this.frames[mid].time <= time) { + left = mid + 1; + } else { + right = mid - 1; + } + } + return this.frames[Math.max(0, left - 1)]; + } +}