From 79ff2c4d063a1a5ff392853ef58f078cf85018f9 Mon Sep 17 00:00:00 2001 From: Alexander Zinchuk Date: Tue, 12 Dec 2023 12:34:50 +0100 Subject: [PATCH] Use MediaSource API in Safari for stories (#4100) --- src/components/story/Story.tsx | 34 +++-- src/components/story/StoryToggler.tsx | 1 + src/components/story/StoryViewer.module.scss | 6 +- src/components/story/helpers/videoFormats.ts | 2 + .../story/hooks/useStoryPreloader.ts | 56 ++++++-- src/hooks/useStreaming.ts | 132 ++++++++++++++++++ src/util/mediaLoader.ts | 4 + src/util/progressieveLoader.ts | 40 ++++++ 8 files changed, 247 insertions(+), 28 deletions(-) create mode 100644 src/components/story/helpers/videoFormats.ts create mode 100644 src/hooks/useStreaming.ts create mode 100644 src/util/progressieveLoader.ts diff --git a/src/components/story/Story.tsx b/src/components/story/Story.tsx index 87afafd51..97955f6eb 100644 --- a/src/components/story/Story.tsx +++ b/src/components/story/Story.tsx @@ -25,6 +25,7 @@ import { formatMediaDuration, formatRelativeTime } from '../../util/dateFormat'; import download from '../../util/download'; import { getServerTime } from '../../util/serverTime'; import renderText from '../common/helpers/renderText'; +import { PRIMARY_VIDEO_MIME, SECONDARY_VIDEO_MIME } from './helpers/videoFormats'; import useUnsupportedMedia from '../../hooks/media/useUnsupportedMedia'; import useAppLayout, { getIsMobile } from '../../hooks/useAppLayout'; @@ -39,6 +40,7 @@ import useLastCallback from '../../hooks/useLastCallback'; import useLongPress from '../../hooks/useLongPress'; import useMediaTransition from '../../hooks/useMediaTransition'; import useShowTransition from '../../hooks/useShowTransition'; +import { useStreaming } from '../../hooks/useStreaming'; import useStoryPreloader from './hooks/useStoryPreloader'; import useStoryProps from './hooks/useStoryProps'; @@ -92,9 +94,6 @@ interface StateProps { const VIDEO_MIN_READY_STATE = 4; const SPACEBAR_CODE = 32; -const PRIMARY_VIDEO_MIME = 'video/mp4; codecs=hvc1.1.6.L63.00'; -const SECONDARY_VIDEO_MIME = 'video/mp4; codecs=avc1.64001E'; - const STEALTH_MODE_NOTIFICATION_DURATION = 4000; function Story({ @@ -211,6 +210,10 @@ function Story({ && !isPausedBySpacebar && !isPausedByLongPress, ); + const duration = isLoadedStory && story.content.video?.duration + ? story.content.video.duration + : undefined; + const shouldShowFooter = isLoadedStory && (isOut || isChannel); const { @@ -236,6 +239,8 @@ function Story({ const { transitionClassNames: appearanceAnimationClassNames } = useShowTransition(true); + useStreaming(videoRef, fullMediaData, PRIMARY_VIDEO_MIME); + useStoryPreloader(peerId, storyId); useEffect(() => { @@ -378,13 +383,6 @@ function Story({ } }, [isComposerHasFocus, isCaptionExpanded, shouldForcePause, isAppFocused, isPausedByLongPress, isPausedBySpacebar]); - const handleVideoStoryTimeUpdate = useLastCallback((e: React.SyntheticEvent) => { - const video = e.currentTarget; - if (video.readyState >= VIDEO_MIN_READY_STATE) { - setCurrentTime(video.currentTime); - } - }); - const handleOpenChat = useLastCallback(() => { onClose(); openChat({ id: peerId }); @@ -405,6 +403,16 @@ function Story({ openNextStory(); }); + const handleVideoStoryTimeUpdate = useLastCallback((e: React.SyntheticEvent) => { + const video = e.currentTarget; + if (video.readyState >= VIDEO_MIN_READY_STATE) { + setCurrentTime(video.currentTime); + } + if (duration && video.currentTime >= duration) { + handleOpenNextStory(); + } + }); + useEffect(() => { return !getIsAnimating() && !isComposerHasFocus ? captureKeyboardListeners({ onRight: handleOpenNextStory, @@ -522,10 +530,6 @@ function Story({ }, [isMobile, lang]); function renderStoriesTabs() { - const duration = isLoadedStory && story.content.video?.duration - ? story.content.video.duration - : undefined; - return (
{(isSingleStory ? [storyId] : orderedIds ?? []).map((id) => ( @@ -738,8 +742,8 @@ function Story({ onPlaying={markStoryPlaying} onPause={unmarkStoryPlaying} onWaiting={unmarkStoryPlaying} + disableRemotePlayback onTimeUpdate={handleVideoStoryTimeUpdate} - onEnded={handleOpenNextStory} > {altMediaData && } diff --git a/src/components/story/StoryToggler.tsx b/src/components/story/StoryToggler.tsx index 8a60ea18a..829fd5008 100644 --- a/src/components/story/StoryToggler.tsx +++ b/src/components/story/StoryToggler.tsx @@ -66,6 +66,7 @@ function StoryToggler({ const preloadPeerIds = useMemo(() => { return orderedPeerIds.slice(0, PRELOAD_PEERS); }, [orderedPeerIds]); + useStoryPreloader(preloadPeerIds); const isVisible = canShow && isShown; diff --git a/src/components/story/StoryViewer.module.scss b/src/components/story/StoryViewer.module.scss index 9caadeb75..82774579b 100644 --- a/src/components/story/StoryViewer.module.scss +++ b/src/components/story/StoryViewer.module.scss @@ -248,7 +248,7 @@ } :global(body.ghost-animating) & { - display: none; + visibility: hidden; } } @@ -262,6 +262,8 @@ height: inherit; border-radius: var(--border-radius-default-small); + transition: opacity 300ms; + @media (max-width: 600px) { bottom: 0; width: 100%; @@ -269,7 +271,7 @@ } :global(body.ghost-animating) .activeSlide & { - display: none; + visibility: hidden; } } diff --git a/src/components/story/helpers/videoFormats.ts b/src/components/story/helpers/videoFormats.ts new file mode 100644 index 000000000..e67b2aaaa --- /dev/null +++ b/src/components/story/helpers/videoFormats.ts @@ -0,0 +1,2 @@ +export const PRIMARY_VIDEO_MIME = 'video/mp4; codecs=hvc1.1.6.L63.00'; +export const SECONDARY_VIDEO_MIME = 'video/mp4; codecs=avc1.64001E'; diff --git a/src/components/story/hooks/useStoryPreloader.ts b/src/components/story/hooks/useStoryPreloader.ts index bd449fa71..7c1920656 100644 --- a/src/components/story/hooks/useStoryPreloader.ts +++ b/src/components/story/hooks/useStoryPreloader.ts @@ -6,29 +6,45 @@ import { ApiMediaFormat } from '../../../api/types'; import { getStoryMediaHash } from '../../../global/helpers'; import { selectPeerStories } from '../../../global/selectors'; import * as mediaLoader from '../../../util/mediaLoader'; +import { getProgressiveUrl } from '../../../util/mediaLoader'; +import { makeProgressiveLoader } from '../../../util/progressieveLoader'; import { pause } from '../../../util/schedulers'; +import { PRIMARY_VIDEO_MIME } from '../helpers/videoFormats'; + +import { checkIfStreamingSupported } from '../../../hooks/useStreaming'; const preloadedStories: Record> = {}; const PEER_STORIES_FOR_PRELOAD = 5; const PROGRESSIVE_PRELOAD_DURATION = 1000; +const STREAM_PRELOAD_SIZE = 1024 * 1024 * 2; // 2 MB const FIRST_PRELOAD_DELAY = 1000; const canPreload = pause(FIRST_PRELOAD_DELAY); +type MediaHash = { hash: string; format: ApiMediaFormat; isStream?: boolean }; + function useStoryPreloader(peerIds: string[]): void; function useStoryPreloader(peerId?: string, aroundStoryId?: number): void; function useStoryPreloader(peerId?: string | string[], aroundStoryId?: number) { useEffect(() => { if (peerId === undefined) return; - const preloadHashes = async (mediaHashes: { hash: string; format: ApiMediaFormat }[]) => { + const preloadHashes = async (mediaHashes: Array) => { await canPreload; - mediaHashes.forEach(({ hash, format }) => { - mediaLoader.fetch(hash, format).then((result) => { - if (format === ApiMediaFormat.Progressive) { - preloadProgressive(result); - } - }); + mediaHashes.forEach(({ hash, format, isStream }) => { + if (isStream) { + preloadStream(hash); + return; + } + mediaLoader.fetch( + hash, + format, + ) + .then((result) => { + if (format === ApiMediaFormat.Progressive) { + preloadProgressive(result); + } + }); }); }; @@ -56,7 +72,7 @@ function getPreloadMediaHashes(peerId: string, storyId: number) { const preloadIds = findIdsAroundCurrentId(peerStories.orderedIds, storyId, PEER_STORIES_FOR_PRELOAD); - const mediaHashes: { hash: string; format: ApiMediaFormat }[] = []; + const mediaHashes: Array = []; preloadIds.forEach((currentStoryId) => { if (preloadedStories[peerId]?.has(currentStoryId)) { return; @@ -71,12 +87,15 @@ function getPreloadMediaHashes(peerId: string, storyId: number) { mediaHashes.push({ hash: getStoryMediaHash(story, 'full'), format: story.content.video ? ApiMediaFormat.Progressive : ApiMediaFormat.BlobUrl, + isStream: checkIfStreamingSupported(PRIMARY_VIDEO_MIME), }); // Thumbnail mediaHashes.push({ hash: getStoryMediaHash(story), format: ApiMediaFormat.BlobUrl }); - // Alt video with different codec if (story.content.altVideo) { - mediaHashes.push({ hash: getStoryMediaHash(story, 'full', true)!, format: ApiMediaFormat.Progressive }); + mediaHashes.push({ + hash: getStoryMediaHash(story, 'full', true)!, + format: ApiMediaFormat.Progressive, + }); } preloadedStories[peerId] = (preloadedStories[peerId] || new Set()).add(currentStoryId); @@ -92,12 +111,27 @@ function preloadProgressive(url: string) { video.src = url; video.muted = true; video.autoplay = true; + video.disableRemotePlayback = true; video.style.display = 'none'; head.appendChild(video); - + video.load(); setTimeout(() => { + video.pause(); + video.src = ''; + video.load(); head.removeChild(video); }, PROGRESSIVE_PRELOAD_DURATION); } +async function preloadStream(hash: string) { + const loader = makeProgressiveLoader(getProgressiveUrl(hash)); + let cachedSize = 0; + for await (const chunk of loader) { + cachedSize += chunk.byteLength; + if (cachedSize >= STREAM_PRELOAD_SIZE) { + break; + } + } +} + export default useStoryPreloader; diff --git a/src/hooks/useStreaming.ts b/src/hooks/useStreaming.ts new file mode 100644 index 000000000..a9a222eee --- /dev/null +++ b/src/hooks/useStreaming.ts @@ -0,0 +1,132 @@ +import type { RefObject } from 'react'; +import { useEffect } from '../lib/teact/teact'; + +import { DEBUG } from '../config'; +import { requestMutation } from '../lib/fasterdom/fasterdom'; +import { applyStyles } from '../util/animation'; +import { makeProgressiveLoader } from '../util/progressieveLoader'; +import { IS_SAFARI } from '../util/windowEnvironment'; + +const VIDEO_REVEAL_DELAY = 100; + +export function useStreaming(videoRef: RefObject, url?: string, mimeType?: string) { + useEffect(() => { + if (!url || !videoRef.current) return undefined; + const MediaSourceClass = getMediaSource(); + const video = videoRef.current; + + if (!IS_SAFARI || !mimeType || !MediaSourceClass?.isTypeSupported(mimeType)) { + return undefined; + } + const mediaSource = new MediaSourceClass(); + + function revealVideo() { + requestMutation(() => { + video.style.display = 'block'; + setTimeout(() => { + requestMutation(() => { + applyStyles(video, { opacity: '1' }); + }); + }, VIDEO_REVEAL_DELAY); + }); + } + + function onSourceOpen() { + if (!url || !mimeType) return; + + const sourceBuffer = mediaSource.addSourceBuffer(mimeType); + const loader = makeProgressiveLoader(url); + + function onUpdateEnded() { + loader.next() + .then(({ + value, + done, + }) => { + if (mediaSource.readyState !== 'open') return; + if (done) { + endOfStream(mediaSource); + return; + } + appendBuffer(sourceBuffer, value); + }); + } + + sourceBuffer.addEventListener('updateend', onUpdateEnded); + + loader.next() + .then(({ + value, + done, + }) => { + if (done || mediaSource.readyState !== 'open') return; + revealVideo(); + appendBuffer(sourceBuffer, value); + }); + } + + mediaSource.addEventListener('sourceopen', onSourceOpen, { once: true }); + + requestMutation(() => { + applyStyles(video, { + display: 'none', + opacity: '0', + }); + video.src = URL.createObjectURL(mediaSource); + }); + + return () => { + mediaSource.removeEventListener('sourceopen', onSourceOpen); + if (mediaSource.readyState === 'open') { + endOfStream(mediaSource); + } + URL.revokeObjectURL(video.src); + requestMutation(() => { + video.src = ''; + applyStyles(video, { + display: 'none', + opacity: '0', + }); + }); + }; + }, [mimeType, url, videoRef]); +} + +export function checkIfStreamingSupported(mimeType: string) { + if (!IS_SAFARI) return false; + const MS = getMediaSource(); + return Boolean(MS && MS.isTypeSupported(mimeType)); +} + +function appendBuffer(sourceBuffer: SourceBuffer, buffer: ArrayBuffer) { + try { + sourceBuffer.appendBuffer(buffer); + } catch (error) { + if (DEBUG) { + // eslint-disable-next-line no-console + console.warn('[Stream] failed to append buffer', error); + } + } +} + +function endOfStream(mediaSource: MediaSource) { + try { + mediaSource.endOfStream(); + } catch (error) { + if (DEBUG) { + // eslint-disable-next-line no-console + console.warn('[Stream] failed to end stream', error); + } + } +} + +function getMediaSource(): typeof MediaSource | undefined { + if ('ManagedMediaSource' in window) { + // @ts-ignore + return ManagedMediaSource; + } + if ('MediaSource' in window) { + return MediaSource; + } + return undefined; +} diff --git a/src/util/mediaLoader.ts b/src/util/mediaLoader.ts index d7949ec76..d67888834 100644 --- a/src/util/mediaLoader.ts +++ b/src/util/mediaLoader.ts @@ -115,6 +115,10 @@ export function removeCallback(url: string, callbackUniqueId: string) { callbacks.delete(callbackUniqueId); } +export function getProgressiveUrl(url: string) { + return `${PROGRESSIVE_URL_PREFIX}${url}`; +} + function getProgressive(url: string) { const progressiveUrl = `${PROGRESSIVE_URL_PREFIX}${url}`; diff --git a/src/util/progressieveLoader.ts b/src/util/progressieveLoader.ts new file mode 100644 index 000000000..2f5018296 --- /dev/null +++ b/src/util/progressieveLoader.ts @@ -0,0 +1,40 @@ +const DEFAULT_CHUNK_SIZE = 256 * 1024; +export async function* makeProgressiveLoader( + url: string, + chunkSize = DEFAULT_CHUNK_SIZE, +): AsyncGenerator { + let start = 0; + let fileSize: number | undefined; + + while (true) { + let end = start + chunkSize - 1; + if (fileSize && end > fileSize) { + end = fileSize - 1; + } + + const res = await fetch(url, { + headers: { Range: `bytes=${start}-${end}` }, + }); + + if (!res.ok) return; + + // If fileSize is not yet defined, retrieve it from the first chunk's response + if (!fileSize) { + const contentRange = res.headers.get('Content-Range'); + const match = contentRange?.match(/\/(\d+)$/); + fileSize = match ? Number(match[1]) : undefined; + + if (!fileSize) return; + } + + // Yield the chunk data + const chunk = await res.arrayBuffer(); + yield chunk; + + start = end + 1; + + if (start >= fileSize) { + return; + } + } +}