Use MediaSource API in Safari for stories (#4100)

src/components/story/Story.tsx

@@ -25,6 +25,7 @@ import { formatMediaDuration, formatRelativeTime } from '../../util/dateFormat';
 import download from '../../util/download';
 import { getServerTime } from '../../util/serverTime';
 import renderText from '../common/helpers/renderText';
+import { PRIMARY_VIDEO_MIME, SECONDARY_VIDEO_MIME } from './helpers/videoFormats';
 
 import useUnsupportedMedia from '../../hooks/media/useUnsupportedMedia';
 import useAppLayout, { getIsMobile } from '../../hooks/useAppLayout';
@@ -39,6 +40,7 @@ import useLastCallback from '../../hooks/useLastCallback';
 import useLongPress from '../../hooks/useLongPress';
 import useMediaTransition from '../../hooks/useMediaTransition';
 import useShowTransition from '../../hooks/useShowTransition';
+import { useStreaming } from '../../hooks/useStreaming';
 import useStoryPreloader from './hooks/useStoryPreloader';
 import useStoryProps from './hooks/useStoryProps';
 
@@ -92,9 +94,6 @@ interface StateProps {
 const VIDEO_MIN_READY_STATE = 4;
 const SPACEBAR_CODE = 32;
 
-const PRIMARY_VIDEO_MIME = 'video/mp4; codecs=hvc1.1.6.L63.00';
-const SECONDARY_VIDEO_MIME = 'video/mp4; codecs=avc1.64001E';
-
 const STEALTH_MODE_NOTIFICATION_DURATION = 4000;
 
 function Story({
@@ -211,6 +210,10 @@ function Story({
     && !isPausedBySpacebar && !isPausedByLongPress,
   );
 
+  const duration = isLoadedStory && story.content.video?.duration
+    ? story.content.video.duration
+    : undefined;
+
   const shouldShowFooter = isLoadedStory && (isOut || isChannel);
 
   const {
@@ -236,6 +239,8 @@ function Story({
 
   const { transitionClassNames: appearanceAnimationClassNames } = useShowTransition(true);
 
+  useStreaming(videoRef, fullMediaData, PRIMARY_VIDEO_MIME);
+
   useStoryPreloader(peerId, storyId);
 
   useEffect(() => {
@@ -378,13 +383,6 @@ function Story({
     }
   }, [isComposerHasFocus, isCaptionExpanded, shouldForcePause, isAppFocused, isPausedByLongPress, isPausedBySpacebar]);
 
-  const handleVideoStoryTimeUpdate = useLastCallback((e: React.SyntheticEvent<HTMLVideoElement>) => {
-    const video = e.currentTarget;
-    if (video.readyState >= VIDEO_MIN_READY_STATE) {
-      setCurrentTime(video.currentTime);
-    }
-  });
-
   const handleOpenChat = useLastCallback(() => {
     onClose();
     openChat({ id: peerId });
@@ -405,6 +403,16 @@ function Story({
     openNextStory();
   });
 
+  const handleVideoStoryTimeUpdate = useLastCallback((e: React.SyntheticEvent<HTMLVideoElement>) => {
+    const video = e.currentTarget;
+    if (video.readyState >= VIDEO_MIN_READY_STATE) {
+      setCurrentTime(video.currentTime);
+    }
+    if (duration && video.currentTime >= duration) {
+      handleOpenNextStory();
+    }
+  });
+
   useEffect(() => {
     return !getIsAnimating() && !isComposerHasFocus ? captureKeyboardListeners({
       onRight: handleOpenNextStory,
@@ -522,10 +530,6 @@ function Story({
   }, [isMobile, lang]);
 
   function renderStoriesTabs() {
-    const duration = isLoadedStory && story.content.video?.duration
-      ? story.content.video.duration
-      : undefined;
-
     return (
       <div className={styles.storyIndicators}>
         {(isSingleStory ? [storyId] : orderedIds ?? []).map((id) => (
@@ -738,8 +742,8 @@ function Story({
             onPlaying={markStoryPlaying}
             onPause={unmarkStoryPlaying}
             onWaiting={unmarkStoryPlaying}
+            disableRemotePlayback
             onTimeUpdate={handleVideoStoryTimeUpdate}
-            onEnded={handleOpenNextStory}
           >
             <source src={fullMediaData} type={PRIMARY_VIDEO_MIME} width="720" />
             {altMediaData && <source src={altMediaData} type={SECONDARY_VIDEO_MIME} width="480" />}

src/components/story/StoryToggler.tsx

@@ -66,6 +66,7 @@ function StoryToggler({
   const preloadPeerIds = useMemo(() => {
     return orderedPeerIds.slice(0, PRELOAD_PEERS);
   }, [orderedPeerIds]);
+
   useStoryPreloader(preloadPeerIds);
 
   const isVisible = canShow && isShown;

src/components/story/StoryViewer.module.scss

@@ -248,7 +248,7 @@
   }
 
   :global(body.ghost-animating) & {
-    display: none;
+    visibility: hidden;
   }
 }
 
@@ -262,6 +262,8 @@
   height: inherit;
   border-radius: var(--border-radius-default-small);
 
+  transition: opacity 300ms;
+
   @media (max-width: 600px) {
     bottom: 0;
     width: 100%;
@@ -269,7 +271,7 @@
   }
 
   :global(body.ghost-animating) .activeSlide & {
-    display: none;
+    visibility: hidden;
   }
 }
 

src/components/story/helpers/videoFormats.ts

@@ -0,0 +1,2 @@
+export const PRIMARY_VIDEO_MIME = 'video/mp4; codecs=hvc1.1.6.L63.00';
+export const SECONDARY_VIDEO_MIME = 'video/mp4; codecs=avc1.64001E';

src/components/story/hooks/useStoryPreloader.ts

@@ -6,29 +6,45 @@ import { ApiMediaFormat } from '../../../api/types';
 import { getStoryMediaHash } from '../../../global/helpers';
 import { selectPeerStories } from '../../../global/selectors';
 import * as mediaLoader from '../../../util/mediaLoader';
+import { getProgressiveUrl } from '../../../util/mediaLoader';
+import { makeProgressiveLoader } from '../../../util/progressieveLoader';
 import { pause } from '../../../util/schedulers';
+import { PRIMARY_VIDEO_MIME } from '../helpers/videoFormats';
+
+import { checkIfStreamingSupported } from '../../../hooks/useStreaming';
 
 const preloadedStories: Record<string, Set<number>> = {};
 const PEER_STORIES_FOR_PRELOAD = 5;
 const PROGRESSIVE_PRELOAD_DURATION = 1000;
+const STREAM_PRELOAD_SIZE = 1024 * 1024 * 2; // 2 MB
 
 const FIRST_PRELOAD_DELAY = 1000;
 const canPreload = pause(FIRST_PRELOAD_DELAY);
 
+type MediaHash = { hash: string; format: ApiMediaFormat; isStream?: boolean };
+
 function useStoryPreloader(peerIds: string[]): void;
 function useStoryPreloader(peerId?: string, aroundStoryId?: number): void;
 function useStoryPreloader(peerId?: string | string[], aroundStoryId?: number) {
   useEffect(() => {
     if (peerId === undefined) return;
 
-    const preloadHashes = async (mediaHashes: { hash: string; format: ApiMediaFormat }[]) => {
+    const preloadHashes = async (mediaHashes: Array<MediaHash>) => {
       await canPreload;
-      mediaHashes.forEach(({ hash, format }) => {
-        mediaLoader.fetch(hash, format).then((result) => {
-          if (format === ApiMediaFormat.Progressive) {
-            preloadProgressive(result);
-          }
-        });
+      mediaHashes.forEach(({ hash, format, isStream }) => {
+        if (isStream) {
+          preloadStream(hash);
+          return;
+        }
+        mediaLoader.fetch(
+          hash,
+          format,
+        )
+          .then((result) => {
+            if (format === ApiMediaFormat.Progressive) {
+              preloadProgressive(result);
+            }
+          });
       });
     };
 
@@ -56,7 +72,7 @@ function getPreloadMediaHashes(peerId: string, storyId: number) {
 
   const preloadIds = findIdsAroundCurrentId(peerStories.orderedIds, storyId, PEER_STORIES_FOR_PRELOAD);
 
-  const mediaHashes: { hash: string; format: ApiMediaFormat }[] = [];
+  const mediaHashes: Array<MediaHash> = [];
   preloadIds.forEach((currentStoryId) => {
     if (preloadedStories[peerId]?.has(currentStoryId)) {
       return;
@@ -71,12 +87,15 @@ function getPreloadMediaHashes(peerId: string, storyId: number) {
     mediaHashes.push({
       hash: getStoryMediaHash(story, 'full'),
       format: story.content.video ? ApiMediaFormat.Progressive : ApiMediaFormat.BlobUrl,
+      isStream: checkIfStreamingSupported(PRIMARY_VIDEO_MIME),
     });
     // Thumbnail
     mediaHashes.push({ hash: getStoryMediaHash(story), format: ApiMediaFormat.BlobUrl });
-    // Alt video with different codec
     if (story.content.altVideo) {
-      mediaHashes.push({ hash: getStoryMediaHash(story, 'full', true)!, format: ApiMediaFormat.Progressive });
+      mediaHashes.push({
+        hash: getStoryMediaHash(story, 'full', true)!,
+        format: ApiMediaFormat.Progressive,
+      });
     }
 
     preloadedStories[peerId] = (preloadedStories[peerId] || new Set()).add(currentStoryId);
@@ -92,12 +111,27 @@ function preloadProgressive(url: string) {
   video.src = url;
   video.muted = true;
   video.autoplay = true;
+  video.disableRemotePlayback = true;
   video.style.display = 'none';
   head.appendChild(video);
-
+  video.load();
   setTimeout(() => {
+    video.pause();
+    video.src = '';
+    video.load();
     head.removeChild(video);
   }, PROGRESSIVE_PRELOAD_DURATION);
 }
 
+async function preloadStream(hash: string) {
+  const loader = makeProgressiveLoader(getProgressiveUrl(hash));
+  let cachedSize = 0;
+  for await (const chunk of loader) {
+    cachedSize += chunk.byteLength;
+    if (cachedSize >= STREAM_PRELOAD_SIZE) {
+      break;
+    }
+  }
+}
+
 export default useStoryPreloader;

src/hooks/useStreaming.ts

@@ -0,0 +1,132 @@
+import type { RefObject } from 'react';
+import { useEffect } from '../lib/teact/teact';
+
+import { DEBUG } from '../config';
+import { requestMutation } from '../lib/fasterdom/fasterdom';
+import { applyStyles } from '../util/animation';
+import { makeProgressiveLoader } from '../util/progressieveLoader';
+import { IS_SAFARI } from '../util/windowEnvironment';
+
+const VIDEO_REVEAL_DELAY = 100;
+
+export function useStreaming(videoRef: RefObject<HTMLVideoElement>, url?: string, mimeType?: string) {
+  useEffect(() => {
+    if (!url || !videoRef.current) return undefined;
+    const MediaSourceClass = getMediaSource();
+    const video = videoRef.current;
+
+    if (!IS_SAFARI || !mimeType || !MediaSourceClass?.isTypeSupported(mimeType)) {
+      return undefined;
+    }
+    const mediaSource = new MediaSourceClass();
+
+    function revealVideo() {
+      requestMutation(() => {
+        video.style.display = 'block';
+        setTimeout(() => {
+          requestMutation(() => {
+            applyStyles(video, { opacity: '1' });
+          });
+        }, VIDEO_REVEAL_DELAY);
+      });
+    }
+
+    function onSourceOpen() {
+      if (!url || !mimeType) return;
+
+      const sourceBuffer = mediaSource.addSourceBuffer(mimeType);
+      const loader = makeProgressiveLoader(url);
+
+      function onUpdateEnded() {
+        loader.next()
+          .then(({
+            value,
+            done,
+          }) => {
+            if (mediaSource.readyState !== 'open') return;
+            if (done) {
+              endOfStream(mediaSource);
+              return;
+            }
+            appendBuffer(sourceBuffer, value);
+          });
+      }
+
+      sourceBuffer.addEventListener('updateend', onUpdateEnded);
+
+      loader.next()
+        .then(({
+          value,
+          done,
+        }) => {
+          if (done || mediaSource.readyState !== 'open') return;
+          revealVideo();
+          appendBuffer(sourceBuffer, value);
+        });
+    }
+
+    mediaSource.addEventListener('sourceopen', onSourceOpen, { once: true });
+
+    requestMutation(() => {
+      applyStyles(video, {
+        display: 'none',
+        opacity: '0',
+      });
+      video.src = URL.createObjectURL(mediaSource);
+    });
+
+    return () => {
+      mediaSource.removeEventListener('sourceopen', onSourceOpen);
+      if (mediaSource.readyState === 'open') {
+        endOfStream(mediaSource);
+      }
+      URL.revokeObjectURL(video.src);
+      requestMutation(() => {
+        video.src = '';
+        applyStyles(video, {
+          display: 'none',
+          opacity: '0',
+        });
+      });
+    };
+  }, [mimeType, url, videoRef]);
+}
+
+export function checkIfStreamingSupported(mimeType: string) {
+  if (!IS_SAFARI) return false;
+  const MS = getMediaSource();
+  return Boolean(MS && MS.isTypeSupported(mimeType));
+}
+
+function appendBuffer(sourceBuffer: SourceBuffer, buffer: ArrayBuffer) {
+  try {
+    sourceBuffer.appendBuffer(buffer);
+  } catch (error) {
+    if (DEBUG) {
+      // eslint-disable-next-line no-console
+      console.warn('[Stream] failed to append buffer', error);
+    }
+  }
+}
+
+function endOfStream(mediaSource: MediaSource) {
+  try {
+    mediaSource.endOfStream();
+  } catch (error) {
+    if (DEBUG) {
+      // eslint-disable-next-line no-console
+      console.warn('[Stream] failed to end stream', error);
+    }
+  }
+}
+
+function getMediaSource(): typeof MediaSource | undefined {
+  if ('ManagedMediaSource' in window) {
+    // @ts-ignore
+    return ManagedMediaSource;
+  }
+  if ('MediaSource' in window) {
+    return MediaSource;
+  }
+  return undefined;
+}

src/util/mediaLoader.ts

@@ -115,6 +115,10 @@ export function removeCallback(url: string, callbackUniqueId: string) {
   callbacks.delete(callbackUniqueId);
 }
 
+export function getProgressiveUrl(url: string) {
+  return `${PROGRESSIVE_URL_PREFIX}${url}`;
+}
+
 function getProgressive(url: string) {
   const progressiveUrl = `${PROGRESSIVE_URL_PREFIX}${url}`;
 

src/util/progressieveLoader.ts

@@ -0,0 +1,40 @@
+const DEFAULT_CHUNK_SIZE = 256 * 1024;
+export async function* makeProgressiveLoader(
+  url: string,
+  chunkSize = DEFAULT_CHUNK_SIZE,
+): AsyncGenerator<ArrayBuffer, void, undefined> {
+  let start = 0;
+  let fileSize: number | undefined;
+
+  while (true) {
+    let end = start + chunkSize - 1;
+    if (fileSize && end > fileSize) {
+      end = fileSize - 1;
+    }
+
+    const res = await fetch(url, {
+      headers: { Range: `bytes=${start}-${end}` },
+    });
+
+    if (!res.ok) return;
+
+    // If fileSize is not yet defined, retrieve it from the first chunk's response
+    if (!fileSize) {
+      const contentRange = res.headers.get('Content-Range');
+      const match = contentRange?.match(/\/(\d+)$/);
+      fileSize = match ? Number(match[1]) : undefined;
+
+      if (!fileSize) return;
+    }
+
+    // Yield the chunk data
+    const chunk = await res.arrayBuffer();
+    yield chunk;
+
+    start = end + 1;
+
+    if (start >= fileSize) {
+      return;
+    }
+  }
+}