ztr/TelegramPWA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Media Viewer: Introduce Video Previews (#2689)

Browse Source
This commit is contained in:
Alexander Zinchuk 2023-04-25 17:24:18 +04:00
parent 485a956d59
commit 8535fcff66
46 changed files with 4675 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.eslintignore
View File

@ -1,5 +1,7 @@
src/lib/rlottie/rlottie-wasm.js
src/lib/video-preview/libav*
src/lib/video-preview/polyfill/*
src/lib/webp/webp_wasm.js
src/lib/fasttextweb/fasttext-wasm.js

2
deploy/copy_to_dist.sh
View File

@ -3,6 +3,8 @@
cp -R ./public/* ${1:-"dist"}
cp ./src/lib/rlottie/rlottie-wasm.wasm ${1:-"dist"}
cp ./src/lib/video-preview/libav-3.10.5.1.2-webcodecs.wasm.js ${1:-"dist"}
cp ./src/lib/video-preview/libav-3.10.5.1.2-webcodecs.wasm.wasm ${1:-"dist"}
cp ./src/lib/webp/webp_wasm.wasm ${1:-"dist"}
cp ./node_modules/opus-recorder/dist/decoderWorker.min.wasm ${1:-"dist"}

19
package-lock.json generated
View File

@ -17,6 +17,7 @@
"emoji-data-ios": "git+https://github.com/korenskoy/emoji-data-ios#3c401da02c6792cea0b6c758795da406377bf667",
"idb-keyval": "^6.2.0",
"lowlight": "^2.8.1",
"mp4box": "^0.5.2",
"opus-recorder": "github:Ajaxy/opus-recorder",
"os-browserify": "^0.3.0",
"pako": "^2.1.0",
@ -106,7 +107,6 @@
}
},
"dev/eslint-multitab": {
"name": "eslint-plugin-eslint-multitab-tt",
"version": "0.0.0",
"dev": true,
"license": "ISC",
@ -7438,9 +7438,9 @@
}
},
"node_modules/eslint-doc-generator/node_modules/cosmiconfig": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.1.0.tgz",
"integrity": "sha512-0tLZ9URlPGU7JsKq0DQOQ3FoRsYX8xDZ7xMiATQfaiGMz7EHowNkbU9u1coAOmnh9p/1ySpm0RB3JNWRXM5GCg==",
"version": "8.1.3",
"resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.1.3.tgz",
"integrity": "sha512-/UkO2JKI18b5jVMJUp0lvKFMpa/Gye+ZgZjKD+DGEN9y7NRcf/nK1A0sp67ONmKtnDCNMS44E6jrk0Yc3bDuUw==",
"dev": true,
"dependencies": {
"import-fresh": "^3.2.1",
@ -13151,6 +13151,11 @@
"node": ">=10"
}
},
"node_modules/mp4box": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/mp4box/-/mp4box-0.5.2.tgz",
"integrity": "sha512-zRmGlvxy+YdW3Dmt+TR4xPHynbxwXtAQDTN/Fo9N3LMxaUlB2C5KmZpzYyGKy4c7k4Jf3RCR0A2pm9SZELOLXw=="
},
"node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@ -16699,9 +16704,9 @@
}
},
"node_modules/type-fest": {
"version": "3.6.1",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.6.1.tgz",
"integrity": "sha512-htXWckxlT6U4+ilVgweNliPqlsVSSucbxVexRYllyMVJDtf5rTjv6kF/s+qAd4QSL1BZcnJPEJavYBPQiWuZDA==",
"version": "3.8.0",
"resolved": "https://registry.npmjs.org/type-fest/-/type-fest-3.8.0.tgz",
"integrity": "sha512-FVNSzGQz9Th+/9R6Lvv7WIAkstylfHN2/JYxkyhhmKFYh9At2DST8t6L6Lref9eYO8PXFTfG9Sg1Agg0K3vq3Q==",
"dev": true,
"engines": {
"node": ">=14.16"

1
package.json
View File

@ -124,6 +124,7 @@
"emoji-data-ios": "git+https://github.com/korenskoy/emoji-data-ios#3c401da02c6792cea0b6c758795da406377bf667",
"idb-keyval": "^6.2.0",
"lowlight": "^2.8.1",
"mp4box": "^0.5.2",
"opus-recorder": "github:Ajaxy/opus-recorder",
"os-browserify": "^0.3.0",
"pako": "^2.1.0",

2
src/components/mediaViewer/MediaViewer.tsx
View File

@ -213,7 +213,7 @@ const MediaViewer: FC<StateProps> = ({
useEffect(() => {
if (isGhostAnimation && isOpen && (!prevMessage || shouldAnimateOpening) && !prevAvatarOwner) {
dispatchHeavyAnimationEvent(ANIMATION_DURATION + ANIMATION_END_DELAY);
animateOpening(hasFooter, origin!, bestImageData!, dimensions, isVideo, message);
animateOpening(hasFooter, origin!, bestImageData!, dimensions!, isVideo, message);
}
if (isGhostAnimation && !isOpen && (prevMessage || prevAvatarOwner)) {

10
src/components/mediaViewer/MediaViewerContent.tsx
View File

@ -8,7 +8,9 @@ import type {
import type { AnimationLevel } from '../../types';
import { MediaViewerOrigin } from '../../types';
import { IS_TOUCH_ENV } from '../../util/windowEnvironment';
import {
IS_TOUCH_ENV, IS_IOS, IS_ANDROID, ARE_WEBCODECS_SUPPORTED,
} from '../../util/windowEnvironment';
import {
selectChat, selectChatMessage, selectTabState, selectIsMessageProtected, selectScheduledMessage, selectUser,
} from '../../global/selectors';
@ -57,6 +59,7 @@ type StateProps = {
const ANIMATION_DURATION = 350;
const MOBILE_VERSION_CONTROL_WIDTH = 350;
const IS_PREVIEW_DISABLED = (IS_IOS || IS_ANDROID) && !ARE_WEBCODECS_SUPPORTED;
const MediaViewerContent: FC<OwnProps & StateProps> = (props) => {
const {
@ -89,6 +92,7 @@ const MediaViewerContent: FC<OwnProps & StateProps> = (props) => {
bestData,
dimensions,
isGif,
isLocal,
isVideoAvatar,
videoSize,
loadProgress,
@ -111,7 +115,7 @@ const MediaViewerContent: FC<OwnProps & StateProps> = (props) => {
<div key={chatId} className="MediaViewerContent">
{renderPhoto(
bestData,
calculateMediaViewerDimensions(dimensions, false),
calculateMediaViewerDimensions(dimensions!, false),
!isMobile && !isProtected,
isProtected,
)}
@ -130,6 +134,7 @@ const MediaViewerContent: FC<OwnProps & StateProps> = (props) => {
fileSize={videoSize!}
isMediaViewerOpen={isOpen && isActive}
isProtected={isProtected}
isPreviewDisabled={IS_PREVIEW_DISABLED || isLocal}
noPlay={!isActive}
onClose={onClose}
isMuted
@ -179,6 +184,7 @@ const MediaViewerContent: FC<OwnProps & StateProps> = (props) => {
fileSize={videoSize!}
isMediaViewerOpen={isOpen && isActive}
noPlay={!isActive}
isPreviewDisabled={IS_PREVIEW_DISABLED || isLocal}
onClose={onClose}
isMuted={isMuted}
isHidden={isHidden}

4
src/components/mediaViewer/MediaViewerSlides.tsx
View File

@ -409,7 +409,7 @@ const MediaViewerSlides: FC<OwnProps> = ({
const cleanup = captureEvents(containerRef.current, {
isNotPassive: true,
withNativeDrag: true,
excludedClosestSelector: '.MediaViewerFooter, .ZoomControls',
excludedClosestSelector: '.MediaViewerFooter, .ZoomControls, .VideoPlayerControls',
minZoom: MIN_ZOOM,
maxZoom: MAX_ZOOM,
doubleTapZoom: DOUBLE_TAP_ZOOM,
@ -775,7 +775,7 @@ function checkIfControlTarget(e: TouchEvent | MouseEvent) {
if (checkIfInsideSelector(target, '.VideoPlayerControls')) {
if (checkIfInsideSelector(
target,
'.play, .fullscreen, .volume, .volume-slider, .playback-rate, .playback-rate-menu',
'.play, .fullscreen, .volume, .volume-slider, .playback-rate, .playback-rate-menu, .SeekLine',
)) {
return true;
}

93
src/components/mediaViewer/SeekLine.module.scss Normal file
View File

@ -0,0 +1,93 @@
.container {
position: absolute;
left: 1rem;
right: 1rem;
top: 1rem;
height: 1rem;
touch-action: none;
cursor: pointer;
}
.preview {
position: absolute;
left: 0;
z-index: 1;
bottom: calc(100% + 0.5rem);
border-radius: 0.25rem;
overflow: hidden;
background: #000;
}
.previewCanvas {
width: 100%;
height: 100%;
display: block;
}
body:global(.is-touch-env) .preview {
bottom: calc(100% + 0.75rem);
}
.previewTime {
position: absolute;
left: 0;
right: 0;
bottom: 0;
text-align: center;
}
.previewTimeText {
border-top-left-radius: 0.125rem;
border-top-right-radius: 0.125rem;
background: rgba(0, 0, 0, 0.5);
color: rgba(255, 255, 255, 0.8);
padding: 0.25rem 0.5rem;
}
.track {
position: absolute;
top: 50%;
height: 5px;
transform: translateY(-50%);
background-color: rgba(255, 255, 255, 0.16);
border-radius: var(--border-radius-default);
left: -0.25rem;
right: -0.25rem;
}
.buffered {
position: absolute;
background-color: rgba(255, 255, 255, 0.5);
top: 0;
left: 0;
height: 100%;
border-radius: var(--border-radius-default);
}
.played {
background: var(--color-primary);
position: absolute;
top: 0;
left: 0;
height: 100%;
border-radius: var(--border-radius-default);
&::after {
content: "";
position: absolute;
width: 0.75rem;
height: 0.75rem;
border-radius: 50%;
background-color: var(--color-primary);
right: 0;
top: 50%;
transform: translate(50%, -50%) scale(1);
transition: transform 0.2s ease;
}
}
body:global(.is-touch-env) .seeking {
&::after {
transform: translate(50%, -50%) scale(2);
}
}

233
src/components/mediaViewer/SeekLine.tsx Normal file
View File

@ -0,0 +1,233 @@
import React, {
useRef, useState, useCallback, useEffect, memo, useMemo, useLayoutEffect,
} from '../../lib/teact/teact';
import type { BufferedRange } from '../../hooks/useBuffering';
import type { ApiDimensions } from '../../api/types';
import useSignal from '../../hooks/useSignal';
import useCurrentTimeSignal from './hooks/currentTimeSignal';
import { captureEvents } from '../../util/captureEvents';
import { IS_TOUCH_ENV } from '../../util/windowEnvironment';
import buildClassName from '../../util/buildClassName';
import { formatMediaDuration } from '../../util/dateFormat';
import { clamp, round } from '../../util/math';
import { createVideoPreviews, renderVideoPreview, getPreviewDimensions } from '../../lib/video-preview/VideoPreview';
import ShowTransition from '../ui/ShowTransition';
import styles from './SeekLine.module.scss';
type OwnProps = {
url?: string;
duration: number;
bufferedRanges: BufferedRange[];
isActive?: boolean;
isPreviewDisabled?: boolean;
isReady: boolean;
posterSize?: ApiDimensions;
onSeek: (position: number) => void;
onSeekStart: () => void;
};
const SeekLine: React.FC<OwnProps> = ({
duration,
bufferedRanges,
isReady,
posterSize,
url,
isActive,
isPreviewDisabled,
onSeek,
onSeekStart,
}) => {
// eslint-disable-next-line no-null/no-null
const seekerRef = useRef<HTMLDivElement>(null);
const [getCurrentTime] = useCurrentTimeSignal();
const [getSelectedTime, setSelectedTime] = useSignal(getCurrentTime());
const [getPreviewOffset, setPreviewOffset] = useSignal(0);
const [getPreviewTime, setPreviewTime] = useSignal(0);
const isLockedRef = useRef<boolean>(false);
const [isPreviewVisible, setPreviewVisible] = useState(false);
const [isSeeking, setIsSeeking] = useState(false);
// eslint-disable-next-line no-null/no-null
const previewCanvasRef = useRef<HTMLCanvasElement>(null);
// eslint-disable-next-line no-null/no-null
const previewRef = useRef<HTMLDivElement>(null);
// eslint-disable-next-line no-null/no-null
const progressRef = useRef<HTMLDivElement>(null);
// eslint-disable-next-line no-null/no-null
const previewTimeRef = useRef<HTMLDivElement>(null);
const previewSize = useMemo(() => {
return getPreviewDimensions(posterSize?.width || 0, posterSize?.height || 0);
}, [posterSize]);
const setPreview = useCallback((time: number) => {
time = Math.floor(time);
setPreviewTime(time);
renderVideoPreview(time);
}, [setPreviewTime]);
useEffect(() => {
if (isPreviewDisabled || !url || !isReady) return undefined;
return createVideoPreviews(url, previewCanvasRef.current!);
}, [url, isReady, isPreviewDisabled]);
useEffect(() => {
setPreviewVisible(false);
}, [isActive]);
useEffect(() => {
if (!isLockedRef.current && !isSeeking) {
setSelectedTime(getCurrentTime());
}
}, [getCurrentTime, isSeeking, setSelectedTime]);
useLayoutEffect(() => {
if (!progressRef.current) return;
const progress = round((getSelectedTime() / duration) * 100, 2);
progressRef.current.style.width = `${progress}%`;
}, [getSelectedTime, duration]);
useLayoutEffect(() => {
if (!previewRef.current) return;
previewRef.current.style.left = `${getPreviewOffset()}px`;
}, [getPreviewOffset]);
useLayoutEffect(() => {
if (!previewTimeRef.current) return;
previewTimeRef.current.innerText = formatMediaDuration(getPreviewTime());
}, [getPreviewTime]);
useEffect(() => {
if (!seekerRef.current || !isActive) return undefined;
const seeker = seekerRef.current;
let seekerSize = seeker.getBoundingClientRect();
let time = 0;
let offset = 0;
const getPreviewProps = (e: MouseEvent | TouchEvent) => {
const pageX = e instanceof MouseEvent ? e.pageX : e.touches[0].pageX;
const t = clamp(duration * ((pageX - seekerSize.left) / seekerSize.width), 0, duration);
if (isPreviewDisabled) return [t, 0];
if (!seekerSize.width) seekerSize = seeker.getBoundingClientRect();
const preview = previewRef.current!;
const o = clamp(
pageX - seekerSize.left - preview.clientWidth / 2, -4, seekerSize.width - preview.clientWidth + 4,
);
return [t, o];
};
const handleSeek = (e: MouseEvent | TouchEvent) => {
setPreviewVisible(true);
([time, offset] = getPreviewProps(e));
void setPreview(time);
setPreviewOffset(offset);
setSelectedTime(time);
};
const handleStartSeek = () => {
setPreviewVisible(true);
setIsSeeking(true);
onSeekStart();
};
const handleStopSeek = () => {
isLockedRef.current = true;
setPreviewVisible(false);
setIsSeeking(false);
setSelectedTime(time);
onSeek(time);
// Prevent current time updates from overriding the selected time
setTimeout(() => {
isLockedRef.current = false;
}, 500);
};
const cleanup = captureEvents(seeker, {
onCapture: handleStartSeek,
onRelease: handleStopSeek,
onClick: handleStopSeek,
onDrag: handleSeek,
});
if (IS_TOUCH_ENV || isPreviewDisabled) {
return cleanup;
}
const handleSeekMouseMove = (e: MouseEvent) => {
setPreviewVisible(true);
([time, offset] = getPreviewProps(e));
setPreviewOffset(offset);
void setPreview(time);
};
const handleSeekMouseLeave = () => {
setPreviewVisible(false);
};
seeker.addEventListener('mousemove', handleSeekMouseMove);
seeker.addEventListener('mouseenter', handleSeekMouseMove);
seeker.addEventListener('mouseleave', handleSeekMouseLeave);
return () => {
cleanup();
seeker.removeEventListener('mousemove', handleSeekMouseMove);
seeker.removeEventListener('mouseenter', handleSeekMouseMove);
seeker.removeEventListener('mouseleave', handleSeekMouseLeave);
};
}, [
duration,
setPreview,
isActive,
onSeek,
onSeekStart,
setPreviewOffset,
setSelectedTime,
setIsSeeking,
isPreviewDisabled,
]);
return (
<div className={styles.container} ref={seekerRef}>
{!isPreviewDisabled && (
<ShowTransition
isOpen
isHidden={!isPreviewVisible}
className={styles.preview}
style={`width: ${previewSize.width}px; height: ${previewSize.height}px`}
ref={previewRef}
>
<canvas className={styles.previewCanvas} ref={previewCanvasRef} />
<div className={styles.previewTime}>
<span className={styles.previewTimeText} ref={previewTimeRef} />
</div>
</ShowTransition>
)}
<div className={styles.track}>
{bufferedRanges.map(({
start,
end,
}) => (
<div
key={`${start}-${end}`}
className={styles.buffered}
style={`left: ${start * 100}%; right: ${100 - end * 100}%`}
/>
))}
</div>
<div className={styles.track}>
<div
ref={progressRef}
className={buildClassName(styles.played, isSeeking && styles.seeking)}
/>
</div>
</div>
);
};
export default memo(SeekLine);

7
src/components/mediaViewer/VideoPlayer.scss
View File

@ -3,12 +3,7 @@
display: inline-flex;
flex-direction: column;
overflow: hidden;
@media (min-width: 601px) {
// Safari: custom controls are not displayed after exiting full screen mode.
z-index: 1;
}
@media (max-width: 600px) {
overflow: visible;
}

51
src/components/mediaViewer/VideoPlayer.tsx
View File

@ -16,6 +16,7 @@ import usePictureInPicture from '../../hooks/usePictureInPicture';
import useShowTransition from '../../hooks/useShowTransition';
import useVideoCleanup from '../../hooks/useVideoCleanup';
import useAppLayout from '../../hooks/useAppLayout';
import useCurrentTimeSignal from './hooks/currentTimeSignal';
import useControlsSignal from './hooks/useControlsSignal';
import Button from '../ui/Button';
@ -31,6 +32,7 @@ type OwnProps = {
posterSize?: ApiDimensions;
loadProgress?: number;
fileSize: number;
isPreviewDisabled?: boolean;
isMediaViewerOpen?: boolean;
noPlay?: boolean;
volume: number;
@ -45,6 +47,7 @@ type OwnProps = {
};
const MAX_LOOP_DURATION = 30; // Seconds
const MIN_READY_STATE = 4;
const REWIND_STEP = 5; // Seconds
const VideoPlayer: FC<OwnProps> = ({
@ -64,6 +67,7 @@ const VideoPlayer: FC<OwnProps> = ({
shouldCloseOnClick,
isProtected,
isClickDisabled,
isPreviewDisabled,
}) => {
const {
setMediaViewerVolume,
@ -74,9 +78,10 @@ const VideoPlayer: FC<OwnProps> = ({
// eslint-disable-next-line no-null/no-null
const videoRef = useRef<HTMLVideoElement>(null);
const [isPlaying, setIsPlaying] = useState(!IS_TOUCH_ENV || !IS_IOS);
const [currentTime, setCurrentTime] = useState(0);
const [isFullscreen, setFullscreen, exitFullscreen] = useFullscreen(videoRef, setIsPlaying);
const { isMobile } = useAppLayout();
const duration = videoRef.current?.duration || 0;
const isLooped = isGif || duration <= MAX_LOOP_DURATION;
const handleEnterFullscreen = useCallback(() => {
// Yandex browser doesn't support PIP when video is hidden
@ -95,7 +100,7 @@ const VideoPlayer: FC<OwnProps> = ({
isInPictureInPicture,
] = usePictureInPicture(videoRef, handleEnterFullscreen, handleLeaveFullscreen);
const [, toggleControls] = useControlsSignal();
const [, toggleControls, lockControls] = useControlsSignal();
const handleVideoMove = useCallback(() => {
toggleControls(true);
@ -110,8 +115,9 @@ const VideoPlayer: FC<OwnProps> = ({
}, [toggleControls]);
const {
isBuffered, bufferedRanges, bufferingHandlers, bufferedProgress,
isReady, isBuffered, bufferedRanges, bufferingHandlers, bufferedProgress,
} = useBuffering();
const {
shouldRender: shouldRenderSpinner,
transitionClassNames: spinnerClassNames,
@ -121,6 +127,10 @@ const VideoPlayer: FC<OwnProps> = ({
transitionClassNames: playButtonClassNames,
} = useShowTransition(IS_IOS && !isPlaying && !shouldRenderSpinner, undefined, undefined, 'slow');
useEffect(() => {
lockControls(shouldRenderSpinner);
}, [lockControls, shouldRenderSpinner]);
useEffect(() => {
if (noPlay || !isMediaViewerOpen) {
videoRef.current!.pause();
@ -132,15 +142,6 @@ const VideoPlayer: FC<OwnProps> = ({
}
}, [noPlay, isMediaViewerOpen, url, setMediaViewerMuted]);
useEffect(() => {
if (videoRef.current!.currentTime === videoRef.current!.duration) {
setCurrentTime(0);
setIsPlaying(false);
} else {
setCurrentTime(videoRef.current!.currentTime);
}
}, [currentTime]);
useEffect(() => {
videoRef.current!.volume = volume;
}, [volume]);
@ -164,7 +165,6 @@ const VideoPlayer: FC<OwnProps> = ({
if (isClickDisabled) {
return;
}
if (shouldCloseOnClick) {
onClose(e);
} else {
@ -173,16 +173,25 @@ const VideoPlayer: FC<OwnProps> = ({
}, [onClose, shouldCloseOnClick, togglePlayState, isClickDisabled]);
useVideoCleanup(videoRef, []);
const [, setCurrentTime] = useCurrentTimeSignal();
const handleTimeUpdate = useCallback((e: React.SyntheticEvent<HTMLVideoElement>) => {
setCurrentTime(e.currentTarget.currentTime);
}, []);
const video = e.currentTarget;
if (video.readyState >= MIN_READY_STATE) {
setCurrentTime(video.currentTime);
}
if (!isLooped && video.currentTime === video.duration) {
setCurrentTime(0);
setIsPlaying(false);
}
}, [isLooped, setCurrentTime]);
const handleEnded = useCallback(() => {
if (isLooped) return;
setCurrentTime(0);
setIsPlaying(false);
toggleControls(true);
}, [toggleControls]);
}, [isLooped, setCurrentTime, toggleControls]);
const handleFullscreenChange = useCallback(() => {
if (isFullscreen && exitFullscreen) {
@ -251,7 +260,6 @@ const VideoPlayer: FC<OwnProps> = ({
const wrapperStyle = posterSize && `width: ${posterSize.width}px; height: ${posterSize.height}px`;
const videoStyle = `background-image: url(${posterData})`;
const shouldToggleControls = !IS_TOUCH_ENV && !isForceMobileVersion;
const duration = videoRef.current?.duration || 0;
return (
// eslint-disable-next-line jsx-a11y/mouse-events-have-key-events
@ -277,7 +285,7 @@ const VideoPlayer: FC<OwnProps> = ({
autoPlay={IS_TOUCH_ENV}
controlsList="nodownload"
playsInline
loop={isGif || duration <= MAX_LOOP_DURATION}
loop={isLooped}
// This is to force autoplaying on mobiles
muted={isGif || isMuted}
id="media-viewer-video"
@ -313,18 +321,21 @@ const VideoPlayer: FC<OwnProps> = ({
/>
</div>
)}
{!isGif && !shouldRenderSpinner && (
{!isGif && (
<VideoPlayerControls
url={url}
isPlaying={isPlaying}
bufferedRanges={bufferedRanges}
bufferedProgress={bufferedProgress}
isBuffered={isBuffered}
currentTime={currentTime}
isFullscreenSupported={Boolean(setFullscreen)}
isPictureInPictureSupported={isPictureInPictureSupported}
isFullscreen={isFullscreen}
isPreviewDisabled={isPreviewDisabled}
fileSize={fileSize}
duration={duration}
isReady={isReady}
posterSize={posterSize}
isForceMobileVersion={isForceMobileVersion}
onSeek={handleSeek}
onChangeFullscreen={handleFullscreenChange}

53
src/components/mediaViewer/VideoPlayerControls.scss
View File

@ -88,6 +88,7 @@
.player-time {
color: rgba(255, 255, 255, 0.5);
font-variant-numeric: tabular-nums;
white-space: nowrap;
}
@ -101,58 +102,6 @@
}
}
.player-seekline {
position: absolute;
left: 1rem;
right: 1rem;
top: 1rem;
height: 1rem;
touch-action: none;
cursor: var(--custom-cursor, pointer);
&-track {
position: absolute;
top: 50%;
left: -0.25rem;
right: -0.25rem;
height: 5px;
transform: translateY(-50%);
background-color: rgba(255, 255, 255, 0.16);
border-radius: var(--border-radius-default);
}
&-buffered,
&-played,
&-input {
position: absolute;
top: 0;
left: 0;
height: 100%;
border-radius: var(--border-radius-default);
}
&-buffered {
position: absolute;
background-color: rgba(255, 255, 255, 0.5);
}
&-played {
background: var(--color-primary);
&::after {
content: "";
position: absolute;
width: 0.75rem;
height: 0.75rem;
border-radius: 50%;
background-color: var(--color-primary);
right: 0;
top: 50%;
transform: translate(0.325rem, -50%);
}
}
}
.playback-rate-menu {
color: var(--color-text);
.bubble {

114
src/components/mediaViewer/VideoPlayerControls.tsx
View File

@ -1,21 +1,25 @@
import type { FC } from '../../lib/teact/teact';
import React, {
useEffect, useRef, useCallback, useMemo,
useCallback, memo, useEffect, useMemo, useLayoutEffect,
} from '../../lib/teact/teact';
import type { BufferedRange } from '../../hooks/useBuffering';
import type { ApiDimensions } from '../../api/types';
import { IS_IOS, IS_TOUCH_ENV } from '../../util/windowEnvironment';
import buildClassName from '../../util/buildClassName';
import { formatMediaDuration } from '../../util/dateFormat';
import { formatFileSize } from '../../util/textFormat';
import { captureEvents } from '../../util/captureEvents';
import useLang from '../../hooks/useLang';
import useFlag from '../../hooks/useFlag';
import useAppLayout from '../../hooks/useAppLayout';
import useControlsSignal from './hooks/useControlsSignal';
import useDerivedState from '../../hooks/useDerivedState';
import useSignal from '../../hooks/useSignal';
import useCurrentTimeSignal from './hooks/currentTimeSignal';
import useControlsSignal from './hooks/useControlsSignal';
import buildClassName from '../../util/buildClassName';
import { IS_IOS, IS_TOUCH_ENV } from '../../util/windowEnvironment';
import { formatMediaDuration } from '../../util/dateFormat';
import { formatFileSize } from '../../util/textFormat';
import SeekLine from './SeekLine';
import Button from '../ui/Button';
import RangeSlider from '../ui/RangeSlider';
import Menu from '../ui/Menu';
@ -24,20 +28,23 @@ import MenuItem from '../ui/MenuItem';
import './VideoPlayerControls.scss';
type OwnProps = {
url?: string;
bufferedRanges: BufferedRange[];
bufferedProgress: number;
currentTime: number;
duration: number;
isReady: boolean;
fileSize: number;
isForceMobileVersion?: boolean;
isPlaying: boolean;
isFullscreenSupported: boolean;
isPictureInPictureSupported: boolean;
isFullscreen: boolean;
isPreviewDisabled?: boolean;
isBuffered: boolean;
volume: number;
isMuted: boolean;
playbackRate: number;
posterSize?: ApiDimensions;
onChangeFullscreen: (e: React.MouseEvent<HTMLButtonElement, MouseEvent>) => void;
onPictureInPictureChange?: () => void ;
onVolumeClick: () => void;
@ -61,19 +68,22 @@ const PLAYBACK_RATES = [
const HIDE_CONTROLS_TIMEOUT_MS = 3000;
const VideoPlayerControls: FC<OwnProps> = ({
url,
bufferedRanges,
bufferedProgress,
currentTime,
duration,
isReady,
fileSize,
isForceMobileVersion,
isPlaying,
isFullscreenSupported,
isFullscreen,
isBuffered,
isPreviewDisabled,
volume,
isMuted,
playbackRate,
posterSize,
onChangeFullscreen,
onVolumeClick,
onVolumeChange,
@ -84,10 +94,9 @@ const VideoPlayerControls: FC<OwnProps> = ({
onSeek,
}) => {
const [isPlaybackMenuOpen, openPlaybackMenu, closePlaybackMenu] = useFlag();
// eslint-disable-next-line no-null/no-null
const seekerRef = useRef<HTMLDivElement>(null);
const isSeekingRef = useRef<boolean>(false);
const isSeeking = isSeekingRef.current;
const [getCurrentTime] = useCurrentTimeSignal();
const currentTime = useDerivedState(() => Math.trunc(getCurrentTime()), [getCurrentTime]);
const [getIsSeeking, setIsSeeking] = useSignal(false);
const { isMobile } = useAppLayout();
const [getIsVisible, setVisibility] = useControlsSignal();
@ -96,7 +105,7 @@ const VideoPlayerControls: FC<OwnProps> = ({
useEffect(() => {
if (!IS_TOUCH_ENV && !isForceMobileVersion) return undefined;
let timeout: number | undefined;
if (!isVisible || !isPlaying || isSeeking || isPlaybackMenuOpen) {
if (!isVisible || !isPlaying || isPlaybackMenuOpen || getIsSeeking()) {
if (timeout) window.clearTimeout(timeout);
return undefined;
}
@ -106,9 +115,9 @@ const VideoPlayerControls: FC<OwnProps> = ({
return () => {
if (timeout) window.clearTimeout(timeout);
};
}, [isPlaying, isVisible, isSeeking, setVisibility, isPlaybackMenuOpen, isForceMobileVersion]);
}, [isPlaying, isVisible, setVisibility, isPlaybackMenuOpen, getIsSeeking, isForceMobileVersion]);
useEffect(() => {
useLayoutEffect(() => {
if (isVisible) {
document.body.classList.add('video-controls-visible');
} else {
@ -127,35 +136,14 @@ const VideoPlayerControls: FC<OwnProps> = ({
const lang = useLang();
const handleSeek = useCallback((e: MouseEvent | TouchEvent) => {
if (isSeekingRef.current && seekerRef.current) {
const {
width,
left,
} = seekerRef.current.getBoundingClientRect();
const clientX = e instanceof MouseEvent ? e.clientX : e.targetTouches[0].clientX;
onSeek(Math.max(Math.min(duration * ((clientX - left) / width), duration), 0));
}
}, [duration, onSeek]);
const handleSeek = useCallback((position: number) => {
setIsSeeking(false);
onSeek(position);
}, [onSeek, setIsSeeking]);
const handleStartSeek = useCallback((e: MouseEvent | TouchEvent) => {
isSeekingRef.current = true;
handleSeek(e);
}, [handleSeek]);
const handleStopSeek = useCallback(() => {
isSeekingRef.current = false;
}, []);
useEffect(() => {
if (!seekerRef.current || !isVisible) return undefined;
return captureEvents(seekerRef.current, {
onCapture: handleStartSeek,
onRelease: handleStopSeek,
onClick: handleStopSeek,
onDrag: handleSeek,
});
}, [isVisible, handleStartSeek, handleSeek, handleStopSeek]);
const handleSeekStart = useCallback(() => {
setIsSeeking(true);
}, [setIsSeeking]);
const volumeIcon = useMemo(() => {
if (volume === 0 || isMuted) return 'icon-muted';
@ -169,7 +157,17 @@ const VideoPlayerControls: FC<OwnProps> = ({
className={buildClassName('VideoPlayerControls', isForceMobileVersion && 'mobile', isVisible && 'active')}
onClick={stopEvent}
>
{renderSeekLine(currentTime, duration, bufferedRanges, seekerRef)}
<SeekLine
url={url}
duration={duration}
isReady={isReady}
isPreviewDisabled={isPreviewDisabled}
posterSize={posterSize}
bufferedRanges={bufferedRanges}
onSeek={handleSeek}
onSeekStart={handleSeekStart}
isActive={isVisible}
/>
<div className="buttons">
<Button
ariaLabel={lang('AccActionPlay')}
@ -267,28 +265,4 @@ function renderTime(currentTime: number, duration: number) {
</div>
);
}
function renderSeekLine(
currentTime: number, duration: number, bufferedRanges: BufferedRange[], seekerRef: React.RefObject<HTMLDivElement>,
) {
const percentagePlayed = (currentTime / duration) * 100;
return (
<div className="player-seekline" ref={seekerRef}>
<div className="player-seekline-track">
{bufferedRanges.map(({ start, end }) => (
<div
className="player-seekline-buffered"
style={`left: ${start * 100}%; right: ${100 - end * 100}%`}
/>
))}
<div
className="player-seekline-played"
style={`width: ${percentagePlayed || 0}%`}
/>
</div>
</div>
);
}
export default VideoPlayerControls;
export default memo(VideoPlayerControls);

13
src/components/mediaViewer/hooks/currentTimeSignal.ts Normal file
View File

@ -0,0 +1,13 @@
import { createSignal } from '../../../util/signals';
import { useEffect } from '../../../lib/teact/teact';
export const [getCurrentTime, setCurrentTime] = createSignal(0);
export default function useCurrentTimeSignal() {
useEffect(() => {
return () => {
setCurrentTime(0);
};
}, []);
return [getCurrentTime, setCurrentTime] as const;
}

41
src/components/mediaViewer/hooks/useMediaProps.ts
View File

@ -1,5 +1,5 @@
import type {
ApiMessage, ApiChat, ApiUser, ApiDimensions,
ApiMessage, ApiChat, ApiUser,
} from '../../../api/types';
import { ApiMediaFormat } from '../../../api/types';
import {
@ -18,7 +18,8 @@ import {
getMessageDocument,
getPhotoFullDimensions,
getVideoDimensions,
getMessageFileSize, getMessageActionPhoto,
getMessageFileSize,
getMessageActionPhoto,
} from '../../../global/helpers';
import { useMemo } from '../../../lib/teact/teact';
import useMedia from '../../../hooks/useMedia';
@ -121,25 +122,36 @@ export const useMediaProps = ({
const bestData = localBlobUrl || fullMediaBlobUrl || (
!isVideo ? previewBlobUrl || pictogramBlobUrl || bestImageData : undefined
);
const isLocal = Boolean(localBlobUrl);
const fileName = message
? getMessageFileName(message)
: avatarOwner
? `avatar${avatarOwner!.id}.${avatarOwner?.hasVideoAvatar ? 'mp4' : 'jpg'}`
: undefined;
let dimensions!: ApiDimensions;
if (message) {
if (isDocumentPhoto || isDocumentVideo) {
dimensions = getMessageDocument(message)!.mediaSize!;
} else if (photo || webPagePhoto || actionPhoto) {
dimensions = getPhotoFullDimensions((photo || webPagePhoto || actionPhoto)!)!;
} else if (video || webPageVideo) {
dimensions = getVideoDimensions((video || webPageVideo)!)!;
const dimensions = useMemo(() => {
if (message) {
if (isDocumentPhoto || isDocumentVideo) {
return getMessageDocument(message)!.mediaSize!;
} else if (photo || webPagePhoto) {
return getPhotoFullDimensions((photo || webPagePhoto)!)!;
} else if (video || webPageVideo) {
return getVideoDimensions((video || webPageVideo)!)!;
}
} else {
return isVideoAvatar ? VIDEO_AVATAR_FULL_DIMENSIONS : AVATAR_FULL_DIMENSIONS;
}
} else {
dimensions = isVideoAvatar ? VIDEO_AVATAR_FULL_DIMENSIONS : AVATAR_FULL_DIMENSIONS;
}
return undefined;
}, [
isDocumentPhoto,
isDocumentVideo,
isVideoAvatar,
message,
photo,
video,
webPagePhoto,
webPageVideo,
]);
return {
getMediaHash,
@ -160,6 +172,7 @@ export const useMediaProps = ({
isFromSharedMedia,
avatarPhoto: avatarMedia,
isVideoAvatar,
isLocal,
loadProgress,
videoSize,
};

12
src/components/ui/ShowTransition.tsx
View File

@ -14,7 +14,9 @@ type OwnProps = {
onClick?: (e: React.MouseEvent<HTMLElement, MouseEvent>) => void;
noCloseTransition?: boolean;
shouldAnimateFirstRender?: boolean;
style?: string;
children: React.ReactNode;
ref?: React.LegacyRef<HTMLDivElement>;
};
const ShowTransition: FC<OwnProps> = ({
@ -27,6 +29,8 @@ const ShowTransition: FC<OwnProps> = ({
children,
noCloseTransition,
shouldAnimateFirstRender,
style,
ref,
}) => {
const prevIsOpen = usePrevious(isOpen);
const prevChildren = usePrevious(children);
@ -49,7 +53,13 @@ const ShowTransition: FC<OwnProps> = ({
return (
(shouldRender || isHidden) && (
<div id={id} className={buildClassName(className, transitionClassNames)} onClick={onClick}>
<div
id={id}
ref={ref}
className={buildClassName(className, transitionClassNames)}
onClick={onClick}
style={style}
>
{isOpen ? children : fromChildrenRef.current!}
</div>
)

3
src/hooks/useBuffering.ts
View File

@ -17,6 +17,7 @@ export type BufferedRange = { start: number; end: number };
const useBuffering = (noInitiallyBuffered = false, onTimeUpdate?: AnyToVoidFunction) => {
const [isBuffered, setIsBuffered] = useState(!noInitiallyBuffered);
const [isReady, setIsReady] = useState(false);
const [bufferedProgress, setBufferedProgress] = useState(0);
const [bufferedRanges, setBufferedRanges] = useState<BufferedRange[]>([]);
@ -42,6 +43,7 @@ const useBuffering = (noInitiallyBuffered = false, onTimeUpdate?: AnyToVoidFunct
}
setIsBufferedDebounced(media.readyState >= MIN_READY_STATE || media.currentTime > 0);
setIsReady((current) => current || media.readyState > MIN_READY_STATE);
}
}, [onTimeUpdate, setIsBufferedDebounced]);
@ -55,6 +57,7 @@ const useBuffering = (noInitiallyBuffered = false, onTimeUpdate?: AnyToVoidFunct
};
return {
isReady,
isBuffered,
bufferedProgress,
bufferedRanges,

3
src/lib/gramjs/Utils.js
View File

@ -326,6 +326,9 @@ function getInputLocation(location) {
* @returns {Number}
*/
function getDownloadPartSize(fileSize) {
if (fileSize <= 65536) { // 64KB
return 64;
}
if (fileSize <= 104857600) { // 100MB
return 128;
}

8
src/lib/gramjs/client/downloadFile.ts
View File

@ -140,11 +140,11 @@ async function downloadFile2(
end = end && end < fileSize ? end : fileSize - 1;
if (!partSizeKb) {
partSizeKb = fileSize ? getDownloadPartSize(fileSize) : DEFAULT_CHUNK_SIZE;
partSizeKb = fileSize ? getDownloadPartSize(start ? (end - start + 1) : fileSize) : DEFAULT_CHUNK_SIZE;
}
const partSize = partSizeKb * 1024;
const partsCount = end ? Math.ceil((end - start) / partSize) : 1;
const partsCount = end ? Math.ceil((end + 1 - start + 1) / partSize) : 1;
const noParallel = !end;
const shouldUseMultipleConnections = fileSize
&& fileSize >= MULTIPLE_CONNECTIONS_MIN_FILE_SIZE
@ -186,6 +186,10 @@ async function downloadFile2(
isPrecise = true;
}
if (offset % MIN_CHUNK_SIZE !== 0 || limit % MIN_CHUNK_SIZE !== 0) {
isPrecise = true;
}
// Use only first connection for avatars, because no size is known and we don't want to
// download empty parts using all connections at once
const senderIndex = !shouldUseMultipleConnections ? 0 : currentForemanIndex % (

9
src/lib/mediaWorker/index.worker.ts Normal file
View File

@ -0,0 +1,9 @@
import type { RLottieApi } from '../rlottie/rlottie.worker';
import type { VideoPreviewApi } from '../video-preview/video-preview.worker';
import '../rlottie/rlottie.worker';
import '../video-preview/video-preview.worker';
export type MediaWorkerApi =
RLottieApi
& VideoPreviewApi;

17
src/lib/rlottie/RLottie.ts
View File

@ -1,14 +1,12 @@
import { requestMeasure, requestMutation } from '../fasterdom/fasterdom';
import type { RLottieApi } from './rlottie.worker';
import {
DPR, IS_SAFARI, IS_ANDROID, IS_IOS,
} from '../../util/windowEnvironment';
import { createConnector } from '../../util/PostMessageConnector';
import { animate } from '../../util/animation';
import cycleRestrict from '../../util/cycleRestrict';
import generateIdFor from '../../util/generateIdFor';
import launchMediaWorkers, { MAX_WORKERS } from '../../util/launchMediaWorkers';
interface Params {
size: number;
@ -24,7 +22,6 @@ type Frame =
| typeof WAITING
| ImageBitmap;
const MAX_WORKERS = Math.min(navigator.hardwareConcurrency || 4, 4);
const HIGH_PRIORITY_QUALITY = (IS_ANDROID || IS_IOS) ? 0.75 : 1;
const LOW_PRIORITY_QUALITY = IS_ANDROID ? 0.5 : 0.75;
const LOW_PRIORITY_QUALITY_SIZE_THRESHOLD = 24;
@ -32,11 +29,9 @@ const HIGH_PRIORITY_CACHE_MODULO = IS_SAFARI ? 2 : 4;
const LOW_PRIORITY_CACHE_MODULO = 0;
const ID_STORE = {};
const workers = launchMediaWorkers().map(({ connector }) => connector);
const instancesByRenderId = new Map<string, RLottie>();
const workers = new Array(MAX_WORKERS).fill(undefined).map(
() => createConnector<RLottieApi>(new Worker(new URL('./rlottie.worker.ts', import.meta.url))),
);
let lastWorkerIndex = -1;
class RLottie {
@ -371,7 +366,7 @@ class RLottie {
this.workerIndex = cycleRestrict(MAX_WORKERS, ++lastWorkerIndex);
workers[this.workerIndex].request({
name: 'init',
name: 'rlottie:init',
args: [
this.renderId,
this.tgsUrl,
@ -385,7 +380,7 @@ class RLottie {
private destroyRenderer() {
workers[this.workerIndex].request({
name: 'destroy',
name: 'rlottie:destroy',
args: [this.renderId],
});
}
@ -407,7 +402,7 @@ class RLottie {
this.initConfig();
workers[this.workerIndex].request({
name: 'changeData',
name: 'rlottie:changeData',
args: [
this.renderId,
this.tgsUrl,
@ -567,7 +562,7 @@ class RLottie {
this.frames[frameIndex] = WAITING;
workers[this.workerIndex].request({
name: 'renderFrames',
name: 'rlottie:renderFrames',
args: [this.renderId, frameIndex, this.onFrameLoad.bind(this)],
});
}

8
src/lib/rlottie/rlottie.worker.ts
View File

@ -165,10 +165,10 @@ function destroy(key: string, isRepeated = false) {
}
const api = {
init,
changeData,
renderFrames,
destroy,
'rlottie:init': init,
'rlottie:changeData': changeData,
'rlottie:renderFrames': renderFrames,
'rlottie:destroy': destroy,
};
createWorkerInterface(api);

214
src/lib/video-preview/MP4Demuxer.ts Normal file
View File

@ -0,0 +1,214 @@
import type { MP4ArrayBuffer, MP4VideoTrack, MP4Info } from 'mp4box';
import MP4Box, { DataStream } from 'mp4box';
import { requestPart } from './requestPart';
const META_PART_SIZE = 64 * 1024;
const MIN_PART_SIZE = 1024;
enum Status {
loading = 'loading',
ready = 'ready',
closed = 'closed',
}
export type MP4DecoderConfig = {
codec: string;
codedHeight: number;
codedWidth: number;
description: Uint8Array;
};
type MP4DemuxerConfig = {
framesPerVideo: number;
stepOffset: number;
stepMultiplier: number;
onConfig: (config: any) => void;
onChunk: (chunk: any) => void;
};
export class MP4Demuxer {
private readonly url: string;
private file: MP4Box.MP4File;
private status = Status.loading;
private readonly framesPerVideo: number;
private readonly stepOffset: number;
private readonly stepMultiplier: number;
private decodedSamples = new Set<string>();
private lastSample = 0;
private readonly onConfig: (config: MP4DecoderConfig) => void;
private readonly onChunk: (chunk: any) => void;
constructor(url: string, {
onConfig,
onChunk,
framesPerVideo,
stepOffset,
stepMultiplier,
}: MP4DemuxerConfig) {
this.url = url;
this.framesPerVideo = framesPerVideo;
this.stepOffset = stepOffset;
this.stepMultiplier = stepMultiplier;
this.onConfig = onConfig;
this.onChunk = onChunk;
this.file = MP4Box.createFile();
this.file.onError = (e) => {
// eslint-disable-next-line no-console
console.error(e);
};
this.file.onReady = this.onReady.bind(this);
this.file.onSamples = this.onSamples.bind(this);
void this.loadMetadata();
}
private async loadMetadata() {
let offset: number | undefined = 0;
while (offset !== undefined) {
try {
offset = await this.requestPart(offset, META_PART_SIZE);
} catch (e) {
// eslint-disable-next-line no-console
console.error(e);
}
if (this.status === Status.ready) break;
}
}
private async loadNextFrames(step: number, size: number, partSize: number) {
let tick = step * this.stepOffset;
let lastSample = 0;
let rap = this.file.seek(tick, true);
while (this.status !== Status.closed && rap.offset < size) {
try {
await this.requestPart(rap.offset, partSize);
if (this.lastSample > 1 && lastSample < this.lastSample) {
tick += step * this.stepMultiplier;
lastSample = this.lastSample;
}
rap = this.file.seek(tick, true);
} catch (e) {
// eslint-disable-next-line no-console
console.error(e);
}
}
this.file.flush();
}
private async requestPart(offset: number, partSize: number, useRap = true) {
const reminder = (offset % MIN_PART_SIZE);
const start = offset - reminder;
const end = start + partSize - 1;
let arrayBuffer = await requestPart({ url: this.url, start, end }) as MP4ArrayBuffer;
if (!arrayBuffer) {
return undefined;
}
if (reminder) {
arrayBuffer = arrayBuffer.slice(reminder) as MP4ArrayBuffer;
}
arrayBuffer.fileStart = offset;
const nextOffset = this.file.appendBuffer(arrayBuffer);
if (!useRap) return offset + arrayBuffer.byteLength;
return nextOffset;
}
private description(track: MP4VideoTrack) {
const t = this.file.getTrackById(track.id);
for (const entry of t.mdia.minf.stbl.stsd.entries) {
if (entry.avcC || entry.hvcC || entry.av1C) {
const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
if (entry.avcC) {
entry.avcC.write(stream);
} else if (entry.hvcC) {
entry.hvcC.write(stream);
} else if (entry.av1C) {
entry.av1C.write(stream);
}
return new Uint8Array(stream.buffer, 8); // Remove the box header.
}
}
throw new Error('avcC, hvcC ro av1C not found');
}
private onReady(info: MP4Info) {
const track = info.videoTracks[0];
let codec = track.codec;
if (codec.startsWith('avc1')) {
// Somehow this is the only avc1 codec that works.
codec = 'avc1.4d001f';
}
// Generate and emit an appropriate VideoDecoderConfig.
this.onConfig({
codec,
codedHeight: track.video.height,
codedWidth: track.video.width,
description: this.description(track),
});
const duration = info.duration / info.timescale;
// If we set a part size too small, the onSamples callback is not called.
const partSize = roundPartSize(track.bitrate / 24);
const minStep = duration < 30 ? 2 : 3;
const step = Math.max(Math.floor(duration / this.framesPerVideo), minStep);
// Start demuxing.
this.file.setExtractionOptions(track.id, undefined, { nbSamples: 1 });
this.file.start();
this.status = Status.ready;
// // Load frames
void this.loadNextFrames(step, track.size, partSize);
}
private onSamples(trackId: number, ref: any, samples: any) {
if (this.status !== Status.ready) return;
// Generate and emit an EncodedVideoChunk for each demuxed sample.
for (const sample of samples) {
const time = sample.cts / sample.timescale;
const type = sample.is_sync ? 'key' : 'delta';
const id = `${type}${sample.number}`;
// Skip already decoded samples.
if (this.decodedSamples.has(id)) continue;
// @ts-ignore
this.onChunk(new EncodedVideoChunk({
type,
timestamp: (1e6 * time),
duration: (1e6 * sample.duration) / sample.timescale,
data: sample.data,
}));
this.decodedSamples.add(id);
this.lastSample = parseInt(sample.number, 10);
if (sample.is_sync) {
this.file.releaseUsedSamples(trackId, sample.number);
}
}
}
close() {
this.file.flush();
this.file.stop();
this.status = Status.closed;
}
}
function roundPartSize(size: number) {
return size + MIN_PART_SIZE - (size % MIN_PART_SIZE);
}

128
src/lib/video-preview/VideoPreview.ts Normal file
View File

@ -0,0 +1,128 @@
import { requestMutation } from '../fasterdom/fasterdom';
import { callApi } from '../../api/gramjs';
import { ApiMediaFormat } from '../../api/types';
import { IS_ANDROID, IS_IOS } from '../../util/windowEnvironment';
import launchMediaWorkers, { MAX_WORKERS } from '../../util/launchMediaWorkers';
const PREVIEW_SIZE_RATIO = (IS_ANDROID || IS_IOS) ? 0.3 : 0.25;
const PREVIEW_MAX_SIDE = 200;
const connections = launchMediaWorkers();
let videoPreview: VideoPreview | undefined;
export class VideoPreview {
frames: Map<number, ImageBitmap> = new Map();
currentTime = 0;
canvas: HTMLCanvasElement;
constructor(url: string, canvas: HTMLCanvasElement) {
this.canvas = canvas;
connections.forEach(({ connector }, index) => {
void connector.request({
name: 'video-preview:init',
args: [
url,
index,
MAX_WORKERS,
this.onFrame.bind(this),
],
});
});
}
private onFrame(time: number, frame: ImageBitmap) {
this.frames.set(time, frame);
if (time === this.currentTime) {
this.render(time);
}
}
private clearCache() {
this.frames.forEach((frame) => {
frame.close();
});
this.frames.clear();
}
render(time: number) {
this.currentTime = time;
const frame = this.frames.get(time);
if (!frame) return false;
requestMutation(() => {
this.canvas.width = frame.width;
this.canvas.height = frame.height;
const ctx = this.canvas.getContext('2d')!;
ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
});
return true;
}
destroy() {
this.clearCache();
connections.forEach(({ connector }) => {
void connector.request({
name: 'video-preview:destroy',
args: [],
});
});
}
}
export function getPreviewDimensions(width: number, height: number) {
width = Math.round(width * PREVIEW_SIZE_RATIO);
height = Math.round(height * PREVIEW_SIZE_RATIO);
const ratio = width / height;
if (width > PREVIEW_MAX_SIDE) {
width = PREVIEW_MAX_SIDE;
height = Math.round(width / ratio);
}
if (height > PREVIEW_MAX_SIDE) {
height = PREVIEW_MAX_SIDE;
width = Math.round(height * ratio);
}
return { width, height };
}
connections.forEach(({ worker }) => {
worker.addEventListener('message', async (e) => {
const { type, messageId, params } = e.data as {
type: string;
messageId: string;
params: { url: string; start: number; end: number };
};
if (type !== 'requestPart') {
return;
}
const result = await callApi('downloadMedia', { mediaFormat: ApiMediaFormat.Progressive, ...params });
if (!result) {
return;
}
const { arrayBuffer } = result;
worker.postMessage({
type: 'partResponse',
messageId,
result: arrayBuffer,
}, [arrayBuffer!]);
});
});
export function createVideoPreviews(url: string, canvas: HTMLCanvasElement) {
if (videoPreview) {
videoPreview.destroy();
}
videoPreview = new VideoPreview(url, canvas);
return () => videoPreview?.destroy();
}
export function renderVideoPreview(time: number) {
if (!videoPreview) return false;
return videoPreview.render(time);
}

1
src/lib/video-preview/libav-3.10.5.1.2-webcodecs.js Normal file
View File

File diff suppressed because one or more lines are too long

814
src/lib/video-preview/libav-3.10.5.1.2-webcodecs.wasm.js Normal file
View File

File diff suppressed because one or more lines are too long

BIN
src/lib/video-preview/libav-3.10.5.1.2-webcodecs.wasm.wasm Normal file
View File

Binary file not shown.

848
src/lib/video-preview/libav.types.d.ts vendored Normal file
View File

@ -0,0 +1,848 @@
/*
* Copyright (C) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/**
* Frames, as taken/given by libav.js.
*/
export interface Frame {
/**
* The actual frame data. For non-planar audio data, this is a typed array.
* For planar audio data, this is an array of typed arrays, one per plane.
* For video data, this is an array of planes, where each plane is in turn
* an array of typed arrays, one per line (because of how libav buffers
* lines).
*/
data: any[];
/**
* Sample format or pixel format.
*/
format: number;
/**
* Presentation timestamp for this frame. Units depends on surrounding
* context. Will always be set by libav.js, but libav.js will accept frames
* from outside that do not have this set.
*/
pts?: number, ptshi?: number;
/**
* Audio only. Channel layout. It is possible for only one of this and
* channels to be set.
*/
channel_layout?: number;
/**
* Audio only. Number of channels. It is possible for only one of this and
* channel_layout to be set.
*/
channels?: number;
/**
* Audio only. Number of samples in the frame.
*/
nb_samples?: number;
/**
* Audio only. Sample rate.
*/
sample_rate?: number;
/**
* Video only. Width of frame.
*/
width?: number;
/**
* Video only. Height of frame.
*/
height?: number;
/**
* Video only. Sample aspect ratio (pixel aspect ratio), as a numerator and
* denominator. 0 is interpreted as 1 (square pixels).
*/
sample_aspect_ratio?: [number, number];
/**
* Is this a keyframe? (1=yes, 0=maybe)
*/
key_frame?: number;
/**
* Picture type (libav-specific value)
*/
pict_type?: number;
}
/**
* Packets, as taken/given by libav.js.
*/
export interface Packet {
/**
* The actual data represented by this packet.
*/
data: Uint8Array;
/**
* Presentation timestamp.
*/
pts?: number, ptshi?: number;
/**
* Decoding timestamp.
*/
dts?: number, dtshi?: number;
/**
* Index of this stream within a surrounding muxer/demuxer.
*/
stream_index?: number;
/**
* Packet flags, as defined by ffmpeg.
*/
flags?: number;
/**
* Duration of this packet. Rarely used.
*/
duration?: number, durationhi?: number;
/**
* Side data. Codec-specific.
*/
side_data?: any;
}
/**
* Stream information, as returned by ff_init_demuxer_file.
*/
export interface Stream {
/**
* Index of this stream.
*/
index: number;
/**
* Codec parameters.
*/
codecpar: number;
/**
* Type of codec (audio or video, typically)
*/
codec_type: number;
/**
* Codec identifier.
*/
codec_id: number;
/**
* Base for timestamps of packets in this stream.
*/
time_base_num: number, time_base_den: number;
/**
* Duration of this stream in time_base units.
*/
duration_time_base: number;
/**
* Duration of this stream in seconds.
*/
duration: number;
}
/**
* Settings used to set up a filter.
*/
export interface FilterIOSettings {
/**
* Audio only. Sample rate of the input.
*/
sample_rate?: number;
/**
* Audio only. Sample format of the input.
*/
sample_fmt?: number;
/**
* Audio only. Channel layout of the input. Note that there is no
* "channels"; you must describe a layout.
*/
channel_layout?: number;
/**
* Audio only, output only, optional. Size of an audio frame.
*/
frame_size?: number;
}
/**
* Supported properties of an AVCodecContext, used by ff_init_encoder.
*/
export interface AVCodecContextProps {
bit_rate?: number;
bit_ratehi?: number;
channel_layout?: number;
channel_layouthi?: number;
channels?: number;
frame_size?: number;
framerate_num?: number;
framerate_den?: number;
gop_size?: number;
height?: number;
keyint_min?: number;
level?: number;
pix_fmt?: number;
profile?: number;
rc_max_rate?: number;
rc_max_ratehi?: number;
rc_min_rate?: number;
rc_min_ratehi?: number;
sample_aspect_ratio_num?: number;
sample_aspect_ratio_den?: number;
sample_fmt?: number;
sample_rate?: number;
qmax?: number;
qmin?: number;
width?: number;
}
export interface LibAV {
av_get_bytes_per_sample(a0: number): Promise<number>;
av_opt_set_int_list_js(a0: number,a1: string,a2: number,a3: number,a4: number,a5: number): Promise<number>;
av_frame_alloc(): Promise<number>;
av_frame_free(a0: number): Promise<void>;
av_frame_get_buffer(a0: number,a1: number): Promise<number>;
av_frame_make_writable(a0: number): Promise<number>;
av_frame_unref(a0: number): Promise<void>;
av_packet_alloc(): Promise<number>;
av_packet_free(a0: number): Promise<void>;
av_packet_new_side_data(a0: number,a1: number,a2: number): Promise<number>;
av_packet_unref(a0: number): Promise<void>;
av_strdup(a0: string): Promise<number>;
av_buffersink_get_frame(a0: number,a1: number): Promise<number>;
av_buffersink_set_frame_size(a0: number,a1: number): Promise<void>;
av_buffersrc_add_frame_flags(a0: number,a1: number,a2: number): Promise<number>;
avfilter_free(a0: number): Promise<void>;
avfilter_get_by_name(a0: string): Promise<number>;
avfilter_graph_alloc(): Promise<number>;
avfilter_graph_config(a0: number,a1: number): Promise<number>;
avfilter_graph_create_filter_js(a0: number,a1: string,a2: string,a3: number,a4: number): Promise<number>;
avfilter_graph_free(a0: number): Promise<void>;
avfilter_graph_parse(a0: number,a1: string,a2: number,a3: number,a4: number): Promise<number>;
avfilter_inout_alloc(): Promise<number>;
avfilter_inout_free(a0: number): Promise<void>;
avfilter_link(a0: number,a1: number,a2: number,a3: number): Promise<number>;
avcodec_alloc_context3(a0: number): Promise<number>;
avcodec_close(a0: number): Promise<number>;
avcodec_find_decoder(a0: number): Promise<number>;
avcodec_find_decoder_by_name(a0: string): Promise<number>;
avcodec_find_encoder(a0: number): Promise<number>;
avcodec_find_encoder_by_name(a0: string): Promise<number>;
avcodec_free_context(a0: number): Promise<void>;
avcodec_get_name(a0: number): Promise<string>;
avcodec_open2(a0: number,a1: number,a2: number): Promise<number>;
ff_calloc_AVCodecParameters(): Promise<number>;
avcodec_parameters_from_context(a0: number,a1: number): Promise<number>;
avcodec_parameters_to_context(a0: number,a1: number): Promise<number>;
avcodec_receive_frame(a0: number,a1: number): Promise<number>;
avcodec_receive_packet(a0: number,a1: number): Promise<number>;
avcodec_send_frame(a0: number,a1: number): Promise<number>;
avcodec_send_packet(a0: number,a1: number): Promise<number>;
av_find_input_format(a0: string): Promise<number>;
avformat_alloc_context(): Promise<number>;
avformat_alloc_output_context2_js(a0: number,a1: string,a2: string): Promise<number>;
avformat_close_input(a0: number): Promise<void>;
avformat_find_stream_info(a0: number,a1: number): Promise<number>;
avformat_free_context(a0: number): Promise<void>;
avformat_new_stream(a0: number,a1: number): Promise<number>;
avformat_open_input(a0: number,a1: string,a2: number,a3: number): Promise<number>;
avformat_open_input_js(a0: string,a1: number,a2: number): Promise<number>;
avformat_write_header(a0: number,a1: number): Promise<number>;
avio_open2_js(a0: string,a1: number,a2: number,a3: number): Promise<number>;
avio_close(a0: number): Promise<number>;
av_find_best_stream(a0: number,a1: number,a2: number,a3: number,a4: number,a5: number): Promise<number>;
av_grow_packet(a0: number,a1: number): Promise<number>;
av_interleaved_write_frame(a0: number,a1: number): Promise<number>;
av_packet_make_writable(a0: number): Promise<number>;
av_pix_fmt_desc_get(a0: number): Promise<number>;
av_read_frame(a0: number,a1: number): Promise<number>;
av_shrink_packet(a0: number,a1: number): Promise<void>;
av_write_frame(a0: number,a1: number): Promise<number>;
av_write_trailer(a0: number): Promise<number>;
av_dict_set(a0: number,a1: string,a2: string,a3: number): Promise<number>;
av_dict_free(a0: number): Promise<void>;
sws_getContext(a0: number,a1: number,a2: number,a3: number,a4: number,a5: number,a6: number,a7: number,a8: number,a9: number): Promise<number>;
sws_freeContext(a0: number): Promise<void>;
sws_scale_frame(a0: number,a1: number,a2: number): Promise<number>;
AVFrame_sample_aspect_ratio_num(a0: number): Promise<number>;
AVFrame_sample_aspect_ratio_den(a0: number): Promise<number>;
AVFrame_sample_aspect_ratio_s(a0: number,a1: number,a2: number): Promise<void>;
AVCodecContext_framerate_num(a0: number): Promise<number>;
AVCodecContext_framerate_den(a0: number): Promise<number>;
AVCodecContext_framerate_num_s(a0: number,a1: number): Promise<void>;
AVCodecContext_framerate_den_s(a0: number,a1: number): Promise<void>;
AVCodecContext_framerate_s(a0: number,a1: number,a2: number): Promise<void>;
AVCodecContext_sample_aspect_ratio_num(a0: number): Promise<number>;
AVCodecContext_sample_aspect_ratio_den(a0: number): Promise<number>;
AVCodecContext_sample_aspect_ratio_num_s(a0: number,a1: number): Promise<void>;
AVCodecContext_sample_aspect_ratio_den_s(a0: number,a1: number): Promise<void>;
AVCodecContext_sample_aspect_ratio_s(a0: number,a1: number,a2: number): Promise<void>;
AVCodecContext_time_base_s(a0: number,a1: number,a2: number): Promise<void>;
AVStream_time_base_num(a0: number): Promise<number>;
AVStream_time_base_den(a0: number): Promise<number>;
AVStream_time_base_s(a0: number,a1: number,a2: number): Promise<void>;
AVPacketSideData_data(a0: number,a1: number): Promise<number>;
AVPacketSideData_size(a0: number,a1: number): Promise<number>;
AVPacketSideData_type(a0: number,a1: number): Promise<number>;
ff_error(a0: number): Promise<string>;
ff_nothing(): Promise<void>;
calloc(a0: number,a1: number): Promise<number>;
free(a0: number): Promise<void>;
malloc(a0: number): Promise<number>;
mallinfo_uordblks(): Promise<number>;
libavjs_with_swscale(): Promise<number>;
AVFrame_channel_layout(ptr: number): Promise<number>;
AVFrame_channel_layout_s(ptr: number, val: number): Promise<void>;
AVFrame_channel_layouthi(ptr: number): Promise<number>;
AVFrame_channel_layouthi_s(ptr: number, val: number): Promise<void>;
AVFrame_channels(ptr: number): Promise<number>;
AVFrame_channels_s(ptr: number, val: number): Promise<void>;
AVFrame_data_a(ptr: number, idx: number): Promise<number>;
AVFrame_data_a_s(ptr: number, idx: number, val: number): Promise<void>;
AVFrame_format(ptr: number): Promise<number>;
AVFrame_format_s(ptr: number, val: number): Promise<void>;
AVFrame_height(ptr: number): Promise<number>;
AVFrame_height_s(ptr: number, val: number): Promise<void>;
AVFrame_key_frame(ptr: number): Promise<number>;
AVFrame_key_frame_s(ptr: number, val: number): Promise<void>;
AVFrame_linesize_a(ptr: number, idx: number): Promise<number>;
AVFrame_linesize_a_s(ptr: number, idx: number, val: number): Promise<void>;
AVFrame_nb_samples(ptr: number): Promise<number>;
AVFrame_nb_samples_s(ptr: number, val: number): Promise<void>;
AVFrame_pict_type(ptr: number): Promise<number>;
AVFrame_pict_type_s(ptr: number, val: number): Promise<void>;
AVFrame_pts(ptr: number): Promise<number>;
AVFrame_pts_s(ptr: number, val: number): Promise<void>;
AVFrame_ptshi(ptr: number): Promise<number>;
AVFrame_ptshi_s(ptr: number, val: number): Promise<void>;
AVFrame_sample_rate(ptr: number): Promise<number>;
AVFrame_sample_rate_s(ptr: number, val: number): Promise<void>;
AVFrame_width(ptr: number): Promise<number>;
AVFrame_width_s(ptr: number, val: number): Promise<void>;
AVPixFmtDescriptor_log2_chroma_h(ptr: number): Promise<number>;
AVPixFmtDescriptor_log2_chroma_h_s(ptr: number, val: number): Promise<void>;
AVCodecContext_bit_rate(ptr: number): Promise<number>;
AVCodecContext_bit_rate_s(ptr: number, val: number): Promise<void>;
AVCodecContext_bit_ratehi(ptr: number): Promise<number>;
AVCodecContext_bit_ratehi_s(ptr: number, val: number): Promise<void>;
AVCodecContext_channel_layout(ptr: number): Promise<number>;
AVCodecContext_channel_layout_s(ptr: number, val: number): Promise<void>;
AVCodecContext_channel_layouthi(ptr: number): Promise<number>;
AVCodecContext_channel_layouthi_s(ptr: number, val: number): Promise<void>;
AVCodecContext_channels(ptr: number): Promise<number>;
AVCodecContext_channels_s(ptr: number, val: number): Promise<void>;
AVCodecContext_extradata(ptr: number): Promise<number>;
AVCodecContext_extradata_s(ptr: number, val: number): Promise<void>;
AVCodecContext_extradata_size(ptr: number): Promise<number>;
AVCodecContext_extradata_size_s(ptr: number, val: number): Promise<void>;
AVCodecContext_frame_size(ptr: number): Promise<number>;
AVCodecContext_frame_size_s(ptr: number, val: number): Promise<void>;
AVCodecContext_gop_size(ptr: number): Promise<number>;
AVCodecContext_gop_size_s(ptr: number, val: number): Promise<void>;
AVCodecContext_height(ptr: number): Promise<number>;
AVCodecContext_height_s(ptr: number, val: number): Promise<void>;
AVCodecContext_keyint_min(ptr: number): Promise<number>;
AVCodecContext_keyint_min_s(ptr: number, val: number): Promise<void>;
AVCodecContext_level(ptr: number): Promise<number>;
AVCodecContext_level_s(ptr: number, val: number): Promise<void>;
AVCodecContext_pix_fmt(ptr: number): Promise<number>;
AVCodecContext_pix_fmt_s(ptr: number, val: number): Promise<void>;
AVCodecContext_profile(ptr: number): Promise<number>;
AVCodecContext_profile_s(ptr: number, val: number): Promise<void>;
AVCodecContext_rc_max_rate(ptr: number): Promise<number>;
AVCodecContext_rc_max_rate_s(ptr: number, val: number): Promise<void>;
AVCodecContext_rc_max_ratehi(ptr: number): Promise<number>;
AVCodecContext_rc_max_ratehi_s(ptr: number, val: number): Promise<void>;
AVCodecContext_rc_min_rate(ptr: number): Promise<number>;
AVCodecContext_rc_min_rate_s(ptr: number, val: number): Promise<void>;
AVCodecContext_rc_min_ratehi(ptr: number): Promise<number>;
AVCodecContext_rc_min_ratehi_s(ptr: number, val: number): Promise<void>;
AVCodecContext_sample_fmt(ptr: number): Promise<number>;
AVCodecContext_sample_fmt_s(ptr: number, val: number): Promise<void>;
AVCodecContext_sample_rate(ptr: number): Promise<number>;
AVCodecContext_sample_rate_s(ptr: number, val: number): Promise<void>;
AVCodecContext_qmax(ptr: number): Promise<number>;
AVCodecContext_qmax_s(ptr: number, val: number): Promise<void>;
AVCodecContext_qmin(ptr: number): Promise<number>;
AVCodecContext_qmin_s(ptr: number, val: number): Promise<void>;
AVCodecContext_width(ptr: number): Promise<number>;
AVCodecContext_width_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_codec_id(ptr: number): Promise<number>;
AVCodecParameters_codec_id_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_codec_type(ptr: number): Promise<number>;
AVCodecParameters_codec_type_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_extradata(ptr: number): Promise<number>;
AVCodecParameters_extradata_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_extradata_size(ptr: number): Promise<number>;
AVCodecParameters_extradata_size_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_format(ptr: number): Promise<number>;
AVCodecParameters_format_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_bit_rate(ptr: number): Promise<number>;
AVCodecParameters_bit_rate_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_profile(ptr: number): Promise<number>;
AVCodecParameters_profile_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_level(ptr: number): Promise<number>;
AVCodecParameters_level_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_width(ptr: number): Promise<number>;
AVCodecParameters_width_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_height(ptr: number): Promise<number>;
AVCodecParameters_height_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_color_range(ptr: number): Promise<number>;
AVCodecParameters_color_range_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_color_primaries(ptr: number): Promise<number>;
AVCodecParameters_color_primaries_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_color_trc(ptr: number): Promise<number>;
AVCodecParameters_color_trc_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_color_space(ptr: number): Promise<number>;
AVCodecParameters_color_space_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_chroma_location(ptr: number): Promise<number>;
AVCodecParameters_chroma_location_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_channels(ptr: number): Promise<number>;
AVCodecParameters_channels_s(ptr: number, val: number): Promise<void>;
AVCodecParameters_sample_rate(ptr: number): Promise<number>;
AVCodecParameters_sample_rate_s(ptr: number, val: number): Promise<void>;
AVPacket_pts(ptr: number): Promise<number>;
AVPacket_pts_s(ptr: number, val: number): Promise<void>;
AVPacket_ptshi(ptr: number): Promise<number>;
AVPacket_ptshi_s(ptr: number, val: number): Promise<void>;
AVPacket_dts(ptr: number): Promise<number>;
AVPacket_dts_s(ptr: number, val: number): Promise<void>;
AVPacket_dtshi(ptr: number): Promise<number>;
AVPacket_dtshi_s(ptr: number, val: number): Promise<void>;
AVPacket_data(ptr: number): Promise<number>;
AVPacket_data_s(ptr: number, val: number): Promise<void>;
AVPacket_size(ptr: number): Promise<number>;
AVPacket_size_s(ptr: number, val: number): Promise<void>;
AVPacket_stream_index(ptr: number): Promise<number>;
AVPacket_stream_index_s(ptr: number, val: number): Promise<void>;
AVPacket_flags(ptr: number): Promise<number>;
AVPacket_flags_s(ptr: number, val: number): Promise<void>;
AVPacket_side_data(ptr: number): Promise<number>;
AVPacket_side_data_s(ptr: number, val: number): Promise<void>;
AVPacket_side_data_elems(ptr: number): Promise<number>;
AVPacket_side_data_elems_s(ptr: number, val: number): Promise<void>;
AVPacket_duration(ptr: number): Promise<number>;
AVPacket_duration_s(ptr: number, val: number): Promise<void>;
AVPacket_durationhi(ptr: number): Promise<number>;
AVPacket_durationhi_s(ptr: number, val: number): Promise<void>;
AVFormatContext_nb_streams(ptr: number): Promise<number>;
AVFormatContext_nb_streams_s(ptr: number, val: number): Promise<void>;
AVFormatContext_oformat(ptr: number): Promise<number>;
AVFormatContext_oformat_s(ptr: number, val: number): Promise<void>;
AVFormatContext_pb(ptr: number): Promise<number>;
AVFormatContext_pb_s(ptr: number, val: number): Promise<void>;
AVFormatContext_streams_a(ptr: number, idx: number): Promise<number>;
AVFormatContext_streams_a_s(ptr: number, idx: number, val: number): Promise<void>;
AVStream_codecpar(ptr: number): Promise<number>;
AVStream_codecpar_s(ptr: number, val: number): Promise<void>;
AVStream_duration(ptr: number): Promise<number>;
AVStream_duration_s(ptr: number, val: number): Promise<void>;
AVStream_durationhi(ptr: number): Promise<number>;
AVStream_durationhi_s(ptr: number, val: number): Promise<void>;
AVFilterInOut_filter_ctx(ptr: number): Promise<number>;
AVFilterInOut_filter_ctx_s(ptr: number, val: number): Promise<void>;
AVFilterInOut_name(ptr: number): Promise<number>;
AVFilterInOut_name_s(ptr: number, val: number): Promise<void>;
AVFilterInOut_next(ptr: number): Promise<number>;
AVFilterInOut_next_s(ptr: number, val: number): Promise<void>;
AVFilterInOut_pad_idx(ptr: number): Promise<number>;
AVFilterInOut_pad_idx_s(ptr: number, val: number): Promise<void>;
av_frame_free_js(ptr: number);
av_packet_free_js(ptr: number);
avformat_close_input_js(ptr: number);
avcodec_free_context_js(ptr: number);
avfilter_graph_free_js(ptr: number);
avfilter_inout_free_js(ptr: number);
copyin_u8(ptr: number, arr: Uint8Array): Promise<void>;
copyout_u8(ptr: number, len: number): Promise<Uint8Array>;
copyin_s16(ptr: number, arr: Int16Array): Promise<void>;
copyout_s16(ptr: number, len: number): Promise<Int16Array>;
copyin_s32(ptr: number, arr: Int32Array): Promise<void>;
copyout_s32(ptr: number, len: number): Promise<Int32Array>;
copyin_f32(ptr: number, arr: Float32Array): Promise<void>;
copyout_f32(ptr: number, len: number): Promise<Float32Array>;
/**
* Read a complete file from the in-memory filesystem.
* @param name Filename to read
*/
readFile(name: string): Promise<Uint8Array>;
/**
* Write a complete file to the in-memory filesystem.
* @param name Filename to write
* @param content Content to write to the file
*/
writeFile(name: string, content: Uint8Array): Promise<Uint8Array>;
/**
* Delete a file in the in-memory filesystem.
* @param name Filename to delete
*/
unlink(name: string): Promise<void>;
/**
* Make a reader device.
* @param name Filename to create
* @param mode Unix permissions (pointless since this is an in-memory
* filesystem)
*/
mkreaderdev(name: string, mode?: number): Promise<void>;
/**
* Make a writer device.
* @param name Filename to create
* @param mode Unix permissions
*/
mkwriterdev(name: string, mode?: number): Promise<void>;
/**
* Send some data to a reader device
* @param name Filename of the reader device
* @param data Data to sending
*/
ff_reader_dev_send(name: string, data: Uint8Array): Promise<void>;
/**
* Metafunction to determine whether any device has any waiters. This can be
* used to determine whether more data needs to be sent before a previous step
* will be fully resolved.
*/
ff_reader_dev_waiting(): Promise<boolean>;
/**
* Metafunction to initialize an encoder with all the bells and whistles.
* Returns [AVCodec, AVCodecContext, AVFrame, AVPacket, frame_size]
* @param name libav name of the codec
* @param opts Encoder options
*/
ff_init_encoder(
name: string, opts?: {
ctx?: AVCodecContextProps, options?: Record<string, string>
}
): Promise<[number, number, number, number, number]>;
/**
* Metafunction to initialize a decoder with all the bells and whistles.
* Similar to ff_init_encoder but doesn't need to initialize the frame.
* Returns [AVCodec, AVCodecContext, AVPacket, AVFrame]
* @param name libav decoder identifier or name
* @param codecpar Optional AVCodecParameters
*/
ff_init_decoder(
name: string | number, codecpar?: number
): Promise<[number, number, number, number]>;
/**
* Free everything allocated by ff_init_encoder.
* @param c AVCodecContext
* @param frame AVFrame
* @param pkt AVPacket
*/
ff_free_encoder(
c: number, frame: number, pkt: number
): Promise<void>;
/**
* Free everything allocated by ff_init_decoder
* @param c AVCodecContext
* @param pkt AVPacket
* @param frame AVFrame
*/
ff_free_decoder(
c: number, pkt: number, frame: number
): Promise<void>;
/**
* Encode some number of frames at once. Done in one go to avoid excess message
* passing.
* @param ctx AVCodecContext
* @param frame AVFrame
* @param pkt AVPacket
* @param inFrames Array of frames in libav.js format
* @param fin Set to true if this is the end of encoding
*/
ff_encode_multi(
ctx: number, frame: number, pkt: number, inFrames: Frame[],
fin?: boolean
): Promise<Packet[]>;
/**
* Decode some number of packets at once. Done in one go to avoid excess
* message passing.
* @param ctx AVCodecContext
* @param pkt AVPacket
* @param frame AVFrame
* @param inPackets Incoming packets to decode
* @param config Decoding options. May be "true" to indicate end of stream.
*/
ff_decode_multi(
ctx: number, pkt: number, frame: number, inPackets: Packet[],
config?: boolean | {
fin?: boolean,
ignoreErrors?: boolean
}
): Promise<Frame[]>;
/**
* Initialize a muxer format, format context and some number of streams.
* Returns [AVFormatContext, AVOutputFormat, AVIOContext, AVStream[]]
* @param opts Muxer options
* @param stramCtxs Context info for each stream to mux
*/
ff_init_muxer(
opts: {
oformat?: number, // format pointer
format_name?: string, // libav name
filename?: string,
device?: boolean, // Create a writer device
open?: boolean // Open the file for writing
},
streamCtxs: [number, number, number][] // AVCodecContext, time_base_num, time_base_den
): Promise<[number, number, number, number[]]>;
/**
* Free up a muxer format and/or file
* @param oc AVFormatContext
* @param pb AVIOContext
*/
ff_free_muxer(oc: number, pb: number): Promise<void>;
/**
* Initialize a demuxer from a file and format context, and get the list of
* codecs/types.
* Returns [AVFormatContext, Stream[]]
* @param filename Filename to open
* @param fmt Format to use (optional)
*/
ff_init_demuxer_file(
filename: string, fmt?: string
): Promise<[number, Stream[]]>;
/**
* Write some number of packets at once.
* @param oc AVFormatContext
* @param pkt AVPacket
* @param inPackets Packets to write
* @param interleave Set to false to *not* use the interleaved writer.
* Interleaving is the default.
*/
ff_write_multi(
oc: number, pkt: number, inPackets: Packet[], interleave?: boolean
): Promise<void>;
/**
* Read many packets at once. If you don't set any limits, this function will
* block (asynchronously) until the whole file is read, so make sure you set
* some limits if you want to read a bit at a time. Returns a pair [result,
* packets], where the result indicates whether an error was encountered, an
* EOF, or simply limits (EAGAIN), and packets is a dictionary indexed by the
* stream number in which each element is an array of packets from that stream.
* @param fmt_ctx AVFormatContext
* @param pkt AVPacket
* @param devfile Name of the device file being read from, if applicable. Used
* to set limits on when to read based on available data.
* @param opts Other options
*/
ff_read_multi(
fmt_ctx: number, pkt: number, devfile?: string, opts?: {
limit?: number, // OUTPUT limit, in bytes
devLimit?: number // INPUT limit, in bytes (don't read if less than this much data is available)
}
): Promise<[number, Record<number, Packet[]>]>;
/**
* Initialize a filter graph. No equivalent free since you just need to free
* the graph itself (av_filter_graph_free) and everything under it will be
* freed automatically.
* Returns [AVFilterGraph, AVFilterContext, AVFilterContext], where the second
* and third are the input and output buffer source/sink. For multiple
* inputs/outputs, the second and third will be arrays, as appropriate.
* @param filters_descr Filtergraph description
* @param input Input settings, or array of input settings for multiple inputs
* @param output Output settings, or array of output settings for multiple
* outputs
*/
ff_init_filter_graph(
filters_descr: string,
input: FilterIOSettings,
output: FilterIOSettings
): Promise<[number, number, number]>;
ff_init_filter_graph(
filters_descr: string,
input: FilterIOSettings[],
output: FilterIOSettings
): Promise<[number, number[], number]>;
ff_init_filter_graph(
filters_descr: string,
input: FilterIOSettings,
output: FilterIOSettings[]
): Promise<[number, number, number[]]>;
ff_init_filter_graph(
filters_descr: string,
input: FilterIOSettings[],
output: FilterIOSettings[]
): Promise<[number, number[], number[]]>;
/**
* Filter some number of frames, possibly corresponding to multiple sources.
* @param srcs AVFilterContext(s), input
* @param buffersink_ctx AVFilterContext, output
* @param framePtr AVFrame
* @param inFrames Input frames, either as an array of frames or with frames
* per input
* @param fin Indicate end-of-stream(s)
*/
ff_filter_multi(
srcs: number, buffersink_ctx: number, framePtr: number,
inFrames: Frame[], fin?: boolean
): Promise<Frame[]>;
ff_filter_multi(
srcs: number[], buffersink_ctx: number, framePtr: number,
inFrames: Frame[][], fin?: boolean[]
): Promise<Frame[]>;
/**
* Copy out a frame.
* @param frame AVFrame
*/
ff_copyout_frame(frame: number): Promise<Frame>;
/**
* Copy in a frame.
* @param framePtr AVFrame
* @param frame Frame to copy in
*/
ff_copyin_frame(framePtr: number, frame: Frame): Promise<void>;
/**
* Copy out a packet.
* @param pkt AVPacket
*/
ff_copyout_packet(pkt: number): Promise<Packet>;
/**
* Copy in a packet.
* @param pktPtr AVPacket
* @param packet Packet to copy in.
*/
ff_copyin_packet(pktPtr: number, packet: Packet): Promise<void>;
/**
* Allocate and copy in a 32-bit int list.
* @param list List of numbers to copy in
*/
ff_malloc_int32_list(list: number[]): Promise<number>;
/**
* Allocate and copy in a 64-bit int list.
* @param list List of numbers to copy in
*/
ff_malloc_int64_list(list: number[]): Promise<number>;
/**
* Callback when writes occur. Set by the user.
*/
onwrite?: (filename: string, position: number, buffer: Uint8Array | Int8Array) => void;
/**
* Terminate the worker associated with this libav.js instance, rendering
* it inoperable and freeing its memory.
*/
terminate(): void;
// Enumerations:
AV_OPT_SEARCH_CHILDREN: number;
AVMEDIA_TYPE_UNKNOWN: number;
AVMEDIA_TYPE_VIDEO: number;
AVMEDIA_TYPE_AUDIO: number;
AVMEDIA_TYPE_DATA: number;
AVMEDIA_TYPE_SUBTITLE: number;
AVMEDIA_TYPE_ATTACHMENT: number;
AV_SAMPLE_FMT_NONE: number;
AV_SAMPLE_FMT_U8: number;
AV_SAMPLE_FMT_S16: number;
AV_SAMPLE_FMT_S32: number;
AV_SAMPLE_FMT_FLT: number;
AV_SAMPLE_FMT_DBL: number;
AV_SAMPLE_FMT_U8P: number;
AV_SAMPLE_FMT_S16P: number;
AV_SAMPLE_FMT_S32P: number;
AV_SAMPLE_FMT_FLTP: number;
AV_SAMPLE_FMT_DBLP: number;
AV_SAMPLE_FMT_S64: number;
AV_SAMPLE_FMT_S64P: number;
AV_SAMPLE_FMT_NB: number;
AV_PIX_FMT_NONE: number;
AV_PIX_FMT_YUV420P: number;
AV_PIX_FMT_YUYV422: number;
AV_PIX_FMT_RGB24: number;
AV_PIX_FMT_BGR24: number;
AV_PIX_FMT_YUV422P: number;
AV_PIX_FMT_YUV444P: number;
AV_PIX_FMT_YUV410P: number;
AV_PIX_FMT_YUV411P: number;
AV_PIX_FMT_GRAY8: number;
AV_PIX_FMT_MONOWHITE: number;
AV_PIX_FMT_MONOBLACK: number;
AV_PIX_FMT_PAL8: number;
AV_PIX_FMT_YUVJ420P: number;
AV_PIX_FMT_YUVJ422P: number;
AV_PIX_FMT_YUVJ444P: number;
AV_PIX_FMT_UYVY422: number;
AV_PIX_FMT_UYYVYY411: number;
AV_PIX_FMT_BGR8: number;
AV_PIX_FMT_BGR4: number;
AV_PIX_FMT_BGR4_BYTE: number;
AV_PIX_FMT_RGB8: number;
AV_PIX_FMT_RGB4: number;
AV_PIX_FMT_RGB4_BYTE: number;
AV_PIX_FMT_NV12: number;
AV_PIX_FMT_NV21: number;
AV_PIX_FMT_ARGB: number;
AV_PIX_FMT_RGBA: number;
AV_PIX_FMT_ABGR: number;
AV_PIX_FMT_BGRA: number;
AV_PIX_FMT_GRAY16BE: number;
AV_PIX_FMT_GRAY16LE: number;
AV_PIX_FMT_YUV440P: number;
AV_PIX_FMT_YUVJ440P: number;
AV_PIX_FMT_YUVA420P: number;
AV_PIX_FMT_RGB48BE: number;
AV_PIX_FMT_RGB48LE: number;
AV_PIX_FMT_RGB565BE: number;
AV_PIX_FMT_RGB565LE: number;
AV_PIX_FMT_RGB555BE: number;
AV_PIX_FMT_RGB555LE: number;
AV_PIX_FMT_BGR565BE: number;
AV_PIX_FMT_BGR565LE: number;
AV_PIX_FMT_BGR555BE: number;
AV_PIX_FMT_BGR555LE: number;
AVIO_FLAG_READ: number;
AVIO_FLAG_WRITE: number;
AVIO_FLAG_READ_WRITE: number;
AVIO_FLAG_NONBLOCK: number;
AVIO_FLAG_DIRECT: number;
EAGAIN: number;
AVERROR_EOF: number;
}
export interface LibAVWrapper {
/**
* URL base from which load workers and modules.
*/
base: string;
/**
* Create a LibAV instance.
* @param opts Options
*/
LibAV(opts?: {
noworker?: boolean,
nowasm?: boolean
}): Promise<LibAV>;
}

93
src/lib/video-preview/mp4box.d.ts vendored Normal file
View File

@ -0,0 +1,93 @@
declare module 'mp4box' {
export interface MP4MediaTrack {
id: number;
created: Date;
modified: Date;
movie_duration: number;
layer: number;
alternate_group: number;
volume: number;
track_width: number;
track_height: number;
timescale: number;
duration: number;
bitrate: number;
codec: string;
language: string;
nb_samples: number;
}
export interface MP4VideoData {
width: number;
height: number;
}
export interface MP4VideoTrack extends MP4MediaTrack {
video: MP4VideoData;
[key: string]: any;
}
export interface MP4AudioData {
sample_rate: number;
channel_count: number;
sample_size: number;
}
export interface MP4AudioTrack extends MP4MediaTrack {
audio: MP4AudioData;
[key: string]: any;
}
export type MP4Track = MP4VideoTrack | MP4AudioTrack;
export class DataStream {
buffer: ArrayBuffer;
static BIG_ENDIAN: number;
constructor(buffer?: ArrayBuffer, offset?: number, endianness?: number);
}
export interface MP4Info {
duration: number;
timescale: number;
fragment_duration: number;
isFragmented: boolean;
isProgressive: boolean;
hasIOD: boolean;
brands: string[];
created: Date;
modified: Date;
tracks: MP4Track[];
videoTracks: MP4VideoTrack[];
}
export type MP4ArrayBuffer = ArrayBuffer & { fileStart: number };
export interface MP4File {
onMoovStart?: () => void;
onReady?: (info: MP4Info) => void;
onSamples?: (trackId: number, ref: any, samples: any) => void;
onError?: (e: string) => void;
processSamples(last: boolean): void;
getTrackById(id: number): MP4Track;
setExtractionOptions(id: number, user?: any, options?: any): void;
appendBuffer(data: MP4ArrayBuffer): number;
start(): void;
stop(): void;
flush(): void;
seek(time: number, useRap?: boolean): { offset: number; time: number };
releaseUsedSamples(id: number, sampleNumber: number): void;
}
export function createFile(): MP4File;
export { };
}

73
src/lib/video-preview/polyfill/config.ts Normal file
View File

@ -0,0 +1,73 @@
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
import type * as eac from './encoded-audio-chunk';
import * as evc from './encoded-video-chunk';
import * as vf from './video-frame';
import * as vdec from './video-decoder';
/**
* A VideoDecoder environment.
*/
export interface VideoDecoderEnvironment {
VideoDecoder: typeof vdec.VideoDecoder;
EncodedVideoChunk: typeof evc.EncodedVideoChunk;
VideoFrame: typeof vf.VideoFrame;
}
/**
* Error thrown to indicate a configuration is unsupported.
*/
export class UnsupportedException extends Error {
constructor() {
super('The requested configuration is not supported');
}
}
/**
* Get an VideoDecoder environment that supports this configuration. Throws an
* UnsupportedException if no environment supports the configuration.
* @param config Video decoder configuration
*/
export async function getVideoDecoder(
config: vdec.VideoDecoderConfig,
): Promise<VideoDecoderEnvironment> {
try {
if (
typeof (<any>global).VideoDecoder !== 'undefined'
&& (await (<any>global).VideoDecoder.isConfigSupported(config)).supported
) {
return {
VideoDecoder: (<any>global).VideoDecoder,
EncodedVideoChunk: (<any>global).EncodedVideoChunk,
VideoFrame: (<any>global).VideoFrame,
};
}
} catch (ex) {}
if ((await vdec.VideoDecoder.isConfigSupported(config)).supported) {
return {
VideoDecoder: vdec.VideoDecoder,
EncodedVideoChunk: evc.EncodedVideoChunk,
VideoFrame: vf.VideoFrame,
};
}
throw new UnsupportedException();
}

62
src/lib/video-preview/polyfill/encoded-audio-chunk.ts Normal file
View File

@ -0,0 +1,62 @@
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
export class EncodedAudioChunk {
constructor(init: EncodedAudioChunkInit) {
this.type = init.type;
this.timestamp = init.timestamp;
this.duration = init.duration || 0;
const data = (this._data = new Uint8Array(
(<any>init.data).buffer || init.data,
(<any>init.data).byteOffset || 0
));
this.byteLength = data.byteLength;
}
readonly type: EncodedAudioChunkType;
readonly timestamp: number; // microseconds
readonly duration?: number; // microseconds
readonly byteLength: number;
private _data: Uint8Array;
// Internal
_libavGetData() {
return this._data;
}
copyTo(destination: BufferSource) {
new Uint8Array(
(<any>destination).buffer || destination,
(<any>destination).byteOffset || 0
).set(this._data);
}
}
export interface EncodedAudioChunkInit {
type: EncodedAudioChunkType;
timestamp: number; // microseconds
duration?: number; // microseconds
data: BufferSource;
}
export type EncodedAudioChunkType = 'key' | 'delta';

25
src/lib/video-preview/polyfill/encoded-video-chunk.ts Normal file
View File

@ -0,0 +1,25 @@
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
import * as eac from './encoded-audio-chunk';
export type EncodedVideoChunk = eac.EncodedAudioChunk;
export const EncodedVideoChunk = eac.EncodedAudioChunk;
export type EncodedVideoChunkInit = eac.EncodedAudioChunkInit;
export type EncodedVideoChunkType = eac.EncodedAudioChunkType;

105
src/lib/video-preview/polyfill/index.ts Normal file
View File

@ -0,0 +1,105 @@
// @ts-nocheck
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
import type * as LibAVJS from '../libav.types';
import * as eac from './encoded-audio-chunk';
import * as evc from './encoded-video-chunk';
import * as vf from './video-frame';
import * as vdec from './video-decoder';
import * as rendering from './rendering';
import * as config from './config';
import * as libav from './libav';
import type * as misc from './misc';
declare let LibAV: LibAVJS.LibAVWrapper;
/**
* Load LibAV-WebCodecs-Polyfill.
*/
export async function load(
options: {
polyfill?: boolean;
libavOptions?: any;
} = {},
) {
// Set up libavOptions
const libavOptions: any = {};
if (options.libavOptions) {
Object.assign(libavOptions, options.libavOptions);
}
// And load the libav handler
libav.setLibAVOptions(libavOptions);
await libav.load();
if (options.polyfill) {
for (const exp of [
'EncodedVideoChunk',
'VideoFrame',
'VideoDecoder',
]) {
if (!(global as any)[exp]) {
(global as any)[exp] = (this as any)[exp];
}
}
}
await rendering.load(libavOptions, !!options.polyfill);
}
// EncodedVideoChunk
export type EncodedVideoChunk = evc.EncodedVideoChunk;
export const EncodedVideoChunk = evc.EncodedVideoChunk;
export type EncodedVideoChunkInit = evc.EncodedVideoChunkInit;
// VideoFrame
export type VideoFrame = vf.VideoFrame;
export const VideoFrame = vf.VideoFrame;
export type VideoFrameInit = vf.VideoFrameInit;
export type VideoFrameBufferInit = vf.VideoFrameBufferInit;
export type VideoPixelFormat = vf.VideoPixelFormat;
export type PlaneLayout = vf.PlaneLayout;
export type VideoFrameCopyToOptions = vf.VideoFrameCopyToOptions;
// VideoDecoder
export type VideoDecoder = vdec.VideoDecoder;
export const VideoDecoder = vdec.VideoDecoder;
export type VideoDecoderInit = vdec.VideoDecoderInit;
export type VideoFrameOutputCallback = vdec.VideoFrameOutputCallback;
export type VideoDecoderConfig = vdec.VideoDecoderConfig;
export type VideoDecoderSupport = vdec.VideoDecoderSupport;
// Rendering
export const createImageBitmap = rendering.createImageBitmap;
// Misc
export type CodecState = misc.CodecState;
export type WebCodecsErrorcallback = misc.WebCodecsErrorCallback;
// Configurations/environments
export type AudioDecoderEnvironment = config.AudioDecoderEnvironment;
export type VideoDecoderEnvironment = config.VideoDecoderEnvironment;
export type UnsupportedException = config.UnsupportedException;
export const UnsupportedException = config.UnsupportedException;
export const getVideoDecoder = config.getVideoDecoder;

136
src/lib/video-preview/polyfill/libav.ts Normal file
View File

@ -0,0 +1,136 @@
// @ts-nocheck
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
import type * as LibAVJS from '../libav.types';
declare let LibAV: LibAVJS.LibAVWrapper;
// Currently available libav instances
const libavs: LibAVJS.LibAV[] = [];
// Options required to create a LibAV instance
let libavOptions: any = {};
/**
* Supported decoders.
*/
export let decoders: string[] = null;
/**
* libav.js-specific codec request, used to bypass the codec registry and use
* anything your implementation of libav.js supports.
*/
export interface LibAVJSCodec {
codec: string;
ctx?: LibAVJS.AVCodecContextProps;
options?: Record<string, string>;
}
/**
* Set the libav loading options.
*/
export function setLibAVOptions(to: any) {
libavOptions = to;
}
/**
* Get a libav instance.
*/
export async function get(): Promise<LibAVJS.LibAV> {
if (libavs.length) {
return libavs.shift();
}
return LibAV.LibAV(libavOptions);
}
/**
* Free a libav instance for later reuse.
*/
export function free(libav: LibAVJS.LibAV) {
libavs.push(libav);
}
/**
* Get the list of encoders/decoders supported by libav (which are also
* supported by this polyfill)
*/
async function codecs(): Promise<string[]> {
const libav = await get();
const ret: string[] = [];
for (const [avname, codec] of [
['libaom-av1', 'av01'],
['h264', 'avc1'],
['hevc', 'hvc1'],
]) {
if (await libav.avcodec_find_decoder_by_name(avname)) {
ret.push(codec);
}
}
free(libav);
return ret;
}
/**
* Load the lists of supported decoders and encoders.
*/
export async function load() {
decoders = await codecs();
}
/**
* Convert a decoder from the codec registry (or libav.js-specific parameters)
* to libav.js. Returns null if unsupported.
*/
export function decoder(codec: string | { libavjs: LibAVJSCodec }): LibAVJSCodec {
if (typeof codec === 'string') {
codec = codec.replace(/\..*/, '');
let outCodec: string = codec;
switch (codec) {
// Video
case 'av01':
outCodec = 'libaom-av1';
break;
case 'avc1':
outCodec = 'h264';
break;
case 'hvc1':
outCodec = 'hevc';
break;
// Unrecognized
default:
return null;
}
// Check whether we actually support this codec
if (!(decoders.indexOf(codec) >= 0)) {
return null;
}
return { codec: outCodec };
} else {
return codec.libavjs;
}
}

35
src/lib/video-preview/polyfill/misc.ts Normal file
View File

@ -0,0 +1,35 @@
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
export type CodecState = 'unconfigured' | 'configured' | 'closed';
export type WebCodecsErrorCallback = (error: DOMException) => void;
/**
* Clone this configuration. Just copies over the supported/recognized fields.
*/
export function cloneConfig(config: any, fields: string[]): any {
const ret: any = {};
for (const field of fields) {
if (field in config) {
ret[field] = config[field];
}
}
return ret;
}

190
src/lib/video-preview/polyfill/rendering.ts Normal file
View File

@ -0,0 +1,190 @@
// @ts-nocheck
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
import type * as LibAVJS from '../libav.types';
import * as vf from './video-frame';
declare let LibAV: LibAVJS.LibAVWrapper;
/* A non-threaded libav.js instance for scaling. This is an any because the
* type definitions only expose the async versions, but this API requires the
* _sync methods. */
let scalerSync: any = null;
// A synchronous libav.js instance for scaling.
let scalerAsync: LibAVJS.LibAV = null;
// The original drawImage
const origDrawImage: any = null;
// The original createImageBitmap
let origCreateImageBitmap: any = null;
/**
* Load rendering capability.
* @param libavOptions Options to use while loading libav, only asynchronous
* @param polyfill Set to polyfill CanvasRenderingContext2D.drawImage
*/
export async function load(libavOptions: any, polyfill: boolean) {
// Get our scalers
scalerSync = await LibAV.LibAV({ noworker: true });
scalerAsync = await LibAV.LibAV(libavOptions);
// Polyfill createImageBitmap
origCreateImageBitmap = global.createImageBitmap;
if (polyfill) {
(<any>global).createImageBitmap = createImageBitmap;
}
}
/**
* Create an ImageBitmap from this drawable, asynchronously. NOTE:
* Sub-rectangles are not implemented for VideoFrames, so only options is
* available, and there, only scaling is available.
* @param image VideoFrame (or anything else) to draw
* @param options Other options
*/
export function createImageBitmap(
image: vf.VideoFrame,
opts: {
resizeWidth?: number;
resizeHeight?: number;
} = {},
): Promise<ImageBitmap> {
if (!(image instanceof vf.VideoFrame)) {
// Just use the original
return origCreateImageBitmap.apply(global, arguments);
}
// Convert the format to libav.js
let format: number = scalerAsync.AV_PIX_FMT_RGBA;
switch (image.format) {
case 'I420':
format = scalerAsync.AV_PIX_FMT_YUV420P;
break;
case 'I420A':
format = scalerAsync.AV_PIX_FMT_YUVA420P;
break;
case 'I422':
format = scalerAsync.AV_PIX_FMT_YUV422P;
break;
case 'I444':
format = scalerAsync.AV_PIX_FMT_YUV444P;
break;
case 'NV12':
format = scalerAsync.AV_PIX_FMT_NV12;
break;
case 'RGBA':
case 'RGBX':
format = scalerAsync.AV_PIX_FMT_RGBA;
break;
case 'BGRA':
case 'BGRX':
format = scalerAsync.AV_PIX_FMT_BGRA;
break;
}
// Normalize arguments
const dWidth = typeof opts.resizeWidth === 'number' ? opts.resizeWidth : image.displayWidth;
const dHeight = typeof opts.resizeHeight === 'number' ? opts.resizeHeight : image.displayHeight;
// Convert the frame
const frameData = new ImageData(dWidth, dHeight);
return (async () => {
const [sctx, inFrame, outFrame] = await Promise.all([
scalerAsync.sws_getContext(
image.codedWidth,
image.codedHeight,
format,
dWidth,
dHeight,
scalerAsync.AV_PIX_FMT_RGBA,
2,
0,
0,
0,
),
scalerAsync.av_frame_alloc(),
scalerAsync.av_frame_alloc(),
]);
// Convert the data (FIXME: duplication)
const rawU8 = image._libavGetData();
let rawIdx = 0;
const raw: Uint8Array[][] = [];
const planes = vf.numPlanes(image.format);
for (let p = 0; p < planes; p++) {
const plane: Uint8Array[] = [];
raw.push(plane);
const sb = vf.sampleBytes(image.format, p);
const hssf = vf.horizontalSubSamplingFactor(image.format, p);
const vssf = vf.verticalSubSamplingFactor(image.format, p);
const w = ~~((image.codedWidth * sb) / hssf);
const h = ~~(image.codedHeight / vssf);
for (let y = 0; y < h; y++) {
plane.push(rawU8.subarray(rawIdx, rawIdx + w));
rawIdx += w;
}
}
const [, , frame] = await Promise.all([
// Copy it in
scalerAsync.ff_copyin_frame(inFrame, {
data: raw,
format,
width: image.codedWidth,
height: image.codedHeight,
}),
// Rescale
scalerAsync.sws_scale_frame(sctx, outFrame, inFrame),
// Get the data back out again
scalerAsync.ff_copyout_frame(outFrame),
// And clean up
scalerAsync.av_frame_free_js(outFrame),
scalerAsync.av_frame_free_js(inFrame),
scalerAsync.sws_freeContext(sctx),
]);
// Transfer all the data
let idx = 0;
for (let i = 0; i < frame.data.length; i++) {
const plane = frame.data[i];
for (let y = 0; y < plane.length; y++) {
const row = plane[y];
frameData.data.set(row, idx);
idx += row.length;
}
}
// And make the ImageBitmap
return await origCreateImageBitmap(frameData);
})();
}

423
src/lib/video-preview/polyfill/video-decoder.ts Normal file
View File

@ -0,0 +1,423 @@
// @ts-nocheck
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
import type * as LibAVJS from '../libav.types';
import type * as evc from './encoded-video-chunk';
import * as libavs from './libav';
import * as misc from './misc';
import * as vf from './video-frame';
export class VideoDecoder {
constructor(init: VideoDecoderInit) {
this._output = init.output;
this._error = init.error;
this.state = 'unconfigured';
this.decodeQueueSize = 0;
this._p = Promise.all([]);
this._libav = null;
this._codec = this._c = this._pkt = this._frame = 0;
}
/* NOTE: These should technically be readonly, but I'm implementing them as
* plain fields, so they're writable */
state: misc.CodecState;
decodeQueueSize: number;
private _output: VideoFrameOutputCallback;
private _error: misc.WebCodecsErrorCallback;
// Event queue
private _p: Promise<unknown>;
// LibAV state
private _libav: LibAVJS.LibAV;
private _codec: number;
private _c: number;
private _pkt: number;
private _frame: number;
configure(config: VideoDecoderConfig): void {
const self = this;
// 1. If config is not a valid VideoDecoderConfig, throw a TypeError.
// NOTE: We don't support sophisticated codec string parsing (yet)
// 2. If [[state]] is “closed”, throw an InvalidStateError DOMException.
if (this.state === 'closed') {
throw new DOMException('Decoder is closed', 'InvalidStateError');
}
// Free any internal state
if (this._libav) {
this._p = this._p.then(() => this._free());
}
// 3. Set [[state]] to "configured".
this.state = 'configured';
// 4. Set [[key chunk required]] to true.
// NOTE: Not implemented
// 5. Queue a control message to configure the decoder with config.
this._p = this._p
.then(async () => {
/* 1. Let supported be the result of running the Check
* Configuration Support algorithm with config. */
const supported = libavs.decoder(config.codec);
/* 2. If supported is true, assign [[codec implementation]] with an
* implementation supporting config. */
if (supported) {
const libav = (self._libav = await libavs.get());
const ptr = await libav.malloc(config.description.length);
await libav.copyin_u8(ptr, config.description);
const parm = await libav.calloc(1, 1024);
await libav.AVCodecParameters_extradata_s(parm, ptr);
await libav.AVCodecParameters_extradata_size_s(parm, config.description.length);
// Initialize
[self._codec, self._c, self._pkt, self._frame] = await libav.ff_init_decoder(
supported.codec,
parm,
);
await libav.AVCodecContext_time_base_s(self._c, 1, 1000);
await libav.free(ptr);
await libav.free(parm);
} else {
/* 3. Otherwise, run the Close VideoDecoder algorithm with
* NotSupportedError DOMException. */
self._closeVideoDecoder(new DOMException('Unsupported codec', 'NotSupportedError'));
}
})
.catch(this._error);
}
// Our own algorithm, close libav
private async _free() {
if (this._c) {
await this._libav.ff_free_decoder(this._c, this._pkt, this._frame);
this._codec = this._c = this._pkt = this._frame = 0;
}
if (this._libav) {
libavs.free(this._libav);
this._libav = null;
}
}
private _closeVideoDecoder(exception: DOMException) {
// 1. Run the Reset VideoDecoder algorithm with exception.
this._resetVideoDecoder(exception);
// 2. Set [[state]] to "closed".
this.state = 'closed';
/* 3. Clear [[codec implementation]] and release associated system
* resources. */
this._p = this._p.then(() => this._free());
/* 4. If exception is not an AbortError DOMException, queue a task on
* the control thread event loop to invoke the [[error callback]] with
* exception. */
if (exception.name !== 'AbortError') {
this._p = this._p.then(() => {
this._error(exception);
});
}
}
private _resetVideoDecoder(exception: DOMException) {
// 1. If [[state]] is "closed", throw an InvalidStateError.
if (this.state === 'closed') {
throw new DOMException('Decoder closed', 'InvalidStateError');
}
// 2. Set [[state]] to "unconfigured".
this.state = 'unconfigured';
// ... really, we're just going to free it now
this._p = this._p.then(() => this._free());
}
decode(chunk: evc.EncodedVideoChunk): void {
const self = this;
// 1. If [[state]] is not "configured", throw an InvalidStateError.
if (this.state !== 'configured') {
throw new DOMException('Unconfigured', 'InvalidStateError');
}
// 2. If [[key chunk required]] is true:
// 1. If chunk.[[type]] is not key, throw a DataError.
/* 2. Implementers SHOULD inspect the chunk’s [[internal data]] to
* verify that it is truly a key chunk. If a mismatch is detected,
* throw a DataError. */
// 3. Otherwise, assign false to [[key chunk required]].
// 3. Increment [[decodeQueueSize]].
this.decodeQueueSize++;
// 4. Queue a control message to decode the chunk.
this._p = this._p
.then(async () => {
const libav = self._libav;
const c = self._c;
const pkt = self._pkt;
const frame = self._frame;
let decodedOutputs: LibAVJS.Frame[] = null;
// 1. Attempt to use [[codec implementation]] to decode the chunk.
try {
// Convert to a libav packet
const ptsFull = Math.floor(chunk.timestamp / 1000);
const pts = ptsFull % 0x100000000;
const ptshi = ~~(ptsFull / 0x100000000);
const packet: LibAVJS.Packet = {
data: chunk._libavGetData(),
pts,
ptshi,
dts: pts,
dtshi: ptshi,
};
if (chunk.duration) {
packet.duration = Math.floor(chunk.duration / 1000);
packet.durationhi = 0;
}
decodedOutputs = await libav.ff_decode_multi(c, pkt, frame, [packet]);
/* 2. If decoding results in an error, queue a task on the control
* thread event loop to run the Close VideoDecoder algorithm with
* EncodingError. */
} catch (ex) {
// console.log('Error decoding', ex);
self._p = self._p.then(() => {
self._closeVideoDecoder(ex);
});
}
/* 3. Queue a task on the control thread event loop to decrement
* [[decodeQueueSize]]. */
self.decodeQueueSize--;
/* 4. Let decoded outputs be a list of decoded audio data outputs
* emitted by [[codec implementation]]. */
/* 5. If decoded outputs is not empty, queue a task on the control
* thread event loop to run the Output VideoData algorithm with
* decoded outputs. */
if (decodedOutputs) {
self._outputVideoFrames(decodedOutputs);
}
})
.catch(this._error);
}
private _outputVideoFrames(frames: LibAVJS.Frame[]) {
const libav = this._libav;
for (const frame of frames) {
// 1. format
let format: vf.VideoPixelFormat;
switch (frame.format) {
case libav.AV_PIX_FMT_YUV420P:
format = 'I420';
break;
case libav.AV_PIX_FMT_YUVA420P:
format = 'I420A';
break;
case libav.AV_PIX_FMT_YUV422P:
format = 'I422';
break;
case libav.AV_PIX_FMT_YUV444P:
format = 'I444';
break;
case libav.AV_PIX_FMT_NV12:
format = 'NV12';
break;
case libav.AV_PIX_FMT_RGBA:
format = 'RGBA';
break;
case libav.AV_PIX_FMT_BGRA:
format = 'BGRA';
break;
default:
throw new DOMException('Unsupported libav format!', 'EncodingError');
}
// 2. width and height
const codedWidth = frame.width;
const codedHeight = frame.height;
// Check for non-square pixels
let displayWidth = codedWidth;
let displayHeight = codedHeight;
if (frame.sample_aspect_ratio[0]) {
const sar = frame.sample_aspect_ratio;
if (sar[0] > sar[1]) {
displayWidth = ~~((codedWidth * sar[0]) / sar[1]);
} else {
displayHeight = ~~((codedHeight * sar[1]) / sar[0]);
}
}
// 3. timestamp
const timestamp = (frame.ptshi * 0x100000000 + frame.pts) * 1000;
// 4. data
let raw: Uint8Array;
{
let size = 0;
const planes = vf.numPlanes(format);
const sbs = [];
const hssfs = [];
const vssfs = [];
for (let i = 0; i < planes; i++) {
sbs.push(vf.sampleBytes(format, i));
hssfs.push(vf.horizontalSubSamplingFactor(format, i));
vssfs.push(vf.verticalSubSamplingFactor(format, i));
}
for (let i = 0; i < planes; i++) {
size += frame.width * frame.height * sbs[i] / hssfs[i]
/ vssfs[i];
}
raw = new Uint8Array(size);
let off = 0;
for (let i = 0; i < planes; i++) {
const fd = frame.data[i];
for (let j = 0; j < frame.height / vssfs[i]; j++) {
const part = fd[j].subarray(0, frame.width / hssfs[i]);
raw.set(part, off);
off += part.length;
}
}
}
const data = new vf.VideoFrame(raw, {
format,
codedWidth,
codedHeight,
displayWidth,
displayHeight,
timestamp,
});
this._output(data);
}
}
flush(): Promise<void> {
const self = this;
const ret = this._p.then(async () => {
if (!self._c) {
return;
}
// Make sure any last data is flushed
const libav = self._libav;
const c = self._c;
const pkt = self._pkt;
const frame = self._frame;
let decodedOutputs: LibAVJS.Frame[] = null;
try {
decodedOutputs = await libav.ff_decode_multi(c, pkt, frame, [], true);
} catch (ex) {
self._p = self._p.then(() => {
self._closeVideoDecoder(ex);
});
}
if (decodedOutputs) {
self._outputVideoFrames(decodedOutputs);
}
});
this._p = ret;
return ret;
}
reset(): void {
this._resetVideoDecoder(new DOMException('Reset', 'AbortError'));
}
close(): void {
this._closeVideoDecoder(new DOMException('Close', 'AbortError'));
}
static async isConfigSupported(config: VideoDecoderConfig): Promise<VideoDecoderSupport> {
const dec = libavs.decoder(config.codec);
let supported = false;
if (dec) {
const libav = await libavs.get();
try {
const [, c, pkt, frame] = await libav.ff_init_decoder(dec.codec);
await libav.ff_free_decoder(c, pkt, frame);
supported = true;
} catch (ex) {}
await libavs.free(libav);
}
return {
supported,
config: misc.cloneConfig(config, ['codec', 'codedWidth', 'codedHeight']),
};
}
}
export interface VideoDecoderInit {
output: VideoFrameOutputCallback;
error: misc.WebCodecsErrorCallback;
}
export type VideoFrameOutputCallback = (output: vf.VideoFrame) => void;
export interface VideoDecoderConfig {
codec: string | { libavjs: libavs.LibAVJSCodec };
description: Uint8Array;
codedWidth?: number;
codedHeight?: number;
displayAspectWidth?: number;
displayAspectHeight?: number;
colorSpace?: vf.VideoColorSpaceInit;
hardwareAcceleration?: string; // Ignored
optimizeForLatency?: boolean;
}
export interface VideoDecoderSupport {
supported: boolean;
config: VideoDecoderConfig;
}

822
src/lib/video-preview/polyfill/video-frame.ts Normal file
View File

@ -0,0 +1,822 @@
// @ts-nocheck
/*
* This file is part of the libav.js WebCodecs Polyfill implementation. The
* interface implemented is derived from the W3C standard. No attribution is
* required when using this library.
*
* Copyright (c) 2021 Yahweasel
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
// A canvas element used to convert CanvasImageSources to buffers
let offscreenCanvas: HTMLCanvasElement = null;
export class VideoFrame {
constructor(data: CanvasImageSource | BufferSource, init: VideoFrameInit | VideoFrameBufferInit) {
if (data instanceof ArrayBuffer || (<any>data).buffer instanceof ArrayBuffer) {
this._constructBuffer(<BufferSource>data, <VideoFrameBufferInit>init);
} else {
this._constructCanvas(<CanvasImageSource>data, <VideoFrameInit>init);
}
}
private _constructCanvas(image: any, init: VideoFrameInit) {
if (offscreenCanvas === null) {
offscreenCanvas = document.createElement('canvas');
offscreenCanvas.style.display = 'none';
document.body.appendChild(offscreenCanvas);
}
// Convert it to a buffer
// Get the width and height
let width = 0;
let height = 0;
if (image.naturalWidth) {
width = image.naturalWidth;
height = image.naturalHeight;
} else if (image.videoWidth) {
width = image.videoWidth;
height = image.videoHeight;
} else if (image.width) {
width = image.width;
height = image.height;
}
if (!width || !height) {
throw new DOMException('Could not determine dimensions', 'InvalidStateError');
}
// Draw it
offscreenCanvas.width = width;
offscreenCanvas.height = height;
const ctx = offscreenCanvas.getContext('2d');
ctx.clearRect(0, 0, width, height);
ctx.drawImage(image, 0, 0);
this._constructBuffer(ctx.getImageData(0, 0, width, height).data, {
format: 'RGBA',
codedWidth: width,
codedHeight: height,
timestamp: init.timestamp,
duration: init.duration || 0,
layout: [
{
offset: 0,
stride: width * 4
}
],
displayWidth: init.displayWidth || width,
displayHeight: init.displayHeight || height
});
}
private _constructBuffer(data: BufferSource, init: VideoFrameBufferInit) {
const format = (this.format = init.format);
const width = (this.codedWidth = init.codedWidth);
const height = (this.codedHeight = init.codedHeight);
this.visibleRect = new DOMRect(0, 0, width, height);
const dWidth = (this.displayWidth = init.displayWidth || init.codedWidth);
const dHeight = (this.displayHeight = init.displayHeight || init.codedHeight);
// Account for non-square pixels
if (dWidth !== width || dHeight !== height) {
// Dubious (but correct) SAR calculation
this._nonSquarePixels = true;
this._sar_num = dWidth * height;
this._sar_den = dHeight * width;
} else {
this._nonSquarePixels = false;
}
this.timestamp = init.timestamp;
if (init.duration) {
this.duration = init.duration;
}
if (init.layout) {
this._layout = init.layout; // FIXME: Make sure it's the right size
} else {
const numPlanes_ = numPlanes(format);
const layout: PlaneLayout[] = [];
let offset = 0;
for (let i = 0; i < numPlanes_; i++) {
const sampleWidth = horizontalSubSamplingFactor(format, i);
const sampleHeight = verticalSubSamplingFactor(format, i);
const stride = ~~(width / sampleWidth);
layout.push({
offset,
stride
});
offset += stride * ~~(height / sampleHeight);
}
this._layout = layout;
}
this._data = new Uint8Array((<any>data).buffer || data, (<any>data).byteOffset || 0);
}
/* NOTE: These should all be readonly, but the constructor style above
* doesn't work with that */
format: VideoPixelFormat;
codedWidth: number;
codedHeight: number;
codedRect: DOMRectReadOnly;
visibleRect: DOMRectReadOnly;
displayWidth: number;
displayHeight: number;
duration: number; // microseconds
timestamp: number; // microseconds
colorSpace: VideoColorSpace;
private _layout: PlaneLayout[];
private _data: Uint8Array;
/**
* (Internal) Does this use non-square pixels?
*/
_nonSquarePixels: boolean;
/**
* (Internal) If non-square pixels, the SAR (sample/pixel aspect ratio)
*/
_sar_num: number;
_sar_den: number;
// Internal
_libavGetData() {
return this._data;
}
allocationSize(options: VideoFrameCopyToOptions = {}): number {
// 1. If [[Detached]] is true, throw an InvalidStateError DOMException.
if (this._data === null) {
throw new DOMException('Detached', 'InvalidStateError');
}
// 2. If [[format]] is null, throw a NotSupportedError DOMException.
if (this.format === null) {
throw new DOMException('Not supported', 'NotSupportedError');
}
/* 3. Let combinedLayout be the result of running the Parse
* VideoFrameCopyToOptions algorithm with options. */
// 4. If combinedLayout is an exception, throw combinedLayout.
const combinedLayout = this._parseVideoFrameCopyToOptions(options);
// 5. Return combinedLayout’s allocationSize.
return combinedLayout.allocationSize;
}
private _parseVideoFrameCopyToOptions(options: VideoFrameCopyToOptions) {
/* 1. Let defaultRect be the result of performing the getter steps for
* visibleRect. */
const defaultRect = this.visibleRect;
// 2. Let overrideRect be undefined.
// 3. If options.rect exists, assign its value to overrideRect.
const overrideRect: DOMRectReadOnly = options.rect
? new DOMRect(options.rect.x, options.rect.y, options.rect.width, options.rect.height)
: null;
/* 4. Let parsedRect be the result of running the Parse Visible Rect
* algorithm with defaultRect, overrideRect, [[coded width]], [[coded
* height]], and [[format]]. */
// 5. If parsedRect is an exception, return parsedRect.
const parsedRect = this._parseVisibleRect(defaultRect, overrideRect);
// 6. Let optLayout be undefined.
// 7. If options.layout exists, assign its value to optLayout.
const optLayout = options.layout || null;
/* 8. Let combinedLayout be the result of running the Compute Layout
* and Allocation Size algorithm with parsedRect, [[format]], and
* optLayout. */
const combinedLayout = this._computeLayoutAndAllocationSize(parsedRect, optLayout);
// 9. Return combinedLayout.
return combinedLayout;
}
private _parseVisibleRect(defaultRect: DOMRectReadOnly, overrideRect: DOMRectReadOnly) {
// 1. Let sourceRect be defaultRect
let sourceRect = defaultRect;
// 2. If overrideRect is not undefined:
if (overrideRect) {
/* 1. If either of overrideRect.width or height is 0, return a
* TypeError. */
if (overrideRect.width === 0 || overrideRect.height === 0) {
throw new TypeError('Invalid rectangle');
}
/* 2. If the sum of overrideRect.x and overrideRect.width is
* greater than [[coded width]], return a TypeError. */
if (overrideRect.x + overrideRect.width > this.codedWidth) {
throw new TypeError('Invalid rectangle');
}
/* 3. If the sum of overrideRect.y and overrideRect.height is
* greater than [[coded height]], return a TypeError. */
if (overrideRect.y + overrideRect.height > this.codedHeight) {
throw new TypeError('Invalid rectangle');
}
// 4. Assign overrideRect to sourceRect.
sourceRect = overrideRect;
}
/* 3. Let validAlignment be the result of running the Verify Rect
* Sample Alignment algorithm with format and sourceRect. */
const validAlignment = this._verifyRectSampleAlignment(sourceRect);
// 4. If validAlignment is false, throw a TypeError.
if (!validAlignment) {
throw new TypeError('Invalid alignment');
}
// 5. Return sourceRect.
return sourceRect;
}
private _computeLayoutAndAllocationSize(parsedRect: DOMRectReadOnly, layout: PlaneLayout[]) {
// 1. Let numPlanes be the number of planes as defined by format.
const numPlanes_ = numPlanes(this.format);
/* 2. If layout is not undefined and its length does not equal
* numPlanes, throw a TypeError. */
if (layout && layout.length !== numPlanes_) {
throw new TypeError('Invalid layout');
}
// 3. Let minAllocationSize be 0.
let minAllocationSize = 0;
// 4. Let computedLayouts be a new list.
const computedLayouts: ComputedPlaneLayout[] = [];
// 5. Let endOffsets be a new list.
const endOffsets = [];
// 6. Let planeIndex be 0.
let planeIndex = 0;
// 7. While planeIndex < numPlanes:
while (planeIndex < numPlanes_) {
/* 1. Let plane be the Plane identified by planeIndex as defined by
* format. */
// 2. Let sampleBytes be the number of bytes per sample for plane.
const sampleBytes_ = sampleBytes(this.format, planeIndex);
/* 3. Let sampleWidth be the horizontal sub-sampling factor of each
* subsample for plane. */
const sampleWidth = horizontalSubSamplingFactor(this.format, planeIndex);
/* 4. Let sampleHeight be the vertical sub-sampling factor of each
* subsample for plane. */
const sampleHeight = verticalSubSamplingFactor(this.format, planeIndex);
/* 5. Let sampleWidthBytes be the product of multiplying
* sampleWidth by sampleBytes. */
const sampleWidthBytes = sampleWidth * sampleBytes_;
// 6. Let computedLayout be a new computed plane layout.
const computedLayout: ComputedPlaneLayout = {
destinationOffset: 0,
destinationStride: 0,
/* 7. Set computedLayout’s sourceTop to the result of the
* integer division of truncated parsedRect.y by sampleHeight. */
sourceTop: ~~(parsedRect.y / sampleHeight),
/* 8. Set computedLayout’s sourceHeight to the result of the
* integer division of truncated parsedRect.height by
* sampleHeight */
sourceHeight: ~~(parsedRect.height / sampleHeight),
/* 9. Set computedLayout’s sourceLeftBytes to the result of the
* integer division of truncated parsedRect.x by
* sampleWidthBytes. */
sourceLeftBytes: ~~(parsedRect.x / sampleWidthBytes),
/* 10. Set computedLayout’s sourceWidthBytes to the result of
* the integer division of truncated parsedRect.width by
* sampleWidthBytes. */
sourceWidthBytes: ~~(parsedRect.width / sampleWidthBytes)
};
// 11. If layout is not undefined:
if (layout) {
/* 1. Let planeLayout be the PlaneLayout in layout at position
* planeIndex. */
const planeLayout = layout[planeIndex];
/* 2. If planeLayout.stride is less than computedLayout’s
* sourceWidthBytes, return a TypeError. */
if (planeLayout.stride < computedLayout.sourceWidthBytes) {
throw new TypeError('Invalid stride');
}
/* 3. Assign planeLayout.offset to computedLayout’s
* destinationOffset. */
computedLayout.destinationOffset = planeLayout.offset;
/* 4. Assign planeLayout.stride to computedLayout’s
* destinationStride. */
computedLayout.destinationStride = planeLayout.stride;
// 12. Otherwise:
} else {
/* 1. Assign minAllocationSize to computedLayout’s
* destinationOffset. */
computedLayout.destinationOffset = minAllocationSize;
/* 2. Assign computedLayout’s sourceWidthBytes to
* computedLayout’s destinationStride. */
computedLayout.destinationStride = computedLayout.sourceWidthBytes;
}
/* 13. Let planeSize be the product of multiplying computedLayout’s
* destinationStride and sourceHeight. */
const planeSize = computedLayout.destinationStride * computedLayout.sourceHeight;
/* 14. Let planeEnd be the sum of planeSize and computedLayout’s
* destinationOffset. */
const planeEnd = planeSize + computedLayout.destinationOffset;
/* 15. If planeSize or planeEnd is greater than maximum range of
* unsigned long, return a TypeError. */
if (planeSize >= 0x100000000 || planeEnd >= 0x100000000) {
throw new TypeError('Plane too large');
}
// 16. Append planeEnd to endOffsets.
endOffsets.push(planeEnd);
/* 17. Assign the maximum of minAllocationSize and planeEnd to
* minAllocationSize. */
if (planeEnd > minAllocationSize) {
minAllocationSize = planeEnd;
}
// 18. Let earlierPlaneIndex be 0.
let earlierPlaneIndex = 0;
// 19. While earlierPlaneIndex is less than planeIndex.
while (earlierPlaneIndex < planeIndex) {
// 1. Let earlierLayout be computedLayouts[earlierPlaneIndex].
const earlierLayout = computedLayouts[earlierPlaneIndex];
/* 2. If endOffsets[planeIndex] is less than or equal to
* earlierLayout’s destinationOffset or if
* endOffsets[earlierPlaneIndex] is less than or equal to
* computedLayout’s destinationOffset, continue. */
if (
planeEnd <= earlierLayout.destinationOffset ||
endOffsets[earlierPlaneIndex] <= computedLayout.destinationOffset
) {
// 3. Otherwise, return a TypeError.
} else {
throw new TypeError('Invalid plane layout');
}
// 4. Increment earlierPlaneIndex by 1.
earlierPlaneIndex++;
}
// 20. Append computedLayout to computedLayouts.
computedLayouts.push(computedLayout);
// 21. Increment planeIndex by 1.
planeIndex++;
}
/* 8. Let combinedLayout be a new combined buffer layout, initialized
* as follows: */
const combinedLayout = {
// 1. Assign computedLayouts to computedLayouts.
computedLayouts,
// 2. Assign minAllocationSize to allocationSize.
allocationSize: minAllocationSize
};
// 9. Return combinedLayout.
return combinedLayout;
}
private _verifyRectSampleAlignment(rect: DOMRectReadOnly) {
// 1. If format is null, return true.
if (!this.format) {
return true;
}
// 2. Let planeIndex be 0.
let planeIndex = 0;
// 3. Let numPlanes be the number of planes as defined by format.
const numPlanes_ = numPlanes(this.format);
// 4. While planeIndex is less than numPlanes:
while (planeIndex < numPlanes_) {
/* 1. Let plane be the Plane identified by planeIndex as defined by
* format. */
/* 2. Let sampleWidth be the horizontal sub-sampling factor of each
* subsample for plane. */
const sampleWidth = horizontalSubSamplingFactor(this.format, planeIndex);
/* 3. Let sampleHeight be the vertical sub-sampling factor of each
* subsample for plane. */
const sampleHeight = verticalSubSamplingFactor(this.format, planeIndex);
/* 4. If rect.x and rect.width are not both multiples of
* sampleWidth, return false. */
const xw = rect.x / sampleWidth;
if (xw !== ~~xw) {
return false;
}
const ww = rect.width / sampleWidth;
if (ww !== ~~ww) {
return false;
}
/* 5. If rect.y and rect.height are not both multiples of
* sampleHeight, return false. */
const yh = rect.y / sampleHeight;
if (yh !== ~~yh) {
return false;
}
const hh = rect.height / sampleHeight;
if (hh !== ~~hh) {
return false;
}
// 6. Increment planeIndex by 1.
planeIndex++;
}
// 5. Return true.
return true;
}
async copyTo(
destination: BufferSource,
options: VideoFrameCopyToOptions = {}
): Promise<PlaneLayout[]> {
const destBuf = new Uint8Array(
(<any>destination).buffer || destination,
(<any>destination).byteOffset || 0
);
// 1. If [[Detached]] is true, throw an InvalidStateError DOMException.
if (this._data === null) {
throw new DOMException('Detached', 'InvalidStateError');
}
// 2. If [[format]] is null, throw a NotSupportedError DOMException.
if (!this.format) {
throw new DOMException('No format', 'NotSupportedError');
}
/* 3. Let combinedLayout be the result of running the Parse
* VideoFrameCopyToOptions algorithm with options. */
/* 4. If combinedLayout is an exception, return a promise rejected with
* combinedLayout. */
const combinedLayout = this._parseVideoFrameCopyToOptions(options);
/* 5. If destination.byteLength is less than combinedLayout’s
* allocationSize, return a promise rejected with a TypeError. */
if (destination.byteLength < combinedLayout.allocationSize) {
throw new TypeError('Insufficient space');
}
// 6. Let p be a new Promise.
/* 7. Let copyStepsQueue be the result of starting a new parallel
* queue. */
// 8. Enqueue the following steps to copyStepsQueue:
// NOTE: This is an async function anyway, so we can just do these.
const ret: PlaneLayout[] = [];
/* 1. Let resource be the media resource referenced by [[resource
* reference]]. */
// 2. Let numPlanes be the number of planes as defined by [[format]].
const numPlanes_ = numPlanes(this.format);
// 3. Let planeIndex be 0.
let planeIndex = 0;
// 4. While planeIndex is less than combinedLayout’s numPlanes:
while (planeIndex < combinedLayout.computedLayouts.length) {
/* 1. Let sourceStride be the stride of the plane in resource as
* identified by planeIndex. */
const sourceStride = this._layout[planeIndex].stride;
/* 2. Let computedLayout be the computed plane layout in
* combinedLayout’s computedLayouts at the position of planeIndex */
const computedLayout = combinedLayout.computedLayouts[planeIndex];
/* 3. Let sourceOffset be the product of multiplying
* computedLayout’s sourceTop by sourceStride */
let sourceOffset = computedLayout.sourceTop * sourceStride;
// 4. Add computedLayout’s sourceLeftBytes to sourceOffset.
sourceOffset += computedLayout.sourceLeftBytes;
// 5. Let destinationOffset be computedLayout’s destinationOffset.
let destinationOffset = computedLayout.destinationOffset;
// 6. Let rowBytes be computedLayout’s sourceWidthBytes.
const rowBytes = computedLayout.sourceWidthBytes;
// 7. Let row be 0.
let row = 0;
// 8. While row is less than computedLayout’s sourceHeight:
while (row < computedLayout.sourceHeight) {
/* 1. Copy rowBytes bytes from resource starting at
* sourceOffset to destination starting at destinationOffset. */
destBuf.set(this._data.subarray(sourceOffset, sourceOffset + rowBytes), destinationOffset);
// 2. Increment sourceOffset by sourceStride.
sourceOffset += sourceStride;
/* 3. Increment destinationOffset by computedLayout’s
* destinationStride. */
destinationOffset += computedLayout.destinationStride;
// 4. Increment row by 1.
row++;
}
// 9. Increment planeIndex by 1.
planeIndex++;
ret.push({
offset: computedLayout.destinationOffset,
stride: computedLayout.destinationStride
});
}
// 5. Queue a task on the control thread event loop to resolve p.
// 6. Return p.
return ret;
}
clone(): VideoFrame {
return new VideoFrame(this._data, {
format: this.format,
codedWidth: this.codedWidth,
codedHeight: this.codedHeight,
timestamp: this.timestamp,
duration: this.duration,
layout: this._layout
});
}
close(): void {
this._data = null;
}
}
export interface VideoFrameInit {
duration?: number; // microseconds
timestamp: number; // microseconds
// FIXME: AlphaOption alpha = "keep";
// Default matches image. May be used to efficiently crop. Will trigger
// new computation of displayWidth and displayHeight using image’s pixel
// aspect ratio unless an explicit displayWidth and displayHeight are given.
visibleRect?: DOMRectInit;
// Default matches image unless visibleRect is provided.
displayWidth?: number;
displayHeight?: number;
}
export interface VideoFrameBufferInit {
format: VideoPixelFormat;
codedWidth: number;
codedHeight: number;
timestamp: number; // microseconds
duration?: number; // microseconds
// Default layout is tightly-packed.
layout?: PlaneLayout[];
// Default visible rect is coded size positioned at (0,0)
visibleRect?: DOMRectInit;
// Default display dimensions match visibleRect.
displayWidth?: number;
displayHeight?: number;
// FIXME: Not used
colorSpace?: VideoColorSpaceInit;
}
export type VideoPixelFormat =
// 4:2:0 Y, U, V
| 'I420'
// 4:2:0 Y, U, V, A
| 'I420A'
// 4:2:2 Y, U, V
| 'I422'
// 4:4:4 Y, U, V
| 'I444'
// 4:2:0 Y, UV
| 'NV12'
// 32bpp RGBA
| 'RGBA'
// 32bpp RGBX (opaque)
| 'RGBX'
// 32bpp BGRA
| 'BGRA'
// 32bpp BGRX (opaque)
| 'BGRX';
/**
* Number of planes in the given format.
* @param format The format
*/
export function numPlanes(format: VideoPixelFormat) {
switch (format) {
case 'I420':
case 'I422':
case 'I444':
return 3;
case 'I420A':
return 4;
case 'NV12':
return 2;
case 'RGBA':
case 'RGBX':
case 'BGRA':
case 'BGRX':
return 1;
default:
throw new DOMException('Unsupported video pixel format', 'NotSupportedError');
}
}
/**
* Number of bytes per sample in the given format and plane.
* @param format The format
* @param planeIndex The plane index
*/
export function sampleBytes(format: VideoPixelFormat, planeIndex: number) {
switch (format) {
case 'I420':
case 'I420A':
case 'I422':
case 'I444':
return 1;
case 'NV12':
if (planeIndex === 1) {
return 2;
} else {
return 1;
}
case 'RGBA':
case 'RGBX':
case 'BGRA':
case 'BGRX':
return 4;
default:
throw new DOMException('Unsupported video pixel format', 'NotSupportedError');
}
}
/**
* Horizontal sub-sampling factor for the given format and plane.
* @param format The format
* @param planeIndex The plane index
*/
export function horizontalSubSamplingFactor(format: VideoPixelFormat, planeIndex: number) {
// First plane (often luma) is always full
if (planeIndex === 0) {
return 1;
}
switch (format) {
case 'I420':
case 'I422':
return 2;
case 'I420A':
if (planeIndex === 3) {
return 1;
} else {
return 2;
}
case 'I444':
return 1;
case 'NV12':
return 2;
case 'RGBA':
case 'RGBX':
case 'BGRA':
case 'BGRX':
return 1;
default:
throw new DOMException('Unsupported video pixel format', 'NotSupportedError');
}
}
/**
* Vertical sub-sampling factor for the given format and plane.
* @param format The format
* @param planeIndex The plane index
*/
export function verticalSubSamplingFactor(format: VideoPixelFormat, planeIndex: number) {
// First plane (often luma) is always full
if (planeIndex === 0) {
return 1;
}
switch (format) {
case 'I420':
return 2;
case 'I420A':
if (planeIndex === 3) {
return 1;
} else {
return 2;
}
case 'I422':
case 'I444':
return 1;
case 'NV12':
return 2;
case 'RGBA':
case 'RGBX':
case 'BGRA':
case 'BGRX':
return 1;
default:
throw new DOMException('Unsupported video pixel format', 'NotSupportedError');
}
}
/**
* NOTE: Color space is not actually supported
*/
export type VideoColorSpace = any;
export type VideoColorSpaceInit = any;
export interface PlaneLayout {
offset: number;
stride: number;
}
export interface VideoFrameCopyToOptions {
rect?: DOMRectInit;
layout?: PlaneLayout[];
}
interface ComputedPlaneLayout {
destinationOffset: number;
destinationStride: number;
sourceTop: number;
sourceHeight: number;
sourceLeftBytes: number;
sourceWidthBytes: number;
}

62
src/lib/video-preview/requestPart.ts Normal file
View File

@ -0,0 +1,62 @@
import generateIdFor from '../../util/generateIdFor';
import { pause } from '../../util/schedulers';
declare const self: WorkerGlobalScope;
type RequestStates = {
resolve: (response: ArrayBuffer) => void;
reject: () => void;
};
type RequestPartParams = { url: string; start: number; end: number };
const PART_TIMEOUT = 30000;
const requestStates = new Map<string, RequestStates>();
export function requestPart(params: RequestPartParams): Promise<ArrayBuffer | undefined> {
const messageId = generateIdFor(requestStates);
const requestState = {} as RequestStates;
let isResolved = false;
const promise = Promise.race([
pause(PART_TIMEOUT).then(() => (isResolved ? undefined : Promise.reject(new Error('ERROR_PART_TIMEOUT')))),
new Promise<ArrayBuffer>((resolve, reject) => {
Object.assign(requestState, { resolve, reject });
}),
]);
requestStates.set(messageId, requestState);
promise
.catch(() => undefined)
.finally(() => {
requestStates.delete(messageId);
isResolved = true;
});
const message = {
type: 'requestPart',
messageId,
params,
};
postMessage(message);
return promise;
}
self.addEventListener('message', (e) => {
const { type, messageId, result } = (e as any).data as {
type: string;
messageId: string;
result: ArrayBuffer;
};
if (type === 'partResponse') {
const requestState = requestStates.get(messageId);
if (requestState) {
requestState.resolve(result);
}
}
});

88
src/lib/video-preview/video-preview.worker.ts Normal file
View File

@ -0,0 +1,88 @@
import { createWorkerInterface } from '../../util/createPostMessageInterface';
import type { CancellableCallback } from '../../util/PostMessageConnector';
import { MP4Demuxer } from './MP4Demuxer';
import * as LibAVWebCodecs from './polyfill';
const MAX_PREVIEWS_PER_VIDEO = 300;
let decoder: any;
let demuxer: any;
let onDestroy: VoidFunction | undefined;
let isLoaded = false;
async function init(url: string, workerIndex: number, workersTotal: number, onFrame: CancellableCallback) {
if (!('VideoDecoder' in globalThis)) {
await loadLibAV();
}
const decodedFrames = new Set<number>();
// @ts-ignore
decoder = new VideoDecoder({
async output(frame: any) {
const time = frame.timestamp / 1e6;
const seconds = Math.floor(time);
// Only render whole second frames
if (!decodedFrames.has(seconds)) {
const bitmap = await createImageBitmap(frame);
decodedFrames.add(seconds);
onFrame(seconds, bitmap);
}
frame.close();
},
error(e: any) {
// eslint-disable-next-line no-console
console.error('[Video Preview] error', e);
},
});
demuxer = new MP4Demuxer(url, {
framesPerVideo: Math.round(MAX_PREVIEWS_PER_VIDEO / workersTotal),
stepOffset: workerIndex,
stepMultiplier: workersTotal,
onConfig(config) {
decoder?.configure(config);
},
onChunk(chunk) {
if (decoder?.state !== 'configured') return;
decoder?.decode(chunk);
},
});
return new Promise<void>((resolve) => {
onDestroy = resolve;
});
}
function destroy() {
try {
decoder?.close();
demuxer?.close();
} catch {
// Ignore
}
decoder = undefined;
demuxer = undefined;
onDestroy?.();
}
async function loadLibAV() {
if (isLoaded) return;
// @ts-ignore
await import('script-loader!./libav-3.10.5.1.2-webcodecs');
await LibAVWebCodecs.load({
polyfill: true,
libavOptions: { noworker: true, nosimd: true },
});
isLoaded = true;
}
const api = {
'video-preview:init': init,
'video-preview:destroy': destroy,
};
createWorkerInterface(api);
export type VideoPreviewApi = typeof api;

4
src/serviceWorker/download.ts
View File

@ -59,7 +59,9 @@ export async function respondForDownload(e: FetchEvent) {
const queue = new FilePartQueue<ArrayBuffer | undefined>();
const enqueue = (offset: number) => {
queue.push(requestPart(e, { url, start: offset, end: offset + DOWNLOAD_PART_SIZE - 1 })
queue.push(requestPart(e, {
url, start: offset, end: offset + DOWNLOAD_PART_SIZE - 1,
})
.then((part) => part?.arrayBuffer));
return offset + DOWNLOAD_PART_SIZE;
};

2
src/util/PostMessageConnector.ts
View File

@ -213,4 +213,4 @@ export function createConnector<T extends InputRequestTypes>(
return connector;
}
export type Connector<T extends InputRequestTypes> = ReturnType<typeof createConnector<T>>;
export type Connector<T extends InputRequestTypes = InputRequestTypes> = ReturnType<typeof createConnector<T>>;

11
src/util/createPostMessageInterface.ts
View File

@ -3,6 +3,7 @@ import type {
} from './PostMessageConnector';
import { DEBUG } from '../config';
import { createCallbackManager } from './callbacks';
declare const self: WorkerGlobalScope;
@ -13,6 +14,9 @@ type ApiConfig =
| Record<string, Function>;
type SendToOrigin = (data: WorkerMessageData, transferables?: Transferable[]) => void;
const messageHandlers = createCallbackManager();
onmessage = messageHandlers.runCallbacks;
export function createWorkerInterface(api: ApiConfig, channel?: string) {
function sendToOrigin(data: WorkerMessageData, transferables?: Transferable[]) {
data.channel = channel;
@ -26,11 +30,11 @@ export function createWorkerInterface(api: ApiConfig, channel?: string) {
handleErrors(sendToOrigin);
onmessage = (message: OriginMessageEvent) => {
messageHandlers.addCallback((message: OriginMessageEvent) => {
if (message.data?.channel === channel) {
onMessage(api, message.data, sendToOrigin);
}
};
});
}
async function onMessage(
@ -62,7 +66,10 @@ async function onMessage(
const {
messageId, name, args, withCallback,
} = data;
try {
if (typeof api !== 'function' && !api[name]) return;
if (messageId && withCallback) {
const callback = (...callbackArgs: any[]) => {
const lastArg = callbackArgs[callbackArgs.length - 1];

25
src/util/launchMediaWorkers.ts Normal file
View File

@ -0,0 +1,25 @@
import type { MediaWorkerApi } from '../lib/mediaWorker/index.worker';
import type { Connector } from './PostMessageConnector';
import { createConnector } from './PostMessageConnector';
export const MAX_WORKERS = Math.min(navigator.hardwareConcurrency || 4, 4);
let instances: {
worker: Worker;
connector: Connector<MediaWorkerApi>;
}[] | undefined;
export default function launchMediaWorkers() {
if (!instances) {
instances = new Array(MAX_WORKERS).fill(undefined).map(
() => {
const worker = new Worker(new URL('../lib/mediaWorker/index.worker.ts', import.meta.url));
const connector = createConnector<MediaWorkerApi>(worker);
return { worker, connector };
},
);
}
return instances;
}

2
src/util/windowEnvironment.ts
View File

@ -90,6 +90,8 @@ if (IS_MOV_SUPPORTED) {
export const IS_WEBM_SUPPORTED = Boolean(TEST_VIDEO.canPlayType('video/webm; codecs="vp9"').replace('no', ''));
export const ARE_WEBCODECS_SUPPORTED = 'VideoDecoder' in window;
export const DPR = window.devicePixelRatio || 1;
export const MASK_IMAGE_DISABLED = true;

3
webpack.config.js
View File

@ -59,6 +59,9 @@ module.exports = (_env, { mode = 'production' }) => {
{
directory: path.resolve(__dirname, 'src/lib/rlottie'),
},
{
directory: path.resolve(__dirname, 'src/lib/video-preview'),
},
{
directory: path.resolve(__dirname, 'src/lib/secret-sauce'),
},