[Perf] RLottie: Avoid redundant JSON data (de)serialization

This commit is contained in:
Alexander Zinchuk 2022-01-24 04:41:54 +01:00
parent b6c80b5236
commit 29d5ef3ef7
25 changed files with 119 additions and 101 deletions

View File

@ -1,5 +1,3 @@
import { inflate } from 'pako/dist/pako_inflate';
import { Api as GramJs, TelegramClient } from '../../../lib/gramjs';
import {
ApiMediaFormat, ApiOnProgress, ApiParsedMedia, ApiPreparedMedia,
@ -20,7 +18,9 @@ type EntityType = (
'msg' | 'sticker' | 'wallpaper' | 'gif' | 'channel' | 'chat' | 'user' | 'photo' | 'stickerSet' | 'webDocument' |
'document'
);
const MEDIA_ENTITY_TYPES = new Set(['msg', 'sticker', 'gif', 'wallpaper', 'photo', 'webDocument', 'document']);
const TGS_MIME_TYPE = 'application/x-tgsticker';
export default async function downloadMedia(
{
@ -53,7 +53,7 @@ export default async function downloadMedia(
void cacheApi.save(cacheName, url, parsed);
}
const prepared = mediaFormat === ApiMediaFormat.Progressive ? '' : prepareMedia(parsed);
const prepared = mediaFormat === ApiMediaFormat.Progressive ? '' : prepareMedia(parsed as string | Blob);
const arrayBuffer = mediaFormat === ApiMediaFormat.Progressive ? parsed as ArrayBuffer : undefined;
return {
@ -183,7 +183,7 @@ async function download(
return { mimeType, data, fullSize };
} else if (entityType === 'stickerSet') {
const data = await client.downloadStickerSetThumb(entity);
const mimeType = mediaFormat === ApiMediaFormat.Lottie ? 'application/json' : getMimeType(data);
const mimeType = mediaFormat === ApiMediaFormat.Lottie ? TGS_MIME_TYPE : getMimeType(data);
return { mimeType, data };
} else {
@ -228,10 +228,8 @@ async function parseMedia(
): Promise<ApiParsedMedia | undefined> {
switch (mediaFormat) {
case ApiMediaFormat.BlobUrl:
return new Blob([data], { type: mimeType });
case ApiMediaFormat.Lottie: {
const json = inflate(data, { to: 'string' });
return JSON.parse(json);
return new Blob([data], { type: mimeType });
}
case ApiMediaFormat.Progressive: {
return data.buffer;
@ -241,7 +239,7 @@ async function parseMedia(
return undefined;
}
function prepareMedia(mediaData: ApiParsedMedia): ApiPreparedMedia {
function prepareMedia(mediaData: Exclude<ApiParsedMedia, ArrayBuffer>): ApiPreparedMedia {
if (mediaData instanceof Blob) {
return URL.createObjectURL(mediaData);
}

View File

@ -8,6 +8,5 @@ export enum ApiMediaFormat {
Stream,
}
export type ApiParsedMedia = string | Blob | AnyLiteral | ArrayBuffer;
export type ApiPreparedMedia = string | AnyLiteral;
export type ApiMediaFormatToPrepared<T> = T extends ApiMediaFormat.Lottie ? AnyLiteral : string;
export type ApiParsedMedia = string | Blob | ArrayBuffer;
export type ApiPreparedMedia = string;

View File

@ -89,7 +89,7 @@ const AnimatedEmoji: FC<OwnProps> = ({
<AnimatedSticker
key={localMediaHash}
id={localMediaHash}
animationData={mediaData as AnyLiteral}
animationData={mediaData!}
size={width}
quality={QUALITY}
play={isIntersecting && playKey}

View File

@ -19,7 +19,7 @@ const AnimatedIcon: FC<OwnProps> = ({
playSegment,
color,
}) => {
const [iconData, setIconData] = useState<Record<string, any>>();
const [iconData, setIconData] = useState<string>();
useEffect(() => {
getAnimationData(name).then(setIconData);

View File

@ -10,7 +10,7 @@ import useBackgroundMode from '../../hooks/useBackgroundMode';
type OwnProps = {
className?: string;
id: string;
animationData: AnyLiteral;
animationData?: string;
play?: boolean | string;
playSegment?: [number, number];
speed?: number;
@ -202,7 +202,7 @@ const AnimatedSticker: FC<OwnProps> = ({
if (animation) {
if (isFirstRender.current) {
isFirstRender.current = false;
} else {
} else if (animationData) {
animation.changeData(animationData);
playAnimation();
}

View File

@ -3,7 +3,7 @@ import React, {
} from '../../lib/teact/teact';
import { getDispatch } from '../../lib/teact/teactn';
import { ApiMediaFormat, ApiMessage } from '../../api/types';
import { ApiMessage } from '../../api/types';
import { getDocumentExtension, getDocumentHasPreview } from './helpers/documentInfo';
import {
@ -83,7 +83,7 @@ const Document: FC<OwnProps> = ({
const shouldDownload = Boolean(isDownloading || (isLoadAllowed && wasIntersected));
const documentHash = getMessageMediaHash(message, 'download');
const { loadProgress: downloadProgress, mediaData } = useMediaWithLoadProgress<ApiMediaFormat.BlobUrl>(
const { loadProgress: downloadProgress, mediaData } = useMediaWithLoadProgress(
documentHash, !shouldDownload, undefined, undefined, undefined, true,
);
const isLoaded = Boolean(mediaData);

View File

@ -49,7 +49,7 @@ const LocalAnimatedEmoji: FC<OwnProps> = ({
const isIntersecting = useIsIntersecting(ref, observeIntersection);
const [localStickerAnimationData, setLocalStickerAnimationData] = useState<AnyLiteral>();
const [localStickerAnimationData, setLocalStickerAnimationData] = useState<string>();
useEffect(() => {
if (localSticker) {
getAnimationData(localSticker as keyof typeof ANIMATED_STICKERS_PATHS).then((data) => {

View File

@ -22,8 +22,8 @@ const SEGMENT_COVER_EYE: [number, number] = [20, 0];
const STICKER_SIZE = IS_SINGLE_COLUMN_LAYOUT ? STICKER_SIZE_AUTH_MOBILE : STICKER_SIZE_AUTH;
const PasswordMonkey: FC<OwnProps> = ({ isPasswordVisible, isBig }) => {
const [closeMonkeyData, setCloseMonkeyData] = useState<Record<string, any>>();
const [peekMonkeyData, setPeekMonkeyData] = useState<Record<string, any>>();
const [closeMonkeyData, setCloseMonkeyData] = useState<string>();
const [peekMonkeyData, setPeekMonkeyData] = useState<string>();
const [isFirstMonkeyLoaded, setIsFirstMonkeyLoaded] = useState(false);
const [isPeekShown, setIsPeekShown] = useState(false);

View File

@ -29,8 +29,8 @@ const TrackingMonkey: FC<OwnProps> = ({
isTracking,
isBig,
}) => {
const [idleMonkeyData, setIdleMonkeyData] = useState<Record<string, any>>();
const [trackingMonkeyData, setTrackingMonkeyData] = useState<Record<string, any>>();
const [idleMonkeyData, setIdleMonkeyData] = useState<string>();
const [trackingMonkeyData, setTrackingMonkeyData] = useState<string>();
const [isFirstMonkeyLoaded, setIsFirstMonkeyLoaded] = useState(false);
const TRACKING_FRAMES_PER_SYMBOL = (TRACKING_END_FRAME - TRACKING_START_FRAME) / codeLength;

View File

@ -69,7 +69,7 @@ const SettingsFoldersEdit: FC<OwnProps & StateProps> = ({
loadMoreChats,
} = getDispatch();
const [animationData, setAnimationData] = useState<Record<string, any>>();
const [animationData, setAnimationData] = useState<string>();
const [isAnimationLoaded, setIsAnimationLoaded] = useState(false);
const handleAnimationLoad = useCallback(() => setIsAnimationLoaded(true), []);

View File

@ -64,7 +64,7 @@ const SettingsFoldersMain: FC<OwnProps & StateProps> = ({
showDialog,
} = getDispatch();
const [animationData, setAnimationData] = useState<Record<string, any>>();
const [animationData, setAnimationData] = useState<string>();
const [isAnimationLoaded, setIsAnimationLoaded] = useState(false);
const handleAnimationLoad = useCallback(() => setIsAnimationLoaded(true), []);

View File

@ -37,7 +37,7 @@ const DownloadManager: FC<StateProps> = ({
}
if (!startedDownloads.has(downloadHash)) {
const mediaData = mediaLoader.getFromMemory<ApiMediaFormat.BlobUrl>(downloadHash);
const mediaData = mediaLoader.getFromMemory(downloadHash);
if (mediaData) {
startedDownloads.delete(downloadHash);
download(mediaData, getMessageContentFilename(message));

View File

@ -75,9 +75,9 @@ const EmojiInteractionAnimation: FC<OwnProps & StateProps> = ({
}, PLAYING_DURATION);
}, [stop]);
const mediaDataEffect = useMedia(`sticker${effectAnimationId}`, !effectAnimationId, ApiMediaFormat.Lottie);
const effectAnimationData = useMedia(`sticker${effectAnimationId}`, !effectAnimationId, ApiMediaFormat.Lottie);
const [localEffectAnimationData, setLocalEffectAnimationData] = useState<AnyLiteral>();
const [localEffectAnimationData, setLocalEffectAnimationData] = useState<string | undefined>();
useEffect(() => {
if (localEffectAnimation) {
getAnimationData(localEffectAnimation as keyof typeof ANIMATED_STICKERS_PATHS).then((data) => {
@ -99,7 +99,7 @@ const EmojiInteractionAnimation: FC<OwnProps & StateProps> = ({
<AnimatedSticker
id={`effect_${effectAnimationId}`}
size={EFFECT_SIZE}
animationData={(localEffectAnimationData || mediaDataEffect) as AnyLiteral}
animationData={localEffectAnimationData || effectAnimationData}
play={isPlaying}
isLowPriority={IS_ANDROID}
forceOnHeavyAnimation

View File

@ -67,7 +67,7 @@ const ReactionAnimatedEmoji: FC<OwnProps> = ({
id={`reaction_emoji_${centerIconId}`}
className={animationClassNames}
size={CENTER_ICON_SIZE}
animationData={mediaDataCenterIcon as AnyLiteral}
animationData={mediaDataCenterIcon}
play
noLoop
onLoad={markAnimationLoaded}
@ -78,7 +78,7 @@ const ReactionAnimatedEmoji: FC<OwnProps> = ({
id={`reaction_effect_${effectId}`}
className={buildClassName('effect', animationClassNames)}
size={EFFECT_SIZE}
animationData={mediaDataEffect as AnyLiteral}
animationData={mediaDataEffect}
play
noLoop
onEnded={handleEnded}

View File

@ -65,7 +65,7 @@ const AvailableReaction: FC<{
<AnimatedSticker
id={`select_${reaction.reaction}`}
className={animatedClassNames}
animationData={mediaData as AnyLiteral}
animationData={mediaData}
play={isActivated}
noLoop
size={REACTION_SIZE}

View File

@ -45,7 +45,7 @@ const Sticker: FC<OwnProps> = ({
const mediaData = useMedia(
mediaHash,
!shouldLoad,
getMessageMediaFormat(message, 'inline', true),
getMessageMediaFormat(message, 'inline'),
lastSyncTime,
);
@ -89,7 +89,7 @@ const Sticker: FC<OwnProps> = ({
key={mediaHash}
className={buildClassName('full-media', transitionClassNames)}
id={mediaHash}
animationData={mediaData as AnyLiteral}
animationData={mediaData}
size={width}
play={shouldPlay}
noLoop={!shouldLoop}

View File

@ -55,7 +55,7 @@ const ManageDiscussion: FC<OwnProps & StateProps> = ({
} = getDispatch();
const [linkedGroupId, setLinkedGroupId] = useState<string>();
const [animationData, setAnimationData] = useState<Record<string, any>>();
const [animationData, setAnimationData] = useState<string>();
const [isAnimationLoaded, setIsAnimationLoaded] = useState(false);
const handleAnimationLoad = useCallback(() => setIsAnimationLoaded(true), []);
const [isConfirmUnlinkGroupDialogOpen, openConfirmUnlinkGroupDialog, closeConfirmUnlinkGroupDialog] = useFlag();

View File

@ -13,7 +13,7 @@ export default <T extends ApiMediaFormat = ApiMediaFormat.BlobUrl>(
cacheBuster?: number,
delay?: number | false,
) => {
const mediaData = mediaHash ? mediaLoader.getFromMemory<T>(mediaHash) : undefined;
const mediaData = mediaHash ? mediaLoader.getFromMemory(mediaHash) : undefined;
const forceUpdate = useForceUpdate();
useEffect(() => {

View File

@ -22,7 +22,7 @@ export default function useMediaWithLoadProgress<T extends ApiMediaFormat = ApiM
delay?: number | false,
isHtmlAllowed = false,
) {
const mediaData = mediaHash ? mediaLoader.getFromMemory<T>(mediaHash) : undefined;
const mediaData = mediaHash ? mediaLoader.getFromMemory(mediaHash) : undefined;
const isStreaming = mediaFormat === ApiMediaFormat.Stream || (
IS_PROGRESSIVE_SUPPORTED && mediaFormat === ApiMediaFormat.Progressive
);

View File

@ -22,8 +22,6 @@ const CHUNK_SIZE = 1;
const MAX_WORKERS = 4;
const HIGH_PRIORITY_QUALITY = IS_SINGLE_COLUMN_LAYOUT ? 0.75 : 1;
const LOW_PRIORITY_QUALITY = 0.75;
const HIGH_PRIORITY_MAX_FPS = 60;
const LOW_PRIORITY_MAX_FPS = 30;
const HIGH_PRIORITY_CACHE_MODULO = IS_SAFARI ? 2 : 4;
const LOW_PRIORITY_CACHE_MODULO = 0;
@ -39,9 +37,9 @@ class RLottie {
private key!: string;
private msPerFrame!: number;
private msPerFrame = 1000 / 60;
private reduceFactor!: number;
private reduceFactor = 1;
private cacheModulo!: number;
@ -86,7 +84,7 @@ class RLottie {
constructor(
private id: string,
private container: HTMLDivElement,
private animationData: AnyLiteral,
private animationData: string,
private params: Params = {},
private onLoad?: () => void,
private customColor?: [number, number, number],
@ -189,10 +187,6 @@ class RLottie {
const { isLowPriority } = this.params;
const maxFps = isLowPriority ? LOW_PRIORITY_MAX_FPS : HIGH_PRIORITY_MAX_FPS;
const sourceFps = this.animationData.fr || maxFps;
this.reduceFactor = sourceFps % maxFps === 0 ? sourceFps / maxFps : 1;
this.msPerFrame = 1000 / (sourceFps / this.reduceFactor);
this.cacheModulo = isLowPriority ? LOW_PRIORITY_CACHE_MODULO : HIGH_PRIORITY_CACHE_MODULO;
this.chunkSize = CHUNK_SIZE;
}
@ -201,17 +195,10 @@ class RLottie {
this.canvas.remove();
}
private onChangeData(framesCount: number) {
this.isWaiting = false;
this.framesCount = framesCount;
this.chunksCount = Math.ceil(framesCount / this.chunkSize);
this.isAnimating = false;
this.doPlay();
}
setColor(newColor: [number, number, number] | undefined) {
this.customColor = newColor;
// TODO Remove?
if (this.customColor) {
const imageData = this.ctx.getImageData(0, 0, this.imgSize, this.imgSize);
const arr = imageData.data;
@ -226,21 +213,6 @@ class RLottie {
}
}
changeData(animationData: AnyLiteral) {
this.pause();
this.animationData = animationData;
this.initConfig();
workers[this.workerIndex].request({
name: 'changeData',
args: [
this.key,
this.animationData,
this.onChangeData.bind(this),
],
});
}
private initRenderer() {
this.workerIndex = cycleRestrict(MAX_WORKERS, ++lastWorkerIndex);
@ -251,7 +223,6 @@ class RLottie {
this.animationData,
this.imgSize,
this.params.isLowPriority,
this.reduceFactor,
this.onRendererInit.bind(this),
],
});
@ -264,7 +235,9 @@ class RLottie {
});
}
private onRendererInit(framesCount: number) {
private onRendererInit(reduceFactor: number, msPerFrame: number, framesCount: number) {
this.reduceFactor = reduceFactor;
this.msPerFrame = msPerFrame;
this.framesCount = framesCount;
this.chunksCount = Math.ceil(framesCount / this.chunkSize);
@ -273,6 +246,33 @@ class RLottie {
}
}
changeData(animationData: string) {
this.pause();
this.animationData = animationData;
this.initConfig();
workers[this.workerIndex].request({
name: 'changeData',
args: [
this.key,
this.animationData,
this.params.isLowPriority,
this.onChangeData.bind(this),
],
});
}
private onChangeData(reduceFactor: number, msPerFrame: number, framesCount: number) {
this.reduceFactor = reduceFactor;
this.msPerFrame = msPerFrame;
this.framesCount = framesCount;
this.chunksCount = Math.ceil(framesCount / this.chunkSize);
this.isWaiting = false;
this.isAnimating = false;
this.doPlay();
}
private doPlay() {
if (!this.framesCount) {
return;

View File

@ -1,3 +1,4 @@
import { inflate } from 'pako/dist/pako_inflate';
import createWorkerInterface from '../../util/createWorkerInterface';
import { CancellableCallback } from '../../util/WorkerConnector';
@ -27,6 +28,9 @@ const rLottieApiPromise = new Promise<void>((resolve) => {
};
});
const HIGH_PRIORITY_MAX_FPS = 60;
const LOW_PRIORITY_MAX_FPS = 30;
const renderers = new Map<string, {
imgSize: number;
reduceFactor: number;
@ -35,42 +39,70 @@ const renderers = new Map<string, {
async function init(
key: string,
animationData: AnyLiteral,
animationData: string,
imgSize: number,
isLowPriority: boolean,
reduceFactor: number,
onInit: CancellableCallback,
) {
if (!rLottieApi) {
await rLottieApiPromise;
}
const json = JSON.stringify(animationData);
const json = await extractJson(animationData);
const stringOnWasmHeap = allocate(intArrayFromString(json), 'i8', 0);
const handle = rLottieApi.init();
const framesCount = rLottieApi.loadFromData(handle, stringOnWasmHeap);
rLottieApi.resize(handle, imgSize, imgSize);
renderers.set(key, { imgSize, reduceFactor, handle });
const { reduceFactor, msPerFrame, reducedFramesCount } = calcParams(json, isLowPriority, framesCount);
onInit(Math.ceil(framesCount / reduceFactor));
renderers.set(key, { imgSize, reduceFactor, handle });
onInit(reduceFactor, msPerFrame, reducedFramesCount);
}
async function changeData(
key: string,
animationData: AnyLiteral,
animationData: string,
isLowPriority: boolean,
onInit: CancellableCallback,
) {
if (!rLottieApi) {
await rLottieApiPromise;
}
const json = JSON.stringify(animationData);
const { reduceFactor, handle } = renderers.get(key)!;
const json = await extractJson(animationData);
const stringOnWasmHeap = allocate(intArrayFromString(json), 'i8', 0);
const { handle } = renderers.get(key)!;
const framesCount = rLottieApi.loadFromData(handle, stringOnWasmHeap);
onInit(Math.ceil(framesCount / reduceFactor));
const { reduceFactor, msPerFrame, reducedFramesCount } = calcParams(json, isLowPriority, framesCount);
onInit(reduceFactor, msPerFrame, reducedFramesCount);
}
async function extractJson(animationData: string) {
const response = await fetch(animationData);
const contentType = response.headers.get('Content-Type');
// Support deprecated JSON format cached locally
if (contentType?.startsWith('text/')) {
return response.text();
}
const arrayBuffer = await response.arrayBuffer();
return inflate(arrayBuffer, { to: 'string' });
}
function calcParams(json: string, isLowPriority: boolean, framesCount: number) {
const animationData = JSON.parse(json);
const maxFps = isLowPriority ? LOW_PRIORITY_MAX_FPS : HIGH_PRIORITY_MAX_FPS;
const sourceFps = animationData.fr || maxFps;
const reduceFactor = sourceFps % maxFps === 0 ? sourceFps / maxFps : 1;
return {
reduceFactor,
msPerFrame: 1000 / (sourceFps / reduceFactor),
reducedFramesCount: Math.ceil(framesCount / reduceFactor),
};
}
async function renderFrames(

View File

@ -243,10 +243,6 @@ export function getAudioHasCover(media: ApiAudio) {
return media.thumbnailSizes && media.thumbnailSizes.length > 0;
}
export function getMessageMediaFormat(
message: ApiMessage, target: Target,
): Exclude<ApiMediaFormat, ApiMediaFormat.Lottie>;
export function getMessageMediaFormat(message: ApiMessage, target: Target, canBeLottie: true): ApiMediaFormat;
export function getMessageMediaFormat(
message: ApiMessage, target: Target,
): ApiMediaFormat {

View File

@ -35,12 +35,6 @@ export async function fetch(
}
const blob = await response.blob();
// Safari does not return correct Content-Type header for webp images.
if (key.startsWith('sticker')) {
return new Blob([blob], { type: 'image/webp' });
}
const shouldRecreate = !blob.type || (!isHtmlAllowed && blob.type.includes('html'));
// iOS Safari fails to preserve `type` in cache
let resolvedType = blob.type || contentType;

View File

@ -1,6 +1,5 @@
import {
ApiMediaFormat,
ApiMediaFormatToPrepared,
ApiOnProgress,
ApiParsedMedia,
ApiPreparedMedia,
@ -18,7 +17,7 @@ import { webpToPng } from './webpToPng';
const asCacheApiType = {
[ApiMediaFormat.BlobUrl]: cacheApi.Type.Blob,
[ApiMediaFormat.Lottie]: cacheApi.Type.Json,
[ApiMediaFormat.Lottie]: cacheApi.Type.Blob,
[ApiMediaFormat.Progressive]: undefined,
[ApiMediaFormat.Stream]: undefined,
};
@ -36,13 +35,13 @@ export function fetch<T extends ApiMediaFormat>(
isHtmlAllowed = false,
onProgress?: ApiOnProgress,
callbackUniqueId?: string,
): Promise<ApiMediaFormatToPrepared<T>> {
): Promise<ApiPreparedMedia> {
if (mediaFormat === ApiMediaFormat.Progressive) {
return (
IS_PROGRESSIVE_SUPPORTED
? getProgressive(url)
: fetch(url, ApiMediaFormat.BlobUrl, isHtmlAllowed, onProgress, callbackUniqueId)
) as Promise<ApiMediaFormatToPrepared<T>>;
) as Promise<ApiPreparedMedia>;
}
if (!fetchPromises.has(url)) {
@ -73,11 +72,11 @@ export function fetch<T extends ApiMediaFormat>(
activeCallbacks.set(callbackUniqueId, onProgress);
}
return fetchPromises.get(url) as Promise<ApiMediaFormatToPrepared<T>>;
return fetchPromises.get(url) as Promise<ApiPreparedMedia>;
}
export function getFromMemory<T extends ApiMediaFormat>(url: string) {
return memoryCache.get(url) as ApiMediaFormatToPrepared<T>;
export function getFromMemory(url: string) {
return memoryCache.get(url) as ApiPreparedMedia;
}
export function cancelProgress(progressCallback: ApiOnProgress) {
@ -214,7 +213,7 @@ function makeOnProgress(url: string, mediaSource?: MediaSource, sourceBuffer?: S
return onProgress;
}
function prepareMedia(mediaData: ApiParsedMedia): ApiPreparedMedia {
function prepareMedia(mediaData: Exclude<ApiParsedMedia, ArrayBuffer>): ApiPreparedMedia {
if (mediaData instanceof Blob) {
return URL.createObjectURL(mediaData);
}

View File

@ -308,10 +308,10 @@ function getNotificationContent(chat: ApiChat, message: ApiMessage) {
async function getAvatar(chat: ApiChat) {
const imageHash = getChatAvatarHash(chat);
if (!imageHash) return undefined;
let mediaData = mediaLoader.getFromMemory<ApiMediaFormat.BlobUrl>(imageHash);
let mediaData = mediaLoader.getFromMemory(imageHash);
if (!mediaData) {
await mediaLoader.fetch(imageHash, ApiMediaFormat.BlobUrl);
mediaData = mediaLoader.getFromMemory<ApiMediaFormat.BlobUrl>(imageHash);
mediaData = mediaLoader.getFromMemory(imageHash);
}
return mediaData;
}