From f2e8768123666579fbfac255098fabea8f0515aa Mon Sep 17 00:00:00 2001 From: Alexander Zinchuk Date: Fri, 6 Sep 2024 15:42:57 +0200 Subject: [PATCH] [Perf] Introduce Offscreen Canvas for blurred thumbs --- src/components/common/File.tsx | 2 +- src/components/common/GifButton.tsx | 2 +- src/components/common/MediaSpoiler.tsx | 2 +- src/components/common/ProfilePhoto.tsx | 2 +- src/components/middle/message/BaseStory.tsx | 2 +- src/components/middle/message/Photo.tsx | 4 +-- src/components/middle/message/RoundVideo.tsx | 2 +- src/components/middle/message/Video.tsx | 4 +-- .../message/hooks/useBlurredMediaThumbRef.ts | 14 ++------ src/hooks/useCanvasBlur.ts | 2 -- src/hooks/useOffscreenCanvasBlur.ts | 36 +++++++++++++++++++ src/lib/mediaWorker/index.worker.ts | 5 ++- .../offscreen-canvas.worker.ts | 33 +++++++++++++++++ src/styles/index.scss | 4 --- src/util/PostMessageConnector.ts | 23 ++++++++---- src/util/files.ts | 14 -------- 16 files changed, 102 insertions(+), 49 deletions(-) create mode 100644 src/hooks/useOffscreenCanvasBlur.ts create mode 100644 src/lib/offscreen-canvas/offscreen-canvas.worker.ts diff --git a/src/components/common/File.tsx b/src/components/common/File.tsx index 258734873..3a0fe4993 100644 --- a/src/components/common/File.tsx +++ b/src/components/common/File.tsx @@ -120,7 +120,7 @@ const File: FC = ({ {withThumb && ( )} diff --git a/src/components/common/GifButton.tsx b/src/components/common/GifButton.tsx index eae31a074..8c501f0bb 100644 --- a/src/components/common/GifButton.tsx +++ b/src/components/common/GifButton.tsx @@ -157,7 +157,7 @@ const GifButton: FC = ({ {withThumb && ( )} {previewBlobUrl && !isVideoReady && ( diff --git a/src/components/common/MediaSpoiler.tsx b/src/components/common/MediaSpoiler.tsx index 9fc3fa754..7f10af85d 100644 --- a/src/components/common/MediaSpoiler.tsx +++ b/src/components/common/MediaSpoiler.tsx @@ -63,7 +63,7 @@ const MediaSpoiler: FC = ({ > diff --git a/src/components/common/ProfilePhoto.tsx b/src/components/common/ProfilePhoto.tsx index cad040647..f3024faa4 100644 --- a/src/components/common/ProfilePhoto.tsx +++ b/src/components/common/ProfilePhoto.tsx @@ -123,7 +123,7 @@ const ProfilePhoto: FC = ({ content = ( <> {isBlurredThumb ? ( - + ) : ( )} diff --git a/src/components/middle/message/BaseStory.tsx b/src/components/middle/message/BaseStory.tsx index e67c20685..b5b95ac1b 100644 --- a/src/components/middle/message/BaseStory.tsx +++ b/src/components/middle/message/BaseStory.tsx @@ -84,7 +84,7 @@ function BaseStory({ onClick={isConnected ? handleClick : undefined} > {!isExpired && isPreview && ( - + )} {shouldRender && ( <> diff --git a/src/components/middle/message/Photo.tsx b/src/components/middle/message/Photo.tsx index 18e03beb9..56ee3718d 100644 --- a/src/components/middle/message/Photo.tsx +++ b/src/components/middle/message/Photo.tsx @@ -216,7 +216,7 @@ const Photo = ({ onClick={isUploading ? undefined : handleClick} > {withBlurredBackground && ( - + )} ({ {withThumb && ( )} {isProtected && } diff --git a/src/components/middle/message/RoundVideo.tsx b/src/components/middle/message/RoundVideo.tsx index b9b4edbb3..4dd83b3bb 100644 --- a/src/components/middle/message/RoundVideo.tsx +++ b/src/components/middle/message/RoundVideo.tsx @@ -260,7 +260,7 @@ const RoundVideo: FC = ({ {!shouldRenderSpoiler && ( )} diff --git a/src/components/middle/message/Video.tsx b/src/components/middle/message/Video.tsx index 9b55d0034..e0e827e84 100644 --- a/src/components/middle/message/Video.tsx +++ b/src/components/middle/message/Video.tsx @@ -235,7 +235,7 @@ const Video = ({ onClick={isUploading ? undefined : (e) => handleClick(e)} > {withBlurredBackground && ( - + )} {isInline && ( ({ {hasThumb && !isPreviewPreloaded && ( )} {isProtected && } diff --git a/src/components/middle/message/hooks/useBlurredMediaThumbRef.ts b/src/components/middle/message/hooks/useBlurredMediaThumbRef.ts index fc9bb4536..000b07685 100644 --- a/src/components/middle/message/hooks/useBlurredMediaThumbRef.ts +++ b/src/components/middle/message/hooks/useBlurredMediaThumbRef.ts @@ -1,10 +1,8 @@ import { getMediaThumbUri, type MediaWithThumbs } from '../../../../global/helpers'; -import { IS_CANVAS_FILTER_SUPPORTED } from '../../../../util/windowEnvironment'; -import useAppLayout from '../../../../hooks/useAppLayout'; -import useCanvasBlur from '../../../../hooks/useCanvasBlur'; +import useOffscreenCanvasBlur from '../../../../hooks/useOffscreenCanvasBlur'; -type CanvasBlurReturnType = ReturnType; +type CanvasBlurReturnType = ReturnType; export default function useBlurredMediaThumbRef( forcedUri: string | undefined, isDisabled: boolean, @@ -14,13 +12,7 @@ export default function useBlurredMediaThumbRef( media: MediaWithThumbs | string | undefined, isDisabled?: boolean, ) { - const { isMobile } = useAppLayout(); - const dataUri = media ? typeof media === 'string' ? media : getMediaThumbUri(media) : undefined; - return useCanvasBlur( - dataUri, - isDisabled, - isMobile && !IS_CANVAS_FILTER_SUPPORTED, - ); + return useOffscreenCanvasBlur(dataUri, isDisabled); } diff --git a/src/hooks/useCanvasBlur.ts b/src/hooks/useCanvasBlur.ts index 7ed7a32a0..73183e208 100644 --- a/src/hooks/useCanvasBlur.ts +++ b/src/hooks/useCanvasBlur.ts @@ -52,8 +52,6 @@ export default function useCanvasBlur( ctx.drawImage(img, -radius * 2, -radius * 2, width + radius * 4, height + radius * 4); - canvas.classList.remove('canvas-blur-setup'); - if (!IS_CANVAS_FILTER_SUPPORTED) { fastBlur(ctx, 0, 0, width, height, radius, ITERATIONS); } diff --git a/src/hooks/useOffscreenCanvasBlur.ts b/src/hooks/useOffscreenCanvasBlur.ts new file mode 100644 index 000000000..a023bd2be --- /dev/null +++ b/src/hooks/useOffscreenCanvasBlur.ts @@ -0,0 +1,36 @@ +import { useLayoutEffect, useMemo, useRef } from '../lib/teact/teact'; + +import cycleRestrict from '../util/cycleRestrict'; +import launchMediaWorkers, { MAX_WORKERS } from '../util/launchMediaWorkers'; + +const RADIUS = 7; + +let lastWorkerIndex = -1; + +export default function useOffscreenCanvasBlur( + dataUri?: string, + isDisabled = false, + radius = RADIUS, +) { + // eslint-disable-next-line no-null/no-null + const canvasRef = useRef(null); + const workerIndex = useMemo(() => cycleRestrict(MAX_WORKERS, ++lastWorkerIndex), []); + + useLayoutEffect(() => { + if (!dataUri || isDisabled) return; + + const canvas = canvasRef.current; + if (!canvas) return; + + const offscreen = canvas.transferControlToOffscreen(); + + const { connector } = launchMediaWorkers()[workerIndex]; + connector.request({ + name: 'blurThumb', + args: [offscreen, dataUri, radius], + transferables: [offscreen], + }); + }, [dataUri, isDisabled, radius, workerIndex]); + + return canvasRef; +} diff --git a/src/lib/mediaWorker/index.worker.ts b/src/lib/mediaWorker/index.worker.ts index e16316ee8..38ef94d4f 100644 --- a/src/lib/mediaWorker/index.worker.ts +++ b/src/lib/mediaWorker/index.worker.ts @@ -1,9 +1,12 @@ import '../rlottie/rlottie.worker'; import '../video-preview/video-preview.worker'; +import '../offscreen-canvas/offscreen-canvas.worker'; +import type { OffscreenCanvasApi } from '../offscreen-canvas/offscreen-canvas.worker'; import type { RLottieApi } from '../rlottie/rlottie.worker'; import type { VideoPreviewApi } from '../video-preview/video-preview.worker'; export type MediaWorkerApi = RLottieApi - & VideoPreviewApi; + & VideoPreviewApi + & OffscreenCanvasApi; diff --git a/src/lib/offscreen-canvas/offscreen-canvas.worker.ts b/src/lib/offscreen-canvas/offscreen-canvas.worker.ts new file mode 100644 index 000000000..a0e5fab94 --- /dev/null +++ b/src/lib/offscreen-canvas/offscreen-canvas.worker.ts @@ -0,0 +1,33 @@ +import { createWorkerInterface } from '../../util/createPostMessageInterface'; + +export async function blurThumb(canvas: OffscreenCanvas, dataUri: string, radius: number) { + const imageBitmap = await dataUriToImageBitmap(dataUri); + const { width, height } = canvas; + const ctx = canvas.getContext('2d')!; + + // Draw image twice to battle white-ish edges + ctx.drawImage(imageBitmap, 0, 0, width, height); + ctx.filter = `blur(${radius}px)`; + ctx.drawImage(imageBitmap, 0, 0, width, height); +} + +function dataUriToImageBitmap(dataUri: string) { + const byteString = atob(dataUri.split(',')[1]); + const mimeString = dataUri.split(',')[0].split(':')[1].split(';')[0]; + const buffer = new ArrayBuffer(byteString.length); + const dataArray = new Uint8Array(buffer); + + for (let i = 0; i < byteString.length; i++) { + dataArray[i] = byteString.charCodeAt(i); + } + + const blob = new Blob([buffer], { type: mimeString }); + + return createImageBitmap(blob); +} + +const api = { blurThumb }; + +createWorkerInterface(api, 'media'); + +export type OffscreenCanvasApi = typeof api; diff --git a/src/styles/index.scss b/src/styles/index.scss index 62bc533d5..86026491f 100644 --- a/src/styles/index.scss +++ b/src/styles/index.scss @@ -201,10 +201,6 @@ body:not(.is-ios) { } } -.canvas-blur-setup { - will-change: width, height; -} - .emoji-small { background: no-repeat; background-size: var(--emoji-size); diff --git a/src/util/PostMessageConnector.ts b/src/util/PostMessageConnector.ts index 51878ef87..58861f823 100644 --- a/src/util/PostMessageConnector.ts +++ b/src/util/PostMessageConnector.ts @@ -97,6 +97,7 @@ export type RequestTypes = Values<{ [Name in keyof (T)]: { name: Name & string; args: Parameters; + transferables?: Transferable[]; } }>; @@ -107,6 +108,8 @@ class ConnectorClass { private pendingPayloads: OriginPayload[] = []; + private pendingTransferables: Transferable[] = []; + constructor( public target: Worker, private onUpdate?: (update: ApiUpdate) => void, @@ -127,12 +130,13 @@ class ConnectorClass { request(messageData: RequestTypes) { const { requestStates, requestStatesByCallback } = this; + const { transferables, ...restMessageData } = messageData; const messageId = generateUniqueId(); const payload: CallMethodPayload = { type: 'callMethod', messageId, - ...messageData, + ...restMessageData, }; const requestState = { messageId } as RequestState; @@ -161,7 +165,7 @@ class ConnectorClass { } }); - this.postMessageOnTickEnd(payload); + this.postMessageOnTickEnd(payload, transferables); return promise; } @@ -208,20 +212,25 @@ class ConnectorClass { }); } - private postMessageOnTickEnd(payload: OriginPayload) { + private postMessageOnTickEnd(payload: OriginPayload, transferables?: Transferable[]) { this.pendingPayloads.push(payload); + + if (transferables) { + this.pendingTransferables.push(...transferables); + } + this.postMessagesOnTickEnd(); } private postMessagesOnTickEnd = throttleWithTickEnd(() => { + const { channel } = this; const payloads = this.pendingPayloads; + const transferables = this.pendingTransferables; this.pendingPayloads = []; + this.pendingTransferables = []; - this.target.postMessage({ - channel: this.channel, - payloads, - }); + this.target.postMessage({ channel, payloads }, transferables); }); } diff --git a/src/util/files.ts b/src/util/files.ts index 235f9bcba..8a80abb67 100644 --- a/src/util/files.ts +++ b/src/util/files.ts @@ -21,20 +21,6 @@ if (typeof File === 'undefined') { } as typeof File; } -export function dataUriToBlob(dataUri: string) { - const arr = dataUri.split(','); - const mime = arr[0].match(/:(.*?);/)![1]; - const bstr = atob(arr[1]); - let n = bstr.length; - const u8arr = new Uint8Array(n); - - while (n--) { - u8arr[n] = bstr.charCodeAt(n); - } - - return new Blob([u8arr], { type: mime }); -} - export function blobToDataUri(blob: Blob): Promise { return new Promise((resolve, reject) => { const reader = new FileReader();