Downloads: Fallback to Service Worker when OPFS not supported (#1949)

This commit is contained in:
Alexander Zinchuk 2022-08-05 19:23:12 +02:00
parent e57bf3518d
commit 2a5eeb3111
24 changed files with 227 additions and 46 deletions

View File

@ -50,7 +50,7 @@ export async function init(_onUpdate: OnApiUpdate, initialArgs: ApiInitialArgs)
onUpdate = _onUpdate;
const {
userAgent, platform, sessionData, isTest, isMovSupported, isWebmSupported,
userAgent, platform, sessionData, isTest, isMovSupported, isWebmSupported, maxBufferSize,
} = initialArgs;
const session = new sessions.CallbackSession(sessionData, onSessionUpdate);
@ -60,6 +60,8 @@ export async function init(_onUpdate: OnApiUpdate, initialArgs: ApiInitialArgs)
if (isMovSupported) SUPPORTED_VIDEO_CONTENT_TYPES.add(VIDEO_MOV_TYPE);
// eslint-disable-next-line no-restricted-globals
(self as any).isWebmSupported = isWebmSupported;
// eslint-disable-next-line no-restricted-globals
(self as any).maxBufferSize = maxBufferSize;
client = new TelegramClient(
session,

View File

@ -253,6 +253,7 @@ async function parseMedia(
case ApiMediaFormat.Text:
return data.toString();
case ApiMediaFormat.Progressive:
case ApiMediaFormat.DownloadUrl:
return data.buffer;
}

View File

@ -5,6 +5,7 @@ export enum ApiMediaFormat {
BlobUrl,
Progressive,
Stream,
DownloadUrl,
Text,
}

View File

@ -9,6 +9,7 @@ export interface ApiInitialArgs {
isTest?: boolean;
isMovSupported?: boolean;
isWebmSupported?: boolean;
maxBufferSize?: number;
}
export interface ApiOnProgress {

View File

@ -125,6 +125,7 @@ const Audio: FC<OwnProps> = ({
const { loadProgress: downloadProgress } = useMediaWithLoadProgress(
getMessageMediaHash(message, 'download'),
!isDownloading,
getMessageMediaFormat(message, 'download'),
);
const handleForcePlay = useCallback(() => {

View File

@ -9,6 +9,7 @@ import type { ApiMessage } from '../../api/types';
import { getDocumentExtension, getDocumentHasPreview } from './helpers/documentInfo';
import {
getMediaTransferState,
getMessageMediaFormat,
getMessageMediaHash,
getMessageMediaThumbDataUri,
isMessageDocumentVideo,
@ -86,7 +87,7 @@ const Document: FC<OwnProps> = ({
const documentHash = getMessageMediaHash(message, 'download');
const { loadProgress: downloadProgress, mediaData } = useMediaWithLoadProgress(
documentHash, !shouldDownload, undefined, undefined, undefined, true,
documentHash, !shouldDownload, getMessageMediaFormat(message, 'download'), undefined, undefined, true,
);
const isLoaded = Boolean(mediaData);

View File

@ -6,11 +6,11 @@ import type { Thread } from '../../global/types';
import type { ApiMessage } from '../../api/types';
import { ApiMediaFormat } from '../../api/types';
import { IS_OPFS_SUPPORTED } from '../../util/environment';
import { IS_OPFS_SUPPORTED, IS_SERVICE_WORKER_SUPPORTED, MAX_BUFFER_SIZE } from '../../util/environment';
import * as mediaLoader from '../../util/mediaLoader';
import download from '../../util/download';
import {
getMessageContentFilename, getMessageMediaHash,
getMessageContentFilename, getMessageMediaFormat, getMessageMediaHash,
} from '../../global/helpers';
import useRunDebounced from '../../hooks/useRunDebounced';
@ -25,8 +25,6 @@ type StateProps = {
const GLOBAL_UPDATE_DEBOUNCE = 1000;
const MAX_BLOB_SAFE_SIZE = 2000 * 1024 * 1024;
const processedMessages = new Set<ApiMessage>();
const downloadedMessages = new Set<ApiMessage>();
@ -81,18 +79,30 @@ const DownloadManager: FC<StateProps> = ({
document, video, audio,
} = message.content;
const mediaSize = (document || video || audio)?.size || 0;
if (mediaSize > MAX_BLOB_SAFE_SIZE && !IS_OPFS_SUPPORTED) {
if (mediaSize > MAX_BUFFER_SIZE && !IS_OPFS_SUPPORTED && !IS_SERVICE_WORKER_SUPPORTED) {
showNotification({
message: 'Downloading files bigger than 2GB is currently not supported in your browser.',
message: 'Downloading files bigger than 2GB is not supported in your browser.',
});
handleMessageDownloaded(message);
return;
}
mediaLoader.fetch(downloadHash, ApiMediaFormat.BlobUrl, true).then((result) => {
if (result) {
const mediaFormat = getMessageMediaFormat(message, 'download');
mediaLoader.fetch(downloadHash, mediaFormat, true).then((result) => {
if (mediaFormat === ApiMediaFormat.DownloadUrl) {
const url = new URL(result, window.document.baseURI);
const filename = getMessageContentFilename(message);
url.searchParams.set('filename', encodeURIComponent(filename));
const downloadWindow = window.open(url.toString());
downloadWindow?.addEventListener('beforeunload', () => {
showNotification({
message: 'Download started. Please, do not close the app before it is finished.',
});
});
} else if (result) {
download(result, getMessageContentFilename(message));
}
handleMessageDownloaded(message);
});
});

View File

@ -10,7 +10,7 @@ import { formatFileSize } from '../../../../util/textFormat';
import { getActions, withGlobal } from '../../../../global';
import { selectIsCurrentUserPremium, selectIsPremiumPurchaseBlocked } from '../../../../global/selectors';
import useLang from '../../../../hooks/useLang';
import { FILEPART_SIZE } from '../../../../config';
import { MAX_UPLOAD_FILEPART_SIZE } from '../../../../config';
import useFlag from '../../../../hooks/useFlag';
import Modal from '../../../ui/Modal';
@ -60,7 +60,7 @@ const LIMIT_VALUE_FORMATTER: Partial<Record<ApiLimitTypeWithModal, (...args: any
// The real size is not exactly 4gb, so we need to round it
if (value === 8000) return lang('FileSize.GB', '4');
if (value === 4000) return lang('FileSize.GB', '2');
return formatFileSize(lang, value * FILEPART_SIZE);
return formatFileSize(lang, value * MAX_UPLOAD_FILEPART_SIZE);
},
};

View File

@ -9,7 +9,7 @@ import { getActions, withGlobal } from '../../global';
import type { ApiMessage } from '../../api/types';
import { IS_SINGLE_COLUMN_LAYOUT } from '../../util/environment';
import { getMessageMediaHash } from '../../global/helpers';
import { getMessageMediaFormat, getMessageMediaHash } from '../../global/helpers';
import useLang from '../../hooks/useLang';
import useMediaWithLoadProgress from '../../hooks/useMediaWithLoadProgress';
import { selectIsDownloading, selectIsMessageProtected } from '../../global/selectors';
@ -63,6 +63,7 @@ const MediaViewerActions: FC<OwnProps & StateProps> = ({
const { loadProgress: downloadProgress } = useMediaWithLoadProgress(
message && getMessageMediaHash(message, 'download'),
!isDownloading,
message && getMessageMediaFormat(message, 'download'),
);
const handleDownloadClick = useCallback(() => {

View File

@ -31,7 +31,7 @@ import {
EDITABLE_INPUT_ID,
REPLIES_USER_ID,
SEND_MESSAGE_ACTION_INTERVAL,
EDITABLE_INPUT_CSS_SELECTOR, FILEPART_SIZE,
EDITABLE_INPUT_CSS_SELECTOR, MAX_UPLOAD_FILEPART_SIZE,
} from '../../../config';
import { IS_VOICE_RECORDING_SUPPORTED, IS_SINGLE_COLUMN_LAYOUT, IS_IOS } from '../../../util/environment';
import { MEMO_EMPTY_ARRAY } from '../../../util/memo';
@ -1355,7 +1355,7 @@ export default memo(withGlobal<OwnProps>(
attachMenuBots: global.attachMenu.bots,
attachMenuPeerType: selectAttachMenuPeerType(global, chatId),
theme: selectTheme(global),
fileSizeLimit: selectCurrentLimit(global, 'uploadMaxFileparts') * FILEPART_SIZE,
fileSizeLimit: selectCurrentLimit(global, 'uploadMaxFileparts') * MAX_UPLOAD_FILEPART_SIZE,
captionLimit: selectCurrentLimit(global, 'captionLength'),
isCurrentUserPremium: selectIsCurrentUserPremium(global),
};

View File

@ -14,6 +14,7 @@ import {
getMessageMediaHash,
getMediaTransferState,
isOwnMessage,
getMessageMediaFormat,
} from '../../../global/helpers';
import type { ObserveFn } from '../../../hooks/useIntersectionObserver';
import { useIsIntersecting } from '../../../hooks/useIntersectionObserver';
@ -87,7 +88,9 @@ const Photo: FC<OwnProps> = ({
const {
loadProgress: downloadProgress,
} = useMediaWithLoadProgress(getMessageMediaHash(message, 'download'), !isDownloading);
} = useMediaWithLoadProgress(
getMessageMediaHash(message, 'download'), !isDownloading, getMessageMediaFormat(message, 'download'),
);
const {
isUploading, isTransferring, transferProgress,

View File

@ -3,7 +3,6 @@ import React, { useCallback, useRef, useState } from '../../../lib/teact/teact';
import { getActions } from '../../../global';
import type { ApiMessage } from '../../../api/types';
import { ApiMediaFormat } from '../../../api/types';
import type { IMediaDimensions } from './helpers/calculateAlbumLayout';
import { formatMediaDuration } from '../../../util/dateFormat';
@ -101,7 +100,7 @@ const Video: FC<OwnProps> = ({
const { loadProgress: downloadProgress } = useMediaWithLoadProgress(
getMessageMediaHash(message, 'download'),
!isDownloading,
ApiMediaFormat.BlobUrl,
getMessageMediaFormat(message, 'download'),
lastSyncTime,
);

View File

@ -214,7 +214,7 @@ export const LIGHT_THEME_BG_COLOR = '#99BA92';
export const DARK_THEME_BG_COLOR = '#0F0F0F';
export const DEFAULT_PATTERN_COLOR = '#4A8E3A8C';
export const DARK_THEME_PATTERN_COLOR = '#0A0A0A8C';
export const FILEPART_SIZE = 524288;
export const MAX_UPLOAD_FILEPART_SIZE = 524288;
// Group calls
export const GROUP_CALL_VOLUME_MULTIPLIER = 100;

View File

@ -13,7 +13,9 @@ import {
IS_TEST,
LOCK_SCREEN_ANIMATION_DURATION_MS,
} from '../../../config';
import { IS_MOV_SUPPORTED, IS_WEBM_SUPPORTED, PLATFORM_ENV } from '../../../util/environment';
import {
IS_MOV_SUPPORTED, IS_WEBM_SUPPORTED, MAX_BUFFER_SIZE, PLATFORM_ENV,
} from '../../../util/environment';
import { unsubscribe } from '../../../util/notifications';
import * as cacheApi from '../../../util/cacheApi';
import { updateAppBadge } from '../../../util/appBadge';
@ -42,6 +44,7 @@ addActionHandler('initApi', async (global, actions) => {
isTest: window.location.search.includes('test'),
isMovSupported: IS_MOV_SUPPORTED,
isWebmSupported: IS_WEBM_SUPPORTED,
maxBufferSize: MAX_BUFFER_SIZE,
});
});

View File

@ -12,7 +12,13 @@ import type {
} from '../../api/types';
import { ApiMediaFormat } from '../../api/types';
import { IS_OPUS_SUPPORTED, IS_PROGRESSIVE_SUPPORTED, IS_SAFARI } from '../../util/environment';
import {
IS_OPFS_SUPPORTED,
IS_OPUS_SUPPORTED,
IS_PROGRESSIVE_SUPPORTED,
IS_SAFARI,
MAX_BUFFER_SIZE,
} from '../../util/environment';
import { getMessageKey, isMessageLocal, matchLinkInMessageText } from './messages';
import { getDocumentHasPreview } from '../../components/common/helpers/documentInfo';
@ -306,14 +312,22 @@ export function getAudioHasCover(media: ApiAudio) {
export function getMessageMediaFormat(
message: ApiMessage, target: Target,
): ApiMediaFormat {
const { video, audio, voice } = message.content;
const {
video, audio, voice, document,
} = message.content;
const fullVideo = video || getMessageWebPageVideo(message);
const size = (video || audio || document)?.size!;
if (target === 'download' && IS_PROGRESSIVE_SUPPORTED && size > MAX_BUFFER_SIZE && !IS_OPFS_SUPPORTED) {
return ApiMediaFormat.DownloadUrl;
}
if (fullVideo && IS_PROGRESSIVE_SUPPORTED && (
target === 'viewerFull' || target === 'inline'
)) {
return ApiMediaFormat.Progressive;
} else if (audio || voice) {
}
if (audio || voice) {
// Safari
if (voice && !IS_OPUS_SUPPORTED) {
return ApiMediaFormat.BlobUrl;

View File

@ -320,12 +320,35 @@ function getInputLocation(location) {
*/
/**
* Gets the appropriated part size when uploading or downloading files,
* Gets the appropriated part size when downloading files,
* given an initial file size.
* @param fileSize
* @returns {Number}
*/
function getAppropriatedPartSize(fileSize) {
function getDownloadPartSize(fileSize) {
if (fileSize <= 104857600) { // 100MB
return 128;
}
if (fileSize <= 786432000) { // 750MB
return 256;
}
if (fileSize <= 2097152000) { // 2000MB
return 512;
}
if (fileSize <= 4194304000) { // 4000MB
return 1024;
}
throw new Error('File size too large');
}
/**
* Gets the appropriated part size when uploading files,
* given an initial file size.
* @param fileSize
* @returns {Number}
*/
function getUploadPartSize(fileSize) {
if (fileSize <= 104857600) { // 100MB
return 128;
}
@ -683,7 +706,8 @@ module.exports = {
getDisplayName,
// resolveId,
// isListLike,
getAppropriatedPartSize,
getDownloadPartSize,
getUploadPartSize,
// getInputLocation,
strippedPhotoToJpg,
getDC,

View File

@ -1,9 +1,9 @@
import { Api } from '..';
import type { Api } from '..';
import { BotAuthParams, UserAuthParams } from './auth';
import { uploadFile, UploadFileParams } from './uploadFile';
import { downloadFile, DownloadFileParams } from './downloadFile';
import { TwoFaParams, updateTwoFaSettings } from './2fa';
import type { BotAuthParams, UserAuthParams } from './auth';
import type { uploadFile, UploadFileParams } from './uploadFile';
import type { downloadFile, DownloadFileParams } from './downloadFile';
import type { TwoFaParams, updateTwoFaSettings } from './2fa';
declare class TelegramClient {
constructor(...args: any);

View File

@ -1,8 +1,8 @@
import BigInt from 'big-integer';
import Api from '../tl/api';
import type TelegramClient from './TelegramClient';
import { getAppropriatedPartSize } from '../Utils';
import { sleep, createDeferred } from '../Helpers';
import { getDownloadPartSize } from '../Utils';
import errors from '../errors';
interface OnProgress {
@ -35,7 +35,6 @@ const MIN_CHUNK_SIZE = 4096;
const DEFAULT_CHUNK_SIZE = 64; // kb
const ONE_MB = 1024 * 1024;
const DISCONNECT_SLEEP = 1000;
const MAX_BUFFER_SAFE_SIZE = 2000 * 1024 * 1024;
// when the sender requests hangs for 60 second we will reimport
const SENDER_TIMEOUT = 60 * 1000;
@ -83,7 +82,8 @@ class FileView {
constructor(size?: number) {
this.size = size;
this.type = (size && size > MAX_BUFFER_SAFE_SIZE) ? 'opfs' : 'memory';
// eslint-disable-next-line no-restricted-globals
this.type = (size && size > (self as any).maxBufferSize) ? 'opfs' : 'memory';
}
async init() {
@ -158,7 +158,7 @@ async function downloadFile2(
end = end && end < fileSize ? end : fileSize - 1;
if (!partSizeKb) {
partSizeKb = fileSize ? getAppropriatedPartSize(fileSize) : DEFAULT_CHUNK_SIZE;
partSizeKb = fileSize ? getDownloadPartSize(fileSize) : DEFAULT_CHUNK_SIZE;
}
const partSize = partSizeKb * 1024;

View File

@ -3,7 +3,7 @@ import { default as Api } from '../tl/api';
import type TelegramClient from './TelegramClient';
import { generateRandomBytes, readBigIntFromBuffer, sleep } from '../Helpers';
import { getAppropriatedPartSize } from '../Utils';
import { getUploadPartSize } from '../Utils';
import errors from '../errors';
interface OnProgress {
@ -34,7 +34,7 @@ export async function uploadFile(
const fileId = readBigIntFromBuffer(generateRandomBytes(8), true, true);
const isLarge = size > LARGE_FILE_THRESHOLD;
const partSize = getAppropriatedPartSize(size) * KB_TO_BYTES;
const partSize = getUploadPartSize(size) * KB_TO_BYTES;
const partCount = Math.floor((size + partSize - 1) / partSize);
// Make sure a new sender can be created before starting upload

View File

@ -1,5 +1,6 @@
import { DEBUG } from './config';
import { respondForProgressive } from './serviceWorker/progressive';
import { respondForDownload } from './serviceWorker/download';
import { respondWithCache, clearAssetCache } from './serviceWorker/assetCache';
import { handlePush, handleNotificationClick, handleClientMessage } from './serviceWorker/pushNotification';
import { pause } from './util/schedulers';
@ -38,7 +39,6 @@ self.addEventListener('activate', (e) => {
);
});
// eslint-disable-next-line no-restricted-globals
self.addEventListener('fetch', (e: FetchEvent) => {
const { url } = e.request;
@ -47,6 +47,11 @@ self.addEventListener('fetch', (e: FetchEvent) => {
return true;
}
if (url.includes('/download/')) {
e.respondWith(respondForDownload(e));
return true;
}
if (url.startsWith('http') && url.match(ASSET_CACHE_PATTERN)) {
e.respondWith(respondWithCache(e));
return true;

View File

@ -0,0 +1,99 @@
import { DEBUG } from '../config';
import { requestPart } from './progressive';
const DOWNLOAD_PART_SIZE = 1024 * 1024;
const TEST_PART_SIZE = 64 * 1024;
const QUEUE_SIZE = 5;
class FilePartQueue<T> {
queue: Promise<T>[];
constructor() {
this.queue = [];
}
push(task: Promise<T>) {
this.queue.push(task);
}
async pop(): Promise<T> {
const result = await this.queue.shift()!;
return result;
}
get size() {
return this.queue.length;
}
}
export async function respondForDownload(e: FetchEvent) {
const { url } = e.request;
let partInfo;
try {
partInfo = await requestPart(e, { url, start: 0, end: TEST_PART_SIZE });
} catch (err) {
if (DEBUG) {
// eslint-disable-next-line no-console
console.error('FETCH DOWNLOAD', err);
}
}
if (!partInfo) {
return new Response('', {
status: 500,
statusText: 'Failed to fetch file to download',
});
}
const matchedFilename = e.request.url.match(/filename=(.*)/);
const filenameHeader = matchedFilename ? `filename="${decodeURIComponent(matchedFilename[1])}"` : '';
const { fullSize, mimeType } = partInfo;
const headers = [
['Content-Length', String(fullSize)],
['Content-Type', mimeType],
['Content-Disposition', `attachment; ${filenameHeader}`],
];
const queue = new FilePartQueue<ArrayBuffer | undefined>();
const enqueue = (offset: number) => {
queue.push(requestPart(e, { url, start: offset, end: offset + DOWNLOAD_PART_SIZE - 1 })
.then((part) => part?.arrayBuffer));
return offset + DOWNLOAD_PART_SIZE;
};
let lastOffset = 0;
const stream = new ReadableStream({
start() {
for (let i = 0; i < QUEUE_SIZE; i++) {
if (lastOffset >= fullSize) break;
lastOffset = enqueue(lastOffset);
}
},
async pull(controller) {
const buffer = await queue.pop();
if (!buffer) {
controller.close();
return;
}
controller.enqueue(new Uint8Array(buffer));
if (buffer.byteLength < DOWNLOAD_PART_SIZE) {
controller.close();
return;
}
if (lastOffset < fullSize) {
lastOffset = enqueue(lastOffset);
}
},
});
return new Response(stream, {
status: 200,
statusText: 'OK',
headers,
});
}

View File

@ -137,16 +137,14 @@ async function saveToCache(cacheKey: string, arrayBuffer: ArrayBuffer, headers:
]);
}
async function requestPart(
export async function requestPart(
e: FetchEvent,
params: { url: string; start: number; end: number },
): Promise<PartInfo | undefined> {
if (!e.clientId) {
return undefined;
}
// eslint-disable-next-line no-restricted-globals
const client = await self.clients.get(e.clientId);
const isDownload = params.url.includes('/download/');
const client = isDownload ? (await self.clients.matchAll())
.find((c) => c.type === 'window' && c.frameType === 'top-level')
: await (self.clients.get(e.clientId));
if (!client) {
return undefined;
}

View File

@ -114,6 +114,8 @@ export const IS_INSTALL_PROMPT_SUPPORTED = 'onbeforeinstallprompt' in window;
// Smaller area reduces scroll jumps caused by `patchChromiumScroll`
export const MESSAGE_LIST_SENSITIVE_AREA = IS_SCROLL_PATCH_NEEDED ? 300 : 750;
export const MAX_BUFFER_SIZE = (IS_ANDROID || IS_IOS ? 512 : 2000) * 1024 ** 2; // 512 OR 2000 MB
function isLastEmojiVersionSupported() {
const ALLOWABLE_CALCULATION_ERROR_SIZE = 5;
const inlineEl = document.createElement('span');

View File

@ -13,18 +13,22 @@ import {
import { callApi, cancelApiProgress } from '../api/gramjs';
import * as cacheApi from './cacheApi';
import { fetchBlob } from './files';
import { IS_OPUS_SUPPORTED, IS_PROGRESSIVE_SUPPORTED, isWebpSupported } from './environment';
import {
IS_OPUS_SUPPORTED, IS_PROGRESSIVE_SUPPORTED, isWebpSupported,
} from './environment';
import { oggToWav } from './oggToWav';
import { webpToPng } from './webpToPng';
const asCacheApiType = {
[ApiMediaFormat.BlobUrl]: cacheApi.Type.Blob,
[ApiMediaFormat.Text]: cacheApi.Type.Text,
[ApiMediaFormat.DownloadUrl]: undefined,
[ApiMediaFormat.Progressive]: undefined,
[ApiMediaFormat.Stream]: undefined,
};
const PROGRESSIVE_URL_PREFIX = './progressive/';
const URL_DOWNLOAD_PREFIX = './download/';
const memoryCache = new Map<string, ApiPreparedMedia>();
const fetchPromises = new Map<string, Promise<ApiPreparedMedia | undefined>>();
@ -46,6 +50,14 @@ export function fetch<T extends ApiMediaFormat>(
) as Promise<ApiPreparedMedia>;
}
if (mediaFormat === ApiMediaFormat.DownloadUrl) {
return (
IS_PROGRESSIVE_SUPPORTED
? getDownloadUrl(url)
: fetch(url, ApiMediaFormat.BlobUrl, isHtmlAllowed, onProgress, callbackUniqueId)
) as Promise<ApiPreparedMedia>;
}
if (!fetchPromises.has(url)) {
const promise = fetchFromCacheOrRemote(url, mediaFormat, isHtmlAllowed)
.catch((err) => {
@ -110,6 +122,10 @@ function getProgressive(url: string) {
return Promise.resolve(progressiveUrl);
}
function getDownloadUrl(url: string) {
return Promise.resolve(`${URL_DOWNLOAD_PREFIX}${url}`);
}
async function fetchFromCacheOrRemote(
url: string, mediaFormat: ApiMediaFormat, isHtmlAllowed: boolean,
) {