Avatar: Speed up loading (#5288)

public/compatTest.js

@@ -9,9 +9,11 @@ function compatTest() {
   var hasDisplayNames = hasIntl && typeof Intl.DisplayNames !== 'undefined';
   var hasPluralRules = hasIntl && typeof Intl.PluralRules !== 'undefined';
   var hasNumberFormat = hasIntl && typeof Intl.NumberFormat !== 'undefined';
+  var hasWebLocks = typeof navigator.locks !== 'undefined';
+  var hasBigInt = typeof BigInt !== 'undefined';
 
   var isCompatible = hasPromise && hasWebSockets && hasWebCrypto && hasObjectFromEntries && hasResizeObserver
-    && hasCssSupports && hasDisplayNames && hasPluralRules && hasNumberFormat;
+    && hasCssSupports && hasDisplayNames && hasPluralRules && hasNumberFormat && hasWebLocks && hasBigInt;
 
   if (isCompatible || (window.localStorage && window.localStorage.getItem('tt-ignore-compat'))) {
     window.isCompatTestPassed = true;
@@ -29,6 +31,8 @@ function compatTest() {
     console.warn('Intl.DisplayNames', hasDisplayNames);
     console.warn('Intl.PluralRules', hasPluralRules);
     console.warn('Intl.NumberFormat', hasNumberFormat);
+    console.warn('WebLocks', hasWebLocks);
+    console.warn('BigInt', hasBigInt);
   }
 
   // Hardcoded page because server forbids iframe embedding

src/lib/gramjs/client/TelegramClient.js

@@ -480,8 +480,11 @@ class TelegramClient {
                 await this._waitingForAuthKey[dcId];
 
                 const authKey = this.session.getAuthKey(dcId);
-                await sender.authKey.setKey(authKey.getKey());
+
-                hasAuthKey = Boolean(sender.authKey.getKey());
+                hasAuthKey = Boolean(sender.authKey?.getKey());
+                if (hasAuthKey) {
+                    await sender.authKey.setKey(authKey.getKey());
+                }
             } else {
                 this._waitingForAuthKey[dcId] = new Promise((resolve) => {
                     firstConnectResolver = resolve;
@@ -512,15 +515,18 @@ class TelegramClient {
                 ));
 
                 if (this.session.dcId !== dcId && !sender._authenticated) {
-                    this._log.info(`Exporting authorization for data center ${dc.ipAddress}`);
+                    // Prevent another connection from trying to export the auth key while we're doing it
-                    const auth = await this.invoke(new requests.auth.ExportAuthorization({ dcId }));
+                    await navigator.locks.request('GRAMJS_AUTH_EXPORT', async () => {
+                        this._log.info(`Exporting authorization for data center ${dc.ipAddress}`);
+                        const auth = await this.invoke(new requests.auth.ExportAuthorization({ dcId }));
 
-                    const req = this._initWith(new requests.auth.ImportAuthorization({
+                        const req = this._initWith(new requests.auth.ImportAuthorization({
-                        id: auth.id,
+                            id: auth.id,
-                        bytes: auth.bytes,
+                            bytes: auth.bytes,
-                    }));
+                        }));
-                    await sender.send(req);
+                        await sender.send(req);
-                    sender._authenticated = true;
+                        sender._authenticated = true;
+                    });
                 }
 
                 sender.dcId = dcId;
@@ -669,6 +675,7 @@ class TelegramClient {
      * @param [args[end] {number}]
      * @param [args[dcId] {number}]
      * @param [args[workers] {number}]
+     * @param [args[isPriority] {boolean}]
      * @returns {Promise<Buffer>}
      */
     downloadFile(inputLocation, args = {}) {
@@ -721,6 +728,7 @@ class TelegramClient {
 
         return this.downloadFile(loc, {
             dcId,
+            isPriority: true,
         });
     }
 

src/lib/gramjs/client/downloadFile.ts

@@ -27,6 +27,7 @@ export interface DownloadFileParams {
     start?: number;
     end?: number;
     progressCallback?: OnProgress;
+    isPriority?: boolean;
 }
 
 // Chunk sizes for `upload.getFile` must be multiple of the smallest size
@@ -35,6 +36,8 @@ const DEFAULT_CHUNK_SIZE = 64; // kb
 const ONE_MB = 1024 * 1024;
 const DISCONNECT_SLEEP = 1000;
 
+const NEW_CONNECTION_QUEUE_THRESHOLD = 5;
+
 // when the sender requests hangs for 60 second we will reimport
 const SENDER_TIMEOUT = 60 * 1000;
 // Telegram may have server issues so we try several times
@@ -136,7 +139,7 @@ async function downloadFile2(
         partSizeKb, end,
     } = fileParams;
     const {
-        fileSize,
+        fileSize, dcId, progressCallback, isPriority, start = 0,
     } = fileParams;
 
     const fileId = 'id' in inputLocation ? inputLocation.id : undefined;
@@ -148,7 +151,6 @@ async function downloadFile2(
 
     logWithId('Downloading file...');
     const isPremium = Boolean(client.isPremium);
-    const { dcId, progressCallback, start = 0 } = fileParams;
 
     end = end && end < fileSize ? end : fileSize - 1;
 
@@ -159,7 +161,7 @@ async function downloadFile2(
     const partSize = partSizeKb * 1024;
     const partsCount = end ? Math.ceil((end + 1 - start + 1) / partSize) : 1;
     const noParallel = !end;
-    const shouldUseMultipleConnections = fileSize
+    const shouldUseMultipleConnections = Boolean(fileSize)
         && fileSize >= MULTIPLE_CONNECTIONS_MIN_FILE_SIZE
         && !noParallel;
     let deferred: Deferred | undefined;
@@ -173,11 +175,6 @@ async function downloadFile2(
     const fileView = new FileView(end - start + 1);
     const promises: Promise<any>[] = [];
     let offset = start;
-    // Pick the least busy foreman
-    // For some reason, fresh connections give out a higher speed for the first couple of seconds
-    // I have no idea why, but this may speed up the download of small files
-    const activeCounts = foremans.map(({ activeWorkers }) => activeWorkers);
-    let currentForemanIndex = activeCounts.indexOf(Math.min(...activeCounts));
     // Used for files with unknown size and for manual cancellations
     let hasEnded = false;
 
@@ -206,13 +203,9 @@ async function downloadFile2(
             isPrecise = true;
         }
 
-        // Use only first connection for avatars, because no size is known and we don't want to
+        const senderIndex = getFreeForemanIndex(isPremium, shouldUseMultipleConnections);
-        // download empty parts using all connections at once
-        const senderIndex = !shouldUseMultipleConnections ? 0 : currentForemanIndex % (
-            isPremium ? MAX_CONCURRENT_CONNECTIONS_PREMIUM : MAX_CONCURRENT_CONNECTIONS
-        );
 
-        await foremans[senderIndex].requestWorker();
+        await foremans[senderIndex].requestWorker(isPriority);
 
         if (deferred) await deferred.promise;
 
@@ -316,7 +309,6 @@ async function downloadFile2(
         })(offset));
 
         offset += limit;
-        currentForemanIndex++;
 
         if (end && (offset > end)) {
             break;
@@ -325,3 +317,27 @@ async function downloadFile2(
     await Promise.all(promises);
     return fileView.getData();
 }
+
+function getFreeForemanIndex(isPremium: boolean, forceNewConnection?: boolean) {
+    const availableConnections = isPremium ? MAX_CONCURRENT_CONNECTIONS_PREMIUM : MAX_CONCURRENT_CONNECTIONS;
+    let foremanIndex = 0;
+    let minQueueLength = Infinity;
+    for (let i = 0; i < availableConnections; i++) {
+        const foreman = foremans[i];
+        // If worker is free, return it
+        if (!foreman.queueLength) return i;
+
+        // Potentially create a new connection if the current queue is too long
+        if (!forceNewConnection && foreman.queueLength <= NEW_CONNECTION_QUEUE_THRESHOLD) {
+            return i;
+        }
+
+        // If every connection is equally busy, prefer the last one in the list
+        if (foreman.queueLength <= minQueueLength) {
+            foremanIndex = i;
+            minQueueLength = foreman.activeWorkers;
+        }
+    }
+
+    return foremanIndex;
+}

src/util/foreman.ts

@@ -3,29 +3,38 @@ import Deferred from './Deferred';
 export class Foreman {
   private deferreds: Deferred[] = [];
 
+  private priorityDeferreds: Deferred[] = [];
+
   activeWorkers = 0;
 
   constructor(private maxWorkers: number) {
   }
 
-  requestWorker() {
+  requestWorker(isPriority?: boolean) {
     if (this.activeWorkers === this.maxWorkers) {
       const deferred = new Deferred();
-      this.deferreds.push(deferred);
+      if (isPriority) {
+        this.priorityDeferreds.push(deferred);
+      } else {
+        this.deferreds.push(deferred);
+      }
       return deferred.promise;
-    } else {
-      this.activeWorkers++;
     }
 
+    this.activeWorkers++;
     return Promise.resolve();
   }
 
   releaseWorker() {
-    if (this.deferreds.length && (this.activeWorkers === this.maxWorkers)) {
+    if (this.queueLength) {
-      const deferred = this.deferreds.shift()!;
+      const deferred = (this.priorityDeferreds.shift() || this.deferreds.shift())!;
       deferred.resolve();
     } else {
       this.activeWorkers--;
     }
   }
+
+  get queueLength() {
+    return this.deferreds.length + this.priorityDeferreds.length;
+  }
 }

src/util/init.ts

@@ -31,6 +31,9 @@ export async function initGlobal(force: boolean = false, prevGlobal?: GlobalStat
 
   if (force) {
     global.byTabId = prevGlobal.byTabId;
+
+    // Keep the theme if it was set before
+    global.settings.byKey.theme = prevGlobal.settings.byKey.theme;
   }
 
   setGlobal(global);

src/util/mediaLoader.ts

@@ -28,8 +28,7 @@ const asCacheApiType = {
 
 const PROGRESSIVE_URL_PREFIX = `${IS_PACKAGED_ELECTRON ? ELECTRON_HOST_URL : '.'}/progressive/`;
 const URL_DOWNLOAD_PREFIX = './download/';
-const RETRY_MEDIA_AFTER = 2000;
+const MAX_MEDIA_RETRIES = 5;
-const MAX_MEDIA_RETRIES = 3;
 
 const memoryCache = new Map<string, ApiPreparedMedia>();
 const fetchPromises = new Map<string, Promise<ApiPreparedMedia | undefined>>();
@@ -162,7 +161,7 @@ async function fetchFromCacheOrRemote(
       throw new Error(`Failed to fetch media ${url}`);
     }
     await new Promise((resolve) => {
-      setTimeout(resolve, RETRY_MEDIA_AFTER);
+      setTimeout(resolve, getRetryTimeout(retryNumber));
     });
     // eslint-disable-next-line no-console
     if (DEBUG) console.debug(`Retrying to fetch media ${url}`);
@@ -234,3 +233,8 @@ if (IS_PROGRESSIVE_SUPPORTED) {
     }, [arrayBuffer!]);
   });
 }
+
+function getRetryTimeout(retryNumber: number) {
+  // 250ms, 500ms, 1s, 2s, 4s
+  return 250 * 2 ** retryNumber;
+}