Emoji Hint: Proper support with keywords (#1152)

package-lock.json

@@ -7660,8 +7660,8 @@
       "dev": true
     },
     "emoji-data-ios": {
-      "version": "github:korenskoy/emoji-data-ios#e644adb357e37683e91985d873f629c91d31bc7e",
+      "version": "github:korenskoy/emoji-data-ios#d3efbb05d3860148b45faf4164d58298a171a7f9",
-      "from": "github:korenskoy/emoji-data-ios#e644adb"
+      "from": "github:korenskoy/emoji-data-ios#d3efbb0"
     },
     "emojis-list": {
       "version": "2.1.0",

package.json

@@ -101,7 +101,7 @@
     "async-mutex": "^0.1.4",
     "big-integer": "painor/BigInteger.js",
     "croppie": "^2.6.4",
-    "emoji-data-ios": "github:korenskoy/emoji-data-ios#e644adb",
+    "emoji-data-ios": "github:korenskoy/emoji-data-ios#d3efbb0",
     "events": "^3.0.0",
     "idb-keyval": "^5.0.5",
     "opus-recorder": "^6.2.0",

src/api/gramjs/methods/index.ts

@@ -34,7 +34,7 @@ export {
 export {
   fetchStickerSets, fetchRecentStickers, fetchFavoriteStickers, fetchFeaturedStickers,
   faveSticker, fetchStickers, fetchSavedGifs, searchStickers, installStickerSet, uninstallStickerSet,
-  searchGifs, fetchAnimatedEmojis, fetchStickersForEmoji,
+  searchGifs, fetchAnimatedEmojis, fetchStickersForEmoji, fetchEmojiKeywords,
 } from './symbols';
 
 export {

src/api/gramjs/methods/symbols.ts

@@ -246,6 +246,30 @@ export async function fetchStickersForEmoji({
   };
 }
 
+export async function fetchEmojiKeywords({ language, fromVersion }: {
+  language: string;
+  fromVersion?: number;
+}) {
+  const result = await invokeRequest(new GramJs.messages.GetEmojiKeywordsDifference({
+    langCode: language,
+    fromVersion,
+  }));
+
+  if (!result) {
+    return undefined;
+  }
+
+  return {
+    language: result.langCode,
+    version: result.version,
+    keywords: result.keywords.reduce((acc, emojiKeyword) => {
+      acc[emojiKeyword.keyword] = emojiKeyword.emoticons;
+
+      return acc;
+    }, {} as Record<string, string[]>),
+  };
+}
+
 function processStickerResult(stickers: GramJs.TypeDocument[]) {
   return stickers
     .map((document) => {

src/components/middle/composer/AttachmentModal.tsx

@@ -3,8 +3,9 @@ import React, {
 } from '../../../lib/teact/teact';
 
 import { ApiAttachment, ApiChatMember, ApiUser } from '../../../api/types';
-import { CONTENT_TYPES_FOR_QUICK_UPLOAD, EDITABLE_INPUT_MODAL_ID } from '../../../config';
+import { LangCode } from '../../../types';
 
+import { CONTENT_TYPES_FOR_QUICK_UPLOAD, EDITABLE_INPUT_MODAL_ID } from '../../../config';
 import { getFileExtension } from '../../common/helpers/documentInfo';
 import captureEscKeyListener from '../../../util/captureEscKeyListener';
 import usePrevious from '../../../hooks/usePrevious';
@@ -31,7 +32,9 @@ export type OwnProps = {
   groupChatMembers?: ApiChatMember[];
   usersById?: Record<number, ApiUser>;
   recentEmojis: string[];
+  language: LangCode;
   addRecentEmoji: AnyToVoidFunction;
+  loadEmojiKeywords: AnyToVoidFunction;
   onCaptionUpdate: (html: string) => void;
   onSend: () => void;
   onFileAppend: (files: File[], isQuick: boolean) => void;
@@ -48,8 +51,10 @@ const AttachmentModal: FC<OwnProps> = ({
   currentUserId,
   usersById,
   recentEmojis,
+  language,
   onCaptionUpdate,
   addRecentEmoji,
+  loadEmojiKeywords,
   onSend,
   onFileAppend,
   onClear,
@@ -229,8 +234,10 @@ const AttachmentModal: FC<OwnProps> = ({
             isOpen={isEmojiTooltipOpen}
             emojis={filteredEmojis}
             onClose={closeEmojiTooltip}
+            language={language}
             onEmojiSelect={insertEmoji}
             addRecentEmoji={addRecentEmoji}
+            loadEmojiKeywords={loadEmojiKeywords}
           />
           <MessageInput
             id="caption-input-text"

src/components/middle/composer/Composer.tsx

@@ -16,6 +16,7 @@ import {
   ApiUser,
   MAIN_THREAD_ID,
 } from '../../../api/types';
+import { LangCode } from '../../../types';
 
 import { EDITABLE_INPUT_ID, SCHEDULED_WHEN_ONLINE } from '../../../config';
 import { IS_VOICE_RECORDING_SUPPORTED, IS_MOBILE_SCREEN, IS_EMOJI_SUPPORTED } from '../../../util/environment';
@@ -30,6 +31,7 @@ import {
   selectEditingMessage,
   selectIsChatWithSelf,
   selectChatUser,
+  selectEmojiKeywords,
 } from '../../../modules/selectors';
 import {
   getAllowedAttachmentOptions,
@@ -118,13 +120,15 @@ type StateProps = {
   lastSyncTime?: number;
   contentToBeScheduled?: GlobalState['messages']['contentToBeScheduled'];
   shouldSuggestStickers?: boolean;
+  language: LangCode;
+  emojiKeywords?: Record<string, string[]>;
 } & Pick<GlobalState, 'connectionState'>;
 
 type DispatchProps = Pick<GlobalActions, (
   'sendMessage' | 'editMessage' | 'saveDraft' | 'forwardMessages' |
   'clearDraft' | 'showError' | 'setStickerSearchQuery' | 'setGifSearchQuery' |
   'openPollModal' | 'closePollModal' | 'loadScheduledHistory' | 'openChat' | 'closePaymentModal' |
-  'clearReceipt' | 'addRecentEmoji'
+  'clearReceipt' | 'addRecentEmoji' | 'loadEmojiKeywords'
 )>;
 
 enum MainButtonState {
@@ -174,6 +178,8 @@ const Composer: FC<OwnProps & StateProps & DispatchProps> = ({
   lastSyncTime,
   contentToBeScheduled,
   shouldSuggestStickers,
+  language,
+  emojiKeywords,
   recentEmojis,
   sendMessage,
   editMessage,
@@ -190,6 +196,7 @@ const Composer: FC<OwnProps & StateProps & DispatchProps> = ({
   openChat,
   clearReceipt,
   addRecentEmoji,
+  loadEmojiKeywords,
 }) => {
   // eslint-disable-next-line no-null/no-null
   const appendixRef = useRef<HTMLDivElement>(null);
@@ -299,6 +306,7 @@ const Composer: FC<OwnProps & StateProps & DispatchProps> = ({
     recentEmojis,
     undefined,
     setHtml,
+    emojiKeywords,
   );
 
   const insertTextAndUpdateCursor = useCallback((text: string, inputId: string = EDITABLE_INPUT_ID) => {
@@ -687,7 +695,9 @@ const Composer: FC<OwnProps & StateProps & DispatchProps> = ({
         usersById={usersById}
         recentEmojis={recentEmojis}
         onCaptionUpdate={setHtml}
+        language={language}
         addRecentEmoji={addRecentEmoji}
+        loadEmojiKeywords={loadEmojiKeywords}
         onSend={shouldSchedule ? openCalendar : handleSend}
         onFileAppend={handleAppendFiles}
         onClear={handleClearAttachment}
@@ -821,6 +831,8 @@ const Composer: FC<OwnProps & StateProps & DispatchProps> = ({
             onClose={closeEmojiTooltip}
             onEmojiSelect={insertEmoji}
             addRecentEmoji={addRecentEmoji}
+            loadEmojiKeywords={loadEmojiKeywords}
+            language={language}
           />
           <AttachMenu
             isOpen={isAttachMenuOpen}
@@ -909,6 +921,8 @@ export default memo(withGlobal<OwnProps>(
     const isChatWithSelf = selectIsChatWithSelf(global, chatId);
     const messageWithActualBotKeyboard = isChatWithBot && selectNewestMessageWithBotKeyboardButtons(global, chatId);
     const scheduledIds = selectScheduledIds(global, chatId);
+    const { language } = global.settings.byKey;
+    const emojiKeywords = selectEmojiKeywords(global, language);
 
     return {
       editingMessage: selectEditingMessage(global, chatId, threadId, messageListType),
@@ -943,6 +957,8 @@ export default memo(withGlobal<OwnProps>(
       isReceiptModalOpen: Boolean(global.payment.receipt),
       shouldSuggestStickers: global.settings.byKey.shouldSuggestStickers,
       recentEmojis: global.recentEmojis,
+      language,
+      emojiKeywords: emojiKeywords ? emojiKeywords.keywords : undefined,
     };
   },
   (setGlobal, actions): DispatchProps => pick(actions, [
@@ -961,5 +977,6 @@ export default memo(withGlobal<OwnProps>(
     'loadScheduledHistory',
     'openChat',
     'addRecentEmoji',
+    'loadEmojiKeywords',
   ]),
 )(Composer));

src/components/middle/composer/EmojiTooltip.tsx

@@ -16,6 +16,7 @@ import Loading from '../../ui/Loading';
 import EmojiButton from './EmojiButton';
 
 import './EmojiTooltip.scss';
+import { LangCode } from '../../../types';
 
 const VIEWPORT_MARGIN = 8;
 const EMOJI_BUTTON_WIDTH = 44;
@@ -50,9 +51,11 @@ function setItemVisible(index: number, containerRef: Record<string, any>) {
 
 export type OwnProps = {
   isOpen: boolean;
+  language: LangCode;
   onEmojiSelect: (text: string) => void;
   onClose: NoneToVoidFunction;
   addRecentEmoji: AnyToVoidFunction;
+  loadEmojiKeywords: AnyToVoidFunction;
   emojis: Emoji[];
 };
 
@@ -60,10 +63,12 @@ const CLOSE_DURATION = 350;
 
 const EmojiTooltip: FC<OwnProps> = ({
   isOpen,
+  language,
   emojis,
   onClose,
   onEmojiSelect,
   addRecentEmoji,
+  loadEmojiKeywords,
 }) => {
   // eslint-disable-next-line no-null/no-null
   const containerRef = useRef<HTMLDivElement>(null);
@@ -72,6 +77,10 @@ const EmojiTooltip: FC<OwnProps> = ({
 
   const [selectedIndex, setSelectedIndex] = useState(-1);
 
+  useEffect(() => {
+    loadEmojiKeywords({ language });
+  }, [loadEmojiKeywords, language]);
+
   useEffect(() => {
     setSelectedIndex(0);
   }, [emojis]);

src/components/middle/composer/hooks/useEmojiTooltip.ts

@@ -7,8 +7,11 @@ import { IS_MOBILE_SCREEN } from '../../../../util/environment';
 import {
   EmojiData, EmojiModule, EmojiRawData, uncompressEmoji,
 } from '../../../../util/emoji';
-import useFlag from '../../../../hooks/useFlag';
 import focusEditableElement from '../../../../util/focusEditableElement';
+import {
+  buildCollectionByKey, flatten, mapValues, pickTruthy, unique,
+} from '../../../../util/iteratees';
+import useFlag from '../../../../hooks/useFlag';
 
 let emojiDataPromise: Promise<EmojiModule>;
 let emojiRawData: EmojiRawData;
@@ -23,28 +26,31 @@ export default function useEmojiTooltip(
   recentEmojiIds: string[],
   inputId = EDITABLE_INPUT_ID,
   onUpdateHtml: (html: string) => void,
+  emojiKeywords?: Record<string, string[]>,
 ) {
   const [isOpen, markIsOpen, unmarkIsOpen] = useFlag();
-  const [emojiIds, setEmojiIds] = useState<string[]>([]);
+
+  const [byId, setById] = useState<Record<string, Emoji> | undefined>();
+  const [byKeyword, setByKeyword] = useState<Record<string, Emoji[]>>({});
+  const [byName, setByName] = useState<Record<string, Emoji[]>>({});
+
   const [filteredEmojis, setFilteredEmojis] = useState<Emoji[]>([]);
 
   const recentEmojis = useMemo(
     () => {
-      if (!emojiIds.length || !recentEmojiIds.length) {
+      if (!byId || !recentEmojiIds.length) {
         return [];
       }
 
-      return recentEmojiIds
+      return Object.values(pickTruthy(byId, recentEmojiIds));
-        .map((emojiId) => emojiData.emojis[emojiId])
-        .filter<Emoji>(Boolean as any);
     },
-    [emojiIds, recentEmojiIds],
+    [byId, recentEmojiIds],
   );
 
   // Initialize data on first render.
   useEffect(() => {
     const exec = () => {
-      setEmojiIds(Object.keys(emojiData.emojis));
+      setById(emojiData.emojis);
     };
 
     if (emojiData) {
@@ -56,7 +62,34 @@ export default function useEmojiTooltip(
   }, []);
 
   useEffect(() => {
-    if (!isAllowed || !html || !emojiIds.length) {
+    if (!byId) {
+      return;
+    }
+
+    const emojis = Object.values(byId);
+
+    if (emojiKeywords) {
+      const byNative = buildCollectionByKey(emojis, 'native');
+      setByKeyword(mapValues(emojiKeywords, (natives) => {
+        return Object.values(pickTruthy(byNative, natives));
+      }));
+    }
+
+    setByName(emojis.reduce((result, emoji) => {
+      emoji.names.forEach((name) => {
+        if (!result[name]) {
+          result[name] = [];
+        }
+
+        result[name].push(emoji);
+      });
+
+      return result;
+    }, {} as Record<string, Emoji[]>));
+  }, [byId, emojiKeywords]);
+
+  useEffect(() => {
+    if (!isAllowed || !html || !byId) {
       unmarkIsOpen();
       return;
     }
@@ -69,20 +102,28 @@ export default function useEmojiTooltip(
     }
 
     const filter = code.substr(1);
-    const matched = filter === ''
+    let matched: Emoji[] = [];
-      ? recentEmojis
+
-      : emojiIds
+    if (!filter) {
-        .filter((emojiId) => emojiData.emojis[emojiId].names.find((name) => name.includes(filter)))
+      matched = recentEmojis;
-        .slice(0, EMOJIS_LIMIT)
+    } else {
-        .map((emojiId) => emojiData.emojis[emojiId]);
+      const matchedKeywords = Object.keys(byKeyword).filter((keyword) => keyword.startsWith(filter));
+      matched = matched.concat(flatten(Object.values(pickTruthy(byKeyword, matchedKeywords))));
+
+      // Also search by names, which is useful for non-English languages
+      const matchedNames = Object.keys(byName).filter((name) => name.startsWith(filter));
+      matched = matched.concat(flatten(Object.values(pickTruthy(byName, matchedNames))));
+
+      matched = unique(matched);
+    }
 
     if (matched.length) {
       markIsOpen();
-      setFilteredEmojis(matched);
+      setFilteredEmojis(matched.slice(0, EMOJIS_LIMIT));
     } else {
       unmarkIsOpen();
     }
-  }, [emojiIds, html, isAllowed, markIsOpen, recentEmojis, unmarkIsOpen]);
+  }, [byId, byKeyword, byName, html, isAllowed, markIsOpen, recentEmojis, unmarkIsOpen]);
 
   const insertEmoji = useCallback((textEmoji: string) => {
     const atIndex = html.lastIndexOf(':');

src/global/cache.ts

@@ -117,6 +117,7 @@ function updateCache() {
         'chatFolders',
         'topPeers',
         'recentEmojis',
+        'emojiKeywords',
         'push',
       ]),
       isChatInfoShown: reduceShowChatInfo(global),

src/global/initial.ts

@@ -63,6 +63,8 @@ export const INITIAL_STATE: GlobalState = {
     forEmoji: {},
   },
 
+  emojiKeywords: {},
+
   gifs: {
     saved: {},
     search: {},

src/global/types.ts

@@ -37,6 +37,8 @@ import {
   ThemeKey,
   IThemeSettings,
   NotifyException,
+  LangCode,
+  EmojiKeywords,
 } from '../types';
 
 export type MessageListType = 'thread' | 'pinned' | 'scheduled';
@@ -204,6 +206,7 @@ export type GlobalState = {
   };
 
   animatedEmojis?: ApiStickerSet;
+  emojiKeywords: Partial<Record<LangCode, EmojiKeywords>>;
 
   gifs: {
     saved: {
@@ -448,7 +451,7 @@ export type ActionTypes = (
   'loadStickerSets' | 'loadAddedStickers' | 'loadRecentStickers' | 'loadFavoriteStickers' | 'loadFeaturedStickers' |
   'loadStickers' | 'setStickerSearchQuery' | 'loadSavedGifs' | 'setGifSearchQuery' | 'searchMoreGifs' |
   'faveSticker' | 'unfaveSticker' | 'toggleStickerSet' | 'loadAnimatedEmojis' |
-  'loadStickersForEmoji' | 'clearStickersForEmoji' |
+  'loadStickersForEmoji' | 'clearStickersForEmoji' | 'loadEmojiKeywords' |
   // bots
   'clickInlineButton' | 'sendBotCommand' |
   // misc

src/lib/gramjs/tl/apiTl.js

@@ -1006,6 +1006,7 @@ messages.sendVote#10ea6184 peer:InputPeer msg_id:int options:Vector<bytes> = Upd
 messages.getOnlines#6e2be050 peer:InputPeer = ChatOnlines;
 messages.editChatAbout#def60797 peer:InputPeer about:string = Bool;
 messages.editChatDefaultBannedRights#a5866b41 peer:InputPeer banned_rights:ChatBannedRights = Updates;
+messages.getEmojiKeywordsDifference#1508b6af lang_code:string from_version:int = EmojiKeywordsDifference;
 messages.getScheduledHistory#e2c2685b peer:InputPeer hash:int = messages.Messages;
 messages.sendScheduledMessages#bd38850a peer:InputPeer id:Vector<int> = Updates;
 messages.deleteScheduledMessages#59ae2b16 peer:InputPeer id:Vector<int> = Updates;

src/lib/gramjs/tl/static/api.reduced.tl

@@ -1006,6 +1006,7 @@ messages.sendVote#10ea6184 peer:InputPeer msg_id:int options:Vector<bytes> = Upd
 messages.getOnlines#6e2be050 peer:InputPeer = ChatOnlines;
 messages.editChatAbout#def60797 peer:InputPeer about:string = Bool;
 messages.editChatDefaultBannedRights#a5866b41 peer:InputPeer banned_rights:ChatBannedRights = Updates;
+messages.getEmojiKeywordsDifference#1508b6af lang_code:string from_version:int = EmojiKeywordsDifference;
 messages.getScheduledHistory#e2c2685b peer:InputPeer hash:int = messages.Messages;
 messages.sendScheduledMessages#bd38850a peer:InputPeer id:Vector<int> = Updates;
 messages.deleteScheduledMessages#59ae2b16 peer:InputPeer id:Vector<int> = Updates;

src/modules/actions/api/symbols.ts

@@ -1,6 +1,7 @@
 import { addReducer, getGlobal, setGlobal } from '../../../lib/teact/teactn';
 
 import { ApiSticker } from '../../../api/types';
+import { LangCode } from '../../../types';
 import { callApi } from '../../../api/gramjs';
 import { pause, throttle } from '../../../util/schedulers';
 import {
@@ -12,7 +13,7 @@ import {
   rebuildStickersForEmoji,
 } from '../../reducers';
 import searchWords from '../../../util/searchWords';
-import { selectStickerSet } from '../../selectors';
+import { selectEmojiKeywords, selectStickerSet } from '../../selectors';
 
 const ADDED_SETS_THROTTLE = 500;
 const ADDED_SETS_THROTTLE_CHUNK = 50;
@@ -109,6 +110,66 @@ addReducer('toggleStickerSet', (global, actions, payload) => {
   void callApi(!installedDate ? 'installStickerSet' : 'uninstallStickerSet', { stickerSetId, accessHash });
 });
 
+addReducer('loadEmojiKeywords', (global, actions, payload: { language: LangCode }) => {
+  const { language } = payload;
+  let currentEmojiKeywords = selectEmojiKeywords(global, language);
+
+  if (currentEmojiKeywords && currentEmojiKeywords.isLoading) {
+    return;
+  }
+
+  setGlobal({
+    ...global,
+    emojiKeywords: {
+      ...global.emojiKeywords,
+      [language]: {
+        ...currentEmojiKeywords,
+        isLoading: true,
+      },
+    },
+  });
+
+  (async () => {
+    const emojiKeywords = await callApi('fetchEmojiKeywords', {
+      language,
+      fromVersion: currentEmojiKeywords ? currentEmojiKeywords.version : 0,
+    });
+
+    global = getGlobal();
+    currentEmojiKeywords = selectEmojiKeywords(global, language);
+
+    if (!emojiKeywords) {
+      setGlobal({
+        ...global,
+        emojiKeywords: {
+          ...global.emojiKeywords,
+          [language]: {
+            ...currentEmojiKeywords,
+            isLoading: false,
+          },
+        },
+      });
+
+      return;
+    }
+
+    setGlobal({
+      ...global,
+      emojiKeywords: {
+        ...global.emojiKeywords,
+        [language]: {
+          isLoading: false,
+          version: emojiKeywords.version,
+          keywords: {
+            ...(currentEmojiKeywords && currentEmojiKeywords.keywords),
+            ...emojiKeywords.keywords,
+          },
+        },
+      },
+    });
+  })();
+});
+
 async function loadStickerSets(hash = 0) {
   const addedStickers = await callApi('fetchStickerSets', { hash });
   if (!addedStickers) {

src/modules/selectors/symbols.ts

@@ -1,5 +1,6 @@
 import { GlobalState } from '../../global/types';
 import { ApiSticker } from '../../api/types';
+import { LangCode, EmojiKeywords } from '../../types';
 
 export function selectIsStickerFavorite(global: GlobalState, sticker: ApiSticker) {
   const { stickers } = global.stickers.favorite;
@@ -44,3 +45,9 @@ export function selectAnimatedEmoji(global: GlobalState, emoji: string) {
 
   return animatedEmojis.stickers.find((sticker) => sticker.emoji === emoji || sticker.emoji === cleanedEmoji);
 }
+
+export function selectEmojiKeywords(global: GlobalState, language: LangCode): EmojiKeywords | undefined {
+  return global.emojiKeywords[language] && global.emojiKeywords[language] !== undefined
+    ? global.emojiKeywords[language] as EmojiKeywords
+    : undefined;
+}

src/types/index.ts

@@ -38,6 +38,11 @@ export type NotifySettings = {
   hasContactJoinedNotifications?: boolean;
 };
 
+export type LangCode = (
+  'en' | 'ar' | 'be' | 'ca' | 'nl' | 'fr' | 'de' | 'id' | 'it' | 'ko' | 'ms' | 'fa' | 'pl' | 'pt-br' | 'ru' | 'es'
+  | 'tr' | 'uk' | 'uz'
+);
+
 export interface ISettings extends NotifySettings, Record<string, any> {
   theme: ThemeKey;
   messageTextSize: number;
@@ -53,10 +58,7 @@ export interface ISettings extends NotifySettings, Record<string, any> {
   shouldLoopStickers: boolean;
   hasPassword?: boolean;
   languages?: ApiLanguage[];
-  language: (
+  language: LangCode;
-    'en' | 'ar' | 'be' | 'ca' | 'nl' | 'fr' | 'de' | 'id' | 'it' | 'ko' | 'ms' | 'fa' | 'pl' | 'pt-br' | 'ru' | 'es'
-    | 'tr' | 'uk' | 'uz'
-  );
 }
 
 export interface ApiPrivacySettings {
@@ -286,3 +288,9 @@ export type NotifyException = {
   isSilent?: boolean;
   shouldShowPreviews?: boolean;
 };
+
+export type EmojiKeywords = {
+  isLoading?: boolean;
+  version: number;
+  keywords: Record<string, string[]>;
+};