Blurhash support (#701)

* Generate blurhash client side

* Make blurhash generation faster

* Simple blurhash display support

* Make image display simpler

* Support non square images

* Don't attach video blurhash to thumbnail

* Add video display support

* Ignore alt tag missing warning

Co-authored-by: Ajay Bura <32841439+ajbura@users.noreply.github.com>

package-lock.json

@@ -14,6 +14,7 @@
         "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
         "@tippyjs/react": "^4.2.6",
         "babel-polyfill": "^6.26.0",
+        "blurhash": "^1.1.5",
         "browser-encrypt-attachment": "^0.3.0",
         "dateformat": "^5.0.3",
         "emojibase-data": "^7.0.1",
@@ -34,6 +35,7 @@
         "prop-types": "^15.8.1",
         "react": "^17.0.2",
         "react-autosize-textarea": "^7.1.0",
+        "react-blurhash": "^0.1.3",
         "react-dnd": "^15.1.2",
         "react-dnd-html5-backend": "^15.1.3",
         "react-dom": "^17.0.2",
@@ -3014,6 +3016,8 @@
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
       "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==",
       "dev": true,
+      "optional": true,
+      "peer": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.1",
         "json-schema-traverse": "^1.0.0",
@@ -3029,7 +3033,9 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
       "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
-      "dev": true
+      "dev": true,
+      "optional": true,
+      "peer": true
     },
     "node_modules/ajv-keywords": {
       "version": "3.5.2",
@@ -3558,6 +3564,11 @@
       "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=",
       "dev": true
     },
+    "node_modules/blurhash": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/blurhash/-/blurhash-1.1.5.tgz",
+      "integrity": "sha512-a+LO3A2DfxTaTztsmkbLYmUzUeApi0LZuKalwbNmqAHR6HhJGMt1qSV/R3wc+w4DL28holjqO3Bg74aUGavGjg=="
+    },
     "node_modules/bmp-js": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
@@ -11540,6 +11551,15 @@
         "react-dom": "^0.14.0 || ^15.0.0 || ^16.0.0"
       }
     },
+    "node_modules/react-blurhash": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/react-blurhash/-/react-blurhash-0.1.3.tgz",
+      "integrity": "sha512-Q9lqbXg92NU6/2DoIl/cBM8YWL+Z4X66OiG4aT9ozOgjBwx104LHFCH5stf6aF+s0Q9Wf310Ul+dG+VXJltmPg==",
+      "peerDependencies": {
+        "blurhash": "^1.1.1",
+        "react": ">=15"
+      }
+    },
     "node_modules/react-dnd": {
       "version": "15.1.2",
       "resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-15.1.2.tgz",
@@ -16495,15 +16515,14 @@
       "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz",
       "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==",
       "dev": true,
-      "requires": {
+      "requires": {},
-        "ajv": "^8.0.0"
-      },
       "dependencies": {
         "ajv": {
-          "version": "8.9.0",
+          "version": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
           "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==",
           "dev": true,
+          "optional": true,
+          "peer": true,
           "requires": {
             "fast-deep-equal": "^3.1.1",
             "json-schema-traverse": "^1.0.0",
@@ -16515,7 +16534,9 @@
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
           "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
-          "dev": true
+          "dev": true,
+          "optional": true,
+          "peer": true
         }
       }
     },
@@ -16950,6 +16971,11 @@
         }
       }
     },
+    "blurhash": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/blurhash/-/blurhash-1.1.5.tgz",
+      "integrity": "sha512-a+LO3A2DfxTaTztsmkbLYmUzUeApi0LZuKalwbNmqAHR6HhJGMt1qSV/R3wc+w4DL28holjqO3Bg74aUGavGjg=="
+    },
     "bmp-js": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
@@ -22933,6 +22959,12 @@
         "prop-types": "^15.5.6"
       }
     },
+    "react-blurhash": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/react-blurhash/-/react-blurhash-0.1.3.tgz",
+      "integrity": "sha512-Q9lqbXg92NU6/2DoIl/cBM8YWL+Z4X66OiG4aT9ozOgjBwx104LHFCH5stf6aF+s0Q9Wf310Ul+dG+VXJltmPg==",
+      "requires": {}
+    },
     "react-dnd": {
       "version": "15.1.2",
       "resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-15.1.2.tgz",

package.json

@@ -20,6 +20,7 @@
     "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
     "@tippyjs/react": "^4.2.6",
     "babel-polyfill": "^6.26.0",
+    "blurhash": "^1.1.5",
     "browser-encrypt-attachment": "^0.3.0",
     "dateformat": "^5.0.3",
     "emojibase-data": "^7.0.1",
@@ -40,6 +41,7 @@
     "prop-types": "^15.8.1",
     "react": "^17.0.2",
     "react-autosize-textarea": "^7.1.0",
+    "react-blurhash": "^0.1.3",
     "react-dnd": "^15.1.2",
     "react-dnd-html5-backend": "^15.1.3",
     "react-dom": "^17.0.2",

src/app/molecules/media/Media.jsx

@@ -4,6 +4,7 @@ import './Media.scss';
 
 import encrypt from 'browser-encrypt-attachment';
 
+import { BlurhashCanvas } from 'react-blurhash';
 import Text from '../../atoms/text/Text';
 import IconButton from '../../atoms/button/IconButton';
 import Spinner from '../../atoms/spinner/Spinner';
@@ -154,7 +155,7 @@ File.propTypes = {
 };
 
 function Image({
-  name, width, height, link, file, type,
+  name, width, height, link, file, type, blurhash,
 }) {
   const [url, setUrl] = useState(null);
 
@@ -175,6 +176,7 @@ function Image({
     <div className="file-container">
       <FileHeader name={name} link={url || link} type={type} external />
       <div style={{ height: width !== null ? getNativeHeight(width, height) : 'unset' }} className="image-container">
+        { blurhash && <BlurhashCanvas hash={blurhash} punch={1} />}
         { url !== null && <img src={url || link} alt={name} />}
       </div>
     </div>
@@ -185,6 +187,7 @@ Image.defaultProps = {
   width: null,
   height: null,
   type: '',
+  blurhash: '',
 };
 Image.propTypes = {
   name: PropTypes.string.isRequired,
@@ -193,6 +196,7 @@ Image.propTypes = {
   link: PropTypes.string.isRequired,
   file: PropTypes.shape({}),
   type: PropTypes.string,
+  blurhash: PropTypes.string,
 };
 
 function Sticker({
@@ -278,8 +282,8 @@ Audio.propTypes = {
 };
 
 function Video({
-  name, link, thumbnail,
+  name, link, thumbnail, thumbnailFile, thumbnailType,
-  width, height, file, type, thumbnailFile, thumbnailType,
+  width, height, file, type, blurhash,
 }) {
   const [isLoading, setIsLoading] = useState(false);
   const [url, setUrl] = useState(null);
@@ -315,10 +319,14 @@ function Video({
       <div
         style={{
           height: width !== null ? getNativeHeight(width, height) : 'unset',
-          backgroundImage: thumbUrl === null ? 'none' : `url(${thumbUrl}`,
         }}
         className="video-container"
       >
+        { url === null && blurhash && <BlurhashCanvas hash={blurhash} punch={1} />}
+        { url === null && thumbUrl !== null && (
+          /* eslint-disable-next-line jsx-a11y/alt-text */
+          <img src={thumbUrl} />
+        )}
         { url === null && isLoading && <Spinner size="small" /> }
         { url === null && !isLoading && <IconButton onClick={handlePlayVideo} tooltip="Play video" src={PlaySVG} />}
         { url !== null && (
@@ -336,20 +344,22 @@ Video.defaultProps = {
   height: null,
   file: null,
   thumbnail: null,
-  type: '',
   thumbnailType: null,
   thumbnailFile: null,
+  type: '',
+  blurhash: null,
 };
 Video.propTypes = {
   name: PropTypes.string.isRequired,
   link: PropTypes.string.isRequired,
   thumbnail: PropTypes.string,
+  thumbnailFile: PropTypes.shape({}),
+  thumbnailType: PropTypes.string,
   width: PropTypes.number,
   height: PropTypes.number,
   file: PropTypes.shape({}),
   type: PropTypes.string,
-  thumbnailFile: PropTypes.shape({}),
+  blurhash: PropTypes.string,
-  thumbnailType: PropTypes.string,
 };
 
 export {

src/app/molecules/media/Media.scss

@@ -33,6 +33,8 @@
   font-size: 0;
   line-height: 0;
 
+  position: relative;
+
   display: flex;
   justify-content: center;
   align-items: center;
@@ -42,6 +44,19 @@
   background-size: cover;
 }
 
+.image-container,
+.video-container {
+  & img,
+  & canvas {
+    position: absolute;
+    max-width: unset !important;
+    width: 100% !important;
+    height: 100%;
+    border-radius: 0 !important;
+    margin: 0 !important;
+  }
+}
+
 .sticker-container {
   display: inline-flex;
   max-width: 128px;
@@ -51,25 +66,17 @@
   }
 }
 
-.image-container {
-  & img {
-    max-width: unset !important;
-    width: 100% !important;
-    border-radius: 0 !important;
-    margin: 0 !important;
-  }
-}
-
 .video-container {
   & .ic-btn-surface {
     background-color: var(--bg-surface-low);
+    position: absolute;
   }
   video {
-    width: 100%
+    width: 100%;
   }
 }
 .audio-container {
   audio {
-    width: 100%
+    width: 100%;
   }
 }

src/app/molecules/message/Message.jsx

@@ -610,6 +610,8 @@ function genMediaContent(mE) {
   let msgType = mE.getContent()?.msgtype;
   if (mE.getType() === 'm.sticker') msgType = 'm.sticker';
 
+  const blurhash = mContent?.info?.['xyz.amorgan.blurhash'];
+
   switch (msgType) {
     case 'm.file':
       return (
@@ -629,6 +631,7 @@ function genMediaContent(mE) {
           link={mx.mxcUrlToHttp(mediaMXC)}
           file={isEncryptedFile ? mContent.file : null}
           type={mContent.info?.mimetype}
+          blurhash={blurhash}
         />
       );
     case 'm.sticker':
@@ -666,6 +669,7 @@ function genMediaContent(mE) {
           height={typeof mContent.info?.h === 'number' ? mContent.info?.h : null}
           file={isEncryptedFile ? mContent.file : null}
           type={mContent.info?.mimetype}
+          blurhash={blurhash}
         />
       );
     default:

src/client/state/RoomsInput.js

@@ -3,12 +3,34 @@ import { micromark } from 'micromark';
 import { gfm, gfmHtml } from 'micromark-extension-gfm';
 import encrypt from 'browser-encrypt-attachment';
 import { math } from 'micromark-extension-math';
+import { encode } from 'blurhash';
 import { getShortcodeToEmoji } from '../../app/organisms/emoji-board/custom-emoji';
 import { mathExtensionHtml, spoilerExtension, spoilerExtensionHtml } from '../../util/markdown';
 import { getImageDimension } from '../../util/common';
 import cons from './cons';
 import settings from './settings';
 
+const blurhashField = 'xyz.amorgan.blurhash';
+
+function encodeBlurhash(img) {
+  const canvas = document.createElement('canvas');
+  canvas.width = 100;
+  canvas.height = 100;
+  const context = canvas.getContext('2d');
+  context.drawImage(img, 0, 0, canvas.width, canvas.height);
+  const data = context.getImageData(0, 0, canvas.width, canvas.height);
+  return encode(data.data, data.width, data.height, 4, 4);
+}
+
+function loadImage(url) {
+  return new Promise((resolve, reject) => {
+    const img = new Image();
+    img.onload = () => resolve(img);
+    img.onerror = (err) => reject(err);
+    img.src = url;
+  });
+}
+
 function loadVideo(videoFile) {
   return new Promise((resolve, reject) => {
     const video = document.createElement('video');
@@ -300,10 +322,11 @@ class RoomsInput extends EventEmitter {
     let uploadData = null;
 
     if (fileType === 'image') {
-      const imgDimension = await getImageDimension(file);
+      const img = await loadImage(URL.createObjectURL(file));
 
-      info.w = imgDimension.w;
+      info.w = img.width;
-      info.h = imgDimension.h;
+      info.h = img.height;
+      info[blurhashField] = encodeBlurhash(img);
 
       content.msgtype = 'm.image';
       content.body = file.name || 'Image';
@@ -313,8 +336,11 @@ class RoomsInput extends EventEmitter {
 
       try {
         const video = await loadVideo(file);
+
         info.w = video.videoWidth;
         info.h = video.videoHeight;
+        info[blurhashField] = encodeBlurhash(video);
+
         const thumbnailData = await getVideoThumbnail(video, video.videoWidth, video.videoHeight, 'image/jpeg');
         const thumbnailUploadData = await this.uploadFile(roomId, thumbnailData.thumbnail);
         info.thumbnail_info = thumbnailData.info;