feat: added caching support, added fallbackDir and fallbackPrefix

Author: niftylettuce

README.md

@@ -81,6 +81,9 @@ Accepts the following arguments and returns a [Nodemailer plugin][nodemailer-plu
   * `aws` (Object) **Required** - configuration options for Amazon Web Services
     * `params` (Object) **Required**
       * `Bucket` (String) **Required** - AWS Bucket Name
+  * `fallbackDir` (String) - a fallback directory to write to in case Amazon S3 upload fails (automatically set to `os.tmpdir()` otherwise if `NODE_ENV` is production then it is set to false and disabled) - you may want to specify the full path to your build directory so files are stored there (e.g. `fallbackDir: path.join(__dirname, 'build', 'img', 'nodemailer')`)
+  * `fallbackPrefix` (String or Boolean) - the prefix to use for relative paths, e.g. you don't want to have `file:///some/tmp/dir/foo.png`, but you want to have `https://example.com/img/foo.png` instead - so specify that prefix here (e.g. `fallbackPrefix: 'http://localhost:3000/img/nodemailer/'` if you have a build directory `fallbackDir` of `path.join(__dirname, 'build', 'img', 'nodemailer')` and `path.join(__dirname, 'build')` is being served by your web server).  The default value is `false` and therefore `file:///` relative path will be used instead.
+  * `logger` (Object) - a logger to use in the event of an error while uploading to S3 (defaults to `console`)
 
 
 ## Gmail Example

index.js

@@ -1,26 +1,37 @@
-const { promisify } = require('util');
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
 const zlib = require('zlib');
+const { promisify } = require('util');
+
+const AWS = require('aws-sdk');
+const Lipo = require('lipo');
+const _ = require('lodash');
+const debug = require('debug')('nodemailer-base64-to-s3');
 const isSANB = require('is-string-and-not-blank');
 const mime = require('mime-types');
-const _ = require('lodash');
 const ms = require('ms');
-const AWS = require('aws-sdk');
 const revHash = require('rev-hash');
-const Lipo = require('lipo');
 
 const regexp = new RegExp(
   /(<img[\s\S]*? src=")data:(image\/(?:png|jpe?g|gif|svg\+xml));base64,([\s\S]*?)("[\s\S]*?>)/g
 );
-
+const PROD = process.env.NODE_ENV === 'production';
+const mkdir = promisify(fs.mkdir).bind(fs);
+const writeFile = promisify(fs.writeFile).bind(fs);
 const gzip = promisify(zlib.gzip).bind(zlib);
+const cache = {};
 
 const base64ToS3 = opts => {
   // set defaults
   opts = _.defaults(opts, {
     aws: {},
     maxAge: ms('1yr'),
     dir: '/',
-    cloudFrontDomainName: process.env.AWS_CLOUDFRONT_DOMAIN || ''
+    cloudFrontDomainName: process.env.AWS_CLOUDFRONT_DOMAIN || '',
+    fallbackDir: process.env.NODE_ENV === 'production' ? false : os.tmpdir(),
+    fallbackPrefix: false,
+    logger: console
   });
 
   if (!_.isNumber(opts.maxAge))
@@ -34,6 +45,12 @@ const base64ToS3 = opts => {
 
   // prepare AWS upload using config
   const s3 = new AWS.S3(opts.aws);
+
+  // we cannot currently use this since it does not return a promise
+  // <https://github.com/aws/aws-sdk-js/pull/1079>
+  // await s3obj.upload({ Body }).promise();
+  //
+  // so instead we use promisify to convert it to a promise
   const upload = promisify(s3.upload).bind(s3);
 
   async function compile(mail, fn) {
@@ -79,28 +96,34 @@ const base64ToS3 = opts => {
   }
 
   async function transformImage({ original, start, mimeType, base64, end }) {
-    // create a buffer of the base64 image
-    // and convert it to a png
-    let buffer = Buffer.from(base64, 'base64');
-
     // get the image extension
     let extension = mime.extension(mimeType);
-
     // convert and optimize the image if it is an SVG file
-    if (extension === 'svg') {
+    if (extension === 'svg') extension = 'png';
+    // if we already cached the base64 then return it
+    const hash = revHash(`${extension}:${base64}`);
+    let buffer;
+    if (cache[hash]) {
+      buffer = cache[hash];
+      debug(`hitting cache for ${hash}`);
+    } else {
+      // create a buffer of the base64 image
+      // and convert it to a png
+      buffer = Buffer.from(base64, 'base64');
       const lipo = new Lipo();
       buffer = await lipo(buffer)
         .png()
         .toBuffer();
-      extension = 'png';
+      cache[hash] = buffer;
     }
 
     // apply transformation and gzip file
     const Body = await gzip(buffer);
 
     // generate random filename
     // get the file extension based on mimeType
-    const Key = `${opts.dir}${revHash(base64)}.${extension}`;
+    const fileName = `${hash}.${extension}`;
+    const Key = `${opts.dir}${fileName}`;
 
     const obj = {
       Key,
@@ -111,18 +134,38 @@ const base64ToS3 = opts => {
       ContentType: 'image/png'
     };
 
-    // we cannot currently use this since it does not return a promise
-    // <https://github.com/aws/aws-sdk-js/pull/1079>
-    // await s3obj.upload({ Body }).promise();
-    //
-    // so instead we use promisify to convert it to a promise
-    const data = await upload(obj);
+    // use a fallback dir if the upload fails
+    // but only if the environment is not production
+    try {
+      const data = cache[Key] ? cache[Key] : await upload(obj);
+      if (cache[Key]) debug(`hitting cache for ${Key}`);
+
+      const replacement = isSANB(opts.cloudFrontDomainName)
+        ? `${start}https://${opts.cloudFrontDomainName}/${data.key}${end}`
+        : `${start}${data.Location}${end}`;
 
-    const replacement = isSANB(opts.cloudFrontDomainName)
-      ? `${start}https://${opts.cloudFrontDomainName}/${data.key}${end}`
-      : `${start}${data.Location}${end}`;
+      cache[Key] = data;
 
-    return [original, replacement];
+      return [original, replacement];
+    } catch (err) {
+      // fallback in case upload to S3 fails for whatever reason
+      if (opts.fallbackDir) {
+        if (PROD) opts.logger.error(err);
+        if (!_.isString(opts.fallbackPrefix) && PROD)
+          throw new Error(
+            'fallbackPrefix was not specified, you cannot use file:/// in production mode'
+          );
+        const filePath = path.join(opts.fallbackDir, fileName);
+        await mkdir(opts.fallbackDir, { recursive: true });
+        await writeFile(filePath, buffer);
+        const replacement = _.isString(opts.fallbackPrefix)
+          ? `${start}${opts.fallbackPrefix}${fileName}${end}`
+          : `${start}file://${filePath}${end}`;
+        return [original, replacement];
+      }
+
+      throw err;
+    }
   }
 
   return compile;

package.json

@@ -21,6 +21,7 @@
   ],
   "dependencies": {
     "aws-sdk": "^2.528.0",
+    "debug": "^4.1.1",
     "is-string-and-not-blank": "^0.0.2",
     "lipo": "^0.0.10",
     "lodash": "^4.17.15",

test/test.js

@@ -1,11 +1,13 @@
 const path = require('path');
-const test = require('ava');
-const nodemailer = require('nodemailer');
-const ms = require('ms');
-const dotenv = require('dotenv');
+
 const cheerio = require('cheerio');
-const validator = require('validator');
+const dotenv = require('dotenv');
 const imageToUri = require('image-to-uri');
+const ms = require('ms');
+const nodemailer = require('nodemailer');
+const test = require('ava');
+const validator = require('validator');
+
 const base64ToS3 = require('..');
 
 const html = {};
@@ -81,3 +83,28 @@ Object.keys(html).forEach(key => {
     t.is(url, clone);
   });
 });
+
+test('writes to a fallback directory if AWS upload failed (e.g. no bucket param)', async t => {
+  const customTransport = nodemailer.createTransport({ jsonTransport: true });
+  customTransport.use(
+    'compile',
+    base64ToS3({
+      maxAge: ms('1d')
+    })
+  );
+  const res = await customTransport.sendMail({
+    html: html.png,
+    subject: 'subject',
+    to: 'niftylettuce@gmail.com',
+    from: 'niftylettuce@gmail.com'
+  });
+
+  const message = JSON.parse(res.message);
+  const $ = cheerio.load(message.html);
+  t.true(
+    validator.isURL($('img').attr('src'), {
+      protocols: ['file'],
+      require_host: false
+    })
+  );
+});