Defer ALL The Things

JavaScript was one of the first languages I ever learned, but as I drifted more toward Python development, I got increasingly out-of-touch with the best ways of doing things.

Then a friend asked me to help on a project. The server was in Node.js and the datastore was MongoDB. Async, webscale, buzzwords, we had it all!

Callback Hell

One of the downsides of everything being asynchronous is that it's very easy to fall into the trap of writing everything as an anonymous callback. You can generally tell this happened by looking at the end of the file.

                            });
                        });
                    });
                // OH GOD WHY
                });
            });
        });
    // CLOSE EVERYTHING
    });
});

This just hurts me. It's from the end of a view that got objects from two collections. As a Python developer, I can do the same thing in four lines of code!

def home(request):
    foos = Foo.objects.all().orderby('-created')[0:5]
    bars = Bar.objects.all().orderby('-created')[0:5]
    return render(request, 'home.html', {'foos': foos, 'bars': bars})

No, it's not asynchronous (well...), but I think I'd rather spend the CPU cycles and save my brain.

I actually delayed coming back to JavaScript because so much of it looks like this.

Why?

Why do I hate this so much?

When callbacks are nested like this, they usually rely on lexical closure for access to variables, e.g.:

app.get('/', function(req, res) {
    // Open the connection.
    db.open(function(err, db) {
        // Get one collection.
        db.collection('users', function(err, usersColl) {
            // Search the first collection.
            usersColl.find({}, {'limit': 3}, function(err, usersCursor) {
                // Convert the result into an array.
                usersCursor.toArray(function(err, users) {
                    // Get the second collection.
                    db.collection('articles', function(err, artColl) {
                        // Search the second collection.
                        artColl.find({}, {'limit': 3}, function(err, artCursor) {
                            // Convert the result into an array.
                            artCursor.toArray(function(err, articles) {
                                // Now we have two arrays, users and articles, in scope.
                                // Render the home page.
                                res.render('home.ejs', {'users': users, 'articles': articles});

The innermost function only has access to the array users because it's closed over several other functions.

There are at least 3 things wrong with this:

1. Both queries are asynchronous, but we do them in serial instead of parallel. That's a waste of time and burns most of our "async" advantages right off the bat.
2. Because these inner functions depend on lexical closure, they're impossible to test in smaller fragments and difficult to refactor to become more testable.
3. Rendering a template in the middle of a database query is no better than a database query in the middle of a template:

<h1><?= mysql_query($my, "SELECT title FROM posts WHERE..."); ?></h1>

And finally, #4, OH GOD THE NESTING.

But then!

Months later, at a hack day, researching something completely different, I stumbled across jQuery.Deferred() and it was like I had come out of the cave and seen the light.

I had seen projects like tame.js, which I like in principle, but I find overly complicated in practice. I thought those were the options: callback hell or precompilers that spit out mounds of impenetrable code.

All of a sudden my functions could be functions again! They could take arguments instead of relying on scope. They could be run in parallel. They could be nice and flat!

var mongodb = require('mongodb');
var express = require('express');
var Deferred = require('Deferred');

var app = express.createServer();
var db = new mongodb.Db('mysite',
                        new mongodb.Server('localhost', 27027, {}),
                        {native_parser: true});

var Deferred = require('Deferred');

// Only do this once. Keep it somewhere else.
function getArray(db, coll, search, options) {
    var def = Deferred();
    db.open(function(err, db) {
        if (err) def.reject(err);
        db.collection(coll, function(err, collection) {
            if (err) def.reject(err);
            collection.find(search, options, function(err, cursor) {
                if (err) def.reject(err);
                cursor.toArray(function(err, arr) {
                    if (err) def.reject(err);
                    def.resolve(arr);
                });
            });
        });
    });
    return def.promise();
}

function home(req, res) {
    var uDef = getArray(db, 'users', {}, {'limit': 3});
    var aDef = getArray(db, 'articles', {}, {'limit': 4});
    var lookup = Deferred.when(uDef, aDef);
    lookup.done(function(users, articles)) {
        res.render('home.ejs', {'users': users, 'articles': articles});
    }).fail(function(err) {
        res.render('error.ejs', {'error': err})
    });
}

app.get('/', home);

Ah, That's Better

A lot of library code, like Dojo and jQuery, takes advantage of Promises and Deferreds to make things cleaner. I hope that more and more userland code will do the same.

Finding Deferreds helped me learn to love JavaScript again. It's such a fun, flexible, interesting language, so I'm glad complex code can be easy to test and fun to read.

About James Socol

James Socol is a Python, JavaScript, and web developer and Mozilla's Community Platforms Manager. He is an active member of the Mozilla, Python, and Django communities and maintains several widely-used open source projects.

coffeeonthekeyboard.comjamessocolPosts