14. September 2013

Fast and Thorough Testing with Jasmine Flow

Automated testing helps to ensure that your software does not have regressions as you make changes. These changes include refactorings, new features, bug fixes, etc. A Software product has an intricate set of behavior which includes a large number of edge cases.

I'm a fan of black box testing, because it is decoupled from the implementation. The implication is the what is verified, rather than the how.

Spec frameworks such as rspec and jasmine give you the constructs to perform edge case testing. Unfortunately, the same setup is repeated when verifying different aspects of the state of the system. This leads to long test suites and even to the motivation to skip edge case testing (sacrificing test coverage) or combining tests (sacrificing clarity of intent of the test).

This juxtaposition led me to create jasmine-flow. The premise of jasmine-flow is to organize tests into a series of steps which you can make assertions up reaching a certain state.

flow("visit /login|invalid login|valid login", function(fl) {
  fl.step("visit /login", function() {
      // I'm assuming the usage a JSDOM here
      window.location.href = "http://yoursite.com/login";
      // setupPage is defined elsewhere to set up your page framework.
      setupPage();
    })
    .aver("shows a login form", function() {
      var $login = $("#login");
      expect($login.length).toEqual(1);
    })

    .step("fill out login form with invalid login credentials and submit", function() {
      $("#login .name").val("user@example.com");
      $("#login .password").val("wrong-password");
      $("#login button").click();
      $("#login").submit();
    })
    .aver("sends a POST /login with the name and password", function() {
      // Using http://github.com/pivotal/jasmine-ajax
      expect(ajaxRequests.length).toEqual(1);
      expect(mostRecentAjaxRequest.method).toEqual("POST");
      expect(mostRecentAjaxRequest.url).toEqual("/login");
      expect(JSON.parse(mostRecentAjaxRequest.params)).toEqual({name: "user@example.com", password: "wrong-password"});
    })
    .step("POST /login 403", function() {
      mostRecentAjaxRequest.response({
        status: 403,
        responseText: JSON.stringify({
          message: "Invalid email/password combination"
        })
      })
    })
    .aver("does not redirect to the home page", function() {
      expect(window.location.href).toEqual("http://yoursite.com/login");
    })
    .aver("shows an error", function() {
      expect($("#login .error").text()).toContain("Invalid email/password combination");
    })

    .step("fill out login form with valid login credentials and submit", function() {
      $("#login .name").val("user@example.com");
      $("#login .password").val("12345");
      $("#login button").click();
      $("#login").submit();
    })
    .aver("sends a POST /login with the name and password", function() {
      // Using http://github.com/pivotal/jasmine-ajax
      expect(ajaxRequests.length).toEqual(2);
      expect(mostRecentAjaxRequest.method).toEqual("POST");
      expect(mostRecentAjaxRequest.url).toEqual("/login");
      expect(JSON.parse(mostRecentAjaxRequest.params)).toEqual({name: "user@example.com", password: "12345"});
    })
    .step("POST /login 200", function() {
      mostRecentAjaxRequest.response({
        status: 200,
        responseText: JSON.stringify({
          name: "User",
          email: "user@example.com"
        })
      })
    .aver("redirects to the home page", function() {
      expect(window.location.href).toEqual("/");
    })
  ;
});

Compare this with the traditional nested describe solution in Jasmine…

describe("Someone visits /login", function() {
  beforeEach(function() {
    // I'm assuming the usage a JSDOM here
    window.location.href = "http://yoursite.com/login";
    // setupPage is defined elsewhere to set up your page framework.
    setupPage();
  });

  it("shows a login form", function() {
    var $login = $("#login");
    expect($login.length).toEqual(1);
  });

  describe("fill out and submitting the login form", function() {
    beforeEach(function() {
      $("#login .name").val("user@example.com");
      $("#login .password").val("12345");
      $("#login button").click();
      $("#login").submit();
    });

    it("sends a POST /login with the name and password", function() {
      // Using http://github.com/pivotal/jasmine-ajax
      expect(ajaxRequests.length).toEqual(1);
      expect(mostRecentAjaxRequest.method).toEqual("POST");
      expect(mostRecentAjaxRequest.url).toEqual("/login");
      expect(JSON.parse(mostRecentAjaxRequest.params)).toEqual({name: "user@example.com", password: "12345"});
    });

    describe("POST /login 200", function() {
      beforeEach(function() {
        mostRecentAjaxRequest.response({
          status: 200,
          responseText: JSON.stringify({
            name: "User",
            email: "user@example.com"
          })
        });
      });

      it("redirects to the home page", function() {
        expect(window.location.href).toEqual("/");
      });
    });

    describe("POST /login 403", function() {
      beforeEach(function() {
        mostRecentAjaxRequest.response({
          status: 403,
          responseText: JSON.stringify({
            message: "Invalid email/password combination"
          })
        });
      });

      it("does not redirect to the home page", function() {
        expect(window.location.href).toEqual("http://yoursite.com/login");
      });

      it("shows an error", function() {
        expect($("#login .error").text()).toContain("Invalid email/password combination");
      });
    });
  });
});

In the nested describe solution, each it requires identical setup code execution which is repeated for each test. This effectively creates a (n log n) test suite for each setup * it. In the jasmine-flow example, the setup code is executed one time, with assertions interspersed (n).

From a personal experience, I‘ve found that it is also easier to organize all of the edge cases since the structure is simpler as a linear flow, opposed to a tree. It’s a bit less pure, but more relevant to linear user actions.

On my current project, it yielded close to a 10x performance boost to my test suite (400 seconds to 40 seconds). I plan on using this pattern more often, since it also scales in complexity better than a describe (context) tree.

14. September 2013

Automated Black Box Testing

Automated testing encourages your software to behave a certain way going forward into the future.

When you write tests against your user facing requirements (black box and acceptance tests), you locking down how your app acts.

When you use automated tests for white box testing (unit tests), you lock down how your software is designed.

I‘ve worked on products which often has changing requirements, and whenever there were unit tests and I needed to make breaking internal changes, I’ve found myself having to make a decision to delete the tests, refactor them, or change the software in a different way.

That false decision led me to totally eschew white box testing as much as possible in favor of black box testing. It‘s been working great, since I’m free to improve the design of the software without being hobbled by irrelevant tests and false positives.

I also don‘t really have any false negatives, because I often don’t care about what I would have unit tested, just the external behavior.

I do see value of unit tests as a waypoint to help drive along TDD. However these unit tests are not useful, and are often counter productive (because they lock down the current design) as regression tests.

Log messages to help isolate where things break and they work in production environments to boot. This help alleviate the need for unit tests to isolate breakages.

08. September 2013

Re: Doing Good in the Addiction Economy

From my wordpress blog

Doing Good in the Addiction Economy

Interesting but long winded post, IMO. But then, maybe my impatience toward long posts is telling in the short term rewards culture we live in. Of course time is valuable, and trying to get to the core concept of some the writing is often the goal. We don’t seem to savor the journey as much, or the journey has changed to be our lives toward success…

America is a culture of accomplishment. We feel the not so subtle urge to be successful, especially in the eyes of our peers. Time is money and there are lots of shiny distractions…

Of course Japanese also has lots of shiny toys. Japanese culture also has a history toward mastery and zen. That may factor into why students sought to solve the impossible math problem. And then, what is life, perhaps an impossible problem? Maybe I should just enjoy the ride more and not worry about success or what others think of me.

I do like his usage of meditation to train himself to be more patient. My recent meditation sessions have been shortened by dissatisfaction to time elapsing. There's always room to improve. We could look at life like an impossible problem, or a journey.