diff --git a/features/support/env.rb b/features/support/env.rb
index 635d3735..0f513754 100644
--- a/features/support/env.rb
+++ b/features/support/env.rb
@@ -11,6 +11,6 @@ JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll')
 
 def run_jekyll(opts = {})
   command = JEKYLL_PATH
-  command << " >> /dev/null" if opts[:debug].nil?
+  command << " >> /dev/null 2>&1" if opts[:debug].nil?
   system command
 end
diff --git a/lib/jekyll.rb b/lib/jekyll.rb
index 3480137e..2d429c6c 100644
--- a/lib/jekyll.rb
+++ b/lib/jekyll.rb
@@ -63,14 +63,15 @@ module Jekyll
     source = override['source'] || Jekyll::DEFAULTS['source']
 
     # Get configuration from <source>/_config.yml
-    config = {}
     config_file = File.join(source, '_config.yml')
     begin
       config = YAML.load_file(config_file)
-      puts "Configuration from #{config_file}"
+      raise "Invalid configuration - #{config_file}" if !config.is_a?(Hash)
+      STDOUT.puts "Configuration from #{config_file}"
     rescue => err
-      puts "WARNING: Could not read configuration. Using defaults (and options)."
-      puts "\t" + err
+      STDERR.puts "WARNING: Could not read configuration. Using defaults (and options)."
+      STDERR.puts "\t" + err
+      config = {}
     end
 
     # Merge DEFAULTS < _config.yml < override
diff --git a/test/test_configuration.rb b/test/test_configuration.rb
new file mode 100644
index 00000000..3b05ed51
--- /dev/null
+++ b/test/test_configuration.rb
@@ -0,0 +1,29 @@
+require File.dirname(__FILE__) + '/helper'
+
+class TestConfiguration < Test::Unit::TestCase
+  context "loading configuration" do
+    setup do
+      @path = './_config.yml'
+    end
+
+    should "fire warning with no _config.yml" do
+      mock(YAML).load_file(@path) { raise "No such file or directory - #{@path}" }
+      mock(STDERR).puts("WARNING: Could not read configuration. Using defaults (and options).")
+      mock(STDERR).puts("\tNo such file or directory - #{@path}")
+      assert_equal Jekyll::DEFAULTS, Jekyll.configuration({})
+    end
+
+    should "load configuration as hash" do
+      mock(YAML).load_file(@path) { Hash.new }
+      mock(STDOUT).puts("Configuration from #{@path}")
+      assert_equal Jekyll::DEFAULTS, Jekyll.configuration({})
+    end
+
+    should "fire warning with bad config" do
+      mock(YAML).load_file(@path) { Array.new }
+      mock(STDERR).puts("WARNING: Could not read configuration. Using defaults (and options).")
+      mock(STDERR).puts("\tInvalid configuration - #{@path}")
+      assert_equal Jekyll::DEFAULTS, Jekyll.configuration({})
+    end
+  end
+end