From 11917645f2159e7c94449e1103237dddee0f42de Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Sat, 15 Nov 2014 22:47:22 -0800
Subject: [PATCH 01/17] Incremental regeneration

---
 lib/jekyll.rb              |  1 +
 lib/jekyll/cleaner.rb      |  9 +++-
 lib/jekyll/command.rb      |  1 +
 lib/jekyll/convertible.rb  |  6 +++
 lib/jekyll/layout.rb       |  4 ++
 lib/jekyll/metadata.rb     | 87 ++++++++++++++++++++++++++++++++++++++
 lib/jekyll/renderer.rb     |  6 +++
 lib/jekyll/site.rb         | 21 +++++++--
 lib/jekyll/tags/include.rb |  9 +++-
 site/.gitignore            |  1 +
 10 files changed, 139 insertions(+), 6 deletions(-)
 create mode 100644 lib/jekyll/metadata.rb

diff --git a/lib/jekyll.rb b/lib/jekyll.rb
index 03724100..0a7c2b69 100644
--- a/lib/jekyll.rb
+++ b/lib/jekyll.rb
@@ -48,6 +48,7 @@ module Jekyll
   autoload :Layout,              'jekyll/layout'
   autoload :LayoutReader,        'jekyll/layout_reader'
   autoload :LogAdapter,          'jekyll/log_adapter'
+  autoload :Metadata,            'jekyll/metadata'
   autoload :Page,                'jekyll/page'
   autoload :PluginManager,       'jekyll/plugin_manager'
   autoload :Post,                'jekyll/post'
diff --git a/lib/jekyll/cleaner.rb b/lib/jekyll/cleaner.rb
index 6dd59ea0..79f97f47 100644
--- a/lib/jekyll/cleaner.rb
+++ b/lib/jekyll/cleaner.rb
@@ -21,7 +21,14 @@ module Jekyll
       #
       # Returns an Array of the file and directory paths
       def obsolete_files
-        (existing_files - new_files - new_dirs + replaced_files).to_a
+        (existing_files - new_files - new_dirs + replaced_files + metadata_file).to_a
+      end
+
+      # Private: The metadata file storing dependency tree and build history
+      #
+      # Returns an Array with the metdata file as the only item
+      def metadata_file
+        [site.metadata.metadata_file]
       end
 
       # Private: The list of existing files, apart from those included in keep_files and hidden files.
diff --git a/lib/jekyll/command.rb b/lib/jekyll/command.rb
index a1cc8c01..d463d235 100644
--- a/lib/jekyll/command.rb
+++ b/lib/jekyll/command.rb
@@ -58,6 +58,7 @@ module Jekyll
         c.option 'unpublished', '--unpublished', 'Render posts that were marked as unpublished'
         c.option 'quiet',   '-q', '--quiet', 'Silence output.'
         c.option 'verbose', '-V', '--verbose', 'Print verbose output.'
+        c.option 'clean', '-c', '--clean', 'Clean the site before rebuilding.'
       end
 
     end
diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb
index 4b21a66b..4b1f3761 100644
--- a/lib/jekyll/convertible.rb
+++ b/lib/jekyll/convertible.rb
@@ -207,6 +207,12 @@ module Jekyll
                                          info,
                                          File.join(site.config['layouts'], layout.name))
 
+        # Add layout to dependency tree
+        site.metadata.add_dependency(
+          Jekyll.sanitized_path(site.source, path),
+          Jekyll.sanitized_path(site.source, layout.path)
+        )
+
         if layout = layouts[layout.data["layout"]]
           if used.include?(layout)
             layout = nil # avoid recursive chain
diff --git a/lib/jekyll/layout.rb b/lib/jekyll/layout.rb
index 4dde59b6..f973019f 100644
--- a/lib/jekyll/layout.rb
+++ b/lib/jekyll/layout.rb
@@ -8,6 +8,9 @@ module Jekyll
     # Gets the name of this layout.
     attr_reader :name
 
+    # Gets the path to this layout.
+    attr_reader :path
+
     # Gets/Sets the extension of this layout.
     attr_accessor :ext
 
@@ -26,6 +29,7 @@ module Jekyll
       @site = site
       @base = base
       @name = name
+      @path = Jekyll.sanitized_path(site.source, File.join(base, name))
 
       self.data = {}
 
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
new file mode 100644
index 00000000..abda35e5
--- /dev/null
+++ b/lib/jekyll/metadata.rb
@@ -0,0 +1,87 @@
+require 'set'
+
+module Jekyll
+  class Metadata
+    attr_reader :site, :metadata
+
+    def initialize(site)
+      @site = site
+
+      # Initialize metadata store by reading YAML file,
+      # or an empty hash if file does not exist
+      @metadata = (File.file?(metadata_file) && !(site.config['clean'])) ? SafeYAML.load(File.read(metadata_file)) : {}
+
+      # Initialize cache to an empty hash
+      @cache = {}
+    end
+
+    # Add a path to the metadata
+    #
+    # Returns true.
+    def add(path)
+      @metadata[path] = {
+        "mtime" => File.mtime(path),
+        "deps" => []
+      }
+      @cache[path] = true
+    end
+
+    # Force a path to regenerate
+    #
+    # Returns true.
+    def force(path)
+      @cache[path] = true
+    end
+
+    # Checks if a path should be regenerated
+    #
+    # Returns a boolean.
+    def regenerate?(path)
+      # Check for path in cache
+      if @cache.has_key? path
+        return @cache[path]
+      end
+
+      # Check path that exists in metadata
+      if (data = @metadata[path])
+        data["deps"].each do |dependency|
+          if regenerate?(dependency)
+            return @cache[dependency] = @cache[path] = true
+          end
+        end
+        if data["mtime"] == File.mtime(path)
+          return @cache[path] = false
+        else
+          return add(path)
+        end
+      end
+
+      # Path does not exist in metadata, add it
+      return add(path)
+    end
+
+    # Add a dependency of a path
+    #
+    # Returns true.
+    def add_dependency(path, dependency)
+      @metadata[path]["deps"] << dependency unless @metadata[path]["deps"].include? dependency
+      add(dependency)
+    end
+
+    # Write the metadata to disk
+    #
+    # Returns nothing.
+    def write
+      File.open(metadata_file, 'w') do |f|
+        f.write(@metadata.to_yaml)
+      end
+    end
+
+    # Produce the absolute path of the metadata file
+    #
+    # Returns the String path of the file.
+    def metadata_file
+      Jekyll.sanitized_path(site.source, '.jekyll-metadata')
+    end
+  end
+end
diff --git a/lib/jekyll/renderer.rb b/lib/jekyll/renderer.rb
index f88a4187..61839353 100644
--- a/lib/jekyll/renderer.rb
+++ b/lib/jekyll/renderer.rb
@@ -138,6 +138,12 @@ module Jekyll
           File.join(site.config['layouts'], layout.name)
         )
 
+        # Add layout to dependency tree
+        site.metadata.add_dependency(
+          Jekyll.sanitized_path(site.source, document.path),
+          Jekyll.sanitized_path(site.source, layout.path)
+        )
+
         if layout = site.layouts[layout.data["layout"]]
           if used.include?(layout)
             layout = nil # avoid recursive chain
diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index 115b4133..38c61661 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -11,6 +11,7 @@ module Jekyll
                   :gems, :plugin_manager
 
     attr_accessor :converters, :generators
+    attr_accessor :metadata
 
     # Public: Initialize a new Site.
     #
@@ -27,6 +28,9 @@ module Jekyll
       @source              = File.expand_path(config['source']).freeze
       @dest                = File.expand_path(config['destination']).freeze
 
+      # Build metadata
+      @metadata = Metadata.new(self)
+
       self.plugin_manager = Jekyll::PluginManager.new(self)
       self.plugins        = plugin_manager.plugins_path
 
@@ -49,7 +53,7 @@ module Jekyll
       read
       generate
       render
-      cleanup
+      cleanup if config['clean']
       write
     end
 
@@ -289,13 +293,17 @@ module Jekyll
 
       collections.each do |label, collection|
         collection.docs.each do |document|
-          document.output = Jekyll::Renderer.new(self, document).run
+          if @metadata.regenerate?(document.path)
+            document.output = Jekyll::Renderer.new(self, document).run
+          end
         end
       end
 
       payload = site_payload
       [posts, pages].flatten.each do |page_or_post|
-        page_or_post.render(layouts, payload)
+        if @metadata.regenerate?(Jekyll.sanitized_path(source, page_or_post.relative_path))
+          page_or_post.render(layouts, payload)
+        end
       end
     rescue Errno::ENOENT => e
       # ignore missing layout dir
@@ -312,7 +320,12 @@ module Jekyll
     #
     # Returns nothing.
     def write
-      each_site_file { |item| item.write(dest) }
+      each_site_file { |item|
+        if @metadata.regenerate? Jekyll.sanitized_path(source, item.path)
+          item.write(dest)
+        end
+      }
+      @metadata.write
     end
 
     # Construct a Hash of Posts indexed by the specified Post attribute.
diff --git a/lib/jekyll/tags/include.rb b/lib/jekyll/tags/include.rb
index 3eb4d7c0..8f7da2ca 100644
--- a/lib/jekyll/tags/include.rb
+++ b/lib/jekyll/tags/include.rb
@@ -105,13 +105,20 @@ eos
       end
 
       def render(context)
+        site = context.registers[:site]
         dir = resolved_includes_dir(context)
 
         file = render_variable(context) || @file
         validate_file_name(file)
 
         path = File.join(dir, file)
-        validate_path(path, dir, context.registers[:site].safe)
+        validate_path(path, dir, site.safe)
+
+        # Add include to dependency tree
+        site.metadata.add_dependency(
+          Jekyll.sanitized_path(site.source, context.registers[:page]["path"]),
+          path
+        )
 
         begin
           partial = Liquid::Template.parse(source(path, context))
diff --git a/site/.gitignore b/site/.gitignore
index 79bd74f7..c1bd964c 100644
--- a/site/.gitignore
+++ b/site/.gitignore
@@ -2,3 +2,4 @@ _site/
 *.swp
 pkg/
 test/
+.jekyll-metadata

From 842470b0c4d3a1e738d6226cd5800779ffab6870 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Sun, 16 Nov 2014 15:41:58 -0800
Subject: [PATCH 02/17] Refinements

---
 .gitignore                  | 1 +
 lib/jekyll/configuration.rb | 2 +-
 lib/jekyll/metadata.rb      | 4 ++--
 site/.gitignore             | 1 -
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index aa6c14f8..d9effd5b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ coverage
 .ruby-version
 .sass-cache
 tmp/stackprof-*
+.jekyll-metadata
diff --git a/lib/jekyll/configuration.rb b/lib/jekyll/configuration.rb
index 7f1148a6..24e96898 100644
--- a/lib/jekyll/configuration.rb
+++ b/lib/jekyll/configuration.rb
@@ -17,7 +17,7 @@ module Jekyll
       # Handling Reading
       'safe'          => false,
       'include'       => ['.htaccess'],
-      'exclude'       => [],
+      'exclude'       => ['.jekyll-metadata'],
       'keep_files'    => ['.git','.svn'],
       'encoding'      => 'utf-8',
       'markdown_ext'  => 'markdown,mkdown,mkdn,mkd,md',
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index abda35e5..72cf8d15 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -62,10 +62,10 @@ module Jekyll
 
     # Add a dependency of a path
     #
-    # Returns true.
+    # Returns nothing.
     def add_dependency(path, dependency)
       @metadata[path]["deps"] << dependency unless @metadata[path]["deps"].include? dependency
-      add(dependency)
+      regenerate? dependency
     end
 
     # Write the metadata to disk
diff --git a/site/.gitignore b/site/.gitignore
index c1bd964c..79bd74f7 100644
--- a/site/.gitignore
+++ b/site/.gitignore
@@ -2,4 +2,3 @@ _site/
 *.swp
 pkg/
 test/
-.jekyll-metadata

From d438362971b8a8ceaaa8d5cea6207e7db398aea8 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Sun, 16 Nov 2014 16:04:11 -0800
Subject: [PATCH 03/17] Add regenerate front-matter variable

---
 lib/jekyll/site.rb | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index 38c61661..3894a2ba 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -293,17 +293,20 @@ module Jekyll
 
       collections.each do |label, collection|
         collection.docs.each do |document|
-          if @metadata.regenerate?(document.path)
-            document.output = Jekyll::Renderer.new(self, document).run
+          document.output = Jekyll::Renderer.new(self, document).run if (
+            @metadata.regenerate?(document.path) ||
+            document.data['regenerate']
+          )
           end
         end
       end
 
       payload = site_payload
       [posts, pages].flatten.each do |page_or_post|
-        if @metadata.regenerate?(Jekyll.sanitized_path(source, page_or_post.relative_path))
-          page_or_post.render(layouts, payload)
-        end
+        page_or_post.render(layouts, payload) if (
+          @metadata.regenerate?(Jekyll.sanitized_path(source, page_or_post.relative_path)) ||
+          page_or_post.data['regenerate']
+        )
       end
     rescue Errno::ENOENT => e
       # ignore missing layout dir
@@ -321,9 +324,10 @@ module Jekyll
     # Returns nothing.
     def write
       each_site_file { |item|
-        if @metadata.regenerate? Jekyll.sanitized_path(source, item.path)
-          item.write(dest)
-        end
+        item.write(dest) if (
+          @metadata.regenerate?(Jekyll.sanitized_path(source, item.path)) ||
+          (item.respond_to?(:data) && item.data['regenerate'])
+        )
       }
       @metadata.write
     end

From fe6bfc6f1b2540dad111153d855a5ec91b2fbf33 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Fri, 21 Nov 2014 21:54:18 -0800
Subject: [PATCH 04/17] Fix failing tests

---
 lib/jekyll/cleaner.rb      |  3 ++-
 lib/jekyll/metadata.rb     | 15 ++++++++++++---
 lib/jekyll/renderer.rb     |  2 +-
 lib/jekyll/site.rb         |  5 ++---
 lib/jekyll/tags/include.rb | 10 ++++++----
 test/helper.rb             |  1 +
 test/test_document.rb      |  3 ++-
 test/test_site.rb          | 15 +++++++++++----
 8 files changed, 37 insertions(+), 17 deletions(-)

diff --git a/lib/jekyll/cleaner.rb b/lib/jekyll/cleaner.rb
index 79f97f47..e7ee44a0 100644
--- a/lib/jekyll/cleaner.rb
+++ b/lib/jekyll/cleaner.rb
@@ -13,6 +13,7 @@ module Jekyll
       # Cleans up the site's destination directory
       def cleanup!
         FileUtils.rm_rf(obsolete_files)
+        FileUtils.rm_rf(metadata_file) if @site.config["clean"]
       end
 
       private
@@ -21,7 +22,7 @@ module Jekyll
       #
       # Returns an Array of the file and directory paths
       def obsolete_files
-        (existing_files - new_files - new_dirs + replaced_files + metadata_file).to_a
+        (existing_files - new_files - new_dirs + replaced_files).to_a
       end
 
       # Private: The metadata file storing dependency tree and build history
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index 72cf8d15..275ee9da 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -33,10 +33,18 @@ module Jekyll
       @cache[path] = true
     end
 
+    # Clear the metadata and cache
+    #
+    # Returns nothing
+    def clear
+      @metadata = {}
+      @cache = {}
+    end
+
     # Checks if a path should be regenerated
     #
     # Returns a boolean.
-    def regenerate?(path)
+    def regenerate?(path, add = true)
       # Check for path in cache
       if @cache.has_key? path
         return @cache[path]
@@ -52,18 +60,19 @@ module Jekyll
         if data["mtime"] == File.mtime(path)
           return @cache[path] = false
         else
-          return add(path)
+          return !add || add(path)
         end
       end
 
       # Path does not exist in metadata, add it
-      return add(path)
+      return !add || add(path)
     end
 
     # Add a dependency of a path
     #
     # Returns nothing.
     def add_dependency(path, dependency)
+      add(path) if @metadata[path].nil?
       @metadata[path]["deps"] << dependency unless @metadata[path]["deps"].include? dependency
       regenerate? dependency
     end
diff --git a/lib/jekyll/renderer.rb b/lib/jekyll/renderer.rb
index 61839353..d2dd095f 100644
--- a/lib/jekyll/renderer.rb
+++ b/lib/jekyll/renderer.rb
@@ -142,7 +142,7 @@ module Jekyll
         site.metadata.add_dependency(
           Jekyll.sanitized_path(site.source, document.path),
           Jekyll.sanitized_path(site.source, layout.path)
-        )
+        ) if document.write?
 
         if layout = site.layouts[layout.data["layout"]]
           if used.include?(layout)
diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index 3894a2ba..f00e24c9 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -53,7 +53,7 @@ module Jekyll
       read
       generate
       render
-      cleanup if config['clean']
+      cleanup
       write
     end
 
@@ -294,10 +294,9 @@ module Jekyll
       collections.each do |label, collection|
         collection.docs.each do |document|
           document.output = Jekyll::Renderer.new(self, document).run if (
-            @metadata.regenerate?(document.path) ||
+            @metadata.regenerate?(document.path, document.write?) ||
             document.data['regenerate']
           )
-          end
         end
       end
 
diff --git a/lib/jekyll/tags/include.rb b/lib/jekyll/tags/include.rb
index 8f7da2ca..49e7357e 100644
--- a/lib/jekyll/tags/include.rb
+++ b/lib/jekyll/tags/include.rb
@@ -115,10 +115,12 @@ eos
         validate_path(path, dir, site.safe)
 
         # Add include to dependency tree
-        site.metadata.add_dependency(
-          Jekyll.sanitized_path(site.source, context.registers[:page]["path"]),
-          path
-        )
+        if context.registers[:page] and context.registers[:page].has_key? "path"
+          site.metadata.add_dependency(
+            Jekyll.sanitized_path(site.source, context.registers[:page]["path"]),
+            path
+          )
+        end
 
         begin
           partial = Liquid::Template.parse(source(path, context))
diff --git a/test/helper.rb b/test/helper.rb
index ea3777ed..58d9e24a 100644
--- a/test/helper.rb
+++ b/test/helper.rb
@@ -46,6 +46,7 @@ class Test::Unit::TestCase
 
   def clear_dest
     FileUtils.rm_rf(dest_dir)
+    FileUtils.rm_rf(source_dir('.jekyll-metadata'))
   end
 
   def test_dir(*subdirs)
diff --git a/test/test_document.rb b/test/test_document.rb
index 90c16b82..cf6c2cbf 100644
--- a/test/test_document.rb
+++ b/test/test_document.rb
@@ -246,7 +246,8 @@ class TestDocument < Test::Unit::TestCase
           }
         },
         "source"      => source_dir,
-        "destination" => dest_dir
+        "destination" => dest_dir,
+        "clean"       => true
       }))
       @site.process
       @document = @site.collections["slides"].files.find { |doc| doc.relative_path == "_slides/octojekyll.png" }
diff --git a/test/test_site.rb b/test/test_site.rb
index 4b8409e7..2fd6ad36 100644
--- a/test/test_site.rb
+++ b/test/test_site.rb
@@ -99,6 +99,7 @@ class TestSite < Test::Unit::TestCase
     should "write only modified static files" do
       clear_dest
       StaticFile.reset_cache
+      @site.metadata.clear
 
       @site.process
       some_static_file = @site.static_files[0].path
@@ -128,6 +129,7 @@ class TestSite < Test::Unit::TestCase
     should "write static files if not modified but missing in destination" do
       clear_dest
       StaticFile.reset_cache
+      @site.metadata.clear
 
       @site.process
       some_static_file = @site.static_files[0].path
@@ -241,6 +243,7 @@ class TestSite < Test::Unit::TestCase
     context 'with orphaned files in destination' do
       setup do
         clear_dest
+        @site.metadata.clear
         @site.process
         # generate some orphaned files:
         # single file
@@ -328,7 +331,7 @@ class TestSite < Test::Unit::TestCase
         end
 
         bad_processor = "Custom::Markdown"
-        s = Site.new(site_configuration('markdown' => bad_processor))
+        s = Site.new(site_configuration('markdown' => bad_processor, 'clean' => true))
         assert_raise Jekyll::Errors::FatalException do
           s.process
         end
@@ -348,7 +351,7 @@ class TestSite < Test::Unit::TestCase
 
       should 'throw FatalException at process time' do
         bad_processor = 'not a processor name'
-        s = Site.new(site_configuration('markdown' => bad_processor))
+        s = Site.new(site_configuration('markdown' => bad_processor, 'clean' => true))
         assert_raise Jekyll::Errors::FatalException do
           s.process
         end
@@ -418,7 +421,9 @@ class TestSite < Test::Unit::TestCase
 
     context "manipulating the Jekyll environment" do
       setup do
-        @site = Site.new(site_configuration)
+        @site = Site.new(site_configuration({
+          "clean" => true
+        }))
         @site.process
         @page = @site.pages.find { |p| p.name == "environment.html" }
       end
@@ -430,7 +435,9 @@ class TestSite < Test::Unit::TestCase
       context "in production" do
         setup do
           ENV["JEKYLL_ENV"] = "production"
-          @site = Site.new(site_configuration)
+          @site = Site.new(site_configuration({
+            "clean" => true
+          }))
           @site.process
           @page = @site.pages.find { |p| p.name == "environment.html" }
         end

From ac03af3229777ddd3fbfb43a646339789e32850a Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Fri, 21 Nov 2014 22:10:08 -0800
Subject: [PATCH 05/17] Implement @mattr-'s suggestions

---
 lib/jekyll/cleaner.rb       |  2 +-
 lib/jekyll/command.rb       |  2 +-
 lib/jekyll/configuration.rb |  2 +-
 lib/jekyll/metadata.rb      | 15 ++++++++++++---
 lib/jekyll/site.rb          |  8 ++++----
 test/test_document.rb       |  2 +-
 test/test_site.rb           |  8 ++++----
 7 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/lib/jekyll/cleaner.rb b/lib/jekyll/cleaner.rb
index e7ee44a0..00191db7 100644
--- a/lib/jekyll/cleaner.rb
+++ b/lib/jekyll/cleaner.rb
@@ -13,7 +13,7 @@ module Jekyll
       # Cleans up the site's destination directory
       def cleanup!
         FileUtils.rm_rf(obsolete_files)
-        FileUtils.rm_rf(metadata_file) if @site.config["clean"]
+        FileUtils.rm_rf(metadata_file) if @site.config["full_rebuild"]
       end
 
       private
diff --git a/lib/jekyll/command.rb b/lib/jekyll/command.rb
index d463d235..423c3202 100644
--- a/lib/jekyll/command.rb
+++ b/lib/jekyll/command.rb
@@ -58,7 +58,7 @@ module Jekyll
         c.option 'unpublished', '--unpublished', 'Render posts that were marked as unpublished'
         c.option 'quiet',   '-q', '--quiet', 'Silence output.'
         c.option 'verbose', '-V', '--verbose', 'Print verbose output.'
-        c.option 'clean', '-c', '--clean', 'Clean the site before rebuilding.'
+        c.option 'full_rebuild', '-f', '--full-rebuild', 'Clean the site before rebuilding.'
       end
 
     end
diff --git a/lib/jekyll/configuration.rb b/lib/jekyll/configuration.rb
index 24e96898..7f1148a6 100644
--- a/lib/jekyll/configuration.rb
+++ b/lib/jekyll/configuration.rb
@@ -17,7 +17,7 @@ module Jekyll
       # Handling Reading
       'safe'          => false,
       'include'       => ['.htaccess'],
-      'exclude'       => ['.jekyll-metadata'],
+      'exclude'       => [],
       'keep_files'    => ['.git','.svn'],
       'encoding'      => 'utf-8',
       'markdown_ext'  => 'markdown,mkdown,mkdn,mkd,md',
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index 275ee9da..2012afc6 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -7,9 +7,8 @@ module Jekyll
     def initialize(site)
       @site = site
 
-      # Initialize metadata store by reading YAML file,
-      # or an empty hash if file does not exist
-      @metadata = (File.file?(metadata_file) && !(site.config['clean'])) ? SafeYAML.load(File.read(metadata_file)) : {}
+      # Read metadata from file
+      read_metadata
 
       # Initialize cache to an empty hash
       @cache = {}
@@ -92,5 +91,15 @@ module Jekyll
     def metadata_file
       Jekyll.sanitized_path(site.source, '.jekyll-metadata')
     end
+
+    private
+
+    # Read metadata from the metadata file, if no file is found,
+    # initialize with an empty hash
+    #
+    # Returns the read metadata.
+    def read_metadata
+      @metadata = (File.file?(metadata_file) && !(site.config['full_rebuild'])) ? SafeYAML.load(File.read(metadata_file)) : {}
+    end
   end
 end
diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index f00e24c9..738cae0e 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -294,7 +294,7 @@ module Jekyll
       collections.each do |label, collection|
         collection.docs.each do |document|
           document.output = Jekyll::Renderer.new(self, document).run if (
-            @metadata.regenerate?(document.path, document.write?) ||
+            metadata.regenerate?(document.path, document.write?) ||
             document.data['regenerate']
           )
         end
@@ -303,7 +303,7 @@ module Jekyll
       payload = site_payload
       [posts, pages].flatten.each do |page_or_post|
         page_or_post.render(layouts, payload) if (
-          @metadata.regenerate?(Jekyll.sanitized_path(source, page_or_post.relative_path)) ||
+          metadata.regenerate?(Jekyll.sanitized_path(source, page_or_post.relative_path)) ||
           page_or_post.data['regenerate']
         )
       end
@@ -324,11 +324,11 @@ module Jekyll
     def write
       each_site_file { |item|
         item.write(dest) if (
-          @metadata.regenerate?(Jekyll.sanitized_path(source, item.path)) ||
+          metadata.regenerate?(Jekyll.sanitized_path(source, item.path)) ||
           (item.respond_to?(:data) && item.data['regenerate'])
         )
       }
-      @metadata.write
+      metadata.write
     end
 
     # Construct a Hash of Posts indexed by the specified Post attribute.
diff --git a/test/test_document.rb b/test/test_document.rb
index cf6c2cbf..2ff688ba 100644
--- a/test/test_document.rb
+++ b/test/test_document.rb
@@ -247,7 +247,7 @@ class TestDocument < Test::Unit::TestCase
         },
         "source"      => source_dir,
         "destination" => dest_dir,
-        "clean"       => true
+        "full_rebuild"       => true
       }))
       @site.process
       @document = @site.collections["slides"].files.find { |doc| doc.relative_path == "_slides/octojekyll.png" }
diff --git a/test/test_site.rb b/test/test_site.rb
index 2fd6ad36..a6ef5af8 100644
--- a/test/test_site.rb
+++ b/test/test_site.rb
@@ -331,7 +331,7 @@ class TestSite < Test::Unit::TestCase
         end
 
         bad_processor = "Custom::Markdown"
-        s = Site.new(site_configuration('markdown' => bad_processor, 'clean' => true))
+        s = Site.new(site_configuration('markdown' => bad_processor, 'full_rebuild' => true))
         assert_raise Jekyll::Errors::FatalException do
           s.process
         end
@@ -351,7 +351,7 @@ class TestSite < Test::Unit::TestCase
 
       should 'throw FatalException at process time' do
         bad_processor = 'not a processor name'
-        s = Site.new(site_configuration('markdown' => bad_processor, 'clean' => true))
+        s = Site.new(site_configuration('markdown' => bad_processor, 'full_rebuild' => true))
         assert_raise Jekyll::Errors::FatalException do
           s.process
         end
@@ -422,7 +422,7 @@ class TestSite < Test::Unit::TestCase
     context "manipulating the Jekyll environment" do
       setup do
         @site = Site.new(site_configuration({
-          "clean" => true
+          'full_rebuild' => true
         }))
         @site.process
         @page = @site.pages.find { |p| p.name == "environment.html" }
@@ -436,7 +436,7 @@ class TestSite < Test::Unit::TestCase
         setup do
           ENV["JEKYLL_ENV"] = "production"
           @site = Site.new(site_configuration({
-            "clean" => true
+            'full_rebuild' => true
           }))
           @site.process
           @page = @site.pages.find { |p| p.name == "environment.html" }

From 4acf343fea062e26def0cdef77c45a596a1a30ca Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Sun, 23 Nov 2014 12:51:19 -0800
Subject: [PATCH 06/17] Add clean command

---
 lib/jekyll/commands/clean.rb | 42 ++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 lib/jekyll/commands/clean.rb

diff --git a/lib/jekyll/commands/clean.rb b/lib/jekyll/commands/clean.rb
new file mode 100644
index 00000000..7d787d30
--- /dev/null
+++ b/lib/jekyll/commands/clean.rb
@@ -0,0 +1,42 @@
+module Jekyll
+  module Commands
+    class Clean < Command
+      class << self
+
+        def init_with_program(prog)
+          prog.command(:clean) do |c|
+            c.syntax 'clean [subcommand]'
+            c.description 'Clean the site (removes site output and metadata file) without building.'
+
+            c.action do |args, _|
+              Jekyll::Commands::Clean.process({})
+            end
+          end
+        end
+
+        def process(options)
+          options = configuration_from_options(options)
+          destination = options['destination']
+          metadata_file = File.join(options['source'], '.jekyll-metadata')
+
+          if File.directory? destination
+            Jekyll.logger.info "Cleaning #{destination}..."
+            FileUtils.rm_rf(destination)
+            Jekyll.logger.info "", "done."
+          else
+            Jekyll.logger.info "Nothing to do for #{destination}."
+          end
+
+          if File.file? metadata_file
+            Jekyll.logger.info "Removing #{metadata_file}..."
+            FileUtils.rm_rf(metadata_file)
+            Jekyll.logger.info "", "done."
+          else
+            Jekyll.logger.info "Nothing to do for #{metadata_file}."
+          end
+        end
+
+      end
+    end
+  end
+end

From 75c5c162979b7cd729131e682b4ac7a299ca9067 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Sun, 23 Nov 2014 14:06:29 -0800
Subject: [PATCH 07/17] Handle path overrides

---
 lib/jekyll/metadata.rb | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index 2012afc6..68a31a05 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -16,8 +16,10 @@ module Jekyll
 
     # Add a path to the metadata
     #
-    # Returns true.
+    # Returns true, also on failure.
     def add(path)
+      return true if not File.exist? path
+
       @metadata[path] = {
         "mtime" => File.mtime(path),
         "deps" => []
@@ -71,7 +73,8 @@ module Jekyll
     #
     # Returns nothing.
     def add_dependency(path, dependency)
-      add(path) if @metadata[path].nil?
+      return if @metadata[path].nil?
+
       @metadata[path]["deps"] << dependency unless @metadata[path]["deps"].include? dependency
       regenerate? dependency
     end

From 2a5cf11ee27c0faeda1009d6a8099882230038fc Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Sun, 23 Nov 2014 15:38:00 -0800
Subject: [PATCH 08/17] Add --no-metadata option

---
 lib/jekyll/command.rb  |  1 +
 lib/jekyll/metadata.rb | 10 ++++++++--
 lib/jekyll/site.rb     |  2 +-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/jekyll/command.rb b/lib/jekyll/command.rb
index 423c3202..658ce597 100644
--- a/lib/jekyll/command.rb
+++ b/lib/jekyll/command.rb
@@ -59,6 +59,7 @@ module Jekyll
         c.option 'quiet',   '-q', '--quiet', 'Silence output.'
         c.option 'verbose', '-V', '--verbose', 'Print verbose output.'
         c.option 'full_rebuild', '-f', '--full-rebuild', 'Clean the site before rebuilding.'
+        c.option 'no_metadata', '--no-metadata', 'Disable incremental regeneration.'
       end
 
     end
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index 68a31a05..3e242d69 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -46,6 +46,8 @@ module Jekyll
     #
     # Returns a boolean.
     def regenerate?(path, add = true)
+      return true if site.config['no_metadata']
+
       # Check for path in cache
       if @cache.has_key? path
         return @cache[path]
@@ -73,7 +75,7 @@ module Jekyll
     #
     # Returns nothing.
     def add_dependency(path, dependency)
-      return if @metadata[path].nil?
+      return if (@metadata[path].nil? || site.config['no_metadata'])
 
       @metadata[path]["deps"] << dependency unless @metadata[path]["deps"].include? dependency
       regenerate? dependency
@@ -102,7 +104,11 @@ module Jekyll
     #
     # Returns the read metadata.
     def read_metadata
-      @metadata = (File.file?(metadata_file) && !(site.config['full_rebuild'])) ? SafeYAML.load(File.read(metadata_file)) : {}
+      @metadata = if !(site.config['full_rebuild'] || site.config['no_metadata']) && File.file?(metadata_file)
+        SafeYAML.load(File.read(metadata_file))
+      else
+        {}
+      end
     end
   end
 end
diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index 738cae0e..69f11a75 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -328,7 +328,7 @@ module Jekyll
           (item.respond_to?(:data) && item.data['regenerate'])
         )
       }
-      metadata.write
+      metadata.write unless site.config['no_metadata']
     end
 
     # Construct a Hash of Posts indexed by the specified Post attribute.

From dc30114605bd4c8f121a02e3d4b1aa5e700fde22 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Sun, 23 Nov 2014 15:45:49 -0800
Subject: [PATCH 09/17] Use site.in_source_dir

---
 lib/jekyll.rb              |  1 +
 lib/jekyll/convertible.rb  | 12 +++++++++--
 lib/jekyll/document.rb     |  8 ++++++++
 lib/jekyll/layout.rb       |  2 +-
 lib/jekyll/metadata.rb     | 42 ++++++++++++++++++++------------------
 lib/jekyll/renderer.rb     |  4 ++--
 lib/jekyll/site.rb         | 19 +++++------------
 lib/jekyll/static_file.rb  |  2 ++
 lib/jekyll/tags/include.rb |  2 +-
 test/test_document.rb      |  6 +++---
 10 files changed, 55 insertions(+), 43 deletions(-)

diff --git a/lib/jekyll.rb b/lib/jekyll.rb
index 0a7c2b69..911b3ddb 100644
--- a/lib/jekyll.rb
+++ b/lib/jekyll.rb
@@ -21,6 +21,7 @@ require 'time'
 require 'English'
 require 'pathname'
 require 'logger'
+require 'set'
 
 # 3rd party
 require 'safe_yaml/load'
diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb
index 4b1f3761..2c0d240e 100644
--- a/lib/jekyll/convertible.rb
+++ b/lib/jekyll/convertible.rb
@@ -168,6 +168,14 @@ module Jekyll
       true
     end
 
+    # Determine whether to regenerate the file based on metadata.
+    #
+    # Returns true if file needs to be regenerated
+    def regenerate?
+      site.metadata.regenerate?(site.in_source_dir(relative_path)) ||
+        data['regenerate']
+    end
+
     # Determine whether the file should be placed into layouts.
     #
     # Returns false if the document is an asset file.
@@ -209,8 +217,8 @@ module Jekyll
 
         # Add layout to dependency tree
         site.metadata.add_dependency(
-          Jekyll.sanitized_path(site.source, path),
-          Jekyll.sanitized_path(site.source, layout.path)
+          site.in_source_dir(path),
+          site.in_source_dir(layout.path)
         )
 
         if layout = layouts[layout.data["layout"]]
diff --git a/lib/jekyll/document.rb b/lib/jekyll/document.rb
index 003c04eb..356ce397 100644
--- a/lib/jekyll/document.rb
+++ b/lib/jekyll/document.rb
@@ -105,6 +105,14 @@ module Jekyll
       !(coffeescript_file? || yaml_file?)
     end
 
+    # Determine whether the document should be regenerated based on metadata.
+    #
+    # Returns true if the document needs to be regenerated.
+    def regenerate?
+      site.metadata.regenerate?(path, write?) ||
+        data['regenerate']
+    end
+
     # Determine whether the file should be placed into layouts.
     #
     # Returns false if the document is either an asset file or a yaml file,
diff --git a/lib/jekyll/layout.rb b/lib/jekyll/layout.rb
index f973019f..c29f353f 100644
--- a/lib/jekyll/layout.rb
+++ b/lib/jekyll/layout.rb
@@ -29,7 +29,7 @@ module Jekyll
       @site = site
       @base = base
       @name = name
-      @path = Jekyll.sanitized_path(site.source, File.join(base, name))
+      @path = site.in_source_dir(base, name)
 
       self.data = {}
 
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index 3e242d69..6a31eb25 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -1,12 +1,14 @@
-require 'set'
-
 module Jekyll
   class Metadata
-    attr_reader :site, :metadata
+    attr_reader :site, :metadata, :cache
 
     def initialize(site)
       @site = site
 
+      # Configuration options
+      @full_rebuild = site.config['full_rebuild']
+      @disabled = site.config['no_metadata']
+
       # Read metadata from file
       read_metadata
 
@@ -20,48 +22,48 @@ module Jekyll
     def add(path)
       return true if not File.exist? path
 
-      @metadata[path] = {
+      metadata[path] = {
         "mtime" => File.mtime(path),
         "deps" => []
       }
-      @cache[path] = true
+      cache[path] = true
     end
 
     # Force a path to regenerate
     #
     # Returns true.
     def force(path)
-      @cache[path] = true
+      cache[path] = true
     end
 
     # Clear the metadata and cache
     #
     # Returns nothing
     def clear
-      @metadata = {}
-      @cache = {}
+      metadata = {}
+      cache = {}
     end
 
     # Checks if a path should be regenerated
     #
     # Returns a boolean.
     def regenerate?(path, add = true)
-      return true if site.config['no_metadata']
+      return true if @disabled
 
       # Check for path in cache
-      if @cache.has_key? path
-        return @cache[path]
+      if cache.has_key? path
+        return cache[path]
       end
 
       # Check path that exists in metadata
-      if (data = @metadata[path])
+      if data = metadata[path]
         data["deps"].each do |dependency|
           if regenerate?(dependency)
-            return @cache[dependency] = @cache[path] = true
+            return cache[dependency] = cache[path] = true
           end
         end
-        if data["mtime"] == File.mtime(path)
-          return @cache[path] = false
+        if data["mtime"].eql? File.mtime(path)
+          return cache[path] = false
         else
           return !add || add(path)
         end
@@ -75,9 +77,9 @@ module Jekyll
     #
     # Returns nothing.
     def add_dependency(path, dependency)
-      return if (@metadata[path].nil? || site.config['no_metadata'])
+      return if (metadata[path].nil? || @disabled)
 
-      @metadata[path]["deps"] << dependency unless @metadata[path]["deps"].include? dependency
+      metadata[path]["deps"] << dependency unless metadata[path]["deps"].include? dependency
       regenerate? dependency
     end
 
@@ -86,7 +88,7 @@ module Jekyll
     # Returns nothing.
     def write
       File.open(metadata_file, 'w') do |f|
-        f.write(@metadata.to_yaml)
+        f.write(metadata.to_yaml)
       end
     end
 
@@ -94,7 +96,7 @@ module Jekyll
     #
     # Returns the String path of the file.
     def metadata_file
-      Jekyll.sanitized_path(site.source, '.jekyll-metadata')
+      site.in_source_dir('.jekyll-metadata')
     end
 
     private
@@ -104,7 +106,7 @@ module Jekyll
     #
     # Returns the read metadata.
     def read_metadata
-      @metadata = if !(site.config['full_rebuild'] || site.config['no_metadata']) && File.file?(metadata_file)
+      @metadata = if !(@full_rebuild || @disabled) && File.file?(metadata_file)
         SafeYAML.load(File.read(metadata_file))
       else
         {}
diff --git a/lib/jekyll/renderer.rb b/lib/jekyll/renderer.rb
index d2dd095f..1cdf6c2a 100644
--- a/lib/jekyll/renderer.rb
+++ b/lib/jekyll/renderer.rb
@@ -140,8 +140,8 @@ module Jekyll
 
         # Add layout to dependency tree
         site.metadata.add_dependency(
-          Jekyll.sanitized_path(site.source, document.path),
-          Jekyll.sanitized_path(site.source, layout.path)
+          site.in_source_dir(document.path),
+          site.in_source_dir(layout.path)
         ) if document.write?
 
         if layout = site.layouts[layout.data["layout"]]
diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index 69f11a75..eaf4c3cd 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -11,7 +11,7 @@ module Jekyll
                   :gems, :plugin_manager
 
     attr_accessor :converters, :generators
-    attr_accessor :metadata
+    attr_reader   :metadata
 
     # Public: Initialize a new Site.
     #
@@ -293,19 +293,13 @@ module Jekyll
 
       collections.each do |label, collection|
         collection.docs.each do |document|
-          document.output = Jekyll::Renderer.new(self, document).run if (
-            metadata.regenerate?(document.path, document.write?) ||
-            document.data['regenerate']
-          )
+          document.output = Jekyll::Renderer.new(self, document).run if document.regenerate?
         end
       end
 
       payload = site_payload
       [posts, pages].flatten.each do |page_or_post|
-        page_or_post.render(layouts, payload) if (
-          metadata.regenerate?(Jekyll.sanitized_path(source, page_or_post.relative_path)) ||
-          page_or_post.data['regenerate']
-        )
+        page_or_post.render(layouts, payload) if page_or_post.regenerate?
       end
     rescue Errno::ENOENT => e
       # ignore missing layout dir
@@ -323,12 +317,9 @@ module Jekyll
     # Returns nothing.
     def write
       each_site_file { |item|
-        item.write(dest) if (
-          metadata.regenerate?(Jekyll.sanitized_path(source, item.path)) ||
-          (item.respond_to?(:data) && item.data['regenerate'])
-        )
+        item.write(dest) if item.regenerate?
       }
-      metadata.write unless site.config['no_metadata']
+      metadata.write unless config['no_metadata']
     end
 
     # Construct a Hash of Posts indexed by the specified Post attribute.
diff --git a/lib/jekyll/static_file.rb b/lib/jekyll/static_file.rb
index eae85b54..8e100656 100644
--- a/lib/jekyll/static_file.rb
+++ b/lib/jekyll/static_file.rb
@@ -67,6 +67,8 @@ module Jekyll
       true
     end
 
+    alias_method :regenerate?, :write?
+
     # Write the static file to the destination directory (if modified).
     #
     # dest - The String path to the destination dir.
diff --git a/lib/jekyll/tags/include.rb b/lib/jekyll/tags/include.rb
index 49e7357e..16da6f49 100644
--- a/lib/jekyll/tags/include.rb
+++ b/lib/jekyll/tags/include.rb
@@ -117,7 +117,7 @@ eos
         # Add include to dependency tree
         if context.registers[:page] and context.registers[:page].has_key? "path"
           site.metadata.add_dependency(
-            Jekyll.sanitized_path(site.source, context.registers[:page]["path"]),
+            site.in_source_dir(context.registers[:page]["path"]),
             path
           )
         end
diff --git a/test/test_document.rb b/test/test_document.rb
index 2ff688ba..57c14c22 100644
--- a/test/test_document.rb
+++ b/test/test_document.rb
@@ -245,9 +245,9 @@ class TestDocument < Test::Unit::TestCase
             "output" => true
           }
         },
-        "source"      => source_dir,
-        "destination" => dest_dir,
-        "full_rebuild"       => true
+        "source"       => source_dir,
+        "destination"  => dest_dir,
+        "full_rebuild" => true
       }))
       @site.process
       @document = @site.collections["slides"].files.find { |doc| doc.relative_path == "_slides/octojekyll.png" }

From 8a257aca6b40adc73e22909ffff0ee6e1a4818d0 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Wed, 26 Nov 2014 20:15:02 -0800
Subject: [PATCH 10/17] Implement more suggestions

---
 lib/jekyll/command.rb  |  3 +--
 lib/jekyll/metadata.rb | 19 ++++++++++++-------
 lib/jekyll/site.rb     |  2 +-
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/lib/jekyll/command.rb b/lib/jekyll/command.rb
index 658ce597..12fd66e6 100644
--- a/lib/jekyll/command.rb
+++ b/lib/jekyll/command.rb
@@ -58,8 +58,7 @@ module Jekyll
         c.option 'unpublished', '--unpublished', 'Render posts that were marked as unpublished'
         c.option 'quiet',   '-q', '--quiet', 'Silence output.'
         c.option 'verbose', '-V', '--verbose', 'Print verbose output.'
-        c.option 'full_rebuild', '-f', '--full-rebuild', 'Clean the site before rebuilding.'
-        c.option 'no_metadata', '--no-metadata', 'Disable incremental regeneration.'
+        c.option 'full_rebuild', '-f', '--full-rebuild', 'Disable incremental rebuild.'
       end
 
     end
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index 6a31eb25..de9731d4 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -5,10 +5,6 @@ module Jekyll
     def initialize(site)
       @site = site
 
-      # Configuration options
-      @full_rebuild = site.config['full_rebuild']
-      @disabled = site.config['no_metadata']
-
       # Read metadata from file
       read_metadata
 
@@ -48,7 +44,7 @@ module Jekyll
     #
     # Returns a boolean.
     def regenerate?(path, add = true)
-      return true if @disabled
+      return true if disabled?
 
       # Check for path in cache
       if cache.has_key? path
@@ -56,7 +52,8 @@ module Jekyll
       end
 
       # Check path that exists in metadata
-      if data = metadata[path]
+      data = metadata[path]
+      if data
         data["deps"].each do |dependency|
           if regenerate?(dependency)
             return cache[dependency] = cache[path] = true
@@ -99,6 +96,14 @@ module Jekyll
       site.in_source_dir('.jekyll-metadata')
     end
 
+    # Check if metadata has been disabled
+    #
+    # Returns a Boolean (true for disabled, false for enabled).
+    def disabled?
+      @disabled = site.config['full_rebuild'] if @disabled.nil?
+      @disabled
+    end
+
     private
 
     # Read metadata from the metadata file, if no file is found,
@@ -106,7 +111,7 @@ module Jekyll
     #
     # Returns the read metadata.
     def read_metadata
-      @metadata = if !(@full_rebuild || @disabled) && File.file?(metadata_file)
+      @metadata = if !disabled? && File.file?(metadata_file)
         SafeYAML.load(File.read(metadata_file))
       else
         {}
diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index eaf4c3cd..4b8518c2 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -319,7 +319,7 @@ module Jekyll
       each_site_file { |item|
         item.write(dest) if item.regenerate?
       }
-      metadata.write unless config['no_metadata']
+      metadata.write unless config['full_rebuild']
     end
 
     # Construct a Hash of Posts indexed by the specified Post attribute.

From 02f281eef3ead61af68f38b439eec1f859a2f2ad Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Wed, 26 Nov 2014 21:15:53 -0800
Subject: [PATCH 11/17] Add unit and cucumber tests

---
 features/incremental_rebuild.feature      | 60 +++++++++++++++++++++++
 features/step_definitions/jekyll_steps.rb |  4 ++
 lib/jekyll/metadata.rb                    |  4 +-
 test/test_metadata.rb                     | 38 ++++++++++++++
 4 files changed, 104 insertions(+), 2 deletions(-)
 create mode 100644 features/incremental_rebuild.feature
 create mode 100644 test/test_metadata.rb

diff --git a/features/incremental_rebuild.feature b/features/incremental_rebuild.feature
new file mode 100644
index 00000000..08739d3e
--- /dev/null
+++ b/features/incremental_rebuild.feature
@@ -0,0 +1,60 @@
+Feature: Incremental rebuild
+  As an impatient hacker who likes to blog
+  I want to be able to make a static site
+  Without waiting too long for it to build
+
+  Scenario: Produce correct output site
+    Given I have a _layouts directory
+    And I have a _posts directory
+    And I have the following posts:
+      | title    | date       | layout  | content                               |
+      | Wargames | 2009-03-27 | default | The only winning move is not to play. |
+    And I have a default layout that contains "Post Layout: {{ content }}"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Post Layout: <p>The only winning move is not to play.</p>" in "_site/2009/03/27/wargames.html"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Post Layout: <p>The only winning move is not to play.</p>" in "_site/2009/03/27/wargames.html"
+
+  Scenario: Generate a metadata file
+    Given I have an "index.html" file that contains "Basic Site"
+    When I run jekyll build
+    Then the ".jekyll-metadata" file should exist
+
+  Scenario: Rebuild when content is changed
+    Given I have an "index.html" file that contains "Basic Site"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Basic Site" in "_site/index.html"
+    When I wait 1 second
+    Then I have an "index.html" file that contains "Bacon Site"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Bacon Site" in "_site/index.html"
+
+  Scenario: Rebuild when layout is changed
+    Given I have a _layouts directory
+    And I have an "index.html" page with layout "default" that contains "Basic Site with Layout"
+    And I have a default layout that contains "Page Layout: {{ content }}"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Page Layout: Basic Site with Layout" in "_site/index.html"
+    When I wait 1 second
+    Then I have a default layout that contains "Page Layout Changed: {{ content }}"
+    When I run jekyll build --full-rebuild
+    Then the _site directory should exist
+    And I should see "Page Layout Changed: Basic Site with Layout" in "_site/index.html"
+
+  Scenario: Rebuild when an include is changed
+    Given I have a _includes directory
+    And I have an "index.html" page that contains "Basic Site with include tag: {% include about.textile %}"
+    And I have an "_includes/about.textile" file that contains "Generated by Jekyll"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Basic Site with include tag: Generated by Jekyll" in "_site/index.html"
+    When I wait 1 second
+    Then I have an "_includes/about.textile" file that contains "Regenerated by Jekyll"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Basic Site with include tag: Regenerated by Jekyll" in "_site/index.html"
diff --git a/features/step_definitions/jekyll_steps.rb b/features/step_definitions/jekyll_steps.rb
index b75379f5..069f93b0 100644
--- a/features/step_definitions/jekyll_steps.rb
+++ b/features/step_definitions/jekyll_steps.rb
@@ -133,6 +133,10 @@ Given /^I have fixture collections$/ do
   FileUtils.cp_r File.join(JEKYLL_SOURCE_DIR, "test", "source", "_methods"), source_dir
 end
 
+Given /^I wait (\d+) second(s?)$/ do |time, plural|
+  sleep(time.to_f)
+end
+
 ##################
 #
 # Changing stuff
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index de9731d4..97fc0bf5 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -36,8 +36,8 @@ module Jekyll
     #
     # Returns nothing
     def clear
-      metadata = {}
-      cache = {}
+      @metadata = {}
+      @cache = {}
     end
 
     # Checks if a path should be regenerated
diff --git a/test/test_metadata.rb b/test/test_metadata.rb
new file mode 100644
index 00000000..dcc3a601
--- /dev/null
+++ b/test/test_metadata.rb
@@ -0,0 +1,38 @@
+require 'helper'
+
+class TestMetadata < Test::Unit::TestCase
+  context "The site metadata" do
+    setup do
+      FileUtils.rm_rf(source_dir(".jekyll-metadata"))
+
+      @site = Site.new(Jekyll.configuration({
+        "source" => source_dir,
+        "destination" => dest_dir
+      }))
+
+      @site.process
+      @path = @site.in_source_dir(@site.pages.first.path)
+      @metadata = @site.metadata
+    end
+
+    should "store modification times" do
+      assert_equal @metadata.metadata[@path]["mtime"], File.mtime(@path)
+    end
+
+    should "cache processed entries" do
+      assert @metadata.cache[@path]
+    end
+
+    should "write to the metadata file" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      assert File.file?(source_dir(".jekyll-metadata"))
+    end
+
+    should "read from the metadata file" do
+      @metadata = Metadata.new(@site)
+      assert_equal @metadata.metadata[@path]["mtime"], File.mtime(@path)
+    end
+  end
+end

From b6d81c58df04991fc4b1f8eeaa46a23cb3af6be5 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Thu, 27 Nov 2014 10:00:29 -0800
Subject: [PATCH 12/17] Perform less expensive operation first

---
 lib/jekyll/convertible.rb | 3 +--
 lib/jekyll/document.rb    | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb
index 2c0d240e..d6ddfde6 100644
--- a/lib/jekyll/convertible.rb
+++ b/lib/jekyll/convertible.rb
@@ -172,8 +172,7 @@ module Jekyll
     #
     # Returns true if file needs to be regenerated
     def regenerate?
-      site.metadata.regenerate?(site.in_source_dir(relative_path)) ||
-        data['regenerate']
+      data['regenerate'] || site.metadata.regenerate?(site.in_source_dir(relative_path))
     end
 
     # Determine whether the file should be placed into layouts.
diff --git a/lib/jekyll/document.rb b/lib/jekyll/document.rb
index 356ce397..5b1b79ab 100644
--- a/lib/jekyll/document.rb
+++ b/lib/jekyll/document.rb
@@ -109,8 +109,7 @@ module Jekyll
     #
     # Returns true if the document needs to be regenerated.
     def regenerate?
-      site.metadata.regenerate?(path, write?) ||
-        data['regenerate']
+      data['regenerate'] || site.metadata.regenerate?(path, write?)
     end
 
     # Determine whether the file should be placed into layouts.

From a701e59c070338677a27a5801cfa3d4feb12977c Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Thu, 27 Nov 2014 10:40:31 -0800
Subject: [PATCH 13/17] Add lots more unit tests

---
 test/test_metadata.rb | 106 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 104 insertions(+), 2 deletions(-)

diff --git a/test/test_metadata.rb b/test/test_metadata.rb
index dcc3a601..858b166a 100644
--- a/test/test_metadata.rb
+++ b/test/test_metadata.rb
@@ -16,7 +16,7 @@ class TestMetadata < Test::Unit::TestCase
     end
 
     should "store modification times" do
-      assert_equal @metadata.metadata[@path]["mtime"], File.mtime(@path)
+      assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"]
     end
 
     should "cache processed entries" do
@@ -32,7 +32,109 @@ class TestMetadata < Test::Unit::TestCase
 
     should "read from the metadata file" do
       @metadata = Metadata.new(@site)
-      assert_equal @metadata.metadata[@path]["mtime"], File.mtime(@path)
+      assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"]
+    end
+
+    # Methods
+
+    should "be able to add a path to the metadata" do
+      @metadata.clear
+      @metadata.add(@path)
+      assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"]
+      assert_equal [], @metadata.metadata[@path]["deps"]
+      assert @metadata.cache[@path]
+    end
+
+    should "return true on nonexistent path" do
+      @metadata.clear
+      assert @metadata.add("/bogus/path.md")
+      assert @metadata.regenerate?("/bogus/path.md")
+    end
+
+    should "be able to force a path to regenerate" do
+      @metadata.clear
+      @metadata.force(@path)
+      assert @metadata.cache[@path]
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "be able to clear metadata and cache" do
+      @metadata.clear
+      @metadata.add(@path)
+      assert_equal 1, @metadata.metadata.length
+      assert_equal 1, @metadata.cache.length
+      @metadata.clear
+      assert_equal 0, @metadata.metadata.length
+      assert_equal 0, @metadata.cache.length
+    end
+
+    should "not regenerate a path if it is not modified" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      assert !@metadata.regenerate?(@path)
+    end
+
+    should "not regenerate if path in cache is false" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      assert !@metadata.regenerate?(@path)
+      assert !@metadata.cache[@path]
+      assert !@metadata.regenerate?(@path)
+    end
+
+    should "regenerate if path in not in metadata" do
+      @metadata.clear
+      @metadata.add(@path)
+
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "regenerate if path in cache is true" do
+      @metadata.clear
+      @metadata.add(@path)
+
+      assert @metadata.regenerate?(@path)
+      assert @metadata.cache[@path]
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "regenerate if file is modified" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.metadata[@path]["mtime"] = Time.at(0)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      assert_not_same File.mtime(@path), @metadata.metadata[@path]["mtime"]
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "regenerate if dependency is modified" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      @metadata.add_dependency(@path, "new.dependency")
+      assert_equal ["new.dependency"], @metadata.metadata[@path]["deps"]
+      assert @metadata.regenerate?("new.dependency")
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "regenerate everything if metadata is disabled" do
+      @site.config["full_rebuild"] = true
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      assert @metadata.regenerate?(@path)
     end
   end
 end

From d0e12d69bc73346c156039e569b3d54d2e6ed806 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Fri, 28 Nov 2014 14:05:40 -0800
Subject: [PATCH 14/17] Last few revisions

---
 lib/jekyll/cleaner.rb       | 2 +-
 lib/jekyll/configuration.rb | 1 +
 lib/jekyll/metadata.rb      | 4 ++--
 lib/jekyll/site.rb          | 9 ++++++++-
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/lib/jekyll/cleaner.rb b/lib/jekyll/cleaner.rb
index 00191db7..bc5bd155 100644
--- a/lib/jekyll/cleaner.rb
+++ b/lib/jekyll/cleaner.rb
@@ -13,7 +13,7 @@ module Jekyll
       # Cleans up the site's destination directory
       def cleanup!
         FileUtils.rm_rf(obsolete_files)
-        FileUtils.rm_rf(metadata_file) if @site.config["full_rebuild"]
+        FileUtils.rm_rf(metadata_file) if @site.full_rebuild?
       end
 
       private
diff --git a/lib/jekyll/configuration.rb b/lib/jekyll/configuration.rb
index 7f1148a6..c64f3a89 100644
--- a/lib/jekyll/configuration.rb
+++ b/lib/jekyll/configuration.rb
@@ -22,6 +22,7 @@ module Jekyll
       'encoding'      => 'utf-8',
       'markdown_ext'  => 'markdown,mkdown,mkdn,mkd,md',
       'textile_ext'   => 'textile',
+      'full_rebuild'  => false,
 
       # Filtering Content
       'show_drafts'   => nil,
diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb
index 97fc0bf5..953dd7c0 100644
--- a/lib/jekyll/metadata.rb
+++ b/lib/jekyll/metadata.rb
@@ -16,7 +16,7 @@ module Jekyll
     #
     # Returns true, also on failure.
     def add(path)
-      return true if not File.exist? path
+      return true unless File.exist?(path)
 
       metadata[path] = {
         "mtime" => File.mtime(path),
@@ -100,7 +100,7 @@ module Jekyll
     #
     # Returns a Boolean (true for disabled, false for enabled).
     def disabled?
-      @disabled = site.config['full_rebuild'] if @disabled.nil?
+      @disabled = site.full_rebuild? if @disabled.nil?
       @disabled
     end
 
diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index 4b8518c2..7b8a4a00 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -319,7 +319,7 @@ module Jekyll
       each_site_file { |item|
         item.write(dest) if item.regenerate?
       }
-      metadata.write unless config['full_rebuild']
+      metadata.write unless full_rebuild?
     end
 
     # Construct a Hash of Posts indexed by the specified Post attribute.
@@ -490,6 +490,13 @@ module Jekyll
       @frontmatter_defaults ||= FrontmatterDefaults.new(self)
     end
 
+    # Whether to perform a full rebuild without metadata
+    #
+    # Returns a Boolean: true for a full rebuild, false for normal build
+    def full_rebuild?(override = {})
+      override['full_rebuild'] || config['full_rebuild']
+    end
+
     private
 
     def has_relative_page?

From 52f0b36558e97712f5f0107fbc3b9fe6ac475c32 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Thu, 4 Dec 2014 20:09:49 -0800
Subject: [PATCH 15/17] Add incremental rebuild info to build command output

---
 lib/jekyll/commands/build.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb
index de234328..11f1f75e 100644
--- a/lib/jekyll/commands/build.rb
+++ b/lib/jekyll/commands/build.rb
@@ -50,8 +50,10 @@ module Jekyll
         def build(site, options)
           source      = options['source']
           destination = options['destination']
+          full_build  = options['full_rebuild']
           Jekyll.logger.info "Source:", source
           Jekyll.logger.info "Destination:", destination
+          Jekyll.logger.info "Incremental build:", (full_build ? "disabled" : "enabled")
           Jekyll.logger.info "Generating..."
           process_site(site)
           Jekyll.logger.info "", "done."

From 5d9662f80f2aba39158cd26ea00216f4877e79a8 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Fri, 5 Dec 2014 19:07:18 -0800
Subject: [PATCH 16/17] Always regenerate asset files

---
 lib/jekyll/convertible.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb
index d6ddfde6..ada0fb8e 100644
--- a/lib/jekyll/convertible.rb
+++ b/lib/jekyll/convertible.rb
@@ -172,7 +172,9 @@ module Jekyll
     #
     # Returns true if file needs to be regenerated
     def regenerate?
-      data['regenerate'] || site.metadata.regenerate?(site.in_source_dir(relative_path))
+      asset_file? ||
+      data['regenerate'] ||
+      site.metadata.regenerate?(site.in_source_dir(relative_path))
     end
 
     # Determine whether the file should be placed into layouts.

From 43a28aed967b66de87e2a539ca92c5cb93ce9b66 Mon Sep 17 00:00:00 2001
From: Alfred Xing <alfredxing@users.noreply.github.com>
Date: Fri, 5 Dec 2014 19:38:43 -0800
Subject: [PATCH 17/17] Fix indentation

---
 lib/jekyll/convertible.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb
index ada0fb8e..eb96dd35 100644
--- a/lib/jekyll/convertible.rb
+++ b/lib/jekyll/convertible.rb
@@ -173,8 +173,8 @@ module Jekyll
     # Returns true if file needs to be regenerated
     def regenerate?
       asset_file? ||
-      data['regenerate'] ||
-      site.metadata.regenerate?(site.in_source_dir(relative_path))
+        data['regenerate'] ||
+        site.metadata.regenerate?(site.in_source_dir(relative_path))
     end
 
     # Determine whether the file should be placed into layouts.