Merge pull request #3116 from alfredxing/incremental

.gitignore

@@ -14,3 +14,4 @@ coverage
 .ruby-version
 .sass-cache
 tmp/stackprof-*
+.jekyll-metadata

features/incremental_rebuild.feature

@@ -0,0 +1,60 @@
+Feature: Incremental rebuild
+  As an impatient hacker who likes to blog
+  I want to be able to make a static site
+  Without waiting too long for it to build
+
+  Scenario: Produce correct output site
+    Given I have a _layouts directory
+    And I have a _posts directory
+    And I have the following posts:
+      | title    | date       | layout  | content                               |
+      | Wargames | 2009-03-27 | default | The only winning move is not to play. |
+    And I have a default layout that contains "Post Layout: {{ content }}"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Post Layout: <p>The only winning move is not to play.</p>" in "_site/2009/03/27/wargames.html"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Post Layout: <p>The only winning move is not to play.</p>" in "_site/2009/03/27/wargames.html"
+
+  Scenario: Generate a metadata file
+    Given I have an "index.html" file that contains "Basic Site"
+    When I run jekyll build
+    Then the ".jekyll-metadata" file should exist
+
+  Scenario: Rebuild when content is changed
+    Given I have an "index.html" file that contains "Basic Site"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Basic Site" in "_site/index.html"
+    When I wait 1 second
+    Then I have an "index.html" file that contains "Bacon Site"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Bacon Site" in "_site/index.html"
+
+  Scenario: Rebuild when layout is changed
+    Given I have a _layouts directory
+    And I have an "index.html" page with layout "default" that contains "Basic Site with Layout"
+    And I have a default layout that contains "Page Layout: {{ content }}"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Page Layout: Basic Site with Layout" in "_site/index.html"
+    When I wait 1 second
+    Then I have a default layout that contains "Page Layout Changed: {{ content }}"
+    When I run jekyll build --full-rebuild
+    Then the _site directory should exist
+    And I should see "Page Layout Changed: Basic Site with Layout" in "_site/index.html"
+
+  Scenario: Rebuild when an include is changed
+    Given I have a _includes directory
+    And I have an "index.html" page that contains "Basic Site with include tag: {% include about.textile %}"
+    And I have an "_includes/about.textile" file that contains "Generated by Jekyll"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Basic Site with include tag: Generated by Jekyll" in "_site/index.html"
+    When I wait 1 second
+    Then I have an "_includes/about.textile" file that contains "Regenerated by Jekyll"
+    When I run jekyll build
+    Then the _site directory should exist
+    And I should see "Basic Site with include tag: Regenerated by Jekyll" in "_site/index.html"

features/step_definitions/jekyll_steps.rb

@@ -133,6 +133,10 @@ Given /^I have fixture collections$/ do
   FileUtils.cp_r File.join(JEKYLL_SOURCE_DIR, "test", "source", "_methods"), source_dir
 end
 
+Given /^I wait (\d+) second(s?)$/ do |time, plural|
+  sleep(time.to_f)
+end
+
 ##################
 #
 # Changing stuff

lib/jekyll.rb

@@ -21,6 +21,7 @@ require 'time'
 require 'English'
 require 'pathname'
 require 'logger'
+require 'set'
 
 # 3rd party
 require 'safe_yaml/load'
@@ -48,6 +49,7 @@ module Jekyll
   autoload :Layout,              'jekyll/layout'
   autoload :LayoutReader,        'jekyll/layout_reader'
   autoload :LogAdapter,          'jekyll/log_adapter'
+  autoload :Metadata,            'jekyll/metadata'
   autoload :Page,                'jekyll/page'
   autoload :PluginManager,       'jekyll/plugin_manager'
   autoload :Post,                'jekyll/post'

lib/jekyll/cleaner.rb

@@ -13,6 +13,7 @@ module Jekyll
       # Cleans up the site's destination directory
       def cleanup!
         FileUtils.rm_rf(obsolete_files)
+        FileUtils.rm_rf(metadata_file) if @site.full_rebuild?
       end
 
       private
@@ -24,6 +25,13 @@ module Jekyll
         (existing_files - new_files - new_dirs + replaced_files).to_a
       end
 
+      # Private: The metadata file storing dependency tree and build history
+      #
+      # Returns an Array with the metdata file as the only item
+      def metadata_file
+        [site.metadata.metadata_file]
+      end
+
       # Private: The list of existing files, apart from those included in keep_files and hidden files.
       #
       # Returns a Set with the file paths

lib/jekyll/command.rb

@@ -58,6 +58,7 @@ module Jekyll
         c.option 'unpublished', '--unpublished', 'Render posts that were marked as unpublished'
         c.option 'quiet',   '-q', '--quiet', 'Silence output.'
         c.option 'verbose', '-V', '--verbose', 'Print verbose output.'
+        c.option 'full_rebuild', '-f', '--full-rebuild', 'Disable incremental rebuild.'
       end
 
     end

lib/jekyll/commands/build.rb

@@ -50,8 +50,10 @@ module Jekyll
         def build(site, options)
           source      = options['source']
           destination = options['destination']
+          full_build  = options['full_rebuild']
           Jekyll.logger.info "Source:", source
           Jekyll.logger.info "Destination:", destination
+          Jekyll.logger.info "Incremental build:", (full_build ? "disabled" : "enabled")
           Jekyll.logger.info "Generating..."
           process_site(site)
           Jekyll.logger.info "", "done."

lib/jekyll/commands/clean.rb

@@ -0,0 +1,42 @@
+module Jekyll
+  module Commands
+    class Clean < Command
+      class << self
+
+        def init_with_program(prog)
+          prog.command(:clean) do |c|
+            c.syntax 'clean [subcommand]'
+            c.description 'Clean the site (removes site output and metadata file) without building.'
+
+            c.action do |args, _|
+              Jekyll::Commands::Clean.process({})
+            end
+          end
+        end
+
+        def process(options)
+          options = configuration_from_options(options)
+          destination = options['destination']
+          metadata_file = File.join(options['source'], '.jekyll-metadata')
+
+          if File.directory? destination
+            Jekyll.logger.info "Cleaning #{destination}..."
+            FileUtils.rm_rf(destination)
+            Jekyll.logger.info "", "done."
+          else
+            Jekyll.logger.info "Nothing to do for #{destination}."
+          end
+
+          if File.file? metadata_file
+            Jekyll.logger.info "Removing #{metadata_file}..."
+            FileUtils.rm_rf(metadata_file)
+            Jekyll.logger.info "", "done."
+          else
+            Jekyll.logger.info "Nothing to do for #{metadata_file}."
+          end
+        end
+
+      end
+    end
+  end
+end

lib/jekyll/configuration.rb

@@ -22,6 +22,7 @@ module Jekyll
       'encoding'      => 'utf-8',
       'markdown_ext'  => 'markdown,mkdown,mkdn,mkd,md',
       'textile_ext'   => 'textile',
+      'full_rebuild'  => false,
 
       # Filtering Content
       'show_drafts'   => nil,

lib/jekyll/convertible.rb

@@ -168,6 +168,15 @@ module Jekyll
       true
     end
 
+    # Determine whether to regenerate the file based on metadata.
+    #
+    # Returns true if file needs to be regenerated
+    def regenerate?
+      asset_file? ||
+        data['regenerate'] ||
+        site.metadata.regenerate?(site.in_source_dir(relative_path))
+    end
+
     # Determine whether the file should be placed into layouts.
     #
     # Returns false if the document is an asset file.
@@ -207,6 +216,12 @@ module Jekyll
                                          info,
                                          File.join(site.config['layouts'], layout.name))
 
+        # Add layout to dependency tree
+        site.metadata.add_dependency(
+          site.in_source_dir(path),
+          site.in_source_dir(layout.path)
+        )
+
         if layout = layouts[layout.data["layout"]]
           if used.include?(layout)
             layout = nil # avoid recursive chain

lib/jekyll/document.rb

@@ -105,6 +105,13 @@ module Jekyll
       !(coffeescript_file? || yaml_file?)
     end
 
+    # Determine whether the document should be regenerated based on metadata.
+    #
+    # Returns true if the document needs to be regenerated.
+    def regenerate?
+      data['regenerate'] || site.metadata.regenerate?(path, write?)
+    end
+
     # Determine whether the file should be placed into layouts.
     #
     # Returns false if the document is either an asset file or a yaml file,

lib/jekyll/layout.rb

@@ -8,6 +8,9 @@ module Jekyll
     # Gets the name of this layout.
     attr_reader :name
 
+    # Gets the path to this layout.
+    attr_reader :path
+
     # Gets/Sets the extension of this layout.
     attr_accessor :ext
 
@@ -26,6 +29,7 @@ module Jekyll
       @site = site
       @base = base
       @name = name
+      @path = site.in_source_dir(base, name)
 
       self.data = {}
 

lib/jekyll/metadata.rb

@@ -0,0 +1,121 @@
+module Jekyll
+  class Metadata
+    attr_reader :site, :metadata, :cache
+
+    def initialize(site)
+      @site = site
+
+      # Read metadata from file
+      read_metadata
+
+      # Initialize cache to an empty hash
+      @cache = {}
+    end
+
+    # Add a path to the metadata
+    #
+    # Returns true, also on failure.
+    def add(path)
+      return true unless File.exist?(path)
+
+      metadata[path] = {
+        "mtime" => File.mtime(path),
+        "deps" => []
+      }
+      cache[path] = true
+    end
+
+    # Force a path to regenerate
+    #
+    # Returns true.
+    def force(path)
+      cache[path] = true
+    end
+
+    # Clear the metadata and cache
+    #
+    # Returns nothing
+    def clear
+      @metadata = {}
+      @cache = {}
+    end
+
+    # Checks if a path should be regenerated
+    #
+    # Returns a boolean.
+    def regenerate?(path, add = true)
+      return true if disabled?
+
+      # Check for path in cache
+      if cache.has_key? path
+        return cache[path]
+      end
+
+      # Check path that exists in metadata
+      data = metadata[path]
+      if data
+        data["deps"].each do |dependency|
+          if regenerate?(dependency)
+            return cache[dependency] = cache[path] = true
+          end
+        end
+        if data["mtime"].eql? File.mtime(path)
+          return cache[path] = false
+        else
+          return !add || add(path)
+        end
+      end
+
+      # Path does not exist in metadata, add it
+      return !add || add(path)
+    end
+
+    # Add a dependency of a path
+    #
+    # Returns nothing.
+    def add_dependency(path, dependency)
+      return if (metadata[path].nil? || @disabled)
+
+      metadata[path]["deps"] << dependency unless metadata[path]["deps"].include? dependency
+      regenerate? dependency
+    end
+
+    # Write the metadata to disk
+    #
+    # Returns nothing.
+    def write
+      File.open(metadata_file, 'w') do |f|
+        f.write(metadata.to_yaml)
+      end
+    end
+
+    # Produce the absolute path of the metadata file
+    #
+    # Returns the String path of the file.
+    def metadata_file
+      site.in_source_dir('.jekyll-metadata')
+    end
+
+    # Check if metadata has been disabled
+    #
+    # Returns a Boolean (true for disabled, false for enabled).
+    def disabled?
+      @disabled = site.full_rebuild? if @disabled.nil?
+      @disabled
+    end
+
+    private
+
+    # Read metadata from the metadata file, if no file is found,
+    # initialize with an empty hash
+    #
+    # Returns the read metadata.
+    def read_metadata
+      @metadata = if !disabled? && File.file?(metadata_file)
+        SafeYAML.load(File.read(metadata_file))
+      else
+        {}
+      end
+    end
+  end
+end

lib/jekyll/renderer.rb

@@ -138,6 +138,12 @@ module Jekyll
           File.join(site.config['layouts'], layout.name)
         )
 
+        # Add layout to dependency tree
+        site.metadata.add_dependency(
+          site.in_source_dir(document.path),
+          site.in_source_dir(layout.path)
+        ) if document.write?
+
         if layout = site.layouts[layout.data["layout"]]
           if used.include?(layout)
             layout = nil # avoid recursive chain

lib/jekyll/site.rb

@@ -11,6 +11,7 @@ module Jekyll
                   :gems, :plugin_manager
 
     attr_accessor :converters, :generators
+    attr_reader   :metadata
 
     # Public: Initialize a new Site.
     #
@@ -27,6 +28,9 @@ module Jekyll
       @source              = File.expand_path(config['source']).freeze
       @dest                = File.expand_path(config['destination']).freeze
 
+      # Build metadata
+      @metadata = Metadata.new(self)
+
       self.plugin_manager = Jekyll::PluginManager.new(self)
       self.plugins        = plugin_manager.plugins_path
 
@@ -289,13 +293,13 @@ module Jekyll
 
       collections.each do |label, collection|
         collection.docs.each do |document|
-          document.output = Jekyll::Renderer.new(self, document).run
+          document.output = Jekyll::Renderer.new(self, document).run if document.regenerate?
         end
       end
 
       payload = site_payload
       [posts, pages].flatten.each do |page_or_post|
-        page_or_post.render(layouts, payload)
+        page_or_post.render(layouts, payload) if page_or_post.regenerate?
       end
     rescue Errno::ENOENT => e
       # ignore missing layout dir
@@ -312,7 +316,10 @@ module Jekyll
     #
     # Returns nothing.
     def write
-      each_site_file { |item| item.write(dest) }
+      each_site_file { |item|
+        item.write(dest) if item.regenerate?
+      }
+      metadata.write unless full_rebuild?
     end
 
     # Construct a Hash of Posts indexed by the specified Post attribute.
@@ -483,6 +490,13 @@ module Jekyll
       @frontmatter_defaults ||= FrontmatterDefaults.new(self)
     end
 
+    # Whether to perform a full rebuild without metadata
+    #
+    # Returns a Boolean: true for a full rebuild, false for normal build
+    def full_rebuild?(override = {})
+      override['full_rebuild'] || config['full_rebuild']
+    end
+
     private
 
     def has_relative_page?

lib/jekyll/static_file.rb

@@ -67,6 +67,8 @@ module Jekyll
       true
     end
 
+    alias_method :regenerate?, :write?
+
     # Write the static file to the destination directory (if modified).
     #
     # dest - The String path to the destination dir.

lib/jekyll/tags/include.rb

@@ -105,13 +105,22 @@ eos
       end
 
       def render(context)
+        site = context.registers[:site]
         dir = resolved_includes_dir(context)
 
         file = render_variable(context) || @file
         validate_file_name(file)
 
         path = File.join(dir, file)
-        validate_path(path, dir, context.registers[:site].safe)
+        validate_path(path, dir, site.safe)
+
+        # Add include to dependency tree
+        if context.registers[:page] and context.registers[:page].has_key? "path"
+          site.metadata.add_dependency(
+            site.in_source_dir(context.registers[:page]["path"]),
+            path
+          )
+        end
 
         begin
           partial = Liquid::Template.parse(source(path, context))

test/helper.rb

@@ -49,6 +49,7 @@ class Test::Unit::TestCase
 
   def clear_dest
     FileUtils.rm_rf(dest_dir)
+    FileUtils.rm_rf(source_dir('.jekyll-metadata'))
   end
 
   def test_dir(*subdirs)

test/test_document.rb

@@ -245,8 +245,9 @@ class TestDocument < Test::Unit::TestCase
             "output" => true
           }
         },
-        "source"      => source_dir,
+        "source"       => source_dir,
-        "destination" => dest_dir
+        "destination"  => dest_dir,
+        "full_rebuild" => true
       }))
       @site.process
       @document = @site.collections["slides"].files.find { |doc| doc.relative_path == "_slides/octojekyll.png" }

test/test_metadata.rb

@@ -0,0 +1,140 @@
+require 'helper'
+
+class TestMetadata < Test::Unit::TestCase
+  context "The site metadata" do
+    setup do
+      FileUtils.rm_rf(source_dir(".jekyll-metadata"))
+
+      @site = Site.new(Jekyll.configuration({
+        "source" => source_dir,
+        "destination" => dest_dir
+      }))
+
+      @site.process
+      @path = @site.in_source_dir(@site.pages.first.path)
+      @metadata = @site.metadata
+    end
+
+    should "store modification times" do
+      assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"]
+    end
+
+    should "cache processed entries" do
+      assert @metadata.cache[@path]
+    end
+
+    should "write to the metadata file" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      assert File.file?(source_dir(".jekyll-metadata"))
+    end
+
+    should "read from the metadata file" do
+      @metadata = Metadata.new(@site)
+      assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"]
+    end
+
+    # Methods
+
+    should "be able to add a path to the metadata" do
+      @metadata.clear
+      @metadata.add(@path)
+      assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"]
+      assert_equal [], @metadata.metadata[@path]["deps"]
+      assert @metadata.cache[@path]
+    end
+
+    should "return true on nonexistent path" do
+      @metadata.clear
+      assert @metadata.add("/bogus/path.md")
+      assert @metadata.regenerate?("/bogus/path.md")
+    end
+
+    should "be able to force a path to regenerate" do
+      @metadata.clear
+      @metadata.force(@path)
+      assert @metadata.cache[@path]
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "be able to clear metadata and cache" do
+      @metadata.clear
+      @metadata.add(@path)
+      assert_equal 1, @metadata.metadata.length
+      assert_equal 1, @metadata.cache.length
+      @metadata.clear
+      assert_equal 0, @metadata.metadata.length
+      assert_equal 0, @metadata.cache.length
+    end
+
+    should "not regenerate a path if it is not modified" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      assert !@metadata.regenerate?(@path)
+    end
+
+    should "not regenerate if path in cache is false" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      assert !@metadata.regenerate?(@path)
+      assert !@metadata.cache[@path]
+      assert !@metadata.regenerate?(@path)
+    end
+
+    should "regenerate if path in not in metadata" do
+      @metadata.clear
+      @metadata.add(@path)
+
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "regenerate if path in cache is true" do
+      @metadata.clear
+      @metadata.add(@path)
+
+      assert @metadata.regenerate?(@path)
+      assert @metadata.cache[@path]
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "regenerate if file is modified" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.metadata[@path]["mtime"] = Time.at(0)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      assert_not_same File.mtime(@path), @metadata.metadata[@path]["mtime"]
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "regenerate if dependency is modified" do
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      @metadata.add_dependency(@path, "new.dependency")
+      assert_equal ["new.dependency"], @metadata.metadata[@path]["deps"]
+      assert @metadata.regenerate?("new.dependency")
+      assert @metadata.regenerate?(@path)
+    end
+
+    should "regenerate everything if metadata is disabled" do
+      @site.config["full_rebuild"] = true
+      @metadata.clear
+      @metadata.add(@path)
+      @metadata.write
+      @metadata = Metadata.new(@site)
+
+      assert @metadata.regenerate?(@path)
+    end
+  end
+end

test/test_site.rb

@@ -99,6 +99,7 @@ class TestSite < Test::Unit::TestCase
     should "write only modified static files" do
       clear_dest
       StaticFile.reset_cache
+      @site.metadata.clear
 
       @site.process
       some_static_file = @site.static_files[0].path
@@ -128,6 +129,7 @@ class TestSite < Test::Unit::TestCase
     should "write static files if not modified but missing in destination" do
       clear_dest
       StaticFile.reset_cache
+      @site.metadata.clear
 
       @site.process
       some_static_file = @site.static_files[0].path
@@ -241,6 +243,7 @@ class TestSite < Test::Unit::TestCase
     context 'with orphaned files in destination' do
       setup do
         clear_dest
+        @site.metadata.clear
         @site.process
         # generate some orphaned files:
         # single file
@@ -328,7 +331,7 @@ class TestSite < Test::Unit::TestCase
         end
 
         bad_processor = "Custom::Markdown"
-        s = Site.new(site_configuration('markdown' => bad_processor))
+        s = Site.new(site_configuration('markdown' => bad_processor, 'full_rebuild' => true))
         assert_raise Jekyll::Errors::FatalException do
           s.process
         end
@@ -348,7 +351,7 @@ class TestSite < Test::Unit::TestCase
 
       should 'throw FatalException at process time' do
         bad_processor = 'not a processor name'
-        s = Site.new(site_configuration('markdown' => bad_processor))
+        s = Site.new(site_configuration('markdown' => bad_processor, 'full_rebuild' => true))
         assert_raise Jekyll::Errors::FatalException do
           s.process
         end
@@ -418,7 +421,9 @@ class TestSite < Test::Unit::TestCase
 
     context "manipulating the Jekyll environment" do
       setup do
-        @site = Site.new(site_configuration)
+        @site = Site.new(site_configuration({
+          'full_rebuild' => true
+        }))
         @site.process
         @page = @site.pages.find { |p| p.name == "environment.html" }
       end
@@ -430,7 +435,9 @@ class TestSite < Test::Unit::TestCase
       context "in production" do
         setup do
           ENV["JEKYLL_ENV"] = "production"
-          @site = Site.new(site_configuration)
+          @site = Site.new(site_configuration({
+            'full_rebuild' => true
+          }))
           @site.process
           @page = @site.pages.find { |p| p.name == "environment.html" }
         end