The only winning move is not to play.
" in "_site/2009/03/27/wargames.html" + When I run jekyll build + Then the _site directory should exist + And I should see "Post Layout:The only winning move is not to play.
" in "_site/2009/03/27/wargames.html" + + Scenario: Generate a metadata file + Given I have an "index.html" file that contains "Basic Site" + When I run jekyll build + Then the ".jekyll-metadata" file should exist + + Scenario: Rebuild when content is changed + Given I have an "index.html" file that contains "Basic Site" + When I run jekyll build + Then the _site directory should exist + And I should see "Basic Site" in "_site/index.html" + When I wait 1 second + Then I have an "index.html" file that contains "Bacon Site" + When I run jekyll build + Then the _site directory should exist + And I should see "Bacon Site" in "_site/index.html" + + Scenario: Rebuild when layout is changed + Given I have a _layouts directory + And I have an "index.html" page with layout "default" that contains "Basic Site with Layout" + And I have a default layout that contains "Page Layout: {{ content }}" + When I run jekyll build + Then the _site directory should exist + And I should see "Page Layout: Basic Site with Layout" in "_site/index.html" + When I wait 1 second + Then I have a default layout that contains "Page Layout Changed: {{ content }}" + When I run jekyll build --full-rebuild + Then the _site directory should exist + And I should see "Page Layout Changed: Basic Site with Layout" in "_site/index.html" + + Scenario: Rebuild when an include is changed + Given I have a _includes directory + And I have an "index.html" page that contains "Basic Site with include tag: {% include about.textile %}" + And I have an "_includes/about.textile" file that contains "Generated by Jekyll" + When I run jekyll build + Then the _site directory should exist + And I should see "Basic Site with include tag: Generated by Jekyll" in "_site/index.html" + When I wait 1 second + Then I have an "_includes/about.textile" file that contains "Regenerated by Jekyll" + When I run jekyll build + Then the _site directory should exist + And I should see "Basic Site with include tag: Regenerated by Jekyll" in "_site/index.html" diff --git a/features/step_definitions/jekyll_steps.rb b/features/step_definitions/jekyll_steps.rb index b75379f5..069f93b0 100644 --- a/features/step_definitions/jekyll_steps.rb +++ b/features/step_definitions/jekyll_steps.rb @@ -133,6 +133,10 @@ Given /^I have fixture collections$/ do FileUtils.cp_r File.join(JEKYLL_SOURCE_DIR, "test", "source", "_methods"), source_dir end +Given /^I wait (\d+) second(s?)$/ do |time, plural| + sleep(time.to_f) +end + ################## # # Changing stuff diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 03724100..911b3ddb 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -21,6 +21,7 @@ require 'time' require 'English' require 'pathname' require 'logger' +require 'set' # 3rd party require 'safe_yaml/load' @@ -48,6 +49,7 @@ module Jekyll autoload :Layout, 'jekyll/layout' autoload :LayoutReader, 'jekyll/layout_reader' autoload :LogAdapter, 'jekyll/log_adapter' + autoload :Metadata, 'jekyll/metadata' autoload :Page, 'jekyll/page' autoload :PluginManager, 'jekyll/plugin_manager' autoload :Post, 'jekyll/post' diff --git a/lib/jekyll/cleaner.rb b/lib/jekyll/cleaner.rb index 6dd59ea0..bc5bd155 100644 --- a/lib/jekyll/cleaner.rb +++ b/lib/jekyll/cleaner.rb @@ -13,6 +13,7 @@ module Jekyll # Cleans up the site's destination directory def cleanup! FileUtils.rm_rf(obsolete_files) + FileUtils.rm_rf(metadata_file) if @site.full_rebuild? end private @@ -24,6 +25,13 @@ module Jekyll (existing_files - new_files - new_dirs + replaced_files).to_a end + # Private: The metadata file storing dependency tree and build history + # + # Returns an Array with the metdata file as the only item + def metadata_file + [site.metadata.metadata_file] + end + # Private: The list of existing files, apart from those included in keep_files and hidden files. # # Returns a Set with the file paths diff --git a/lib/jekyll/command.rb b/lib/jekyll/command.rb index a1cc8c01..12fd66e6 100644 --- a/lib/jekyll/command.rb +++ b/lib/jekyll/command.rb @@ -58,6 +58,7 @@ module Jekyll c.option 'unpublished', '--unpublished', 'Render posts that were marked as unpublished' c.option 'quiet', '-q', '--quiet', 'Silence output.' c.option 'verbose', '-V', '--verbose', 'Print verbose output.' + c.option 'full_rebuild', '-f', '--full-rebuild', 'Disable incremental rebuild.' end end diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index de234328..11f1f75e 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -50,8 +50,10 @@ module Jekyll def build(site, options) source = options['source'] destination = options['destination'] + full_build = options['full_rebuild'] Jekyll.logger.info "Source:", source Jekyll.logger.info "Destination:", destination + Jekyll.logger.info "Incremental build:", (full_build ? "disabled" : "enabled") Jekyll.logger.info "Generating..." process_site(site) Jekyll.logger.info "", "done." diff --git a/lib/jekyll/commands/clean.rb b/lib/jekyll/commands/clean.rb new file mode 100644 index 00000000..7d787d30 --- /dev/null +++ b/lib/jekyll/commands/clean.rb @@ -0,0 +1,42 @@ +module Jekyll + module Commands + class Clean < Command + class << self + + def init_with_program(prog) + prog.command(:clean) do |c| + c.syntax 'clean [subcommand]' + c.description 'Clean the site (removes site output and metadata file) without building.' + + c.action do |args, _| + Jekyll::Commands::Clean.process({}) + end + end + end + + def process(options) + options = configuration_from_options(options) + destination = options['destination'] + metadata_file = File.join(options['source'], '.jekyll-metadata') + + if File.directory? destination + Jekyll.logger.info "Cleaning #{destination}..." + FileUtils.rm_rf(destination) + Jekyll.logger.info "", "done." + else + Jekyll.logger.info "Nothing to do for #{destination}." + end + + if File.file? metadata_file + Jekyll.logger.info "Removing #{metadata_file}..." + FileUtils.rm_rf(metadata_file) + Jekyll.logger.info "", "done." + else + Jekyll.logger.info "Nothing to do for #{metadata_file}." + end + end + + end + end + end +end diff --git a/lib/jekyll/configuration.rb b/lib/jekyll/configuration.rb index 7f1148a6..c64f3a89 100644 --- a/lib/jekyll/configuration.rb +++ b/lib/jekyll/configuration.rb @@ -22,6 +22,7 @@ module Jekyll 'encoding' => 'utf-8', 'markdown_ext' => 'markdown,mkdown,mkdn,mkd,md', 'textile_ext' => 'textile', + 'full_rebuild' => false, # Filtering Content 'show_drafts' => nil, diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb index 4b21a66b..eb96dd35 100644 --- a/lib/jekyll/convertible.rb +++ b/lib/jekyll/convertible.rb @@ -168,6 +168,15 @@ module Jekyll true end + # Determine whether to regenerate the file based on metadata. + # + # Returns true if file needs to be regenerated + def regenerate? + asset_file? || + data['regenerate'] || + site.metadata.regenerate?(site.in_source_dir(relative_path)) + end + # Determine whether the file should be placed into layouts. # # Returns false if the document is an asset file. @@ -207,6 +216,12 @@ module Jekyll info, File.join(site.config['layouts'], layout.name)) + # Add layout to dependency tree + site.metadata.add_dependency( + site.in_source_dir(path), + site.in_source_dir(layout.path) + ) + if layout = layouts[layout.data["layout"]] if used.include?(layout) layout = nil # avoid recursive chain diff --git a/lib/jekyll/document.rb b/lib/jekyll/document.rb index 003c04eb..5b1b79ab 100644 --- a/lib/jekyll/document.rb +++ b/lib/jekyll/document.rb @@ -105,6 +105,13 @@ module Jekyll !(coffeescript_file? || yaml_file?) end + # Determine whether the document should be regenerated based on metadata. + # + # Returns true if the document needs to be regenerated. + def regenerate? + data['regenerate'] || site.metadata.regenerate?(path, write?) + end + # Determine whether the file should be placed into layouts. # # Returns false if the document is either an asset file or a yaml file, diff --git a/lib/jekyll/layout.rb b/lib/jekyll/layout.rb index 4dde59b6..c29f353f 100644 --- a/lib/jekyll/layout.rb +++ b/lib/jekyll/layout.rb @@ -8,6 +8,9 @@ module Jekyll # Gets the name of this layout. attr_reader :name + # Gets the path to this layout. + attr_reader :path + # Gets/Sets the extension of this layout. attr_accessor :ext @@ -26,6 +29,7 @@ module Jekyll @site = site @base = base @name = name + @path = site.in_source_dir(base, name) self.data = {} diff --git a/lib/jekyll/metadata.rb b/lib/jekyll/metadata.rb new file mode 100644 index 00000000..953dd7c0 --- /dev/null +++ b/lib/jekyll/metadata.rb @@ -0,0 +1,121 @@ +module Jekyll + class Metadata + attr_reader :site, :metadata, :cache + + def initialize(site) + @site = site + + # Read metadata from file + read_metadata + + # Initialize cache to an empty hash + @cache = {} + end + + # Add a path to the metadata + # + # Returns true, also on failure. + def add(path) + return true unless File.exist?(path) + + metadata[path] = { + "mtime" => File.mtime(path), + "deps" => [] + } + cache[path] = true + end + + # Force a path to regenerate + # + # Returns true. + def force(path) + cache[path] = true + end + + # Clear the metadata and cache + # + # Returns nothing + def clear + @metadata = {} + @cache = {} + end + + # Checks if a path should be regenerated + # + # Returns a boolean. + def regenerate?(path, add = true) + return true if disabled? + + # Check for path in cache + if cache.has_key? path + return cache[path] + end + + # Check path that exists in metadata + data = metadata[path] + if data + data["deps"].each do |dependency| + if regenerate?(dependency) + return cache[dependency] = cache[path] = true + end + end + if data["mtime"].eql? File.mtime(path) + return cache[path] = false + else + return !add || add(path) + end + end + + # Path does not exist in metadata, add it + return !add || add(path) + end + + # Add a dependency of a path + # + # Returns nothing. + def add_dependency(path, dependency) + return if (metadata[path].nil? || @disabled) + + metadata[path]["deps"] << dependency unless metadata[path]["deps"].include? dependency + regenerate? dependency + end + + # Write the metadata to disk + # + # Returns nothing. + def write + File.open(metadata_file, 'w') do |f| + f.write(metadata.to_yaml) + end + end + + # Produce the absolute path of the metadata file + # + # Returns the String path of the file. + def metadata_file + site.in_source_dir('.jekyll-metadata') + end + + # Check if metadata has been disabled + # + # Returns a Boolean (true for disabled, false for enabled). + def disabled? + @disabled = site.full_rebuild? if @disabled.nil? + @disabled + end + + private + + # Read metadata from the metadata file, if no file is found, + # initialize with an empty hash + # + # Returns the read metadata. + def read_metadata + @metadata = if !disabled? && File.file?(metadata_file) + SafeYAML.load(File.read(metadata_file)) + else + {} + end + end + end +end diff --git a/lib/jekyll/renderer.rb b/lib/jekyll/renderer.rb index f88a4187..1cdf6c2a 100644 --- a/lib/jekyll/renderer.rb +++ b/lib/jekyll/renderer.rb @@ -138,6 +138,12 @@ module Jekyll File.join(site.config['layouts'], layout.name) ) + # Add layout to dependency tree + site.metadata.add_dependency( + site.in_source_dir(document.path), + site.in_source_dir(layout.path) + ) if document.write? + if layout = site.layouts[layout.data["layout"]] if used.include?(layout) layout = nil # avoid recursive chain diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index 115b4133..7b8a4a00 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -11,6 +11,7 @@ module Jekyll :gems, :plugin_manager attr_accessor :converters, :generators + attr_reader :metadata # Public: Initialize a new Site. # @@ -27,6 +28,9 @@ module Jekyll @source = File.expand_path(config['source']).freeze @dest = File.expand_path(config['destination']).freeze + # Build metadata + @metadata = Metadata.new(self) + self.plugin_manager = Jekyll::PluginManager.new(self) self.plugins = plugin_manager.plugins_path @@ -289,13 +293,13 @@ module Jekyll collections.each do |label, collection| collection.docs.each do |document| - document.output = Jekyll::Renderer.new(self, document).run + document.output = Jekyll::Renderer.new(self, document).run if document.regenerate? end end payload = site_payload [posts, pages].flatten.each do |page_or_post| - page_or_post.render(layouts, payload) + page_or_post.render(layouts, payload) if page_or_post.regenerate? end rescue Errno::ENOENT => e # ignore missing layout dir @@ -312,7 +316,10 @@ module Jekyll # # Returns nothing. def write - each_site_file { |item| item.write(dest) } + each_site_file { |item| + item.write(dest) if item.regenerate? + } + metadata.write unless full_rebuild? end # Construct a Hash of Posts indexed by the specified Post attribute. @@ -483,6 +490,13 @@ module Jekyll @frontmatter_defaults ||= FrontmatterDefaults.new(self) end + # Whether to perform a full rebuild without metadata + # + # Returns a Boolean: true for a full rebuild, false for normal build + def full_rebuild?(override = {}) + override['full_rebuild'] || config['full_rebuild'] + end + private def has_relative_page? diff --git a/lib/jekyll/static_file.rb b/lib/jekyll/static_file.rb index eae85b54..8e100656 100644 --- a/lib/jekyll/static_file.rb +++ b/lib/jekyll/static_file.rb @@ -67,6 +67,8 @@ module Jekyll true end + alias_method :regenerate?, :write? + # Write the static file to the destination directory (if modified). # # dest - The String path to the destination dir. diff --git a/lib/jekyll/tags/include.rb b/lib/jekyll/tags/include.rb index 3eb4d7c0..16da6f49 100644 --- a/lib/jekyll/tags/include.rb +++ b/lib/jekyll/tags/include.rb @@ -105,13 +105,22 @@ eos end def render(context) + site = context.registers[:site] dir = resolved_includes_dir(context) file = render_variable(context) || @file validate_file_name(file) path = File.join(dir, file) - validate_path(path, dir, context.registers[:site].safe) + validate_path(path, dir, site.safe) + + # Add include to dependency tree + if context.registers[:page] and context.registers[:page].has_key? "path" + site.metadata.add_dependency( + site.in_source_dir(context.registers[:page]["path"]), + path + ) + end begin partial = Liquid::Template.parse(source(path, context)) diff --git a/test/helper.rb b/test/helper.rb index 4b1fa7f8..48bfc746 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -49,6 +49,7 @@ class Test::Unit::TestCase def clear_dest FileUtils.rm_rf(dest_dir) + FileUtils.rm_rf(source_dir('.jekyll-metadata')) end def test_dir(*subdirs) diff --git a/test/test_document.rb b/test/test_document.rb index 90c16b82..57c14c22 100644 --- a/test/test_document.rb +++ b/test/test_document.rb @@ -245,8 +245,9 @@ class TestDocument < Test::Unit::TestCase "output" => true } }, - "source" => source_dir, - "destination" => dest_dir + "source" => source_dir, + "destination" => dest_dir, + "full_rebuild" => true })) @site.process @document = @site.collections["slides"].files.find { |doc| doc.relative_path == "_slides/octojekyll.png" } diff --git a/test/test_metadata.rb b/test/test_metadata.rb new file mode 100644 index 00000000..858b166a --- /dev/null +++ b/test/test_metadata.rb @@ -0,0 +1,140 @@ +require 'helper' + +class TestMetadata < Test::Unit::TestCase + context "The site metadata" do + setup do + FileUtils.rm_rf(source_dir(".jekyll-metadata")) + + @site = Site.new(Jekyll.configuration({ + "source" => source_dir, + "destination" => dest_dir + })) + + @site.process + @path = @site.in_source_dir(@site.pages.first.path) + @metadata = @site.metadata + end + + should "store modification times" do + assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"] + end + + should "cache processed entries" do + assert @metadata.cache[@path] + end + + should "write to the metadata file" do + @metadata.clear + @metadata.add(@path) + @metadata.write + assert File.file?(source_dir(".jekyll-metadata")) + end + + should "read from the metadata file" do + @metadata = Metadata.new(@site) + assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"] + end + + # Methods + + should "be able to add a path to the metadata" do + @metadata.clear + @metadata.add(@path) + assert_equal File.mtime(@path), @metadata.metadata[@path]["mtime"] + assert_equal [], @metadata.metadata[@path]["deps"] + assert @metadata.cache[@path] + end + + should "return true on nonexistent path" do + @metadata.clear + assert @metadata.add("/bogus/path.md") + assert @metadata.regenerate?("/bogus/path.md") + end + + should "be able to force a path to regenerate" do + @metadata.clear + @metadata.force(@path) + assert @metadata.cache[@path] + assert @metadata.regenerate?(@path) + end + + should "be able to clear metadata and cache" do + @metadata.clear + @metadata.add(@path) + assert_equal 1, @metadata.metadata.length + assert_equal 1, @metadata.cache.length + @metadata.clear + assert_equal 0, @metadata.metadata.length + assert_equal 0, @metadata.cache.length + end + + should "not regenerate a path if it is not modified" do + @metadata.clear + @metadata.add(@path) + @metadata.write + @metadata = Metadata.new(@site) + + assert !@metadata.regenerate?(@path) + end + + should "not regenerate if path in cache is false" do + @metadata.clear + @metadata.add(@path) + @metadata.write + @metadata = Metadata.new(@site) + + assert !@metadata.regenerate?(@path) + assert !@metadata.cache[@path] + assert !@metadata.regenerate?(@path) + end + + should "regenerate if path in not in metadata" do + @metadata.clear + @metadata.add(@path) + + assert @metadata.regenerate?(@path) + end + + should "regenerate if path in cache is true" do + @metadata.clear + @metadata.add(@path) + + assert @metadata.regenerate?(@path) + assert @metadata.cache[@path] + assert @metadata.regenerate?(@path) + end + + should "regenerate if file is modified" do + @metadata.clear + @metadata.add(@path) + @metadata.metadata[@path]["mtime"] = Time.at(0) + @metadata.write + @metadata = Metadata.new(@site) + + assert_not_same File.mtime(@path), @metadata.metadata[@path]["mtime"] + assert @metadata.regenerate?(@path) + end + + should "regenerate if dependency is modified" do + @metadata.clear + @metadata.add(@path) + @metadata.write + @metadata = Metadata.new(@site) + + @metadata.add_dependency(@path, "new.dependency") + assert_equal ["new.dependency"], @metadata.metadata[@path]["deps"] + assert @metadata.regenerate?("new.dependency") + assert @metadata.regenerate?(@path) + end + + should "regenerate everything if metadata is disabled" do + @site.config["full_rebuild"] = true + @metadata.clear + @metadata.add(@path) + @metadata.write + @metadata = Metadata.new(@site) + + assert @metadata.regenerate?(@path) + end + end +end diff --git a/test/test_site.rb b/test/test_site.rb index 4b8409e7..a6ef5af8 100644 --- a/test/test_site.rb +++ b/test/test_site.rb @@ -99,6 +99,7 @@ class TestSite < Test::Unit::TestCase should "write only modified static files" do clear_dest StaticFile.reset_cache + @site.metadata.clear @site.process some_static_file = @site.static_files[0].path @@ -128,6 +129,7 @@ class TestSite < Test::Unit::TestCase should "write static files if not modified but missing in destination" do clear_dest StaticFile.reset_cache + @site.metadata.clear @site.process some_static_file = @site.static_files[0].path @@ -241,6 +243,7 @@ class TestSite < Test::Unit::TestCase context 'with orphaned files in destination' do setup do clear_dest + @site.metadata.clear @site.process # generate some orphaned files: # single file @@ -328,7 +331,7 @@ class TestSite < Test::Unit::TestCase end bad_processor = "Custom::Markdown" - s = Site.new(site_configuration('markdown' => bad_processor)) + s = Site.new(site_configuration('markdown' => bad_processor, 'full_rebuild' => true)) assert_raise Jekyll::Errors::FatalException do s.process end @@ -348,7 +351,7 @@ class TestSite < Test::Unit::TestCase should 'throw FatalException at process time' do bad_processor = 'not a processor name' - s = Site.new(site_configuration('markdown' => bad_processor)) + s = Site.new(site_configuration('markdown' => bad_processor, 'full_rebuild' => true)) assert_raise Jekyll::Errors::FatalException do s.process end @@ -418,7 +421,9 @@ class TestSite < Test::Unit::TestCase context "manipulating the Jekyll environment" do setup do - @site = Site.new(site_configuration) + @site = Site.new(site_configuration({ + 'full_rebuild' => true + })) @site.process @page = @site.pages.find { |p| p.name == "environment.html" } end @@ -430,7 +435,9 @@ class TestSite < Test::Unit::TestCase context "in production" do setup do ENV["JEKYLL_ENV"] = "production" - @site = Site.new(site_configuration) + @site = Site.new(site_configuration({ + 'full_rebuild' => true + })) @site.process @page = @site.pages.find { |p| p.name == "environment.html" } end