From 091b5bd6976ae549ffc34eff3216d824f8c0fe87 Mon Sep 17 00:00:00 2001 From: Ashwin Maroli Date: Tue, 19 Feb 2019 22:24:18 +0530 Subject: [PATCH] Always exclude certain paths from being processed (#7188) Merge pull request 7188 --- docs/_docs/upgrading/3-to-4.md | 45 +++++++++++++- features/site_configuration.feature | 93 ++++++++++++++++++++++++----- lib/jekyll/configuration.rb | 22 +++++-- lib/jekyll/entry_filter.rb | 2 +- lib/jekyll/reader.rb | 22 +++++++ lib/site_template/_config.yml | 14 ++++- test/test_configuration.rb | 29 +++++++-- 7 files changed, 198 insertions(+), 29 deletions(-) diff --git a/docs/_docs/upgrading/3-to-4.md b/docs/_docs/upgrading/3-to-4.md index 9ccb776d..de4418f9 100644 --- a/docs/_docs/upgrading/3-to-4.md +++ b/docs/_docs/upgrading/3-to-4.md @@ -19,6 +19,7 @@ If you're using Ruby >= 2.3.0, go ahead and fetch the latest version of Jekyll: gem update jekyll ``` +--- ### Template rendering @@ -40,7 +41,6 @@ You'll therefore have to ensure that *`payload`* is not memoized or cached in yo * If its a requirement that `template` you get from the above step *be different* at all times, you can invoke *`Liquid::Template`* directly: - ```diff - template = site.liquid_renderer.file(path).parse(content) + template = Liquid::Template.parse(content) @@ -48,4 +48,47 @@ you can invoke *`Liquid::Template`* directly: --- +### Exclusion changes + +We've enhanced our default exclusion array. It now looks like the following: + +```yaml +# default excludes +exclude: +- .sass-cache/ +- .jekyll-cache/ +- gemfiles/ +- Gemfile +- Gemfile.lock +- node_modules/ +- vendor/bundle/ +- vendor/cache/ +- vendor/gems/ +- vendor/ruby/ +``` + +What's new is that this array **does not get overridden by the `exclude` array +in the user's config file anymore**. The user's exclude entries simply get +**added** to the above default array (if the entry isn't already excluded). + +To forcibly "process" directories or files that have been excluded, list them +in the `include` array instead: + +```yaml +# overrides your excluded items configuration and the default include array ([".htaccess"]) +include: + - .htaccess + - node_modules/uglifier/index.js +``` + +The above configuration directs Jekyll to handle only `node_modules/uglifier/index.js` +while ignoring every other file in the `node_modules` directory since that directory is +"excluded" by default. + +Note that the default `include` array still gets overridden by the `include` array in your +config file. So, be sure to add `.htaccess` to the list if you need that file to be +present in the generated site. + +--- + *Did we miss something? Please click "Improve this page" above and add a section. Thanks!* diff --git a/features/site_configuration.feature b/features/site_configuration.feature index b2f88440..0778c676 100644 --- a/features/site_configuration.feature +++ b/features/site_configuration.feature @@ -48,7 +48,7 @@ Feature: Site configuration And I have a configuration file with "exclude" set to "['Rakefile', 'README']" When I run jekyll build Then I should see "I want to be included" in "_site/index.html" - And the "_site/Gemfile" file should exist + And the "_site/Gemfile" file should not exist And the "_site/Rakefile" file should not exist And the "_site/README" file should not exist @@ -56,6 +56,7 @@ Feature: Site configuration Given I have an "Rakefile" file that contains "I want to be excluded" And I have an "README" file that contains "I want to be excluded" And I have an "index.html" file that contains "I want to be included" + And I have a "Gemfile" file that contains "gem 'include-me'" And I have a configuration file with "exclude" set to: | value | | README | @@ -64,6 +65,83 @@ Feature: Site configuration Then I should see "I want to be included" in "_site/index.html" And the "_site/Rakefile" file should not exist And the "_site/README" file should not exist + And the "_site/Gemfile" file should not exist + + Scenario: Copy over excluded files when their directory is explicitly included + Given I have a ".gitignore" file that contains ".DS_Store" + And I have an ".htaccess" file that contains "SomeDirective" + And I have a "Gemfile" file that contains "gem 'include-me'" + And I have a node_modules directory + And I have a "node_modules/bazinga.js" file that contains "var c = 'Bazinga!';" + And I have a "node_modules/warning.js" file that contains "var w = 'Winter is coming!';" + And I have a configuration file with "include" set to: + | value | + | .gitignore | + | .foo | + | Gemfile | + | node_modules | + When I run jekyll build + Then I should get a zero exit status + And the _site directory should exist + And the "_site/.htaccess" file should not exist + But I should see ".DS_Store" in "_site/.gitignore" + And I should see "gem 'include-me'" in "_site/Gemfile" + And I should see "var c = 'Bazinga!';" in "_site/node_modules/bazinga.js" + And I should see "var w = 'Winter is coming!';" in "_site/node_modules/warning.js" + + Scenario: Copy over excluded files only when they are explicitly included + Given I have a ".gitignore" file that contains ".DS_Store" + And I have an ".htaccess" file that contains "SomeDirective" + And I have a node_modules directory + And I have a "node_modules/bazinga.js" file that contains "var c = 'Bazinga!';" + And I have a "node_modules/warning.js" file that contains "var w = 'Winter is coming!';" + And I have a configuration file with "include" set to: + | value | + | .gitignore | + | .foo | + | node_modules/bazinga.js | + When I run jekyll build + Then I should get a zero exit status + And the _site directory should exist + And the "_site/.htaccess" file should not exist + But I should see ".DS_Store" in "_site/.gitignore" + And I should see "var c = 'Bazinga!';" in "_site/node_modules/bazinga.js" + But the "_site/node_modules/warning.js" file should not exist + + Scenario: Copy over excluded wild-card files only when they are explicitly included + Given I have a ".gitignore" file that contains ".DS_Store" + And I have an ".htaccess" file that contains "SomeDirective" + And I have an "foo.txt" file that contains "Lorem Ipsum" + And I have an "index.md" page that contains "{{ site.title }}" + And I have an "about.md" page that contains "{{ site.author }}" + And I have a configuration file with: + | key | value | + | title | Barren Site | + | author | John Doe | + | exclude | ["**"] | + When I run jekyll build + Then I should get a zero exit status + And the _site directory should exist + And the "_site/.gitignore" file should not exist + And the "_site/foo.txt" file should not exist + And the "_site/index.html" file should not exist + And the "_site/about.html" file should not exist + But the "_site/.htaccess" file should exist + Given I have a configuration file with: + | key | value | + | title | Barren Site | + | author | John Doe | + | exclude | ["**"] | + | include | ["about.md"] | + When I run jekyll build + Then I should get a zero exit status + And the _site directory should exist + And the "_site/.gitignore" file should not exist + And the "_site/foo.txt" file should not exist + And the "_site/index.html" file should not exist + And the "_site/.htaccess" file should not exist + But the "_site/about.html" file should exist + And I should see "John Doe" in "_site/about.html" Scenario: Use Kramdown for markup Given I have an "index.markdown" page that contains "[Google](https://www.google.com)" @@ -196,19 +274,6 @@ Feature: Site configuration And the "_site/2009/04/01/oranges.html" file should exist And the "_site/2009/03/27/apples.html" file should not exist - Scenario: Copy over normally excluded files when they are explicitly included - Given I have a ".gitignore" file that contains ".DS_Store" - And I have an ".htaccess" file that contains "SomeDirective" - And I have a configuration file with "include" set to: - | value | - | .gitignore | - | .foo | - When I run jekyll build - Then I should get a zero exit status - And the _site directory should exist - And I should see ".DS_Store" in "_site/.gitignore" - And the "_site/.htaccess" file should not exist - Scenario: Using a different layouts directory Given I have a _theme directory And I have a page theme that contains "Page Layout: {{ site.posts.size }} on {{ site.time | date: "%Y-%m-%d" }}" diff --git a/lib/jekyll/configuration.rb b/lib/jekyll/configuration.rb index bd63bb91..759ddae9 100644 --- a/lib/jekyll/configuration.rb +++ b/lib/jekyll/configuration.rb @@ -19,10 +19,7 @@ module Jekyll # Handling Reading "safe" => false, "include" => [".htaccess"], - "exclude" => %w( - Gemfile Gemfile.lock node_modules vendor/bundle/ vendor/cache/ vendor/gems/ - vendor/ruby/ - ), + "exclude" => [], "keep_files" => [".git", ".svn"], "encoding" => "utf-8", "markdown_ext" => "markdown,mkdown,mkdn,mkd,md", @@ -92,7 +89,7 @@ module Jekyll # problems and backwards-compatibility. def from(user_config) Utils.deep_merge_hashes(DEFAULTS, Configuration[user_config].stringify_keys) - .add_default_collections + .add_default_collections.add_default_excludes end end @@ -270,6 +267,21 @@ module Jekyll config end + DEFAULT_EXCLUDES = %w( + .sass-cache .jekyll-cache + gemfiles Gemfile Gemfile.lock + node_modules + vendor/bundle/ vendor/cache/ vendor/gems/ vendor/ruby/ + ).freeze + + def add_default_excludes + config = clone + return config if config["exclude"].nil? + + config["exclude"].concat(DEFAULT_EXCLUDES).uniq! + config + end + def renamed_key(old, new, config, _ = nil) if config.key?(old) Jekyll::Deprecator.deprecation_message "The '#{old}' configuration" \ diff --git a/lib/jekyll/entry_filter.rb b/lib/jekyll/entry_filter.rb index 95f95c2a..5f34cf82 100644 --- a/lib/jekyll/entry_filter.rb +++ b/lib/jekyll/entry_filter.rb @@ -56,7 +56,7 @@ module Jekyll end def excluded?(entry) - glob_include?(site.exclude, relative_to_source(entry)).tap do |excluded| + glob_include?(site.exclude - site.include, relative_to_source(entry)).tap do |excluded| if excluded Jekyll.logger.debug( "EntryFilter:", diff --git a/lib/jekyll/reader.rb b/lib/jekyll/reader.rb index c50ad78a..18730ab3 100644 --- a/lib/jekyll/reader.rb +++ b/lib/jekyll/reader.rb @@ -14,6 +14,7 @@ module Jekyll def read @site.layouts = LayoutReader.new(site).read read_directories + read_included_excludes sort_files! @site.data = DataReader.new(site).read(site.config["data_dir"]) CollectionReader.new(site).read @@ -158,5 +159,26 @@ module Jekyll def post_reader @post_reader ||= PostReader.new(site) end + + def read_included_excludes + site.include.each do |entry| + next if entry == ".htaccess" + + entry_path = site.in_source_dir(entry) + next if File.directory?(entry_path) + + read_included_file(entry_path) if File.file?(entry_path) + end + end + + def read_included_file(entry_path) + dir = File.dirname(entry_path).sub(site.source, "") + file = Array(File.basename(entry_path)) + if Utils.has_yaml_header?(entry_path) + site.pages.concat(PageReader.new(site, dir).read(file)) + else + site.static_files.concat(StaticFileReader.new(site, dir).read(file)) + end + end end end diff --git a/lib/site_template/_config.yml b/lib/site_template/_config.yml index 876310ac..60e14aa7 100644 --- a/lib/site_template/_config.yml +++ b/lib/site_template/_config.yml @@ -35,12 +35,20 @@ plugins: - jekyll-feed # Exclude from processing. -# The following items will not be processed, by default. Create a custom list -# to override the default setting. +# The following items will not be processed, by default. +# Any item listed under the `exclude:` key here will be automatically added to +# the internal "default list". +# +# Excluded items can be processed by explicitly listing the directories or +# their entries' file path in the `include:` list. +# # exclude: +# - .sass-cache/ +# - .jekyll-cache/ +# - gemfiles/ # - Gemfile # - Gemfile.lock -# - node_modules +# - node_modules/ # - vendor/bundle/ # - vendor/cache/ # - vendor/gems/ diff --git a/test/test_configuration.rb b/test/test_configuration.rb index c49a48e5..f09c4a09 100644 --- a/test/test_configuration.rb +++ b/test/test_configuration.rb @@ -41,20 +41,39 @@ class TestConfiguration < JekyllUnitTest end end - context "the defaults" do - should "exclude node_modules" do - assert_includes Configuration.from({})["exclude"], "node_modules" + context "the effective site configuration" do + setup do + @config = Configuration.from( + "exclude" => %w( + README.md Licence + ) + ) end - should "exclude ruby vendor directories" do - exclude = Configuration.from({})["exclude"] + should "always exclude node_modules" do + assert_includes @config["exclude"], "node_modules" + end + + should "always exclude Gemfile and related paths" do + exclude = @config["exclude"] assert_includes exclude, "Gemfile" assert_includes exclude, "Gemfile.lock" + assert_includes exclude, "gemfiles" + end + + should "always exclude ruby vendor directories" do + exclude = @config["exclude"] assert_includes exclude, "vendor/bundle/" assert_includes exclude, "vendor/cache/" assert_includes exclude, "vendor/gems/" assert_includes exclude, "vendor/ruby/" end + + should "always exclude default cache directories" do + exclude = @config["exclude"] + assert_includes exclude, ".sass-cache" + assert_includes exclude, ".jekyll-cache" + end end context "#add_default_collections" do