From 760cbc7f91ecd3dada0d6ce8c214bbc102bebfd9 Mon Sep 17 00:00:00 2001
From: liufengyun <liufengyunchina@gmail.com>
Date: Tue, 1 Oct 2013 11:18:31 +0800
Subject: [PATCH] Autoload yaml files under _data directory

The jekyll engine will autoload all yaml files(ends with .yml or .yaml)
under _data. If there's a file members.yml under the directory, then user
can access contents of the file through site.members.
---
 features/data.feature                     | 65 +++++++++++++++++++++++
 features/step_definitions/jekyll_steps.rb |  6 +++
 jekyll.gemspec                            |  2 +
 lib/jekyll/configuration.rb               |  1 +
 lib/jekyll/site.rb                        | 40 +++++++++++++-
 site/docs/structure.md                    | 17 ++++++
 test/source/_data/languages.yml           |  2 +
 test/source/_data/members.yaml            |  7 +++
 test/source/_data/products.yml            |  1 +
 test/source/products.yml                  |  4 ++
 test/source/symlink-test/_data            |  1 +
 test/test_site.rb                         | 57 ++++++++++++++++++++
 12 files changed, 201 insertions(+), 2 deletions(-)
 create mode 100644 features/data.feature
 create mode 100644 test/source/_data/languages.yml
 create mode 100644 test/source/_data/members.yaml
 create mode 120000 test/source/_data/products.yml
 create mode 100644 test/source/products.yml
 create mode 120000 test/source/symlink-test/_data

diff --git a/features/data.feature b/features/data.feature
new file mode 100644
index 00000000..33adfaad
--- /dev/null
+++ b/features/data.feature
@@ -0,0 +1,65 @@
+Feature: Data
+  In order to use well-formatted data in my blog
+  As a blog's user
+  I want to use _data directory in my site
+
+  Scenario: autoload *.yaml files in _data directory
+    Given I have a _data directory
+    And I have a "_data/products.yaml" file with content:
+      """
+      - name: sugar
+        price: 5.3
+      - name: salt
+        price: 2.5
+      """
+    And I have an "index.html" page that contains "{% for product in site.data.products %}{{product.name}}{% endfor %}"
+    When I run jekyll
+    Then the "_site/index.html" file should exist
+    And I should see "sugar" in "_site/index.html"
+    And I should see "salt" in "_site/index.html"
+
+  Scenario: autoload *.yml files in _data directory
+    Given I have a _data directory
+    And I have a "_data/members.yml" file with content:
+      """
+      - name: Jack
+        age: 28
+      - name: Leon
+        age: 34
+      """
+    And I have an "index.html" page that contains "{% for member in site.data.members %}{{member.name}}{% endfor %}"
+    When I run jekyll
+    Then the "_site/index.html" file should exist
+    And I should see "Jack" in "_site/index.html"
+    And I should see "Leon" in "_site/index.html"
+
+  Scenario: autoload *.yml files in _data directory with space in file name
+    Given I have a _data directory
+    And I have a "_data/team members.yml" file with content:
+      """
+      - name: Jack
+        age: 28
+      - name: Leon
+        age: 34
+      """
+    And I have an "index.html" page that contains "{% for member in site.data.team_members %}{{member.name}}{% endfor %}"
+    When I run jekyll
+    Then the "_site/index.html" file should exist
+    And I should see "Jack" in "_site/index.html"
+    And I should see "Leon" in "_site/index.html"
+
+  Scenario: should be backward compatible with site.data in _config.yml
+    Given I have a "_config.yml" file with content:
+      """
+      data:
+        - name: Jack
+          age: 28
+        - name: Leon
+          age: 34
+      """
+    And I have an "index.html" page that contains "{% for member in site.data %}{{member.name}}{% endfor %}"
+    When I run jekyll
+    Then the "_site/index.html" file should exist
+    And I should see "Jack" in "_site/index.html"
+    And I should see "Leon" in "_site/index.html"
+
diff --git a/features/step_definitions/jekyll_steps.rb b/features/step_definitions/jekyll_steps.rb
index 56da1d11..3604f523 100644
--- a/features/step_definitions/jekyll_steps.rb
+++ b/features/step_definitions/jekyll_steps.rb
@@ -43,6 +43,12 @@ Given /^I have an? (.*) (layout|theme) that contains "(.*)"$/ do |name, type, te
   end
 end
 
+Given /^I have an? "(.*)" file with content:$/ do |file, text|
+  File.open(file, 'w') do |f|
+    f.write(text)
+  end
+end
+
 Given /^I have an? (.*) directory$/ do |dir|
   FileUtils.mkdir_p(dir)
 end
diff --git a/jekyll.gemspec b/jekyll.gemspec
index f726d4d1..a9ddd338 100644
--- a/jekyll.gemspec
+++ b/jekyll.gemspec
@@ -59,6 +59,7 @@ Gem::Specification.new do |s|
     bin/jekyll
     cucumber.yml
     features/create_sites.feature
+    features/data.feature
     features/drafts.feature
     features/embed_filters.feature
     features/include_tag.feature
@@ -203,6 +204,7 @@ Gem::Specification.new do |s|
     test/helper.rb
     test/source/+/foo.md
     test/source/.htaccess
+    test/source/_data/members.yaml
     test/source/_includes/params.html
     test/source/_includes/sig.markdown
     test/source/_layouts/default.html
diff --git a/lib/jekyll/configuration.rb b/lib/jekyll/configuration.rb
index 43244b47..de903a96 100644
--- a/lib/jekyll/configuration.rb
+++ b/lib/jekyll/configuration.rb
@@ -10,6 +10,7 @@ module Jekyll
       'destination'   => File.join(Dir.pwd, '_site'),
       'plugins'       => '_plugins',
       'layouts'       => '_layouts',
+      'data_source'   =>  '_data',
       'keep_files'    => ['.git','.svn'],
 
       'timezone'      => nil,           # use the local timezone
diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb
index 6084631a..5cad11c8 100644
--- a/lib/jekyll/site.rb
+++ b/lib/jekyll/site.rb
@@ -3,7 +3,7 @@ module Jekyll
     attr_accessor :config, :layouts, :posts, :pages, :static_files,
                   :categories, :exclude, :include, :source, :dest, :lsi, :pygments,
                   :permalink_style, :tags, :time, :future, :safe, :plugins, :limit_posts,
-                  :show_drafts, :keep_files, :baseurl, :file_read_opts
+                  :show_drafts, :keep_files, :baseurl, :data, :file_read_opts
 
     attr_accessor :converters, :generators
 
@@ -56,6 +56,7 @@ module Jekyll
       self.static_files    = []
       self.categories      = Hash.new { |hash, key| hash[key] = [] }
       self.tags            = Hash.new { |hash, key| hash[key] = [] }
+      self.data            = {}
 
       if self.limit_posts < 0
         raise ArgumentError, "limit_posts must be a non-negative number"
@@ -110,6 +111,7 @@ module Jekyll
     def read
       self.read_layouts
       self.read_directories
+      self.read_data(config['data_source'])
     end
 
     # Read all the files in <source>/<layouts> and create a new Layout object
@@ -197,6 +199,25 @@ module Jekyll
       end
     end
 
+    # Read and parse all yaml files under <source>/<dir>
+    #
+    # Returns nothing
+    def read_data(dir)
+      base = File.join(self.source, dir)
+      return unless File.directory?(base) && (!self.safe || !File.symlink?(base))
+
+      entries = Dir.chdir(base) { Dir['*.{yaml,yml}'] }
+      entries.delete_if { |e| File.directory?(File.join(base, e)) }
+
+      entries.each do |entry|
+        path = File.join(self.source, dir, entry)
+        next if File.symlink?(path) && self.safe
+
+        key = sanitize_filename(File.basename(entry, '.*'))
+        self.data[key] = YAML.safe_load_file(path)
+      end
+    end
+
     # Run each of the Generators.
     #
     # Returns nothing.
@@ -262,6 +283,14 @@ module Jekyll
       hash
     end
 
+    # Prepare site data for site payload. The method maintains backward compatibility
+    # if the key 'data' is already used in _config.yml.
+    #
+    # Returns the Hash to be hooked to site.data.
+    def site_data
+      self.config['data'] || self.data
+    end
+
     # The Hash payload containing site-wide data.
     #
     # Returns the Hash: { "site" => data } where data is a Hash with keys:
@@ -283,7 +312,8 @@ module Jekyll
           "pages"      => self.pages,
           "html_pages" => self.pages.reject { |page| !page.html? },
           "categories" => post_attr_hash('categories'),
-          "tags"       => post_attr_hash('tags')})}
+          "tags"       => post_attr_hash('tags'),
+          "data"       => site_data})}
     end
 
     # Filter out any files/directories that are hidden or backup files (start
@@ -393,5 +423,11 @@ module Jekyll
     def site_cleaner
       @site_cleaner ||= Cleaner.new(self)
     end
+
+    def sanitize_filename(name)
+      name = name.gsub(/[^\w\s_-]+/, '')
+      name = name.gsub(/(^|\b\s)\s+($|\s?\b)/, '\\1\\2')
+      name = name.gsub(/\s+/, '_')
+    end
   end
 end
diff --git a/site/docs/structure.md b/site/docs/structure.md
index 02b33daf..d03baee0 100644
--- a/site/docs/structure.md
+++ b/site/docs/structure.md
@@ -31,6 +31,8 @@ A basic Jekyll site usually looks something like this:
 ├── _posts
 |   ├── 2007-10-29-why-every-programmer-should-play-nethack.textile
 |   └── 2009-04-26-barcamp-boston-4-roundup.textile
+├── _data
+|   └── members.yml
 ├── _site
 └── index.html
 {% endhighlight %}
@@ -121,6 +123,21 @@ An overview of what each of these does:
         </p>
       </td>
     </tr>
+    <tr>
+      <td>
+        <p><code>_data</code></p>
+      </td>
+      <td>
+        <p>
+
+          Well-formatted site data should be placed here. The jekyll engine will 
+          autoload all yaml files (ends with <code>.yml</code> or <code>.yaml</code>) 
+          in this directory. If there's a file <code>members.yml</code> under the directory, 
+          then you can access contents of the file through <code>site.data.members</code>.
+
+        </p>
+      </td>
+    </tr>
     <tr>
       <td>
         <p><code>_site</code></p>
diff --git a/test/source/_data/languages.yml b/test/source/_data/languages.yml
new file mode 100644
index 00000000..6b0a250c
--- /dev/null
+++ b/test/source/_data/languages.yml
@@ -0,0 +1,2 @@
+- java
+- ruby
diff --git a/test/source/_data/members.yaml b/test/source/_data/members.yaml
new file mode 100644
index 00000000..22e29a9f
--- /dev/null
+++ b/test/source/_data/members.yaml
@@ -0,0 +1,7 @@
+- name: Jack
+  age: 27
+  blog: http://example.com/jack
+
+- name: John
+  age: 32
+  blog: http://example.com/john
diff --git a/test/source/_data/products.yml b/test/source/_data/products.yml
new file mode 120000
index 00000000..bc0c6852
--- /dev/null
+++ b/test/source/_data/products.yml
@@ -0,0 +1 @@
+../products.yml
\ No newline at end of file
diff --git a/test/source/products.yml b/test/source/products.yml
new file mode 100644
index 00000000..21828a09
--- /dev/null
+++ b/test/source/products.yml
@@ -0,0 +1,4 @@
+- name: sugar
+  price: 5.3
+- name: salt
+  price: 2.5
diff --git a/test/source/symlink-test/_data b/test/source/symlink-test/_data
new file mode 120000
index 00000000..37fb8ffe
--- /dev/null
+++ b/test/source/symlink-test/_data
@@ -0,0 +1 @@
+../_data
\ No newline at end of file
diff --git a/test/test_site.rb b/test/test_site.rb
index b4d06239..0ff9185f 100644
--- a/test/test_site.rb
+++ b/test/test_site.rb
@@ -335,5 +335,62 @@ class TestSite < Test::Unit::TestCase
       end
     end
 
+    context 'data directory' do
+      should 'auto load yaml files' do
+        site = Site.new(Jekyll.configuration)
+        site.process
+
+        file_content = YAML.safe_load_file(File.join(source_dir, '_data', 'members.yaml'))
+
+        assert_equal site.data['members'], file_content
+        assert_equal site.site_payload['site']['data']['members'], file_content
+      end
+
+      should 'auto load yml files' do
+        site = Site.new(Jekyll.configuration)
+        site.process
+
+        file_content = YAML.safe_load_file(File.join(source_dir, '_data', 'languages.yml'))
+
+        assert_equal site.data['languages'], file_content
+        assert_equal site.site_payload['site']['data']['languages'], file_content
+      end
+
+      should "load symlink files in unsafe mode" do
+        site = Site.new(Jekyll.configuration.merge({'safe' => false}))
+        site.process
+
+        file_content = YAML.safe_load_file(File.join(source_dir, '_data', 'products.yml'))
+
+        assert_equal site.data['products'], file_content
+        assert_equal site.site_payload['site']['data']['products'], file_content
+      end
+
+      should "not load symlink files in safe mode" do
+        site = Site.new(Jekyll.configuration.merge({'safe' => true}))
+        site.process
+
+        assert_nil site.data['products']
+        assert_nil site.site_payload['site']['data']['products']
+      end
+
+      should "load symlink directory in unsafe mode" do
+        site = Site.new(Jekyll.configuration.merge({'safe' => false, 'data_source' => File.join('symlink-test', '_data')}))
+        site.process
+
+        assert_not_nil site.data['products']
+        assert_not_nil site.data['languages']
+        assert_not_nil site.data['members']
+      end
+
+      should "not load symlink directory in safe mode" do
+        site = Site.new(Jekyll.configuration.merge({'safe' => true, 'data_source' => File.join('symlink-test', '_data')}))
+        site.process
+
+        assert_nil site.data['products']
+        assert_nil site.data['languages']
+        assert_nil site.data['members']
+      end
+    end
   end
 end