Merge pull request #2199 from jekyll/collection-plate

features/collections.feature

@@ -0,0 +1,38 @@
+Feature: Collections
+  As a hacker who likes to structure content
+  I want to be able to create collections of similar information
+  And render them
+
+  Scenario: Unrendered collection
+    Given I have an "index.html" page that contains "Collections: {{ site.methods }}"
+    And I have fixture collections
+    And I have a configuration file with "collections" set to "['methods']"
+    When I run jekyll
+    Then the _site directory should exist
+    And I should see "Collections: Use `{{ page.title }}` to build a full configuration for use w/Jekyll.\n\nWhatever: {{ page.whatever }}\n`{{ page.title }}` is used to make sure your path is in your source.\nRun your generators! {{ page.layout }}\nCreate dat site.\nRun your generators! {{ page.layout }}" in "_site/index.html"
+
+  Scenario: Rendered collection
+    Given I have an "index.html" page that contains "Collections: {{ site.collections }}"
+    And I have fixture collections
+    And I have a configuration file with:
+      | key         | value       |
+      | collections | ['methods'] |
+      | render      | ['methods'] |
+    When I run jekyll
+    Then the _site directory should exist
+    And I should see "Collections: methods" in "_site/index.html"
+    And I should see "<p>Whatever: foo.bar</p>" in "_site/methods/configuration.html"
+
+  Scenario: Rendered document in a layout
+    Given I have an "index.html" page that contains "Collections: {{ site.collections }}"
+    And I have a default layout that contains "<div class='title'>Tom Preston-Werner</div> {{content}}"
+    And I have fixture collections
+    And I have a configuration file with:
+      | key         | value       |
+      | collections | ['methods'] |
+      | render      | ['methods'] |
+    When I run jekyll
+    Then the _site directory should exist
+    And I should see "Collections: methods" in "_site/index.html"
+    And I should see "<p>Run your generators! default</p>" in "_site/methods/site/generate.html"
+    And I should see "<div class='title'>Tom Preston-Werner</div>" in "_site/methods/site/generate.html"

features/step_definitions/jekyll_steps.rb

@@ -16,15 +16,14 @@ def file_content_from_hash(input_hash)
 EOF
 end
 
-
 Before do
   FileUtils.mkdir_p(TEST_DIR) unless File.exist?(TEST_DIR)
   Dir.chdir(TEST_DIR)
 end
 
 After do
-  FileUtils.rm_rf(TEST_DIR)   if File.exist?(TEST_DIR)
-  FileUtils.rm(JEKYLL_COMMAND_OUTPUT_FILE)
+  FileUtils.rm_rf(TEST_DIR)   if File.exists?(TEST_DIR)
+  FileUtils.rm(JEKYLL_COMMAND_OUTPUT_FILE) if File.exists?(JEKYLL_COMMAND_OUTPUT_FILE)
 end
 
 World(Test::Unit::Assertions)
@@ -130,6 +129,16 @@ Given /^I have a configuration file with "([^\"]*)" set to:$/ do |key, table|
   end
 end
 
+Given /^I have fixture collections$/ do
+  FileUtils.cp_r File.join(JEKYLL_SOURCE_DIR, "test", "source", "_methods"), source_dir
+end
+
+##################
+#
+# Changing stuff
+#
+##################
+
 When /^I run jekyll(?: with "(.+)")?$/ do |opt|
   run_jekyll_build(opt)
 end

features/support/env.rb

@@ -6,10 +6,15 @@ require 'rr'
 require 'test/unit'
 require 'time'
 
+JEKYLL_SOURCE_DIR = File.dirname(File.dirname(File.dirname(__FILE__)))
 TEST_DIR    = File.expand_path(File.join('..', '..', 'tmp', 'jekyll'), File.dirname(__FILE__))
 JEKYLL_PATH = File.join(File.dirname(__FILE__), '..', '..', 'bin', 'jekyll')
 JEKYLL_COMMAND_OUTPUT_FILE = File.join(File.dirname(TEST_DIR), 'jekyll_output.txt')
 
+def source_dir(*files)
+  File.join(TEST_DIR, *files)
+end
+
 def jekyll_output_file
   JEKYLL_COMMAND_OUTPUT_FILE
 end
@@ -19,7 +24,7 @@ def jekyll_run_output
 end
 
 def run_jekyll(args, output_file)
-  command = "#{JEKYLL_PATH} #{args} > #{jekyll_output_file} 2>&1"
+  command = "#{JEKYLL_PATH} #{args} --trace > #{jekyll_output_file} 2>&1"
   system command
 end
 

lib/jekyll.rb

@@ -34,6 +34,8 @@ require 'jekyll/utils'
 require 'jekyll/stevenson'
 require 'jekyll/deprecator'
 require 'jekyll/configuration'
+require 'jekyll/document'
+require 'jekyll/collection'
 require 'jekyll/plugin_manager'
 require 'jekyll/site'
 require 'jekyll/convertible'
@@ -51,6 +53,7 @@ require 'jekyll/cleaner'
 require 'jekyll/entry_filter'
 require 'jekyll/layout_reader'
 require 'jekyll/publisher'
+require 'jekyll/renderer'
 
 # extensions
 require 'jekyll/plugin'

lib/jekyll/cleaner.rb

@@ -4,6 +4,8 @@ module Jekyll
   class Site
     # Handles the cleanup of a site's destination before it is built.
     class Cleaner
+      attr_reader :site
+
       def initialize(site)
         @site = site
       end
@@ -27,7 +29,7 @@ module Jekyll
       # Returns a Set with the file paths
       def existing_files
         files = Set.new
-        Dir.glob(File.join(@site.dest, "**", "*"), File::FNM_DOTMATCH) do |file|
+        Dir.glob(File.join(site.dest, "**", "*"), File::FNM_DOTMATCH) do |file|
           files << file unless file =~ /\/\.{1,2}$/ || file =~ keep_file_regex
         end
         files
@@ -38,7 +40,7 @@ module Jekyll
       # Returns a Set with the file paths
       def new_files
         files = Set.new
-        @site.each_site_file { |item| files << item.destination(@site.dest) }
+        site.each_site_file { |item| files << item.destination(site.dest) }
         files
       end
 
@@ -64,7 +66,7 @@ module Jekyll
       #
       # Returns the regular expression
       def keep_file_regex
-        or_list = @site.keep_files.join("|")
+        or_list = site.keep_files.join("|")
         pattern = "\/(#{or_list.gsub(".", "\.")})"
         Regexp.new pattern
       end

lib/jekyll/collection.rb

@@ -0,0 +1,121 @@
+module Jekyll
+  class Collection
+    attr_reader :site, :label
+
+    # Create a new Collection.
+    #
+    # site - the site to which this collection belongs.
+    # label - the name of the collection
+    #
+    # Returns nothing.
+    def initialize(site, label)
+      @site  = site
+      @label = sanitize_label(label)
+    end
+
+    # Fetch the Documents in this collection.
+    # Defaults to an empty array if no documents have been read in.
+    #
+    # Returns an array of Jekyll::Document objects.
+    def docs
+      @docs ||= []
+    end
+
+    # Read the allowed documents into the collection's array of docs.
+    #
+    # Returns the sorted array of docs.
+    def read
+      filtered_entries.each do |file_path|
+        doc = Jekyll::Document.new(Jekyll.sanitized_path(directory, file_path), { site: site, collection: self })
+        doc.read
+        docs << doc
+      end
+      docs.sort!
+    end
+
+    # All the entries in this collection.
+    #
+    # Returns an Array of file paths to the documents in this collection
+    #   relative to the collection's directory
+    def entries
+      return Array.new unless exists?
+      Dir.glob(File.join(directory, "**", "*.*")).map do |entry|
+        entry[File.join(directory, "")] = ''; entry
+      end
+    end
+
+    # Filtered version of the entries in this collection.
+    # See `Jekyll::EntryFilter#filter` for more information.
+    #
+    # Returns a list of filtered entry paths.
+    def filtered_entries
+      return Array.new unless exists?
+      Dir.chdir(directory) do
+        entry_filter.filter(entries)
+      end
+    end
+
+    # The directory for this Collection, relative to the site source.
+    #
+    # Returns a String containing the directory name where the collection
+    #   is stored on the filesystem.
+    def relative_directory
+      "_#{label}"
+    end
+
+    # The full path to the directory containing the
+    #
+    # Returns a String containing th directory name where the collection
+    #   is stored on the filesystem.
+    def directory
+      Jekyll.sanitized_path(site.source, relative_directory)
+    end
+
+    # Checks whether the directory "exists" for this collection.
+    # The directory must exist on the filesystem and must not be a symlink
+    #   if in safe mode.
+    #
+    # Returns false if the directory doesn't exist or if it's a symlink
+    #   and we're in safe mode.
+    def exists?
+      File.directory?(directory) && !(File.symlink?(directory) && site.safe)
+    end
+
+    # The entry filter for this collection.
+    # Creates an instance of Jekyll::EntryFilter.
+    #
+    # Returns the instance of Jekyll::EntryFilter for this collection.
+    def entry_filter
+      @entry_filter ||= Jekyll::EntryFilter.new(site, relative_directory)
+    end
+
+    # An inspect string.
+    #
+    # Returns the inspect string
+    def inspect
+      "#<Jekyll::Collection @label=#{label} docs=#{docs}>"
+    end
+
+    # Produce a sanitized label name
+    # Label names may not contain anything but alphanumeric characters,
+    #   underscores, and hyphens.
+    #
+    # label - the possibly-unsafe label
+    #
+    # Returns a sanitized version of the label.
+    def sanitize_label(label)
+      label.gsub(/[^a-z0-9_\-]/i, '')
+    end
+
+    # Produce a representation of this Collection for use in Liquid.
+    # Exposes two attributes:
+    #   - label
+    #   - docs
+    #
+    # Returns a representation of this collection for use in Liquid.
+    def to_liquid
+      docs
+    end
+
+  end
+end

lib/jekyll/configuration.rb

@@ -13,6 +13,7 @@ module Jekyll
       'data_source'   =>  '_data',
       'keep_files'    => ['.git','.svn'],
       'gems'          => [],
+      'collections'   => nil,
 
       'timezone'      => nil,           # use the local timezone
 

lib/jekyll/document.rb

@@ -0,0 +1,228 @@
+module Jekyll
+  class Document
+    include Comparable
+
+    attr_reader   :path, :site
+    attr_accessor :content, :collection, :output
+
+    # Create a new Document.
+    #
+    # site - the Jekyll::Site instance to which this Document belongs
+    # path - the path to the file
+    #
+    # Returns nothing.
+    def initialize(path, relations)
+      @site = relations[:site]
+      @path = path
+      @collection = relations[:collection]
+    end
+
+    # Fetch the Document's data.
+    #
+    # Returns a Hash containing the data. An empty hash is returned if
+    #   no data was read.
+    def data
+      @data ||= Hash.new
+    end
+
+    # The path to the document, relative to the site source.
+    #
+    # Returns a String path which represents the relative path
+    #   from the site source to this document
+    def relative_path
+      Pathname.new(path).relative_path_from(Pathname.new(site.source)).to_s
+    end
+
+    # The base filename of the document.
+    #
+    # suffix - (optional) the suffix to be removed from the end of the filename
+    #
+    # Returns the base filename of the document.
+    def basename(suffix = "")
+      File.basename(path, suffix)
+    end
+
+    # The extension name of the document.
+    #
+    # Returns the extension name of the document.
+    def extname
+      File.extname(path)
+    end
+
+    # Produces a "cleaned" relative path.
+    # The "cleaned" relative path is the relative path without the extname
+    #   and with the collection's directory removed as well.
+    # This method is useful when building the URL of the document.
+    #
+    # Examples:
+    #   When relative_path is "_methods/site/generate.md":
+    #     cleaned_relative_path
+    #     # => "/site/generate"
+    #
+    # Returns the cleaned relative path of the document.
+    def cleaned_relative_path
+      relative_path[0 .. -extname.length - 1].sub(collection.relative_directory, "")
+    end
+
+    # Determine whether the document is a YAML file.
+    #
+    # Returns true if the extname is either .yml or .yaml, false otherwise.
+    def yaml_file?
+      %w[.yaml .yml].include?(extname)
+    end
+
+    # Determine whether the document is an asset file.
+    # Asset files include CoffeeScript files and Sass/SCSS files.
+    #
+    # Returns true if the extname belongs to the set of extensions
+    #   that asset files use.
+    def asset_file?
+      %w[.sass .scss .coffee].include?(extname)
+    end
+
+    # Determine whether the file should be rendered with Liquid.
+    #
+    # Returns false if the document is either an asset file or a yaml file,
+    #   true otherwise.
+    def render_with_liquid?
+      !(asset_file? || yaml_file?)
+    end
+
+    # The URL template where the document would be accessible.
+    #
+    # Returns the URL template for the document.
+    def url_template
+      "/:collection/:path:output_ext"
+    end
+
+    # Construct a Hash of key-value pairs which contain a mapping between
+    #   a key in the URL template and the corresponding value for this document.
+    #
+    # Returns the Hash of key-value pairs for replacement in the URL.
+    def url_placeholders
+      {
+        collection: collection.label,
+        path:       cleaned_relative_path,
+        output_ext: Jekyll::Renderer.new(site, self).output_ext
+      }
+    end
+
+    # The permalink for this Document.
+    # Permalink is set via the data Hash.
+    #
+    # Returns the permalink or nil if no permalink was set in the data.
+    def permalink
+      data && data['permalink']
+    end
+
+    # The computed URL for the document. See `Jekyll::URL#to_s` for more details.
+    #
+    # Returns the computed URL for the document.
+    def url
+      @url ||= URL.new({
+        template:     url_template,
+        placeholders: url_placeholders,
+        permalink:    permalink
+      }).to_s
+    end
+
+    # The full path to the output file.
+    #
+    # base_directory - the base path of the output directory
+    #
+    # Returns the full path to the output file of this document.
+    def destination(base_directory)
+      path = Jekyll.sanitized_path(base_directory, url)
+      path = File.join(path, "index.html") if url =~ /\/$/
+      path
+    end
+
+    # Write the generated Document file to the destination directory.
+    #
+    # dest - The String path to the destination dir.
+    #
+    # Returns nothing.
+    def write(dest)
+      path = destination(dest)
+      FileUtils.mkdir_p(File.dirname(path))
+      File.open(path, 'wb') do |f|
+        f.write(output)
+      end
+    end
+
+    # Returns merged option hash for File.read of self.site (if exists)
+    # and a given param
+    #
+    # opts - override options
+    #
+    # Return the file read options hash.
+    def merged_file_read_opts(opts)
+      site ? site.file_read_opts.merge(opts) : opts
+    end
+
+    # Whether the file is published or not, as indicated in YAML front-matter
+    #
+    # Returns true if the 'published' key is specified in the YAML front-matter and not `false`.
+    def published?
+      !(data.has_key?('published') && data['published'] == false)
+    end
+
+    # Read in the file and assign the content and data based on the file contents.
+    #
+    # Returns nothing.
+    def read(opts = {})
+      if yaml_file?
+        @data = SafeYAML.load_file(path)
+      else
+        begin
+          @content = File.read(path, merged_file_read_opts(opts))
+          if content =~ /\A(---\s*\n.*?\n?)^(---\s*$\n?)/m
+            @content = $POSTMATCH
+            @data    = SafeYAML.load($1)
+          end
+        rescue SyntaxError => e
+          puts "YAML Exception reading #{path}: #{e.message}"
+        rescue Exception => e
+          puts "Error reading file #{path}: #{e.message}"
+        end
+      end
+    end
+
+    # Create a Liquid-understandable version of this Document.
+    #
+    # Returns a Hash representing this Document's data.
+    def to_liquid
+      Utils.deep_merge_hashes data, {
+        "content"       => content,
+        "path"          => path,
+        "relative_path" => relative_path,
+        "url"           => url
+      }
+    end
+
+    # The inspect string for this document.
+    # Includes the relative path and the collection label.
+    #
+    # Returns the inspect string for this document.
+    def inspect
+      "#<Jekyll::Document #{relative_path} collection=#{collection.label}>"
+    end
+
+    # The string representation for this document.
+    #
+    # Returns the content of the document
+    def to_s
+      output || content
+    end
+
+    # Compare this document against another document.
+    # Comparison is a comparison between the 2 paths of the documents.
+    #
+    # Returns -1, 0, +1 or nil depending on whether this doc's path is less than,
+    #   equal or greater than the other doc's path. See String#<=> for more details.
+    def <=>(anotherDocument)
+      path <=> anotherDocument.path
+    end
+
+  end
+end

lib/jekyll/entry_filter.rb

@@ -1,5 +1,7 @@
 module Jekyll
   class EntryFilter
+    SPECIAL_LEADING_CHARACTERS = ['.', '_', '#'].freeze
+
     attr_reader :site
 
     def initialize(site, base_directory = nil)
@@ -35,7 +37,8 @@ module Jekyll
     end
 
     def special?(entry)
-      ['.', '_', '#'].include?(entry[0..0])
+      SPECIAL_LEADING_CHARACTERS.include?(entry[0..0]) ||
+        SPECIAL_LEADING_CHARACTERS.include?(File.basename(entry)[0..0])
     end
 
     def backup?(entry)

lib/jekyll/renderer.rb

@@ -0,0 +1,132 @@
+module Jekyll
+  class Renderer
+
+    attr_reader :document, :site
+
+    def initialize(site, document)
+      @site     = site
+      @document = document
+    end
+
+    # Determine which converters to use based on this document's
+    # extension.
+    #
+    # Returns an array of Converter instances.
+    def converters
+      @converters ||= site.converters.select { |c| c.matches(document.extname) }
+    end
+
+    # Determine the extname the outputted file should have
+    #
+    # Returns the output extname including the leading period.
+    def output_ext
+      converters.first.output_ext(document.extname)
+    end
+
+    ######################
+    ## DAT RENDER THO
+    ######################
+
+    def run
+      payload = Utils.deep_merge_hashes({
+        "page" => document.to_liquid
+      }, site.site_payload)
+
+      info = {
+        filters:   [Jekyll::Filters],
+        registers: { :site => site, :page => payload['page'] }
+      }
+
+      # render and transform content (this becomes the final content of the object)
+      payload["highlighter_prefix"] = converters.first.highlighter_prefix
+      payload["highlighter_suffix"] = converters.first.highlighter_suffix
+
+      output = document.content
+
+      if document.render_with_liquid?
+        output = render_liquid(output, payload, info)
+      end
+
+      place_in_layouts(
+        convert(output),
+        payload,
+        info
+      )
+    end
+
+    # Convert the given content using the converters which match this renderer's document.
+    #
+    # content - the raw, unconverted content
+    #
+    # Returns the converted content.
+    def convert(content)
+      converters.reduce(content) do |output, converter|
+        begin
+          converter.convert output
+        rescue => e
+          Jekyll.logger.error "Conversion error:", "#{converter.class} encountered an error converting '#{document.relative_path}'."
+          raise e
+        end
+      end
+    end
+
+    # Render the given content with the payload and info
+    #
+    # content -
+    # payload -
+    # info    -
+    # path    - (optional) the path to the file, for use in ex
+    #
+    # Returns the content, rendered by Liquid.
+    def render_liquid(content, payload, info, path = nil)
+      Liquid::Template.parse(content).render!(payload, info)
+    rescue Tags::IncludeTagError => e
+      Jekyll.logger.error "Liquid Exception:", "#{e.message} in #{e.path}, included in #{path || document.relative_path}"
+      raise e
+    rescue Exception => e
+      Jekyll.logger.error "Liquid Exception:", "#{e.message} in #{path || document.relative_path}"
+      raise e
+    end
+
+    # Render layouts and place given content inside.
+    #
+    # content - the content to be placed in the layout
+    #
+    #
+    # Returns the content placed in the Liquid-rendered layouts
+    def place_in_layouts(content, payload, info)
+      output = content.dup
+      layout = site.layouts[document.data["layout"]]
+      used   = Set.new([layout])
+
+      while layout
+        payload = Utils.deep_merge_hashes(
+          payload,
+          {
+            "content" => output,
+            "page"    => document.to_liquid,
+            "layout"  => layout.data
+          }
+        )
+
+        output = render_liquid(
+          layout.content,
+          payload,
+          info,
+          File.join(site.config['layouts'], layout.name)
+        )
+
+        if layout = site.layouts[layout.data["layout"]]
+          if used.include?(layout)
+            layout = nil # avoid recursive chain
+          else
+            used << layout
+          end
+        end
+      end
+
+      output
+    end
+
+  end
+end

lib/jekyll/site.rb

@@ -4,7 +4,7 @@ module Jekyll
                   :exclude, :include, :source, :dest, :lsi, :highlighter,
                   :permalink_style, :time, :future, :unpublished, :safe, :plugins, :limit_posts,
                   :show_drafts, :keep_files, :baseurl, :data, :file_read_opts, :gems,
-                  :plugin_manager
+                  :plugin_manager, :collections
 
     attr_accessor :converters, :generators
 
@@ -14,12 +14,13 @@ module Jekyll
     def initialize(config)
       self.config = config.clone
 
-      %w[safe lsi highlighter baseurl exclude include future unpublished show_drafts limit_posts keep_files gems].each do |opt|
+      %w[safe lsi highlighter baseurl exclude include future unpublished
+        show_drafts limit_posts keep_files gems].each do |opt|
         self.send("#{opt}=", config[opt])
       end
 
-      self.source = File.expand_path(config['source'])
-      self.dest = File.expand_path(config['destination'])
+      self.source          = File.expand_path(config['source'])
+      self.dest            = File.expand_path(config['destination'])
       self.permalink_style = config['permalink'].to_sym
 
       self.plugin_manager = Jekyll::PluginManager.new(self)
@@ -83,6 +84,26 @@ module Jekyll
       end
     end
 
+    # The list of collections and their corresponding Jekyll::Collection instances.
+    # If config['collections'] is set, a new instance is created for each item in the collection.
+    # If config['collections'] is not set, a new hash is returned.
+    #
+    # Returns a Hash containing collection name-to-instance pairs.
+    def collections
+      @collections ||= if config['collections']
+        Hash[config['collections'].map { |coll| [coll, Jekyll::Collection.new(self, coll)] } ]
+      else
+        Hash.new
+      end
+    end
+
+    # The list of collections to render.
+    #
+    # The array of collection labels to render.
+    def to_render
+      @to_render ||= (config['render'] || Array.new)
+    end
+
     # Read Site data from disk and load it into internal data structures.
     #
     # Returns nothing.
@@ -90,6 +111,7 @@ module Jekyll
       self.layouts = LayoutReader.new(self).read
       read_directories
       read_data(config['data_source'])
+      read_collections
     end
 
     # Recursively traverse directories to find posts, pages and static files
@@ -166,19 +188,25 @@ module Jekyll
     #
     # Returns nothing
     def read_data(dir)
-      base = File.join(source, dir)
-      return unless File.directory?(base) && (!safe || !File.symlink?(base))
-
-      entries = Dir.chdir(base) { Dir['*.{yaml,yml}'] }
-      entries.delete_if { |e| File.directory?(File.join(base, e)) }
-
-      entries.each do |entry|
-        path = File.join(source, dir, entry)
-        next if File.symlink?(path) && safe
-
-        key = sanitize_filename(File.basename(entry, '.*'))
-        self.data[key] = SafeYAML.load_file(path)
+      unless dir.to_s.eql?("_data")
+        Jekyll.logger.error "Error:", "Data source directories other than '_data' have been removed.\n" +
+          "Please move your YAML files to `_data` and remove the `data_source` key from your `_config.yml`."
       end
+
+      collections['data'] = Jekyll::Collection.new(self, "data")
+      collections['data'].read
+
+      collections['data'].docs.each do |doc|
+        key = sanitize_filename(doc.basename(".*"))
+        self.data[key] = doc.data
+      end
+    end
+
+    # Read in all collections specified in the configuration
+    #
+    # Returns nothing.
+    def read_collections
+      collections.each { |_, collection| collection.read }
     end
 
     # Run each of the Generators.
@@ -196,6 +224,12 @@ module Jekyll
     def render
       relative_permalinks_deprecation_method
 
+      to_render.each do |label|
+        collections[label].docs.each do |document|
+          document.output = Jekyll::Renderer.new(self, document).run
+        end
+      end
+
       payload = site_payload
       [posts, pages].flatten.each do |page_or_post|
         page_or_post.render(layouts, payload)
@@ -271,7 +305,8 @@ module Jekyll
     #                  See Site#post_attr_hash for type info.
     def site_payload
       {"jekyll" => { "version" => Jekyll::VERSION },
-       "site"   => config.merge({
+       "site"   => Utils.deep_merge_hashes(config,
+         Utils.deep_merge_hashes(collections, {
           "time"         => time,
           "posts"        => posts.sort { |a, b| b <=> a },
           "pages"        => pages,
@@ -279,7 +314,9 @@ module Jekyll
           "html_pages"   => pages.reject { |page| !page.html? },
           "categories"   => post_attr_hash('categories'),
           "tags"         => post_attr_hash('tags'),
-          "data"         => site_data})}
+          "data"         => site_data
+        }))
+      }
     end
 
     # Filter out any files/directories that are hidden or backup files (start
@@ -357,8 +394,18 @@ module Jekyll
       end
     end
 
+    def documents
+      collections.reduce(Set.new) do |docs, (label, coll)|
+        if to_render.include?(label)
+          docs.merge(coll.docs)
+        else
+          docs
+        end
+      end
+    end
+
     def each_site_file
-      %w(posts pages static_files).each do |type|
+      %w(posts pages static_files documents).each do |type|
         send(type).each do |item|
           yield item
         end

lib/jekyll/url.rb

@@ -24,9 +24,9 @@ module Jekyll
     #                           template. Instead, the given permalink will be
     #                           used as URL.
     def initialize(options)
-      @template = options[:template]
+      @template     = options[:template]
       @placeholders = options[:placeholders] || {}
-      @permalink = options[:permalink]
+      @permalink    = options[:permalink]
 
       if (@template || @permalink).nil?
         raise ArgumentError, "One of :template or :permalink must be supplied."

script/console

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'pry'
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), *%w{ .. lib })
+require 'jekyll'
+
+TEST_DIR = File.expand_path(File.join(File.dirname(__FILE__), *%w{ .. test }))
+
+def fixture_site(overrides = {})
+  Jekyll::Site.new(site_configuration(overrides))
+end
+
+def build_configs(overrides, base_hash = Jekyll::Configuration::DEFAULTS)
+  Jekyll::Utils.deep_merge_hashes(base_hash, overrides)
+end
+
+def site_configuration(overrides = {})
+  build_configs({
+    "source"      => source_dir,
+    "destination" => dest_dir
+  }, build_configs(overrides))
+end
+
+def dest_dir(*subdirs)
+  test_dir('dest', *subdirs)
+end
+
+def source_dir(*subdirs)
+  test_dir('source', *subdirs)
+end
+
+def test_dir(*subdirs)
+  File.join(TEST_DIR, *subdirs)
+end
+
+module Jekyll
+  binding.pry
+end

site/_data/docs.yml

@@ -14,6 +14,7 @@
   - drafts
   - pages
   - variables
+  - collections
   - datafiles
   - assets
   - migrations

site/docs/collections.md

@@ -0,0 +1,126 @@
+---
+layout: docs
+title: Collections
+prev_section: variables
+next_section: datafiles
+permalink: /docs/collections/
+---
+
+<div class="note unreleased">
+  <h5>Collections support is currently unreleased.</h5>
+  <p>
+  In order to use this feature, <a href="/docs/installation/#pre-releases">
+  install the latest development version of Jekyll</a>.
+  </p>
+</div>
+
+<div class="note warning">
+  <h5>Collections support is unstable and may change</h5>
+  <p>
+    This is an experimental feature and that the API may likely change until the feature stabilizes.
+  </p>
+</div>
+
+Put some things in a folder and add the folder to your config. It's simple...
+
+Not everything is a post or a page. Maybe you want to document the various methods in your open source project, members of a team, or talks at a conference. Collections allow you to define a new type of document that behave like Pages or Posts do normally, but also have their own unique properties and namespace.
+
+## Using Collections
+
+### Step 1: Tell Jekyll to read in your collection
+
+Add the following to your site's `_config.yml` file, replacing `my_collection` with the name of your collection:
+
+{% highlight yaml %}
+collections:
+- my_collection
+{% endhighlight %}
+
+### Step 2: Add your content
+
+Create a corresponding folder (e.g. `<source>/_my_collection`) and add documents.
+YAML front-matter is read in as data if it exists, if not, then everything is just stuck in the Document's `content` attribute.
+
+Note: the folder must be named identical to the collection you defined in you config.yml file, with the addition of the preceding `_` character.
+
+### Step 3: Optionally render your collection's documents into independent files
+
+If you'd like Jekyll to create a public-facing, rendered version of each document in your collection, add your collection name to the `render` config key in your `_config.yml`:
+
+{% highlight yaml %}
+render:
+- my_collection
+{% endhighlight %}
+
+This will produce a file for each document in the collection.
+For example, if you have `_my_collection/some_subdir/some_doc.md`,
+it will be rendered using Liquid and the Markdown converter of your
+choice and written out to `<dest>/my_collection/some_subdir/some_doc.html`.
+
+## Liquid Attributes
+
+### Collections
+
+Each collection is accessible via the `site` Liquid variable. For example, if you want to access the `albums` collection found in `_albums`, you'd use `site.albums`. Each collection is itself an array of documents (e.g. `site.albums` is an array of documents, much like `site.pages` and `site.posts`). See below for how to access attributes of those documents.
+
+### Documents
+
+In addition to any YAML front-matter provided in the document's corresponding file, each document has the following attributes:
+
+<div class="mobile-side-scroller">
+<table>
+  <thead>
+    <tr>
+      <th>Variable</th>
+      <th>Description</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td>
+        <p><code>content</code></p>
+      </td>
+      <td>
+        <p>
+          The content of the document. If no YAML front-matter is provided,
+          this is the entirety of the file contents. If YAML front-matter
+          is used, then this is all the contents of the file after the terminating
+          `---` of the front-matter.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <p><code>path</code></p>
+      </td>
+      <td>
+        <p>
+          The full path to the document's source file.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <p><code>relative_path</code></p>
+      </td>
+      <td>
+        <p>
+          The path to the document's source file relative to the site source.
+        </p>
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <p><code>url</code></p>
+      </td>
+      <td>
+        <p>
+          The URL of the rendered collection. The file is only written to the
+          destination when the name of the collection to which it belongs is
+          included in the <code>render</code> key in the site's configuration file.
+        </p>
+      </td>
+    </tr>
+  </tbody>
+</table>
+</div>

site/docs/datafiles.md

@@ -1,7 +1,7 @@
 ---
 layout: docs
 title: Data Files
-prev_section: variables
+prev_section: collections
 next_section: assets
 permalink: /docs/datafiles/
 ---

site/docs/variables.md

@@ -2,7 +2,7 @@
 layout: docs
 title: Variables
 prev_section: pages
-next_section: datafiles
+next_section: collections
 permalink: /docs/variables/
 ---
 

test/source/_methods/_do_not_read_me.md

@@ -0,0 +1,5 @@
+---
+title: The unreadable wonder
+---
+
+Don't read me, you fool! FILTER ME

test/source/_methods/configuration.md

@@ -0,0 +1,8 @@
+---
+title: "Jekyll.configuration"
+whatever: foo.bar
+---
+
+Use `{{ page.title }}` to build a full configuration for use w/Jekyll.
+
+Whatever: {{ page.whatever }}

test/source/_methods/sanitized_path.md

@@ -0,0 +1,5 @@
+---
+title: "Jekyll.sanitized_path"
+---
+
+`{{ page.title }}` is used to make sure your path is in your source.

test/source/_methods/site/_dont_include_me_either.md

@@ -0,0 +1,5 @@
+---
+title: Don't Include Me Either
+---
+
+Don't include me either. FILTER ME PLZ

test/source/_methods/site/generate.md

@@ -0,0 +1,6 @@
+---
+title: "Site#generate"
+layout: default
+---
+
+Run your generators! {{ page.layout }}

test/source/_methods/site/initialize.md

@@ -0,0 +1,5 @@
+---
+title: "Site#initialize"
+---
+
+Create dat site.

test/source/_methods/um_hi.md

@@ -0,0 +1 @@
+./site/generate.md

test/test_collections.rb

@@ -0,0 +1,129 @@
+require 'helper'
+
+class TestCollections < Test::Unit::TestCase
+
+  def fixture_site(overrides = {})
+    Jekyll::Site.new(Jekyll.configuration(
+      overrides.merge({
+        "source"      => source_dir,
+        "destination" => dest_dir
+      })
+    ))
+  end
+
+  context "an evil collection" do
+    setup do
+      @collection = Jekyll::Collection.new(fixture_site, "../../etc/password")
+    end
+
+    should "sanitize the label name" do
+      assert_equal @collection.label, "etcpassword"
+    end
+
+    should "have a sanitized relative path name" do
+      assert_equal @collection.relative_directory, "_etcpassword"
+    end
+
+    should "have a sanitized full path" do
+      assert_equal @collection.directory, source_dir("_etcpassword")
+    end
+  end
+
+  context "a simple collection" do
+    setup do
+      @collection = Jekyll::Collection.new(fixture_site, "methods")
+    end
+
+    should "sanitize the label name" do
+      assert_equal @collection.label, "methods"
+    end
+
+    should "contain no docs when initialized" do
+      assert_empty @collection.docs
+    end
+
+    should "know its relative directory" do
+      assert_equal @collection.relative_directory, "_methods"
+    end
+
+    should "know the full path to itself on the filesystem" do
+      assert_equal @collection.directory, source_dir("_methods")
+    end
+  end
+
+  context "with no collections specified" do
+    setup do
+      @site = fixture_site
+      @site.process
+    end
+
+    should "not contain any collections other than the default ones" do
+      collections = @site.collections.dup
+      assert collections.delete("data").is_a?(Jekyll::Collection)
+      assert_equal Hash.new, collections
+    end
+  end
+
+  context "with a collection" do
+    setup do
+      @site = fixture_site({
+        "collections" => ["methods"]
+      })
+      @site.process
+      @collection = @site.collections["methods"]
+    end
+
+    should "create a Hash on Site with the label mapped to the instance of the Collection" do
+      assert @site.collections.is_a?(Hash)
+      assert_not_nil @site.collections["methods"]
+      assert @site.collections["methods"].is_a? Jekyll::Collection
+    end
+
+    should "collects docs in an array on the Collection object" do
+      assert @site.collections["methods"].docs.is_a? Array
+      @site.collections["methods"].docs.each do |doc|
+        assert doc.is_a? Jekyll::Document
+        assert_include %w[
+          _methods/configuration.md
+          _methods/sanitized_path.md
+          _methods/site/generate.md
+          _methods/site/initialize.md
+          _methods/um_hi.md
+        ], doc.relative_path
+      end
+    end
+
+    should "not include files which start with an underscore in the base collection directory" do
+      assert_not_include @collection.filtered_entries, "_do_not_read_me.md"
+    end
+
+    should "not include files which start with an underscore in a subdirectory" do
+      assert_not_include @collection.filtered_entries, "site/_dont_include_me_either.md"
+    end
+
+    should "not include the underscored files in the list of docs" do
+      assert_not_include @collection.docs.map(&:relative_path), "_methods/_do_not_read_me.md"
+      assert_not_include @collection.docs.map(&:relative_path), "_methods/site/_dont_include_me_either.md"
+    end
+  end
+
+  context "in safe mode" do
+    setup do
+      @site = fixture_site({
+        "collections" => ["methods"],
+        "safe"        => true
+      })
+      @site.process
+      @collection = @site.collections["methods"]
+    end
+
+    should "not allow symlinks" do
+      assert_not_include @collection.filtered_entries, "um_hi.md"
+    end
+
+    should "not include the symlinked file in the list of docs" do
+      assert_not_include @collection.docs.map(&:relative_path), "_methods/um_hi.md"
+    end
+  end
+
+end

test/test_document.rb

@@ -0,0 +1,48 @@
+require 'helper'
+
+class TestDocument < Test::Unit::TestCase
+
+  context "a document in a collection" do
+    setup do
+      @site = Site.new(Jekyll.configuration({
+        "collections" => ["methods"],
+        "source"      => source_dir,
+        "destination" => dest_dir
+      }))
+      @site.process
+      @document = @site.collections["methods"].docs.first
+    end
+
+    should "know its relative path" do
+      assert_equal "_methods/configuration.md", @document.relative_path
+    end
+
+    should "knows its extname" do
+      assert_equal ".md", @document.extname
+    end
+
+    should "know its basename" do
+      assert_equal "configuration.md", @document.basename
+    end
+
+    should "allow the suffix to be specified for the basename" do
+      assert_equal "configuration", @document.basename(".*")
+    end
+
+    should "know whether its a yaml file" do
+      assert_equal false, @document.yaml_file?
+    end
+
+    should "know its data" do
+      assert_equal({
+        "title" => "Jekyll.configuration",
+        "whatever" => "foo.bar"
+      }, @document.data)
+    end
+
+  end
+
+  context " a document part of a rendered collection" do
+  end
+
+end