From 5fb801474e6949751d0a7ac8bb8dea2ebeddd7fe Mon Sep 17 00:00:00 2001 From: dunsmoreb Date: Wed, 1 Feb 2012 06:44:01 -0600 Subject: [PATCH 01/69] Truncate post slugs when importing from Tumblr. Fixes #481. --- lib/jekyll/migrators/tumblr.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/jekyll/migrators/tumblr.rb b/lib/jekyll/migrators/tumblr.rb index 367a83c9..8b717d5d 100644 --- a/lib/jekyll/migrators/tumblr.rb +++ b/lib/jekyll/migrators/tumblr.rb @@ -34,6 +34,7 @@ module Jekyll post[:content] = html_to_markdown post[:content] post[:content] = add_syntax_highlights post[:content] if add_highlights end + post[:name] = truncate_post_name post[:name] if post[:name].size > 255 File.open("_posts/tumblr/#{post[:name]}", "w") do |f| f.puts post[:header].to_yaml + "---\n" + post[:content] end @@ -42,6 +43,11 @@ module Jekyll private + def self.truncate_post_name name + post = name.match(/^(.+)\.(.+)$/).captures + post[0][0..(-1 - post[1].size)] + post[1].size + end + # Converts each type of Tumblr post to a hash with all required # data for Jekyll. def self.post_to_hash(post, format) From 3d2664d983cf065ff6083d30c29ea2f794c47dd4 Mon Sep 17 00:00:00 2001 From: hokaccha Date: Thu, 9 Aug 2012 00:34:44 +0900 Subject: [PATCH 02/69] pygments options for pygments.rb --- lib/jekyll/tags/highlight.rb | 9 ++------- test/test_tags.rb | 8 ++++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/jekyll/tags/highlight.rb b/lib/jekyll/tags/highlight.rb index 9ad69b3e..e16602a7 100644 --- a/lib/jekyll/tags/highlight.rb +++ b/lib/jekyll/tags/highlight.rb @@ -15,8 +15,8 @@ module Jekyll super if markup.strip =~ SYNTAX @lang = $1 + @options = {} if defined?($2) && $2 != '' - tmp_options = {} $2.split.each do |opt| key, value = opt.split('=') if value.nil? @@ -26,13 +26,8 @@ module Jekyll value = true end end - tmp_options[key] = value + @options[key] = value end - tmp_options = tmp_options.to_a.sort.collect { |opt| opt.join('=') } - # additional options to pass to Albino - @options = { 'O' => tmp_options.join(',') } - else - @options = {} end else raise SyntaxError.new("Syntax Error in 'highlight' - Valid syntax: highlight [linenos]") diff --git a/test/test_tags.rb b/test/test_tags.rb index ab0f3188..943d980e 100644 --- a/test/test_tags.rb +++ b/test/test_tags.rb @@ -58,16 +58,16 @@ CONTENT assert_equal({}, tag.instance_variable_get(:@options)) tag = Jekyll::HighlightBlock.new('highlight', 'ruby linenos ', ["test", "{% endhighlight %}", "\n"]) - assert_equal({'O' => "linenos=inline"}, tag.instance_variable_get(:@options)) + assert_equal({ 'linenos' => 'inline' }, tag.instance_variable_get(:@options)) tag = Jekyll::HighlightBlock.new('highlight', 'ruby linenos=table ', ["test", "{% endhighlight %}", "\n"]) - assert_equal({'O' => "linenos=table"}, tag.instance_variable_get(:@options)) + assert_equal({ 'linenos' => 'table' }, tag.instance_variable_get(:@options)) tag = Jekyll::HighlightBlock.new('highlight', 'ruby linenos=table nowrap', ["test", "{% endhighlight %}", "\n"]) - assert_equal({'O' => "linenos=table,nowrap=true"}, tag.instance_variable_get(:@options)) + assert_equal({ 'linenos' => 'table', 'nowrap' => true }, tag.instance_variable_get(:@options)) tag = Jekyll::HighlightBlock.new('highlight', 'ruby linenos=table cssclass=hl', ["test", "{% endhighlight %}", "\n"]) - assert_equal({'O' => "cssclass=hl,linenos=table"}, tag.instance_variable_get(:@options)) + assert_equal({ 'cssclass' => 'hl', 'linenos' => 'table' }, tag.instance_variable_get(:@options)) end end From 7c800d3b0753b76e0880853728ef87abd00e01ca Mon Sep 17 00:00:00 2001 From: edeustace Date: Thu, 23 Aug 2012 12:07:30 +0200 Subject: [PATCH 03/69] Added a configuration variable: keep_files (default: ['.git']), based on this pull request: https://github.com/mojombo/jekyll/pull/556 --- bin/jekyll | 5 +++++ jekyll.gemspec | 7 +++++-- lib/jekyll.rb | 1 + lib/jekyll/site.rb | 18 ++++++++++++++++-- test/test_site.rb | 27 +++++++++++++++++++++++++++ 5 files changed, 54 insertions(+), 4 deletions(-) diff --git a/bin/jekyll b/bin/jekyll index 74cb99b0..07769516 100755 --- a/bin/jekyll +++ b/bin/jekyll @@ -145,6 +145,11 @@ opts = OptionParser.new do |opts| puts "Jekyll " + Jekyll::VERSION exit 0 end + + opts.on( "--keep-files filename1,filename2", Array, "Whether to keep files that match the filename (default: .git)") do |names| + puts "keep-files option: #{names}" + options['keep_files'] = names + end end # Read command line options into `options` hash diff --git a/jekyll.gemspec b/jekyll.gemspec index 55201ae8..1c181122 100644 --- a/jekyll.gemspec +++ b/jekyll.gemspec @@ -5,7 +5,7 @@ Gem::Specification.new do |s| s.name = 'jekyll' s.version = '0.11.2' - s.date = '2011-12-27' + s.date = '2012-08-23' s.rubyforge_project = 'jekyll' s.summary = "A simple, blog aware, static site generator." @@ -74,10 +74,12 @@ Gem::Specification.new do |s| lib/jekyll/migrators/csv.rb lib/jekyll/migrators/drupal.rb lib/jekyll/migrators/enki.rb + lib/jekyll/migrators/joomla.rb lib/jekyll/migrators/marley.rb lib/jekyll/migrators/mephisto.rb lib/jekyll/migrators/mt.rb lib/jekyll/migrators/posterous.rb + lib/jekyll/migrators/rss.rb lib/jekyll/migrators/textpattern.rb lib/jekyll/migrators/tumblr.rb lib/jekyll/migrators/typo.rb @@ -90,6 +92,7 @@ Gem::Specification.new do |s| lib/jekyll/static_file.rb lib/jekyll/tags/highlight.rb lib/jekyll/tags/include.rb + lib/jekyll/tags/post_url.rb test/helper.rb test/source/.htaccess test/source/_includes/sig.markdown @@ -141,9 +144,9 @@ Gem::Specification.new do |s| test/test_post.rb test/test_rdiscount.rb test/test_redcarpet.rb + test/test_redcloth.rb test/test_site.rb test/test_tags.rb - test/test_redcloth.rb ] # = MANIFEST = diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 65e16cdb..50125b08 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -59,6 +59,7 @@ module Jekyll 'source' => Dir.pwd, 'destination' => File.join(Dir.pwd, '_site'), 'plugins' => File.join(Dir.pwd, '_plugins'), + 'keep_files' => ['.git'], 'layouts' => '_layouts', 'future' => true, diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index 8a78e915..b3490264 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -5,7 +5,8 @@ module Jekyll class Site attr_accessor :config, :layouts, :posts, :pages, :static_files, :categories, :exclude, :include, :source, :dest, :lsi, :pygments, - :permalink_style, :tags, :time, :future, :safe, :plugins, :limit_posts + :permalink_style, :tags, :time, :future, :safe, :plugins, :limit_posts, + :keep_files attr_accessor :converters, :generators @@ -26,6 +27,7 @@ module Jekyll self.include = config['include'] || [] self.future = config['future'] self.limit_posts = config['limit_posts'] || nil + self.keep_files = config['keep_files'] || [] self.reset self.setup @@ -217,7 +219,11 @@ module Jekyll # all files and directories in destination, including hidden ones dest_files = Set.new Dir.glob(File.join(self.dest, "**", "*"), File::FNM_DOTMATCH) do |file| - dest_files << file unless file =~ /\/\.{1,2}$/ + if self.keep_files.length > 0 + dest_files << file unless file =~ /\/\.{1,2}$/ or file =~ keep_file_regex + else + dest_files << file unless file =~ /\/\.{1,2}$/ + end end # files to be written @@ -242,6 +248,14 @@ module Jekyll FileUtils.rm_rf(obsolete_files.to_a) end + # create a regex from the keep_files array + # ['.git','.svn'] => /\/(\.git|\/.svn)/ + def keep_file_regex + or_list = self.keep_files.map.inject("") { |x,y| "#{x}|#{y}" }[1..-1] + pattern = "\/(#{or_list.gsub(".", "\.")})" + Regexp.new pattern + end + # Write static files, pages, and posts. # # Returns nothing. diff --git a/test/test_site.rb b/test/test_site.rb index 712bf359..26bfaa52 100644 --- a/test/test_site.rb +++ b/test/test_site.rb @@ -180,6 +180,13 @@ class TestSite < Test::Unit::TestCase File.open(dest_dir('qux/obsolete.html'), 'w') # empty directory FileUtils.mkdir(dest_dir('quux')) + FileUtils.mkdir(dest_dir('.git')) + FileUtils.mkdir(dest_dir('.svn')) + FileUtils.mkdir(dest_dir('.hg')) + # single file in repository + File.open(dest_dir('.git/HEAD'), 'w') + File.open(dest_dir('.svn/HEAD'), 'w') + File.open(dest_dir('.hg/HEAD'), 'w') end teardown do @@ -187,6 +194,9 @@ class TestSite < Test::Unit::TestCase FileUtils.rm_f(dest_dir('obsolete.html')) FileUtils.rm_rf(dest_dir('qux')) FileUtils.rm_f(dest_dir('quux')) + FileUtils.rm_rf(dest_dir('.git')) + FileUtils.rm_rf(dest_dir('.svn')) + FileUtils.rm_rf(dest_dir('.hg')) end should 'remove orphaned files in destination' do @@ -195,6 +205,23 @@ class TestSite < Test::Unit::TestCase assert !File.exist?(dest_dir('obsolete.html')) assert !File.exist?(dest_dir('qux')) assert !File.exist?(dest_dir('quux')) + assert File.exist?(dest_dir('.git')) + assert File.exist?(dest_dir('.git/HEAD')) + end + + should 'remove orphaned files in destination - keep_files .svn' do + + config = Jekyll::DEFAULTS.merge({'source' => source_dir, 'destination' => dest_dir, 'keep_files' => ['.svn']}) + @site = Site.new(config) + @site.process + assert !File.exist?(dest_dir('.htpasswd')) + assert !File.exist?(dest_dir('obsolete.html')) + assert !File.exist?(dest_dir('qux')) + assert !File.exist?(dest_dir('quux')) + assert !File.exist?(dest_dir('.git')) + assert !File.exist?(dest_dir('.git/HEAD')) + assert File.exist?(dest_dir('.svn')) + assert File.exist?(dest_dir('.svn/HEAD')) end end From 3da0bb3a7482004093d41fd6c4c2764cfb3cace2 Mon Sep 17 00:00:00 2001 From: edeustace Date: Thu, 23 Aug 2012 12:13:04 +0200 Subject: [PATCH 04/69] removed puts --- bin/jekyll | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/jekyll b/bin/jekyll index 07769516..03c26c22 100755 --- a/bin/jekyll +++ b/bin/jekyll @@ -147,7 +147,6 @@ opts = OptionParser.new do |opts| end opts.on( "--keep-files filename1,filename2", Array, "Whether to keep files that match the filename (default: .git)") do |names| - puts "keep-files option: #{names}" options['keep_files'] = names end end From 85f2dfffa69f33058e643ffd7edd01e6137a5059 Mon Sep 17 00:00:00 2001 From: Jashank Jeremy Date: Wed, 31 Oct 2012 22:19:59 +1100 Subject: [PATCH 05/69] faster_lsi: Massively accelerate LSI performance. Currently, Classifier::LSI rebuilds the index every time an entry is added. This runs into massive performance overheads on my website; theoretically, disabling automatic index rebuilds, and explicitly rebuilding the LSI index at the end of the LSI repopulation should speed things up nicely. As a side note, here, I use pandoc-ruby to provide a more featureful Markdown transformer, so be mindful that the numbers I quote here have artifically imposed I/O overheads. With just the 76 posts I wrote this year (abysmal, I know), I come up with the following figures: Without faster_lsi: jekyll --lsi 16.91s user 0.88s system 97% cpu 18.302 total With faster_lsi: jekyll --lsi 2.72s user 0.77s system 88% cpu 3.940 total With 109 posts, we begin to see even better improvements: Without faster_lsi: jekyll --lsi 51.00s user 1.47s system 98% cpu 53.060 total With faster_lsi: jekyll --lsi 5.04s user 1.12s system 91% cpu 6.735 total At this point, we begin to see I/O overheads being slower than LSI when faster_lsi is active. I call that fairly conclusive. But wait, there's more. I have 273 posts lying around... I wonder what happens if I feed them all in. With faster_lsi, it was nice and clippy. Without it, I simply gave up, and went and refilled my cup of tea. And it was still going. Without faster_lsi: jekyll --lsi 1277.86s user 10.90s system 99% cpu 21:30.29 total With faster_lsi: jekyll --lsi 34.62s user 4.43s system 96% cpu 40.430 total That is, in anyone's books, a major improvement. Note, however, that I don't know just how well this will perform with `jekyll --auto` because I don't know how it does the LSI rebuilds. I _think_ (but please, don't commit me on this) that the LSI is rebuilt every time Jekyll picks up a file change. So, all up, the performance improvement is massive, and scales depending on how many files you have. At the last point, the improvement is just on 3200%. A more optimal solution would be to cache the LSI index and/or content data somehow. I'll leave that to when faster_lsi takes over ten minutes to run. --- lib/jekyll/post.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/jekyll/post.rb b/lib/jekyll/post.rb index a84c9ab9..d028a290 100644 --- a/lib/jekyll/post.rb +++ b/lib/jekyll/post.rb @@ -162,9 +162,12 @@ module Jekyll if self.site.lsi self.class.lsi ||= begin - puts "Running the classifier... this could take a while." - lsi = Classifier::LSI.new + puts "Starting the classifier..." + lsi = Classifier::LSI.new :auto_rebuild => false + $stdout.print(" Populating LSI... ");$stdout.flush posts.each { |x| $stdout.print(".");$stdout.flush;lsi.add_item(x) } + $stdout.print("\n Rebuilding LSI index... ") + lsi.build_index puts "" lsi end From 4bae42a671ba549bf05b2eabbcb8d3fcf26da430 Mon Sep 17 00:00:00 2001 From: edeustace Date: Sun, 9 Dec 2012 11:43:49 +0100 Subject: [PATCH 06/69] use Array.join instead of Array.inject, add .svn to defaults --- lib/jekyll.rb | 2 +- lib/jekyll/site.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 50125b08..f74e428b 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -59,7 +59,7 @@ module Jekyll 'source' => Dir.pwd, 'destination' => File.join(Dir.pwd, '_site'), 'plugins' => File.join(Dir.pwd, '_plugins'), - 'keep_files' => ['.git'], + 'keep_files' => ['.git','.svn'], 'layouts' => '_layouts', 'future' => true, diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index b3490264..058e6e11 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -251,7 +251,7 @@ module Jekyll # create a regex from the keep_files array # ['.git','.svn'] => /\/(\.git|\/.svn)/ def keep_file_regex - or_list = self.keep_files.map.inject("") { |x,y| "#{x}|#{y}" }[1..-1] + or_list = self.keep_files.join("|") pattern = "\/(#{or_list.gsub(".", "\.")})" Regexp.new pattern end From 6c0c5b61876d1cfba3b046b0d5b831cffffff934 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Sun, 16 Dec 2012 21:53:44 +0000 Subject: [PATCH 07/69] Add commander dependency --- jekyll.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/jekyll.gemspec b/jekyll.gemspec index cd395683..49b8d008 100644 --- a/jekyll.gemspec +++ b/jekyll.gemspec @@ -29,6 +29,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency('maruku', "~> 0.5") s.add_runtime_dependency('kramdown', "~> 0.13.4") s.add_runtime_dependency('pygments.rb', "~> 0.3.2") + s.add_runtime_dependency('commander', "~> 4.1.3") s.add_development_dependency('rake', "~> 0.9") s.add_development_dependency('rdoc', "~> 3.11") From 14766497c8f17cf67270574178b9912a2da75ce9 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 20:34:09 +0000 Subject: [PATCH 08/69] Add bin/jekyll2 and initial BuildCommand The `BuildCommand` class is responsible for handling the building of the site. It can also optionally watch for changes to files and regenerate the site if needed. The `Command` class holds any methods which are used by any command implementation. --- bin/jekyll2 | 31 +++++++++++++++ lib/jekyll.rb | 3 ++ lib/jekyll/command.rb | 14 +++++++ lib/jekyll/commands/build.rb | 77 ++++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 bin/jekyll2 create mode 100644 lib/jekyll/command.rb create mode 100644 lib/jekyll/commands/build.rb diff --git a/bin/jekyll2 b/bin/jekyll2 new file mode 100755 index 00000000..ce85f13b --- /dev/null +++ b/bin/jekyll2 @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby + +$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib }) + +require 'commander/import' +require 'jekyll' + +# Details about Jekyll +program :name, 'jekyll' +program :version, Jekyll::VERSION +program :description, 'Jekyll is a blog-aware, static site generator in Ruby' + +# Global options available to every command +global_option '-s', '--source DIR', String, 'Source directory' +global_option '-d', '--destination DIR', String, 'Destination directory' + +# Build command +# +# Args: +# --source +# --destination +# --watch +command :build do |c| + c.syntax = 'jekyll build [options]' + c.description = 'Build...' + c.option '-w', '--watch', 'Watch for changes and rebuild' + c.action do |args, options| + options.default :watch => false + Jekyll::BuildCommand.process(options) + end +end diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 87984ecf..058fd762 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -41,6 +41,9 @@ require 'jekyll/errors' require 'jekyll/plugin' require 'jekyll/converter' require 'jekyll/generator' +require 'jekyll/command' + +require_all 'jekyll/commands' require_all 'jekyll/converters' require_all 'jekyll/generators' require_all 'jekyll/tags' diff --git a/lib/jekyll/command.rb b/lib/jekyll/command.rb new file mode 100644 index 00000000..340d457d --- /dev/null +++ b/lib/jekyll/command.rb @@ -0,0 +1,14 @@ +module Jekyll + + class Command + def self.globs(source) + Dir.chdir(source) do + dirs = Dir['*'].select { |x| File.directory?(x) } + dirs -= ['_site'] + dirs = dirs.map { |x| "#{x}/**/*" } + dirs += ['*'] + end + end + end + +end diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb new file mode 100644 index 00000000..074f5d49 --- /dev/null +++ b/lib/jekyll/commands/build.rb @@ -0,0 +1,77 @@ +module Jekyll + + class BuildCommand < Command + def self.process(options) + opts = {} + options.__hash__.map do |k,v| + opts[k.to_s] = v + end + + opts = Jekyll.configuration(opts) + site = Jekyll::Site.new(opts) + + source = opts['source'] + destination = opts['destination'] + + if opts['watch'] + self.watch(site, source, destination) + else + self.build(site, source, destination) + end + end + + # Private: Build the site from source into destination. + # + # site - A Jekyll::Site instance + # source - A String of the source path + # destination - A String of the destination path + # + # Returns nothing. + def self.build(site, source, destination) + puts "Building site: #{source} -> #{destination}" + begin + site.process + rescue Jekyll::FatalException => e + puts + puts "ERROR: YOUR SITE COULD NOT BE BUILT:" + puts "------------------------------------" + puts e.message + exit(1) + end + puts "Successfully generated site: #{source} -> #{destination}" + end + + # Private: Watch for file changes and rebuild the site. + # + # site - A Jekyll::Site instance + # source - A String of the source path + # destination - A String of the destination path + # + # Returns nothing. + def self.watch(site, source, destination) + require 'directory_watcher' + + puts "Auto-Regenerating enabled: #{source} -> #{destination}" + + dw = DirectoryWatcher.new(source) + dw.interval = 1 + dw.glob = self.globs(source) + + dw.add_observer do |*args| + t = Time.now.strftime("%Y-%m-%d %H:%M:%S") + puts "[#{t} regeneration: #{args.size} files changed" + site.process + end + + dw.start + + trap("SIGINT") do + puts "Stopping auto-regeneration..." + exit 0 + end + + loop { sleep 1000 } + end + end + +end From 3b4feb41f0a2e767b18d26f01d1a3d3a0fc28713 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:00:24 +0000 Subject: [PATCH 09/69] Add initial serve command The `ServeCommand` will let you serve your site locally for development. You can specify `--port`, `--host` and `--baseurl` options if you wish to change the defaults. Additionally the `BuildCommand` will be called before the processing of the serve command, this makes sure that the site is actually built. This means you are able to pass the `--watch` option to auto-regenerate your site, even while serving it locally. --- bin/jekyll2 | 29 +++++++++++++++++++++++++++++ lib/jekyll/commands/build.rb | 31 ++++++++++++++++++------------- lib/jekyll/commands/serve.rb | 28 ++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 13 deletions(-) create mode 100644 lib/jekyll/commands/serve.rb diff --git a/bin/jekyll2 b/bin/jekyll2 index ce85f13b..d99ba866 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -29,3 +29,32 @@ command :build do |c| Jekyll::BuildCommand.process(options) end end + +# Serve command +# +# Args: +# --source +# --destination +# --watch +# +# --port +# --host +# --baseurl +command :serve do |c| + c.syntax = 'jekyll serve [options]' + c.description = 'Serve...' + c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '-p', '--port [PORT]', 'Port to listen on' + c.option '-h', '--host [HOST]', 'Host to bind to' + c.option '-b', '--baseurl [URL]', 'Base URL' + c.action do |args, options| + options.default :watch => false, + :port => '4000', + :host => '0.0.0.0', + :baseurl => '/', + :serving => true + + Jekyll::BuildCommand.process(options) + Jekyll::ServeCommand.process(options) + end +end diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index 074f5d49..2ff8aa80 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -14,20 +14,21 @@ module Jekyll destination = opts['destination'] if opts['watch'] - self.watch(site, source, destination) + self.watch(site, opts) else - self.build(site, source, destination) + self.build(site, opts) end end # Private: Build the site from source into destination. # # site - A Jekyll::Site instance - # source - A String of the source path - # destination - A String of the destination path + # options - A Hash of options passed to the command # # Returns nothing. - def self.build(site, source, destination) + def self.build(site, options) + source = options['source'] + destination = options['destination'] puts "Building site: #{source} -> #{destination}" begin site.process @@ -44,13 +45,15 @@ module Jekyll # Private: Watch for file changes and rebuild the site. # # site - A Jekyll::Site instance - # source - A String of the source path - # destination - A String of the destination path + # options - A Hash of options passed to the command # # Returns nothing. - def self.watch(site, source, destination) + def self.watch(site, options) require 'directory_watcher' + source = options['source'] + destination = options['destination'] + puts "Auto-Regenerating enabled: #{source} -> #{destination}" dw = DirectoryWatcher.new(source) @@ -65,12 +68,14 @@ module Jekyll dw.start - trap("SIGINT") do - puts "Stopping auto-regeneration..." - exit 0 - end + unless options['serving'] + loop { sleep 1000 } - loop { sleep 1000 } + trap("INT") do + puts "Stopping auto-regeneration..." + exit 0 + end + end end end diff --git a/lib/jekyll/commands/serve.rb b/lib/jekyll/commands/serve.rb new file mode 100644 index 00000000..665a26cf --- /dev/null +++ b/lib/jekyll/commands/serve.rb @@ -0,0 +1,28 @@ +module Jekyll + + class ServeCommand < Command + def self.process(options) + require 'webrick' + include WEBrick + + destination = options.destination + + FileUtils.mkdir_p(destination) + + mime_types = WEBrick::HTTPUtils::DefaultMimeTypes + mime_types.store 'js', 'application/javascript' + + s = HTTPServer.new( + :Port => options.port, + :BindAddress => options.host, + :MimeTypes => mime_types + ) + + s.mount(options.baseurl, HTTPServlet::FileHandler, destination) + t = Thread.new { s.start } + trap("INT") { s.shutdown } + t.join() + end + end + +end From 72f99eb8a7c39509753f8e9bec45c42c151a59e4 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:18:05 +0000 Subject: [PATCH 10/69] Update global options and command descriptions --- bin/jekyll2 | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index d99ba866..eba186dd 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -11,8 +11,11 @@ program :version, Jekyll::VERSION program :description, 'Jekyll is a blog-aware, static site generator in Ruby' # Global options available to every command -global_option '-s', '--source DIR', String, 'Source directory' -global_option '-d', '--destination DIR', String, 'Destination directory' +global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' +global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' +global_option '--safe', 'Safe mode (defaults to false)' +global_option '--plugins', 'Plugins directory (defaults to ./_plugins)' +global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' # Build command # @@ -22,10 +25,9 @@ global_option '-d', '--destination DIR', String, 'Destination directory' # --watch command :build do |c| c.syntax = 'jekyll build [options]' - c.description = 'Build...' + c.description = 'Build your site with the option of auto-renegeration' c.option '-w', '--watch', 'Watch for changes and rebuild' c.action do |args, options| - options.default :watch => false Jekyll::BuildCommand.process(options) end end @@ -42,14 +44,13 @@ end # --baseurl command :serve do |c| c.syntax = 'jekyll serve [options]' - c.description = 'Serve...' + c.description = 'Serve your site locally with the option of auto-regeneration' c.option '-w', '--watch', 'Watch for changes and rebuild' c.option '-p', '--port [PORT]', 'Port to listen on' c.option '-h', '--host [HOST]', 'Host to bind to' c.option '-b', '--baseurl [URL]', 'Base URL' c.action do |args, options| - options.default :watch => false, - :port => '4000', + options.default :port => '4000', :host => '0.0.0.0', :baseurl => '/', :serving => true From d8f328b87ce171f210614509c339644b4529aa01 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:41:18 +0000 Subject: [PATCH 11/69] Update loop position so trap handler is used --- lib/jekyll/commands/build.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index 2ff8aa80..2a7348a7 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -69,12 +69,12 @@ module Jekyll dw.start unless options['serving'] - loop { sleep 1000 } - trap("INT") do puts "Stopping auto-regeneration..." exit 0 end + + loop { sleep 1000 } end end end From 5b2e95b44392fa6b6ed94e1f04dbed9bd7bef8c9 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:54:01 +0000 Subject: [PATCH 12/69] Add missing ] in self.watch string --- lib/jekyll/commands/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index 2a7348a7..d5a0e570 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -62,7 +62,7 @@ module Jekyll dw.add_observer do |*args| t = Time.now.strftime("%Y-%m-%d %H:%M:%S") - puts "[#{t} regeneration: #{args.size} files changed" + puts "[#{t}] regeneration: #{args.size} files changed" site.process end From bd1c8fe760d31a67ee9081c3d2767cb60781d909 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:55:00 +0000 Subject: [PATCH 13/69] Update Jekyll.configuration to convert symbol keys Because Commander uses symbol keys in the options hash and I don't want to go back backport every hash string key to symbols in Jekyll. :star: --- bin/jekyll2 | 6 +++--- lib/jekyll.rb | 3 +++ lib/jekyll/commands/build.rb | 19 +++++++------------ lib/jekyll/commands/serve.rb | 8 ++++---- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index eba186dd..34d040aa 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -28,7 +28,7 @@ command :build do |c| c.description = 'Build your site with the option of auto-renegeration' c.option '-w', '--watch', 'Watch for changes and rebuild' c.action do |args, options| - Jekyll::BuildCommand.process(options) + Jekyll::BuildCommand.process(options.__hash__) end end @@ -55,7 +55,7 @@ command :serve do |c| :baseurl => '/', :serving => true - Jekyll::BuildCommand.process(options) - Jekyll::ServeCommand.process(options) + Jekyll::BuildCommand.process(options.__hash__) + Jekyll::ServeCommand.process(options.__hash__) end end diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 058fd762..98f99402 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -123,6 +123,9 @@ module Jekyll # # Returns the final configuration Hash. def self.configuration(override) + # Convert any symbol keys to strings and remove the old key/values + override.keys.each { |k| override[k.to_s] = override.delete(k) } + # _config.yml may override default source location, but until # then, we need to know where to look for _config.yml source = override['source'] || Jekyll::DEFAULTS['source'] diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index d5a0e570..6c8a1254 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -2,21 +2,16 @@ module Jekyll class BuildCommand < Command def self.process(options) - opts = {} - options.__hash__.map do |k,v| - opts[k.to_s] = v - end + options = Jekyll.configuration(options) + site = Jekyll::Site.new(options) - opts = Jekyll.configuration(opts) - site = Jekyll::Site.new(opts) + source = options['source'] + destination = options['destination'] - source = opts['source'] - destination = opts['destination'] - - if opts['watch'] - self.watch(site, opts) + if options['watch'] + self.watch(site, options) else - self.build(site, opts) + self.build(site, options) end end diff --git a/lib/jekyll/commands/serve.rb b/lib/jekyll/commands/serve.rb index 665a26cf..df717bea 100644 --- a/lib/jekyll/commands/serve.rb +++ b/lib/jekyll/commands/serve.rb @@ -5,7 +5,7 @@ module Jekyll require 'webrick' include WEBrick - destination = options.destination + destination = options['destination'] FileUtils.mkdir_p(destination) @@ -13,12 +13,12 @@ module Jekyll mime_types.store 'js', 'application/javascript' s = HTTPServer.new( - :Port => options.port, - :BindAddress => options.host, + :Port => options['port'], + :BindAddress => options['host'], :MimeTypes => mime_types ) - s.mount(options.baseurl, HTTPServlet::FileHandler, destination) + s.mount(options['baseurl'], HTTPServlet::FileHandler, destination) t = Thread.new { s.start } trap("INT") { s.shutdown } t.join() From dc139e2ac97197182b2d036e8794cc1e45306b40 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 22:13:17 +0000 Subject: [PATCH 14/69] Update the sym->str key conversion --- bin/jekyll2 | 9 ++++++--- lib/jekyll.rb | 2 +- lib/jekyll/commands/build.rb | 1 - 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index 34d040aa..f46f2dc7 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -28,7 +28,9 @@ command :build do |c| c.description = 'Build your site with the option of auto-renegeration' c.option '-w', '--watch', 'Watch for changes and rebuild' c.action do |args, options| - Jekyll::BuildCommand.process(options.__hash__) + options.defaults :serving => false + options = Jekyll.configuration(options.__hash__) + Jekyll::BuildCommand.process(options) end end @@ -55,7 +57,8 @@ command :serve do |c| :baseurl => '/', :serving => true - Jekyll::BuildCommand.process(options.__hash__) - Jekyll::ServeCommand.process(options.__hash__) + options = Jekyll.configuration(options.__hash__) + Jekyll::BuildCommand.process(options) + Jekyll::ServeCommand.process(options) end end diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 98f99402..2b008f88 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -124,7 +124,7 @@ module Jekyll # Returns the final configuration Hash. def self.configuration(override) # Convert any symbol keys to strings and remove the old key/values - override.keys.each { |k| override[k.to_s] = override.delete(k) } + override = override.reduce({}) { |hsh,(k,v)| hsh.merge(k.to_s => v) } # _config.yml may override default source location, but until # then, we need to know where to look for _config.yml diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index 6c8a1254..27461101 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -2,7 +2,6 @@ module Jekyll class BuildCommand < Command def self.process(options) - options = Jekyll.configuration(options) site = Jekyll::Site.new(options) source = options['source'] From be20a03bb9ccebe2f74fa5ec4fd3bafebd19dcc7 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 14:09:02 +0000 Subject: [PATCH 15/69] Update to make the help command default --- bin/jekyll2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/jekyll2 b/bin/jekyll2 index f46f2dc7..c8dfde16 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -10,6 +10,8 @@ program :name, 'jekyll' program :version, Jekyll::VERSION program :description, 'Jekyll is a blog-aware, static site generator in Ruby' +default_command :help + # Global options available to every command global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' From 053c2eb0a72c5fc6719cccc76f3d48bdf2e1a1b1 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 14:13:32 +0000 Subject: [PATCH 16/69] Rename Args to Options in command docs --- bin/jekyll2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index c8dfde16..7c99f953 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -21,7 +21,7 @@ global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' # Build command # -# Args: +# Options: # --source # --destination # --watch @@ -38,7 +38,7 @@ end # Serve command # -# Args: +# Options: # --source # --destination # --watch From 4578c15a24c976860b1437a216ff1bc7748b6602 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 16:34:03 +0000 Subject: [PATCH 17/69] Add TODOs for current switches --- bin/jekyll | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/bin/jekyll b/bin/jekyll index 74cb99b0..8c0d2dad 100755 --- a/bin/jekyll +++ b/bin/jekyll @@ -25,88 +25,108 @@ options = {} opts = OptionParser.new do |opts| opts.banner = help + # TODO: delete, migrator related opts.on("--file [PATH]", "File to import from") do |import_file| options['file'] = import_file end + # TODO: delete, migrator related opts.on("--dbname [TEXT]", "DB to import from") do |import_dbname| options['dbname'] = import_dbname end + # TODO: delete, migrator related opts.on("--user [TEXT]", "Username to use when importing") do |import_user| options['user'] = import_user end + # TODO: delete, migrator related opts.on("--pass [TEXT]", "Password to use when importing") do |import_pass| options['pass'] = import_pass end + # TODO: delete, migrator related opts.on("--host [HOST ADDRESS]", "Host to import from") do |import_host| options['host'] = import_host end + # TODO: delete, migrator related opts.on("--site [SITE NAME]", "Site to import from") do |import_site| options['site'] = import_site end - + # TODO: global option opts.on("--[no-]safe", "Safe mode (default unsafe)") do |safe| options['safe'] = safe end + # TODO: option for build and serve command opts.on("--[no-]auto", "Auto-regenerate") do |auto| options['auto'] = auto end + # TODO: new serve command opts.on("--server [PORT]", "Start web server (default port 4000)") do |port| options['server'] = true options['server_port'] = port unless port.nil? end + # TODO: remove opts.on("--no-server", "Do not start a web server") do |part| options['server'] = false end + # TODO: option for serve command opts.on("--base-url [BASE_URL]", "Serve website from a given base URL (default '/'") do |baseurl| options['baseurl'] = baseurl end + # TODO: does anyone actually use this? opts.on("--default-mimetype [MT]", "Mimetype to use when no file extension (if --server)") do |mt| options['default-mimetype'] = mt end + # TODO: option for build and serve command opts.on("--[no-]lsi", "Use LSI for better related posts") do |lsi| options['lsi'] = lsi end + # TODO: remove and just enable all the time? opts.on("--[no-]pygments", "Use pygments to highlight code") do |pygments| options['pygments'] = pygments end + # TODO: read from config opts.on("--rdiscount", "Use rdiscount gem for Markdown") do options['markdown'] = 'rdiscount' end + # TODO: read from config opts.on("--redcarpet", "Use redcarpet gem for Markdown") do options['markdown'] = 'redcarpet' end + # TODO: read from config opts.on("--kramdown", "Use kramdown gem for Markdown") do options['markdown'] = 'kramdown' end + # TODO: remove and just generate the site as is? opts.on("--time [TIME]", "Time to generate the site for") do |time| options['time'] = Time.parse(time) end + # TODO: remove and just render all posts which aren't 'unpublished'? opts.on("--[no-]future", "Render future dated posts") do |future| options['future'] = future end + # TODO: read from config opts.on("--permalink [TYPE]", "Use 'date' (default) for YYYY/MM/DD") do |style| options['permalink'] = style unless style.nil? end + # TODO: read from config opts.on("--paginate [POSTS_PER_PAGE]", "Paginate a blog's posts") do |per_page| begin options['paginate'] = per_page.to_i @@ -117,6 +137,7 @@ opts = OptionParser.new do |opts| end end + # TODO: read from config opts.on("--paginate_path [PAGINATED_URL_FORMAT]", "Leave blank for /page") do |paginate_path| begin options['paginate_path'] = paginate_path @@ -127,6 +148,7 @@ opts = OptionParser.new do |opts| end end + # TODO: read from config opts.on("--limit_posts [MAX_POSTS]", "Limit the number of posts to publish") do |limit_posts| begin options['limit_posts'] = limit_posts.to_i @@ -137,6 +159,7 @@ opts = OptionParser.new do |opts| end end + # TODO: read from config opts.on("--url [URL]", "Set custom site.url") do |url| options['url'] = url end @@ -206,6 +229,7 @@ end # Get source and destination from command line +# TODO: source and destination are now global options case ARGV.size when 0 when 1 From d298e2bd5759514343a4ac64665a8fa853fec0b2 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 16:44:03 +0000 Subject: [PATCH 18/69] Add LSI option for build and serve commands --- bin/jekyll2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/jekyll2 b/bin/jekyll2 index 7c99f953..76e4c94d 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -29,6 +29,7 @@ command :build do |c| c.syntax = 'jekyll build [options]' c.description = 'Build your site with the option of auto-renegeration' c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '--lsi', 'Use LSI for improved related posts' c.action do |args, options| options.defaults :serving => false options = Jekyll.configuration(options.__hash__) @@ -50,6 +51,7 @@ command :serve do |c| c.syntax = 'jekyll serve [options]' c.description = 'Serve your site locally with the option of auto-regeneration' c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '--lsi', 'Use LSI for improved related posts' c.option '-p', '--port [PORT]', 'Port to listen on' c.option '-h', '--host [HOST]', 'Host to bind to' c.option '-b', '--baseurl [URL]', 'Base URL' From 9c65ceb22dea9fa507ea6ece5132a0fabd0915bc Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 16:56:08 +0000 Subject: [PATCH 19/69] Remove comments --- bin/jekyll2 | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index 76e4c94d..29030c47 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -19,17 +19,13 @@ global_option '--safe', 'Safe mode (defaults to false)' global_option '--plugins', 'Plugins directory (defaults to ./_plugins)' global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' -# Build command -# -# Options: -# --source -# --destination -# --watch command :build do |c| c.syntax = 'jekyll build [options]' c.description = 'Build your site with the option of auto-renegeration' + c.option '-w', '--watch', 'Watch for changes and rebuild' c.option '--lsi', 'Use LSI for improved related posts' + c.action do |args, options| options.defaults :serving => false options = Jekyll.configuration(options.__hash__) @@ -37,21 +33,13 @@ command :build do |c| end end -# Serve command -# -# Options: -# --source -# --destination -# --watch -# -# --port -# --host -# --baseurl command :serve do |c| c.syntax = 'jekyll serve [options]' c.description = 'Serve your site locally with the option of auto-regeneration' + c.option '-w', '--watch', 'Watch for changes and rebuild' c.option '--lsi', 'Use LSI for improved related posts' + c.option '-p', '--port [PORT]', 'Port to listen on' c.option '-h', '--host [HOST]', 'Host to bind to' c.option '-b', '--baseurl [URL]', 'Base URL' From b7944c527496a107e4afecb92735604d5cbbe162 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 17:50:21 +0000 Subject: [PATCH 20/69] Add initial MigrateCommand Not all migrators can actually be calld from the comand line. Some require options which are not passed in and have to be called by other means. --- bin/jekyll2 | 16 ++++++++++++ lib/jekyll/commands/migrate.rb | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 lib/jekyll/commands/migrate.rb diff --git a/bin/jekyll2 b/bin/jekyll2 index 29030c47..3df10e4f 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -43,6 +43,7 @@ command :serve do |c| c.option '-p', '--port [PORT]', 'Port to listen on' c.option '-h', '--host [HOST]', 'Host to bind to' c.option '-b', '--baseurl [URL]', 'Base URL' + c.action do |args, options| options.default :port => '4000', :host => '0.0.0.0', @@ -54,3 +55,18 @@ command :serve do |c| Jekyll::ServeCommand.process(options) end end + +command :migrate do |c| + c.syntax = 'jekyll migrate [options]' + c.description = 'Migrate your own blog to Jekyll' + + c.option '--file', 'File to migrate from' + c.option '--dbname', 'Database name to migrate from' + c.option '--user', 'Username to use when migrating' + c.option '--pass', 'Password to use when migrating' + c.option '--host', 'Host address to use when migrating' + + c.action do |args, options| + Jekyll::MigrateCommand.process(args.first, options) + end +end diff --git a/lib/jekyll/commands/migrate.rb b/lib/jekyll/commands/migrate.rb new file mode 100644 index 00000000..26937357 --- /dev/null +++ b/lib/jekyll/commands/migrate.rb @@ -0,0 +1,47 @@ +module Jekyll + + class MigrateCommand < Command + MIGRATORS = { + :csv => 'CSV', + :drupal => 'Drupal', + :enki => 'Enki', + :mephisto => 'Mephisto', + :mt => 'MT', + :posterous => 'Posterous', + :textpattern => 'TextPattern', + :tumblr => 'Tumblr', + :typo => 'Typo', + :wordpressdotcom => 'WordpressDotCom', + :wordpress => 'WordPress' + } + + def self.process(migrator, options) + abort 'missing argument. Please specify a migrator' if migrator.nil? + migrator = migrator.downcase + + cmd_options = [] + [ :file, :dbname, :user, :pass, :host, :site ].each do |p| + cmd_options << "\"#{options[p]}\"" unless options[p].nil? + end + + + if MIGRATORS.keys.include?(migrator) + app_root = File.expand_path( + File.join(File.dirname(__FILE__), '..', '..', '..') + ) + + require "#{app_root}/lib/jekyll/migrators/#{migrator}" + + if Jekyll.const_defiend?(MIGRATORS[migrator.to_sym]) + puts 'Importing...' + migrator_class = Jekyll.const_get(MIGRATORS[migrator.to_sym]) + migrator_class.process(*cmd_options) + exit 0 + end + end + + abort 'invalid migrator. Please specify a valid migrator' + end + end + +end From e24bb02576e9ac727176d383632dad8b899d9aa4 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 17:54:58 +0000 Subject: [PATCH 21/69] Update the Jekyll command for features testing :star: :metal: :star2: --- features/support/env.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/features/support/env.rb b/features/support/env.rb index 3166ce9c..912b735c 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -7,10 +7,11 @@ World do end TEST_DIR = File.join('/', 'tmp', 'jekyll') -JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll') +JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll2') def run_jekyll(opts = {}) command = JEKYLL_PATH + command << " build" command << " >> /dev/null 2>&1" if opts[:debug].nil? system command end From a151a16f09c6b23f331966d1a737c530454f5720 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 18:23:34 +0000 Subject: [PATCH 22/69] Remove command options from default config Removing command line options from the config is a path towards cleaning up the configuration file and not including options which don't really belong there. --- lib/jekyll.rb | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 2b008f88..d7f011d3 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -51,26 +51,22 @@ require_all 'jekyll/tags' module Jekyll VERSION = '0.11.2' - # Default options. Overriden by values in _config.yml or command-line opts. + # Default options. Overriden by values in _config.yml. # Strings rather than symbols are used for compatability with YAML. DEFAULTS = { - 'safe' => false, - 'auto' => false, - 'server' => false, - 'server_port' => 4000, - 'source' => Dir.pwd, 'destination' => File.join(Dir.pwd, '_site'), + 'plugins' => File.join(Dir.pwd, '_plugins'), 'layouts' => '_layouts', - 'future' => true, - 'lsi' => false, - 'pygments' => false, - 'markdown' => 'maruku', - 'permalink' => 'date', - 'include' => ['.htaccess'], - 'paginate_path' => 'page:num', + 'future' => true, # remove and make true just default + 'pygments' => false, # remove and make true just default + + 'markdown' => 'maruku', # no longer a command option + 'permalink' => 'date', # no longer a command option + 'include' => ['.htaccess'], # no longer a command option + 'paginate_path' => 'page:num', # no longer a command option 'markdown_ext' => 'markdown,mkd,mkdn,md', 'textile_ext' => 'textile', From b9da30bc8ffa4a2d0a9280e755d0904609d4fc97 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Sat, 22 Dec 2012 17:49:33 +0000 Subject: [PATCH 23/69] Remove migrators --- Rakefile | 15 -- lib/jekyll/migrators/csv.rb | 26 --- lib/jekyll/migrators/drupal.rb | 103 --------- lib/jekyll/migrators/enki.rb | 49 ---- lib/jekyll/migrators/joomla.rb | 53 ----- lib/jekyll/migrators/marley.rb | 52 ----- lib/jekyll/migrators/mephisto.rb | 84 ------- lib/jekyll/migrators/mt.rb | 86 ------- lib/jekyll/migrators/posterous.rb | 67 ------ lib/jekyll/migrators/rss.rb | 47 ---- lib/jekyll/migrators/textpattern.rb | 58 ----- lib/jekyll/migrators/tumblr.rb | 195 ---------------- lib/jekyll/migrators/typo.rb | 51 ---- lib/jekyll/migrators/wordpress.rb | 294 ------------------------ lib/jekyll/migrators/wordpressdotcom.rb | 70 ------ 15 files changed, 1250 deletions(-) delete mode 100644 lib/jekyll/migrators/csv.rb delete mode 100644 lib/jekyll/migrators/drupal.rb delete mode 100644 lib/jekyll/migrators/enki.rb delete mode 100644 lib/jekyll/migrators/joomla.rb delete mode 100644 lib/jekyll/migrators/marley.rb delete mode 100644 lib/jekyll/migrators/mephisto.rb delete mode 100644 lib/jekyll/migrators/mt.rb delete mode 100644 lib/jekyll/migrators/posterous.rb delete mode 100644 lib/jekyll/migrators/rss.rb delete mode 100644 lib/jekyll/migrators/textpattern.rb delete mode 100644 lib/jekyll/migrators/tumblr.rb delete mode 100644 lib/jekyll/migrators/typo.rb delete mode 100644 lib/jekyll/migrators/wordpress.rb delete mode 100644 lib/jekyll/migrators/wordpressdotcom.rb diff --git a/Rakefile b/Rakefile index 8eb1360d..a267c50d 100644 --- a/Rakefile +++ b/Rakefile @@ -82,21 +82,6 @@ end # ############################################################################# -namespace :migrate do - desc "Migrate from mephisto in the current directory" - task :mephisto do - sh %q(ruby -r './lib/jekyll/migrators/mephisto' -e 'Jekyll::Mephisto.postgres(:database => "#{ENV["DB"]}")') - end - desc "Migrate from Movable Type in the current directory" - task :mt do - sh %q(ruby -r './lib/jekyll/migrators/mt' -e 'Jekyll::MT.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")') - end - desc "Migrate from Typo in the current directory" - task :typo do - sh %q(ruby -r './lib/jekyll/migrators/typo' -e 'Jekyll::Typo.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")') - end -end - begin require 'cucumber/rake/task' Cucumber::Rake::Task.new(:features) do |t| diff --git a/lib/jekyll/migrators/csv.rb b/lib/jekyll/migrators/csv.rb deleted file mode 100644 index ce5203b7..00000000 --- a/lib/jekyll/migrators/csv.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Jekyll - module CSV - # Reads a csv with title, permalink, body, published_at, and filter. - # It creates a post file for each row in the csv - def self.process(file = "posts.csv") - FileUtils.mkdir_p "_posts" - posts = 0 - FasterCSV.foreach(file) do |row| - next if row[0] == "title" - posts += 1 - name = row[3].split(" ")[0]+"-"+row[1]+(row[4] =~ /markdown/ ? ".markdown" : ".textile") - File.open("_posts/#{name}", "w") do |f| - f.puts <<-HEADER ---- -layout: post -title: #{row[0]} ---- - - HEADER - f.puts row[2] - end - end - "Created #{posts} posts!" - end - end -end diff --git a/lib/jekyll/migrators/drupal.rb b/lib/jekyll/migrators/drupal.rb deleted file mode 100644 index 7fd16aef..00000000 --- a/lib/jekyll/migrators/drupal.rb +++ /dev/null @@ -1,103 +0,0 @@ -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module Drupal - # Reads a MySQL database via Sequel and creates a post file for each post - # in wp_posts that has post_status = 'publish'. This restriction is made - # because 'draft' posts are not guaranteed to have valid dates. - QUERY = "SELECT n.nid, \ - n.title, \ - nr.body, \ - n.created, \ - n.status \ - FROM node AS n, \ - node_revisions AS nr \ - WHERE (n.type = 'blog' OR n.type = 'story') \ - AND n.vid = nr.vid" - - def self.process(dbname, user, pass, host = 'localhost', prefix = '') - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - - if prefix != '' - QUERY[" node "] = " " + prefix + "node " - QUERY[" node_revisions "] = " " + prefix + "node_revisions " - end - - FileUtils.mkdir_p "_posts" - FileUtils.mkdir_p "_drafts" - - # Create the refresh layout - # Change the refresh url if you customized your permalink config - File.open("_layouts/refresh.html", "w") do |f| - f.puts < - - - - - - -EOF - end - - db[QUERY].each do |post| - # Get required fields and construct Jekyll compatible name - node_id = post[:nid] - title = post[:title] - content = post[:body] - created = post[:created] - time = Time.at(created) - is_published = post[:status] == 1 - dir = is_published ? "_posts" : "_drafts" - slug = title.strip.downcase.gsub(/(&|&)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '') - name = time.strftime("%Y-%m-%d-") + slug + '.md' - - # Get the relevant fields as a hash, delete empty fields and convert - # to YAML for the header - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'created' => created, - }.delete_if { |k,v| v.nil? || v == ''}.to_yaml - - # Write out the data and content to file - File.open("#{dir}/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - - # Make a file to redirect from the old Drupal URL - if is_published - aliases = db["SELECT dst FROM #{prefix}url_alias WHERE src = ?", "node/#{node_id}"].all - - aliases.push(:dst => "node/#{node_id}") - - aliases.each do |url_alias| - FileUtils.mkdir_p url_alias[:dst] - File.open("#{url_alias[:dst]}/index.md", "w") do |f| - f.puts "---" - f.puts "layout: refresh" - f.puts "refresh_to_post_id: /#{time.strftime("%Y/%m/%d/") + slug}" - f.puts "---" - end - end - end - end - - # TODO: Make dirs & files for nodes of type 'page' - # Make refresh pages for these as well - - # TODO: Make refresh dirs & files according to entries in url_alias table - end - end -end diff --git a/lib/jekyll/migrators/enki.rb b/lib/jekyll/migrators/enki.rb deleted file mode 100644 index 61cb2562..00000000 --- a/lib/jekyll/migrators/enki.rb +++ /dev/null @@ -1,49 +0,0 @@ -# Adapted by Rodrigo Pinto -# Based on typo.rb by Toby DiPasquale - -require 'fileutils' -require 'rubygems' -require 'sequel' - -module Jekyll - module Enki - SQL = <<-EOS - SELECT p.id, - p.title, - p.slug, - p.body, - p.published_at as date, - p.cached_tag_list as tags - FROM posts p - EOS - - # Just working with postgres, but can be easily adapted - # to work with both mysql and postgres. - def self.process(dbname, user, pass, host = 'localhost') - FileUtils.mkdir_p('_posts') - db = Sequel.postgres(:database => dbname, - :user => user, - :password => pass, - :host => host, - :encoding => 'utf8') - - db[SQL].each do |post| - name = [ sprintf("%.04d", post[:date].year), - sprintf("%.02d", post[:date].month), - sprintf("%.02d", post[:date].day), - post[:slug].strip ].join('-') - name += '.textile' - - File.open("_posts/#{name}", 'w') do |f| - f.puts({ 'layout' => 'post', - 'title' => post[:title].to_s, - 'enki_id' => post[:id], - 'categories' => post[:tags] - }.delete_if { |k, v| v.nil? || v == '' }.to_yaml) - f.puts '---' - f.puts post[:body].delete("\r") - end - end - end - end -end diff --git a/lib/jekyll/migrators/joomla.rb b/lib/jekyll/migrators/joomla.rb deleted file mode 100644 index 87f1e105..00000000 --- a/lib/jekyll/migrators/joomla.rb +++ /dev/null @@ -1,53 +0,0 @@ -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This migrator is made for Joomla 1.5 databases. -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module Joomla - def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'jos_', section = '1') - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - - FileUtils.mkdir_p("_posts") - - # Reads a MySQL database via Sequel and creates a post file for each - # post in wp_posts that has post_status = 'publish'. This restriction is - # made because 'draft' posts are not guaranteed to have valid dates. - query = "SELECT `title`, `alias`, CONCAT(`introtext`,`fulltext`) as content, `created`, `id` FROM #{table_prefix}content WHERE state = '0' OR state = '1' AND sectionid = '#{section}'" - - db[query].each do |post| - # Get required fields and construct Jekyll compatible name. - title = post[:title] - slug = post[:alias] - date = post[:created] - content = post[:content] - name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day, - slug] - - # Get the relevant fields as a hash, delete empty fields and convert - # to YAML for the header. - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'joomla_id' => post[:id], - 'joomla_url' => post[:alias], - 'date' => date - }.delete_if { |k,v| v.nil? || v == '' }.to_yaml - - # Write out the data and content to file - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - end - end -end diff --git a/lib/jekyll/migrators/marley.rb b/lib/jekyll/migrators/marley.rb deleted file mode 100644 index 21bcead5..00000000 --- a/lib/jekyll/migrators/marley.rb +++ /dev/null @@ -1,52 +0,0 @@ -require 'yaml' -require 'fileutils' - -module Jekyll - module Marley - def self.regexp - { :id => /^\d{0,4}-{0,1}(.*)$/, - :title => /^#\s*(.*)\s+$/, - :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/, - :published_on => /.*\s+\(([0-9\/]+)\)$/, - :perex => /^([^\#\n]+\n)$/, - :meta => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp - } - end - - def self.process(marley_data_dir) - raise ArgumentError, "marley dir #{marley_data_dir} not found" unless File.directory?(marley_data_dir) - - FileUtils.mkdir_p "_posts" - - posts = 0 - Dir["#{marley_data_dir}/**/*.txt"].each do |f| - next unless File.exists?(f) - - #copied over from marley's app/lib/post.rb - file_content = File.read(f) - meta_content = file_content.slice!( self.regexp[:meta] ) - body = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip - - title = file_content.scan( self.regexp[:title] ).first.to_s.strip - prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip - published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) ) - meta = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {} - meta['title'] = title - meta['layout'] = 'post' - - formatted_date = published_on.strftime('%Y-%m-%d') - post_name = File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '') - - name = "#{formatted_date}-#{post_name}" - File.open("_posts/#{name}.markdown", "w") do |f| - f.puts meta.to_yaml - f.puts "---\n" - f.puts "\n#{prerex}\n\n" if prerex - f.puts body - end - posts += 1 - end - "Created #{posts} posts!" - end - end -end diff --git a/lib/jekyll/migrators/mephisto.rb b/lib/jekyll/migrators/mephisto.rb deleted file mode 100644 index 7622c722..00000000 --- a/lib/jekyll/migrators/mephisto.rb +++ /dev/null @@ -1,84 +0,0 @@ -# Quickly hacked together my Michael Ivey -# Based on mt.rb by Nick Gerakines, open source and publically -# available under the MIT license. Use this module at your own risk. - -require 'rubygems' -require 'sequel' -require 'fastercsv' -require 'fileutils' -require File.join(File.dirname(__FILE__),"csv.rb") - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module Mephisto - #Accepts a hash with database config variables, exports mephisto posts into a csv - #export PGPASSWORD if you must - def self.postgres(c) - sql = <<-SQL - BEGIN; - CREATE TEMP TABLE jekyll AS - SELECT title, permalink, body, published_at, filter FROM contents - WHERE user_id = 1 AND type = 'Article' ORDER BY published_at; - COPY jekyll TO STDOUT WITH CSV HEADER; - ROLLBACK; - SQL - command = %Q(psql -h #{c[:host] || "localhost"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"}) - puts command - `#{command}` - CSV.process - end - - # This query will pull blog posts from all entries across all blogs. If - # you've got unpublished, deleted or otherwise hidden posts please sift - # through the created posts to make sure nothing is accidently published. - QUERY = "SELECT id, \ - permalink, \ - body, \ - published_at, \ - title \ - FROM contents \ - WHERE user_id = 1 AND \ - type = 'Article' AND \ - published_at IS NOT NULL \ - ORDER BY published_at" - - def self.process(dbname, user, pass, host = 'localhost') - db = Sequel.mysql(dbname, :user => user, - :password => pass, - :host => host, - :encoding => 'utf8') - - FileUtils.mkdir_p "_posts" - - db[QUERY].each do |post| - title = post[:title] - slug = post[:permalink] - date = post[:published_at] - content = post[:body] - - # Ideally, this script would determine the post format (markdown, - # html, etc) and create files with proper extensions. At this point - # it just assumes that markdown will be acceptable. - name = [date.year, date.month, date.day, slug].join('-') + ".markdown" - - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'mt_id' => post[:entry_id], - }.delete_if { |k,v| v.nil? || v == ''}.to_yaml - - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - - end - end -end diff --git a/lib/jekyll/migrators/mt.rb b/lib/jekyll/migrators/mt.rb deleted file mode 100644 index 048c84db..00000000 --- a/lib/jekyll/migrators/mt.rb +++ /dev/null @@ -1,86 +0,0 @@ -# Created by Nick Gerakines, open source and publically available under the -# MIT license. Use this module at your own risk. -# I'm an Erlang/Perl/C++ guy so please forgive my dirty ruby. - -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module MT - # This query will pull blog posts from all entries across all blogs. If - # you've got unpublished, deleted or otherwise hidden posts please sift - # through the created posts to make sure nothing is accidently published. - QUERY = "SELECT entry_id, \ - entry_basename, \ - entry_text, \ - entry_text_more, \ - entry_authored_on, \ - entry_title, \ - entry_convert_breaks \ - FROM mt_entry" - - def self.process(dbname, user, pass, host = 'localhost') - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - - FileUtils.mkdir_p "_posts" - - db[QUERY].each do |post| - title = post[:entry_title] - slug = post[:entry_basename].gsub(/_/, '-') - date = post[:entry_authored_on] - content = post[:entry_text] - more_content = post[:entry_text_more] - entry_convert_breaks = post[:entry_convert_breaks] - - # Be sure to include the body and extended body. - if more_content != nil - content = content + " \n" + more_content - end - - # Ideally, this script would determine the post format (markdown, - # html, etc) and create files with proper extensions. At this point - # it just assumes that markdown will be acceptable. - name = [date.year, date.month, date.day, slug].join('-') + '.' + - self.suffix(entry_convert_breaks) - - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'mt_id' => post[:entry_id], - 'date' => date - }.delete_if { |k,v| v.nil? || v == '' }.to_yaml - - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - end - - def self.suffix(entry_type) - if entry_type.nil? || entry_type.include?("markdown") - # The markdown plugin I have saves this as - # "markdown_with_smarty_pants", so I just look for "markdown". - "markdown" - elsif entry_type.include?("textile") - # This is saved as "textile_2" on my installation of MT 5.1. - "textile" - elsif entry_type == "0" || entry_type.include?("richtext") - # Richtext looks to me like it's saved as HTML, so I include it here. - "html" - else - # Other values might need custom work. - entry_type - end - end - end -end diff --git a/lib/jekyll/migrators/posterous.rb b/lib/jekyll/migrators/posterous.rb deleted file mode 100644 index 0a2280f2..00000000 --- a/lib/jekyll/migrators/posterous.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'rubygems' -require 'jekyll' -require 'fileutils' -require 'net/http' -require 'uri' -require "json" - -# ruby -r './lib/jekyll/migrators/posterous.rb' -e 'Jekyll::Posterous.process(email, pass, api_key, blog)' - -module Jekyll - module Posterous - def self.fetch(uri_str, limit = 10) - # You should choose better exception. - raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0 - - response = nil - Net::HTTP.start('posterous.com') do |http| - req = Net::HTTP::Get.new(uri_str) - req.basic_auth @email, @pass - response = http.request(req) - end - - case response - when Net::HTTPSuccess then response - when Net::HTTPRedirection then fetch(response['location'], limit - 1) - else response.error! - end - end - - def self.process(email, pass, api_token, blog = 'primary') - @email, @pass, @api_token = email, pass, api_token - FileUtils.mkdir_p "_posts" - - posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}").body) - page = 1 - - while posts.any? - posts.each do |post| - title = post["title"] - slug = title.gsub(/[^[:alnum:]]+/, '-').downcase - date = Date.parse(post["display_date"]) - content = post["body_html"] - published = !post["is_private"] - name = "%02d-%02d-%02d-%s.html" % [date.year, date.month, date.day, slug] - - # Get the relevant fields as a hash, delete empty fields and convert - # to YAML for the header - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'published' => published - }.delete_if { |k,v| v.nil? || v == ''}.to_yaml - - # Write out the data and content to file - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - - page += 1 - posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}&page=#{page}").body) - end - end - end -end diff --git a/lib/jekyll/migrators/rss.rb b/lib/jekyll/migrators/rss.rb deleted file mode 100644 index 461abd35..00000000 --- a/lib/jekyll/migrators/rss.rb +++ /dev/null @@ -1,47 +0,0 @@ -# Created by Kendall Buchanan (https://github.com/kendagriff) on 2011-12-22. -# Use at your own risk. The end. -# -# Usage: -# (URL) -# ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('http://yourdomain.com/your-favorite-feed.xml')" -# -# (Local file) -# ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('./somefile/on/your/computer.xml')" - -require 'rubygems' -require 'rss/1.0' -require 'rss/2.0' -require 'open-uri' -require 'fileutils' -require 'yaml' - -module Jekyll - module MigrateRSS - - # The `source` argument may be a URL or a local file. - def self.process(source) - content = "" - open(source) { |s| content = s.read } - rss = RSS::Parser.parse(content, false) - - raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss - - rss.items.each do |item| - formatted_date = item.date.strftime('%Y-%m-%d') - post_name = item.title.split(%r{ |!|/|:|&|-|$|,}).map { |i| i.downcase if i != '' }.compact.join('-') - name = "#{formatted_date}-#{post_name}" - - header = { - 'layout' => 'post', - 'title' => item.title - } - - File.open("_posts/#{name}.html", "w") do |f| - f.puts header.to_yaml - f.puts "---\n" - f.puts item.description - end - end - end - end -end \ No newline at end of file diff --git a/lib/jekyll/migrators/textpattern.rb b/lib/jekyll/migrators/textpattern.rb deleted file mode 100644 index 3b370ed9..00000000 --- a/lib/jekyll/migrators/textpattern.rb +++ /dev/null @@ -1,58 +0,0 @@ -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module TextPattern - # Reads a MySQL database via Sequel and creates a post file for each post. - # The only posts selected are those with a status of 4 or 5, which means - # "live" and "sticky" respectively. - # Other statuses are 1 => draft, 2 => hidden and 3 => pending. - QUERY = "SELECT Title, \ - url_title, \ - Posted, \ - Body, \ - Keywords \ - FROM textpattern \ - WHERE Status = '4' OR \ - Status = '5'" - - def self.process(dbname, user, pass, host = 'localhost') - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - - FileUtils.mkdir_p "_posts" - - db[QUERY].each do |post| - # Get required fields and construct Jekyll compatible name. - title = post[:Title] - slug = post[:url_title] - date = post[:Posted] - content = post[:Body] - - name = [date.strftime("%Y-%m-%d"), slug].join('-') + ".textile" - - # Get the relevant fields as a hash, delete empty fields and convert - # to YAML for the header. - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'tags' => post[:Keywords].split(',') - }.delete_if { |k,v| v.nil? || v == ''}.to_yaml - - # Write out the data and content to file. - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - end - end -end diff --git a/lib/jekyll/migrators/tumblr.rb b/lib/jekyll/migrators/tumblr.rb deleted file mode 100644 index 367a83c9..00000000 --- a/lib/jekyll/migrators/tumblr.rb +++ /dev/null @@ -1,195 +0,0 @@ -require 'rubygems' -require 'open-uri' -require 'fileutils' -require 'nokogiri' -require 'date' -require 'json' -require 'uri' -require 'jekyll' - -module Jekyll - module Tumblr - def self.process(url, format = "html", grab_images = false, - add_highlights = false, rewrite_urls = true) - @grab_images = grab_images - FileUtils.mkdir_p "_posts/tumblr" - url += "/api/read/json/" - per_page = 50 - posts = [] - # Two passes are required so that we can rewrite URLs. - # First pass builds up an array of each post as a hash. - begin - current_page = (current_page || -1) + 1 - feed = open(url + "?num=#{per_page}&start=#{current_page * per_page}") - json = feed.readlines.join("\n")[21...-2] # Strip Tumblr's JSONP chars. - blog = JSON.parse(json) - puts "Page: #{current_page + 1} - Posts: #{blog["posts"].size}" - posts += blog["posts"].map { |post| post_to_hash(post, format) } - end until blog["posts"].size < per_page - # Rewrite URLs and create redirects. - posts = rewrite_urls_and_redirects posts if rewrite_urls - # Second pass for writing post files. - posts.each do |post| - if format == "md" - post[:content] = html_to_markdown post[:content] - post[:content] = add_syntax_highlights post[:content] if add_highlights - end - File.open("_posts/tumblr/#{post[:name]}", "w") do |f| - f.puts post[:header].to_yaml + "---\n" + post[:content] - end - end - end - - private - - # Converts each type of Tumblr post to a hash with all required - # data for Jekyll. - def self.post_to_hash(post, format) - case post['type'] - when "regular" - title = post["regular-title"] - content = post["regular-body"] - when "link" - title = post["link-text"] || post["link-url"] - content = "#{title}" - unless post["link-description"].nil? - content << "
" + post["link-description"] - end - when "photo" - title = post["photo-caption"] - max_size = post.keys.map{ |k| k.gsub("photo-url-", "").to_i }.max - url = post["photo-url"] || post["photo-url-#{max_size}"] - ext = "." + post[post.keys.select { |k| - k =~ /^photo-url-/ && post[k].split("/").last =~ /\./ - }.first].split(".").last - content = "" - unless post["photo-link-url"].nil? - content = "#{content}" - end - when "audio" - if !post["id3-title"].nil? - title = post["id3-title"] - content = post.at["audio-player"] + "
" + post["audio-caption"] - else - title = post["audio-caption"] - content = post.at["audio-player"] - end - when "quote" - title = post["quote-text"] - content = "
#{post["quote-text"]}
" - unless post["quote-source"].nil? - content << "—" + post["quote-source"] - end - when "conversation" - title = post["conversation-title"] - content = "
" - post["conversation"]["line"].each do |line| - content << "
#{line['label']}
#{line}
" - end - content << "
" - when "video" - title = post["video-title"] - content = post["video-player"] - unless post["video-caption"].nil? - content << "
" + post["video-caption"] - end - end - date = Date.parse(post['date']).to_s - title = Nokogiri::HTML(title).text - slug = title.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '') - { - :name => "#{date}-#{slug}.#{format}", - :header => { - "layout" => "post", - "title" => title, - "tags" => post["tags"], - }, - :content => content, - :url => post["url"], - :slug => post["url-with-slug"], - } - end - - # Create a Hash of old urls => new urls, for rewriting and - # redirects, and replace urls in each post. Instantiate Jekyll - # site/posts to get the correct permalink format. - def self.rewrite_urls_and_redirects(posts) - site = Jekyll::Site.new(Jekyll.configuration({})) - dir = File.join(File.dirname(__FILE__), "..") - urls = Hash[posts.map { |post| - # Create an initial empty file for the post so that - # we can instantiate a post object. - File.open("_posts/tumblr/#{post[:name]}", "w") - tumblr_url = URI.parse(post[:slug]).path - jekyll_url = Jekyll::Post.new(site, dir, "", "tumblr/" + post[:name]).url - redirect_dir = tumblr_url.sub(/\//, "") + "/" - FileUtils.mkdir_p redirect_dir - File.open(redirect_dir + "index.html", "w") do |f| - f.puts "" - end - [tumblr_url, jekyll_url] - }] - posts.map { |post| - urls.each do |tumblr_url, jekyll_url| - post[:content].gsub!(/#{tumblr_url}/i, jekyll_url) - end - post - } - end - - # Uses Python's html2text to convert a post's content to - # markdown. Preserve HTML tables as per the markdown docs. - def self.html_to_markdown(content) - preserve = ["table", "tr", "th", "td"] - preserve.each do |tag| - content.gsub!(/<#{tag}/i, "$$" + tag) - content.gsub!(/<\/#{tag}/i, "||" + tag) - end - content = %x[echo '#{content.gsub("'", "''")}' | html2text] - preserve.each do |tag| - content.gsub!("$$" + tag, "<" + tag) - content.gsub!("||" + tag, " -require 'fileutils' -require 'rubygems' -require 'sequel' -require 'yaml' - -module Jekyll - module Typo - # This SQL *should* work for both MySQL and PostgreSQL, but I haven't - # tested PostgreSQL yet (as of 2008-12-16). - SQL = <<-EOS - SELECT c.id id, - c.title title, - c.permalink slug, - c.body body, - c.published_at date, - c.state state, - COALESCE(tf.name, 'html') filter - FROM contents c - LEFT OUTER JOIN text_filters tf - ON c.text_filter_id = tf.id - EOS - - def self.process dbname, user, pass, host='localhost' - FileUtils.mkdir_p '_posts' - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - db[SQL].each do |post| - next unless post[:state] =~ /published/ - - name = [ sprintf("%.04d", post[:date].year), - sprintf("%.02d", post[:date].month), - sprintf("%.02d", post[:date].day), - post[:slug].strip ].join('-') - - # Can have more than one text filter in this field, but we just want - # the first one for this. - name += '.' + post[:filter].split(' ')[0] - - File.open("_posts/#{name}", 'w') do |f| - f.puts({ 'layout' => 'post', - 'title' => post[:title].to_s, - 'typo_id' => post[:id] - }.delete_if { |k, v| v.nil? || v == '' }.to_yaml) - f.puts '---' - f.puts post[:body].delete("\r") - end - end - end - - end -end diff --git a/lib/jekyll/migrators/wordpress.rb b/lib/jekyll/migrators/wordpress.rb deleted file mode 100644 index d2d19039..00000000 --- a/lib/jekyll/migrators/wordpress.rb +++ /dev/null @@ -1,294 +0,0 @@ -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module WordPress - - # Main migrator function. Call this to perform the migration. - # - # dbname:: The name of the database - # user:: The database user name - # pass:: The database user's password - # host:: The address of the MySQL database host. Default: 'localhost' - # options:: A hash table of configuration options. - # - # Supported options are: - # - # :table_prefix:: Prefix of database tables used by WordPress. - # Default: 'wp_' - # :clean_entities:: If true, convert non-ASCII characters to HTML - # entities in the posts, comments, titles, and - # names. Requires the 'htmlentities' gem to - # work. Default: true. - # :comments:: If true, migrate post comments too. Comments - # are saved in the post's YAML front matter. - # Default: true. - # :categories:: If true, save the post's categories in its - # YAML front matter. - # :tags:: If true, save the post's tags in its - # YAML front matter. - # :more_excerpt:: If true, when a post has no excerpt but - # does have a tag, use the - # preceding post content as the excerpt. - # Default: true. - # :more_anchor:: If true, convert a tag into - # two HTML anchors with ids "more" and - # "more-NNN" (where NNN is the post number). - # Default: true. - # :status:: Array of allowed post statuses. Only - # posts with matching status will be migrated. - # Known statuses are :publish, :draft, :private, - # and :revision. If this is nil or an empty - # array, all posts are migrated regardless of - # status. Default: [:publish]. - # - def self.process(dbname, user, pass, host='localhost', options={}) - options = { - :table_prefix => 'wp_', - :clean_entities => true, - :comments => true, - :categories => true, - :tags => true, - :more_excerpt => true, - :more_anchor => true, - :status => [:publish] # :draft, :private, :revision - }.merge(options) - - if options[:clean_entities] - begin - require 'htmlentities' - rescue LoadError - STDERR.puts "Could not require 'htmlentities', so the " + - ":clean_entities option is now disabled." - options[:clean_entities] = false - end - end - - FileUtils.mkdir_p("_posts") - - db = Sequel.mysql(dbname, :user => user, :password => pass, - :host => host, :encoding => 'utf8') - - px = options[:table_prefix] - - posts_query = " - SELECT - posts.ID AS `id`, - posts.guid AS `guid`, - posts.post_type AS `type`, - posts.post_status AS `status`, - posts.post_title AS `title`, - posts.post_name AS `slug`, - posts.post_date AS `date`, - posts.post_content AS `content`, - posts.post_excerpt AS `excerpt`, - posts.comment_count AS `comment_count`, - users.display_name AS `author`, - users.user_login AS `author_login`, - users.user_email AS `author_email`, - users.user_url AS `author_url` - FROM #{px}posts AS `posts` - LEFT JOIN #{px}users AS `users` - ON posts.post_author = users.ID" - - if options[:status] and not options[:status].empty? - status = options[:status][0] - posts_query << " - WHERE posts.post_status = '#{status.to_s}'" - options[:status][1..-1].each do |status| - posts_query << " OR - posts.post_status = '#{status.to_s}'" - end - end - - db[posts_query].each do |post| - process_post(post, db, options) - end - end - - - def self.process_post(post, db, options) - px = options[:table_prefix] - - title = post[:title] - if options[:clean_entities] - title = clean_entities(title) - end - - slug = post[:slug] - if !slug or slug.empty? - slug = sluggify(title) - end - - date = post[:date] || Time.now - name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, - date.day, slug] - content = post[:content].to_s - if options[:clean_entities] - content = clean_entities(content) - end - - excerpt = post[:excerpt].to_s - - more_index = content.index(//) - more_anchor = nil - if more_index - if options[:more_excerpt] and - (post[:excerpt].nil? or post[:excerpt].empty?) - excerpt = content[0...more_index] - end - if options[:more_anchor] - more_link = "more" - content.sub!(//, - "" + - "") - end - end - - categories = [] - tags = [] - - if options[:categories] or options[:tags] - - cquery = - "SELECT - terms.name AS `name`, - ttax.taxonomy AS `type` - FROM - #{px}terms AS `terms`, - #{px}term_relationships AS `trels`, - #{px}term_taxonomy AS `ttax` - WHERE - trels.object_id = '#{post[:id]}' AND - trels.term_taxonomy_id = ttax.term_taxonomy_id AND - terms.term_id = ttax.term_id" - - db[cquery].each do |term| - if options[:categories] and term[:type] == "category" - if options[:clean_entities] - categories << clean_entities(term[:name]) - else - categories << term[:name] - end - elsif options[:tags] and term[:type] == "post_tag" - if options[:clean_entities] - tags << clean_entities(term[:name]) - else - tags << term[:name] - end - end - end - end - - comments = [] - - if options[:comments] and post[:comment_count].to_i > 0 - cquery = - "SELECT - comment_ID AS `id`, - comment_author AS `author`, - comment_author_email AS `author_email`, - comment_author_url AS `author_url`, - comment_date AS `date`, - comment_date_gmt AS `date_gmt`, - comment_content AS `content` - FROM #{px}comments - WHERE - comment_post_ID = '#{post[:id]}' AND - comment_approved != 'spam'" - - - db[cquery].each do |comment| - - comcontent = comment[:content].to_s - if comcontent.respond_to?(:force_encoding) - comcontent.force_encoding("UTF-8") - end - if options[:clean_entities] - comcontent = clean_entities(comcontent) - end - comauthor = comment[:author].to_s - if options[:clean_entities] - comauthor = clean_entities(comauthor) - end - - comments << { - 'id' => comment[:id].to_i, - 'author' => comauthor, - 'author_email' => comment[:author_email].to_s, - 'author_url' => comment[:author_url].to_s, - 'date' => comment[:date].to_s, - 'date_gmt' => comment[:date_gmt].to_s, - 'content' => comcontent, - } - end - - comments.sort!{ |a,b| a['id'] <=> b['id'] } - end - - # Get the relevant fields as a hash, delete empty fields and - # convert to YAML for the header. - data = { - 'layout' => post[:type].to_s, - 'status' => post[:status].to_s, - 'published' => (post[:status].to_s == "publish"), - 'title' => title.to_s, - 'author' => post[:author].to_s, - 'author_login' => post[:author_login].to_s, - 'author_email' => post[:author_email].to_s, - 'author_url' => post[:author_url].to_s, - 'excerpt' => excerpt, - 'more_anchor' => more_anchor, - 'wordpress_id' => post[:id], - 'wordpress_url' => post[:guid].to_s, - 'date' => date, - 'categories' => options[:categories] ? categories : nil, - 'tags' => options[:tags] ? tags : nil, - 'comments' => options[:comments] ? comments : nil, - }.delete_if { |k,v| v.nil? || v == '' }.to_yaml - - # Write out the data and content to file - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - - - def self.clean_entities( text ) - if text.respond_to?(:force_encoding) - text.force_encoding("UTF-8") - end - text = HTMLEntities.new.encode(text, :named) - # We don't want to convert these, it would break all - # HTML tags in the post and comments. - text.gsub!("&", "&") - text.gsub!("<", "<") - text.gsub!(">", ">") - text.gsub!(""", '"') - text.gsub!("'", "'") - text - end - - - def self.sluggify( title ) - begin - require 'unidecode' - title = title.to_ascii - rescue LoadError - STDERR.puts "Could not require 'unidecode'. If your post titles have non-ASCII characters, you could get nicer permalinks by installing unidecode." - end - title.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-") - end - - end -end diff --git a/lib/jekyll/migrators/wordpressdotcom.rb b/lib/jekyll/migrators/wordpressdotcom.rb deleted file mode 100644 index 701c2af4..00000000 --- a/lib/jekyll/migrators/wordpressdotcom.rb +++ /dev/null @@ -1,70 +0,0 @@ -# coding: utf-8 - -require 'rubygems' -require 'hpricot' -require 'fileutils' -require 'yaml' -require 'time' - -module Jekyll - # This importer takes a wordpress.xml file, which can be exported from your - # wordpress.com blog (/wp-admin/export.php). - module WordpressDotCom - def self.process(filename = "wordpress.xml") - import_count = Hash.new(0) - doc = Hpricot::XML(File.read(filename)) - - (doc/:channel/:item).each do |item| - title = item.at(:title).inner_text.strip - permalink_title = item.at('wp:post_name').inner_text - # Fallback to "prettified" title if post_name is empty (can happen) - if permalink_title == "" - permalink_title = title.downcase.split.join('-') - end - - date = Time.parse(item.at('wp:post_date').inner_text) - status = item.at('wp:status').inner_text - - if status == "publish" - published = true - else - published = false - end - - type = item.at('wp:post_type').inner_text - tags = (item/:category).map{|c| c.inner_text}.reject{|c| c == 'Uncategorized'}.uniq - - metas = Hash.new - item.search("wp:postmeta").each do |meta| - key = meta.at('wp:meta_key').inner_text - value = meta.at('wp:meta_value').inner_text - metas[key] = value; - end - - name = "#{date.strftime('%Y-%m-%d')}-#{permalink_title}.html" - header = { - 'layout' => type, - 'title' => title, - 'tags' => tags, - 'status' => status, - 'type' => type, - 'published' => published, - 'meta' => metas - } - - FileUtils.mkdir_p "_#{type}s" - File.open("_#{type}s/#{name}", "w") do |f| - f.puts header.to_yaml - f.puts '---' - f.puts item.at('content:encoded').inner_text - end - - import_count[type] += 1 - end - - import_count.each do |key, value| - puts "Imported #{value} #{key}s" - end - end - end -end From 44822a252e2ce1142e3293f91285e0ced3ba2fe1 Mon Sep 17 00:00:00 2001 From: xiaojian cai Date: Sun, 6 Jan 2013 21:23:31 +0800 Subject: [PATCH 24/69] add regexp support for option 'include','exclude' --- lib/jekyll/site.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index 7dbf1336..423d2e91 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -22,8 +22,8 @@ module Jekyll self.lsi = config['lsi'] self.pygments = config['pygments'] self.permalink_style = config['permalink'].to_sym - self.exclude = config['exclude'] || [] - self.include = config['include'] || [] + self.exclude = (config['exclude'] || []).map { |e| Regexp.new("^#{e}$") } + self.include = (config['include'] || []).map { |e| Regexp.new("^#{e}$") } self.future = config['future'] self.limit_posts = config['limit_posts'] || nil @@ -318,15 +318,19 @@ module Jekyll # Returns the Array of filtered entries. def filter_entries(entries) entries.reject do |e| - unless self.include.include?(e) + unless regexp_include?(self.include, e) ['.', '_', '#'].include?(e[0..0]) || e[-1..-1] == '~' || - self.exclude.include?(e) || + regexp_include?(self.exclude, e) || File.symlink?(e) end end end + def regexp_include?(exps, e) + !(exps.index { |exp| !exp.match(e).nil? }).nil? + end + # Get the implementation class for the given Converter. # # klass - The Class of the Converter to fetch. From 6eed91871f24d081e5d9a7013dc1e8cf37e4e408 Mon Sep 17 00:00:00 2001 From: edeustace Date: Tue, 8 Jan 2013 23:17:06 +0100 Subject: [PATCH 25/69] Changes based on @mojombo's feedback --- jekyll.gemspec | 7 ++----- lib/jekyll/site.rb | 10 +++++++--- test/test_site.rb | 1 - 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/jekyll.gemspec b/jekyll.gemspec index 1c181122..55201ae8 100644 --- a/jekyll.gemspec +++ b/jekyll.gemspec @@ -5,7 +5,7 @@ Gem::Specification.new do |s| s.name = 'jekyll' s.version = '0.11.2' - s.date = '2012-08-23' + s.date = '2011-12-27' s.rubyforge_project = 'jekyll' s.summary = "A simple, blog aware, static site generator." @@ -74,12 +74,10 @@ Gem::Specification.new do |s| lib/jekyll/migrators/csv.rb lib/jekyll/migrators/drupal.rb lib/jekyll/migrators/enki.rb - lib/jekyll/migrators/joomla.rb lib/jekyll/migrators/marley.rb lib/jekyll/migrators/mephisto.rb lib/jekyll/migrators/mt.rb lib/jekyll/migrators/posterous.rb - lib/jekyll/migrators/rss.rb lib/jekyll/migrators/textpattern.rb lib/jekyll/migrators/tumblr.rb lib/jekyll/migrators/typo.rb @@ -92,7 +90,6 @@ Gem::Specification.new do |s| lib/jekyll/static_file.rb lib/jekyll/tags/highlight.rb lib/jekyll/tags/include.rb - lib/jekyll/tags/post_url.rb test/helper.rb test/source/.htaccess test/source/_includes/sig.markdown @@ -144,9 +141,9 @@ Gem::Specification.new do |s| test/test_post.rb test/test_rdiscount.rb test/test_redcarpet.rb - test/test_redcloth.rb test/test_site.rb test/test_tags.rb + test/test_redcloth.rb ] # = MANIFEST = diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index 058e6e11..d8b729f9 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -220,7 +220,7 @@ module Jekyll dest_files = Set.new Dir.glob(File.join(self.dest, "**", "*"), File::FNM_DOTMATCH) do |file| if self.keep_files.length > 0 - dest_files << file unless file =~ /\/\.{1,2}$/ or file =~ keep_file_regex + dest_files << file unless file =~ /\/\.{1,2}$/ || file =~ keep_file_regex else dest_files << file unless file =~ /\/\.{1,2}$/ end @@ -248,8 +248,12 @@ module Jekyll FileUtils.rm_rf(obsolete_files.to_a) end - # create a regex from the keep_files array - # ['.git','.svn'] => /\/(\.git|\/.svn)/ + # Private: creates a regular expression from the keep_files array + # + # Examples + # ['.git','.svn'] creates the following regex: /\/(\.git|\/.svn)/ + # + # Returns the regular expression def keep_file_regex or_list = self.keep_files.join("|") pattern = "\/(#{or_list.gsub(".", "\.")})" diff --git a/test/test_site.rb b/test/test_site.rb index 26bfaa52..833d7979 100644 --- a/test/test_site.rb +++ b/test/test_site.rb @@ -210,7 +210,6 @@ class TestSite < Test::Unit::TestCase end should 'remove orphaned files in destination - keep_files .svn' do - config = Jekyll::DEFAULTS.merge({'source' => source_dir, 'destination' => dest_dir, 'keep_files' => ['.svn']}) @site = Site.new(config) @site.process From 1e892678bcd01f419e59bd090d66148cc43ced0b Mon Sep 17 00:00:00 2001 From: xiaojian cai Date: Wed, 9 Jan 2013 16:06:43 +0800 Subject: [PATCH 26/69] Update lib/jekyll/site.rb --- lib/jekyll/site.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index 423d2e91..b20c18c5 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -22,8 +22,8 @@ module Jekyll self.lsi = config['lsi'] self.pygments = config['pygments'] self.permalink_style = config['permalink'].to_sym - self.exclude = (config['exclude'] || []).map { |e| Regexp.new("^#{e}$") } - self.include = (config['include'] || []).map { |e| Regexp.new("^#{e}$") } + self.exclude = config['exclude'] || [] + self.include = config['include'] || [] self.future = config['future'] self.limit_posts = config['limit_posts'] || nil @@ -318,17 +318,17 @@ module Jekyll # Returns the Array of filtered entries. def filter_entries(entries) entries.reject do |e| - unless regexp_include?(self.include, e) + unless glob_include?(self.include, e) ['.', '_', '#'].include?(e[0..0]) || e[-1..-1] == '~' || - regexp_include?(self.exclude, e) || + glob_include?(self.exclude, e) || File.symlink?(e) end end end - def regexp_include?(exps, e) - !(exps.index { |exp| !exp.match(e).nil? }).nil? + def glob_include?(exps, e) + !(exps.index { |exp| File.fnmatch?(exp, e) }).nil? end # Get the implementation class for the given Converter. From 655cf3b3a8e4738d5bd3338b8158604360da3e84 Mon Sep 17 00:00:00 2001 From: xiaojian cai Date: Wed, 9 Jan 2013 16:23:58 +0800 Subject: [PATCH 27/69] Update test/test_site.rb --- test/test_site.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_site.rb b/test/test_site.rb index 03f1886e..fe4fab49 100644 --- a/test/test_site.rb +++ b/test/test_site.rb @@ -159,11 +159,11 @@ class TestSite < Test::Unit::TestCase end should "not filter entries within include" do - includes = %w[_index.html .htaccess] - files = %w[index.html _index.html .htaccess] + includes = %w[_index.html .htaccess include*] + files = %w[index.html _index.html .htaccess includeA] @site.include = includes - assert_equal files, @site.filter_entries(files) + assert_equal files, @site.filter_entries(files + ["includeA"]) end context 'error handling' do From adba1017340097cd8585bbf537aa968943e49632 Mon Sep 17 00:00:00 2001 From: xiaojian cai Date: Wed, 9 Jan 2013 16:44:53 +0800 Subject: [PATCH 28/69] update test for include,exclude glob support --- test/test_site.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_site.rb b/test/test_site.rb index fe4fab49..a5ca035b 100644 --- a/test/test_site.rb +++ b/test/test_site.rb @@ -154,8 +154,8 @@ class TestSite < Test::Unit::TestCase excludes = %w[README TODO] files = %w[index.html site.css .htaccess] - @site.exclude = excludes - assert_equal files, @site.filter_entries(excludes + files) + @site.exclude = excludes + ["exclude*"] + assert_equal files, @site.filter_entries(excludes + files + ["excludeA"]) end should "not filter entries within include" do @@ -163,7 +163,7 @@ class TestSite < Test::Unit::TestCase files = %w[index.html _index.html .htaccess includeA] @site.include = includes - assert_equal files, @site.filter_entries(files + ["includeA"]) + assert_equal files, @site.filter_entries(files) end context 'error handling' do From e3bd250e696c38b54f5fe196e618183b2909d07d Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 9 Jan 2013 23:42:57 +0000 Subject: [PATCH 29/69] Remove old jekyll command --- bin/jekyll | 347 ++++++---------------------------------- bin/jekyll2 | 72 --------- features/support/env.rb | 2 +- 3 files changed, 49 insertions(+), 372 deletions(-) delete mode 100755 bin/jekyll2 diff --git a/bin/jekyll b/bin/jekyll index f94420e8..3df10e4f 100755 --- a/bin/jekyll +++ b/bin/jekyll @@ -1,323 +1,72 @@ #!/usr/bin/env ruby -$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib]) +$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib }) -help = < ./_site - jekyll # . -> - jekyll # -> - jekyll import # imports posts using named import script - - Configuration is read from '/_config.yml' but can be overriden - using the following options: - -HELP - -require 'optparse' +require 'commander/import' require 'jekyll' +# Details about Jekyll +program :name, 'jekyll' +program :version, Jekyll::VERSION +program :description, 'Jekyll is a blog-aware, static site generator in Ruby' -exec = {} -options = {} -opts = OptionParser.new do |opts| - opts.banner = help +default_command :help - # TODO: delete, migrator related - opts.on("--file [PATH]", "File to import from") do |import_file| - options['file'] = import_file - end - - # TODO: delete, migrator related - opts.on("--dbname [TEXT]", "DB to import from") do |import_dbname| - options['dbname'] = import_dbname - end - - # TODO: delete, migrator related - opts.on("--user [TEXT]", "Username to use when importing") do |import_user| - options['user'] = import_user - end - - # TODO: delete, migrator related - opts.on("--pass [TEXT]", "Password to use when importing") do |import_pass| - options['pass'] = import_pass - end - - # TODO: delete, migrator related - opts.on("--host [HOST ADDRESS]", "Host to import from") do |import_host| - options['host'] = import_host - end - - # TODO: delete, migrator related - opts.on("--site [SITE NAME]", "Site to import from") do |import_site| - options['site'] = import_site - end - - # TODO: global option - opts.on("--[no-]safe", "Safe mode (default unsafe)") do |safe| - options['safe'] = safe - end +# Global options available to every command +global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' +global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' +global_option '--safe', 'Safe mode (defaults to false)' +global_option '--plugins', 'Plugins directory (defaults to ./_plugins)' +global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' - # TODO: option for build and serve command - opts.on("--[no-]auto", "Auto-regenerate") do |auto| - options['auto'] = auto - end +command :build do |c| + c.syntax = 'jekyll build [options]' + c.description = 'Build your site with the option of auto-renegeration' - # TODO: new serve command - opts.on("--server [PORT]", "Start web server (default port 4000)") do |port| - options['server'] = true - options['server_port'] = port unless port.nil? - end + c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '--lsi', 'Use LSI for improved related posts' - # TODO: remove - opts.on("--no-server", "Do not start a web server") do |part| - options['server'] = false - end - - # TODO: option for serve command - opts.on("--base-url [BASE_URL]", "Serve website from a given base URL (default '/'") do |baseurl| - options['baseurl'] = baseurl - end - - # TODO: does anyone actually use this? - opts.on("--default-mimetype [MT]", "Mimetype to use when no file extension (if --server)") do |mt| - options['default-mimetype'] = mt - end - - # TODO: option for build and serve command - opts.on("--[no-]lsi", "Use LSI for better related posts") do |lsi| - options['lsi'] = lsi - end - - # TODO: remove and just enable all the time? - opts.on("--[no-]pygments", "Use pygments to highlight code") do |pygments| - options['pygments'] = pygments - end - - # TODO: read from config - opts.on("--rdiscount", "Use rdiscount gem for Markdown") do - options['markdown'] = 'rdiscount' - end - - # TODO: read from config - opts.on("--redcarpet", "Use redcarpet gem for Markdown") do - options['markdown'] = 'redcarpet' - end - - # TODO: read from config - opts.on("--kramdown", "Use kramdown gem for Markdown") do - options['markdown'] = 'kramdown' - end - - # TODO: remove and just generate the site as is? - opts.on("--time [TIME]", "Time to generate the site for") do |time| - options['time'] = Time.parse(time) - end - - # TODO: remove and just render all posts which aren't 'unpublished'? - opts.on("--[no-]future", "Render future dated posts") do |future| - options['future'] = future - end - - # TODO: read from config - opts.on("--permalink [TYPE]", "Use 'date' (default) for YYYY/MM/DD") do |style| - options['permalink'] = style unless style.nil? - end - - # TODO: read from config - opts.on("--paginate [POSTS_PER_PAGE]", "Paginate a blog's posts") do |per_page| - begin - options['paginate'] = per_page.to_i - raise ArgumentError if options['paginate'] == 0 - rescue - puts 'you must specify a number of posts by page bigger than 0' - exit 0 - end - end - - # TODO: read from config - opts.on("--paginate_path [PAGINATED_URL_FORMAT]", "Leave blank for /page") do |paginate_path| - begin - options['paginate_path'] = paginate_path - raise ArgumentError if options['paginate_path'].nil? - rescue - puts 'You must specify a pagination url format' - exit 0 - end - end - - # TODO: read from config - opts.on("--limit_posts [MAX_POSTS]", "Limit the number of posts to publish") do |limit_posts| - begin - options['limit_posts'] = limit_posts.to_i - raise ArgumentError if options['limit_posts'] < 1 - rescue - puts 'you must specify a number of posts by page bigger than 0' - exit 0 - end - end - - # TODO: read from config - opts.on("--url [URL]", "Set custom site.url") do |url| - options['url'] = url - end - - opts.on("--version", "Display current version") do - puts "Jekyll " + Jekyll::VERSION - exit 0 + c.action do |args, options| + options.defaults :serving => false + options = Jekyll.configuration(options.__hash__) + Jekyll::BuildCommand.process(options) end end -# Read command line options into `options` hash -opts.parse! +command :serve do |c| + c.syntax = 'jekyll serve [options]' + c.description = 'Serve your site locally with the option of auto-regeneration' + c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '--lsi', 'Use LSI for improved related posts' -# Check for import stuff -if ARGV.size > 0 - if ARGV[0] == 'import' - migrator = ARGV[1] + c.option '-p', '--port [PORT]', 'Port to listen on' + c.option '-h', '--host [HOST]', 'Host to bind to' + c.option '-b', '--baseurl [URL]', 'Base URL' - if migrator.nil? - puts "Invalid options. Run `jekyll --help` for assistance." - exit(1) - else - migrator = migrator.downcase - end + c.action do |args, options| + options.default :port => '4000', + :host => '0.0.0.0', + :baseurl => '/', + :serving => true - cmd_options = [] - ['file', 'dbname', 'user', 'pass', 'host', 'site'].each do |p| - cmd_options << "\"#{options[p]}\"" unless options[p].nil? - end - - # It's import time - puts "Importing..." - - # Ideally, this shouldn't be necessary. Maybe parse the actual - # src files for the migrator name? - migrators = { - :posterous => 'Posterous', - :wordpressdotcom => 'WordpressDotCom', - :wordpress => 'WordPress', - :csv => 'CSV', - :drupal => 'Drupal', - :enki => 'Enki', - :mephisto => 'Mephisto', - :mt => 'MT', - :textpattern => 'TextPattern', - :tumblr => 'Tumblr', - :typo => 'Typo' - } - - app_root = File.join(File.dirname(__FILE__), '..') - - require "#{app_root}/lib/jekyll/migrators/#{migrator}" - - if Jekyll.const_defined?(migrators[migrator.to_sym]) - migrator_class = Jekyll.const_get(migrators[migrator.to_sym]) - migrator_class.process(*cmd_options) - else - puts "Invalid migrator. Run `jekyll --help` for assistance." - exit(1) - end - - exit(0) + options = Jekyll.configuration(options.__hash__) + Jekyll::BuildCommand.process(options) + Jekyll::ServeCommand.process(options) end end +command :migrate do |c| + c.syntax = 'jekyll migrate [options]' + c.description = 'Migrate your own blog to Jekyll' + c.option '--file', 'File to migrate from' + c.option '--dbname', 'Database name to migrate from' + c.option '--user', 'Username to use when migrating' + c.option '--pass', 'Password to use when migrating' + c.option '--host', 'Host address to use when migrating' -# Get source and destination from command line -# TODO: source and destination are now global options -case ARGV.size - when 0 - when 1 - options['destination'] = ARGV[0] - when 2 - options['source'] = ARGV[0] - options['destination'] = ARGV[1] - else - puts "Invalid options. Run `jekyll --help` for assistance." - exit(1) -end - -options = Jekyll.configuration(options) - -# Get source and destination directories (possibly set by config file) -source = options['source'] -destination = options['destination'] - -# Files to watch -def globs(source, destination) - Dir.chdir(source) do - dirs = Dir['*'].select { |x| File.directory?(x) } - dirs -= [destination] - dirs = dirs.map { |x| "#{x}/**/*" } - dirs += ['*'] + c.action do |args, options| + Jekyll::MigrateCommand.process(args.first, options) end end - -# Create the Site -site = Jekyll::Site.new(options) - -# Run the directory watcher for auto-generation, if required -if options['auto'] - require 'directory_watcher' - - puts "Auto-regenerating enabled: #{source} -> #{destination}" - - dw = DirectoryWatcher.new(source) - dw.interval = 1 - dw.glob = globs(source, destination) - - dw.add_observer do |*args| - t = Time.now.strftime("%Y-%m-%d %H:%M:%S") - puts "[#{t}] regeneration: #{args.size} files changed" - site.process - end - - dw.start - - unless options['server'] - loop { sleep 1000 } - end -else - puts "Building site: #{source} -> #{destination}" - begin - site.process - rescue Jekyll::FatalException => e - puts - puts "ERROR: YOUR SITE COULD NOT BE BUILT:" - puts "------------------------------------" - puts e.message - exit(1) - end - puts "Successfully generated site: #{source} -> #{destination}" -end - -# Run the server on the specified port, if required -if options['server'] - require 'webrick' - include WEBrick - - FileUtils.mkdir_p(destination) - - mime_types = WEBrick::HTTPUtils::DefaultMimeTypes - mime_types.store 'js', 'application/javascript' - if options['default-mimetype'] - mime_types.store(nil, options['default-mimetype']) - end - - s = HTTPServer.new( - :Port => options['server_port'], - :MimeTypes => mime_types - ) - s.mount(options['baseurl'], HTTPServlet::FileHandler, destination) - t = Thread.new { - s.start - } - - trap("INT") { s.shutdown } - t.join() -end diff --git a/bin/jekyll2 b/bin/jekyll2 deleted file mode 100755 index 3df10e4f..00000000 --- a/bin/jekyll2 +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env ruby - -$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib }) - -require 'commander/import' -require 'jekyll' - -# Details about Jekyll -program :name, 'jekyll' -program :version, Jekyll::VERSION -program :description, 'Jekyll is a blog-aware, static site generator in Ruby' - -default_command :help - -# Global options available to every command -global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' -global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' -global_option '--safe', 'Safe mode (defaults to false)' -global_option '--plugins', 'Plugins directory (defaults to ./_plugins)' -global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' - -command :build do |c| - c.syntax = 'jekyll build [options]' - c.description = 'Build your site with the option of auto-renegeration' - - c.option '-w', '--watch', 'Watch for changes and rebuild' - c.option '--lsi', 'Use LSI for improved related posts' - - c.action do |args, options| - options.defaults :serving => false - options = Jekyll.configuration(options.__hash__) - Jekyll::BuildCommand.process(options) - end -end - -command :serve do |c| - c.syntax = 'jekyll serve [options]' - c.description = 'Serve your site locally with the option of auto-regeneration' - - c.option '-w', '--watch', 'Watch for changes and rebuild' - c.option '--lsi', 'Use LSI for improved related posts' - - c.option '-p', '--port [PORT]', 'Port to listen on' - c.option '-h', '--host [HOST]', 'Host to bind to' - c.option '-b', '--baseurl [URL]', 'Base URL' - - c.action do |args, options| - options.default :port => '4000', - :host => '0.0.0.0', - :baseurl => '/', - :serving => true - - options = Jekyll.configuration(options.__hash__) - Jekyll::BuildCommand.process(options) - Jekyll::ServeCommand.process(options) - end -end - -command :migrate do |c| - c.syntax = 'jekyll migrate [options]' - c.description = 'Migrate your own blog to Jekyll' - - c.option '--file', 'File to migrate from' - c.option '--dbname', 'Database name to migrate from' - c.option '--user', 'Username to use when migrating' - c.option '--pass', 'Password to use when migrating' - c.option '--host', 'Host address to use when migrating' - - c.action do |args, options| - Jekyll::MigrateCommand.process(args.first, options) - end -end diff --git a/features/support/env.rb b/features/support/env.rb index 912b735c..1ed330a1 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -7,7 +7,7 @@ World do end TEST_DIR = File.join('/', 'tmp', 'jekyll') -JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll2') +JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll') def run_jekyll(opts = {}) command = JEKYLL_PATH From c627cd35197584007d454142761a3843e6361ba5 Mon Sep 17 00:00:00 2001 From: xiaojian cai Date: Thu, 10 Jan 2013 09:16:09 +0800 Subject: [PATCH 30/69] use any? instead --- lib/jekyll/site.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index b20c18c5..bdab3237 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -328,7 +328,7 @@ module Jekyll end def glob_include?(exps, e) - !(exps.index { |exp| File.fnmatch?(exp, e) }).nil? + exps.any? { |exp| File.fnmatch?(exp, e) }) end # Get the implementation class for the given Converter. From b3e27f2c5d8f2a15b557f931bb4c0725d6246462 Mon Sep 17 00:00:00 2001 From: xiaojian cai Date: Thu, 10 Jan 2013 19:41:11 +0800 Subject: [PATCH 31/69] fixed a syntax error --- lib/jekyll/site.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index bdab3237..8c28ed89 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -328,7 +328,7 @@ module Jekyll end def glob_include?(exps, e) - exps.any? { |exp| File.fnmatch?(exp, e) }) + exps.any? { |exp| File.fnmatch?(exp, e) } end # Get the implementation class for the given Converter. From ba48870eadd48cdd2e2981a5904e164e915b2320 Mon Sep 17 00:00:00 2001 From: Eric Theise Date: Thu, 10 Jan 2013 12:29:04 -0800 Subject: [PATCH 32/69] outputting full path when file does not parse --- lib/jekyll/convertible.rb | 4 ++-- test/test_convertible.rb | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb index dab0545c..26f7a0c9 100644 --- a/lib/jekyll/convertible.rb +++ b/lib/jekyll/convertible.rb @@ -33,9 +33,9 @@ module Jekyll self.data = YAML.load($1) end rescue => e - puts "Error reading file #{name}: #{e.message}" + puts "Error reading file #{File.join(base,name)}: #{e.message}" rescue SyntaxError => e - puts "YAML Exception reading #{name}: #{e.message}" + puts "YAML Exception reading #{File.join(base,name)}: #{e.message}" end self.data ||= {} diff --git a/test/test_convertible.rb b/test/test_convertible.rb index b9a9e41c..3ad3c817 100644 --- a/test/test_convertible.rb +++ b/test/test_convertible.rb @@ -20,20 +20,26 @@ class TestConvertible < Test::Unit::TestCase end should "not parse if there is syntax error in front-matter" do + @name = 'broken_front_matter2.erb' out = capture_stdout do - ret = @convertible.read_yaml(@base, 'broken_front_matter2.erb') + ret = @convertible.read_yaml(@base, @name) assert_equal({}, ret) end assert_match(/YAML Exception|syntax error/, out) + assert_match(/#{@base}/, out) + assert_match(/#{@name}/, out) end if RUBY_VERSION >= '1.9.2' should "not parse if there is encoding error in file" do + @name = 'broken_front_matter3.erb' out = capture_stdout do - ret = @convertible.read_yaml(@base, 'broken_front_matter3.erb') + ret = @convertible.read_yaml(@base, @name) assert_equal({}, ret) end assert_match(/invalid byte sequence in UTF-8/, out) + assert_match(/#{@base}/, out) + assert_match(/#{@name}/, out) end end end From 3e7f00ae1967c9f342e4b0c81707c1ab857f52a6 Mon Sep 17 00:00:00 2001 From: Eric Theise Date: Thu, 10 Jan 2013 13:01:50 -0800 Subject: [PATCH 33/69] test the path, not the parts --- test/test_convertible.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/test_convertible.rb b/test/test_convertible.rb index 3ad3c817..80353a9d 100644 --- a/test/test_convertible.rb +++ b/test/test_convertible.rb @@ -26,8 +26,7 @@ class TestConvertible < Test::Unit::TestCase assert_equal({}, ret) end assert_match(/YAML Exception|syntax error/, out) - assert_match(/#{@base}/, out) - assert_match(/#{@name}/, out) + assert_match(/#{File.join(@base,@name)}/, out) end if RUBY_VERSION >= '1.9.2' @@ -38,8 +37,7 @@ class TestConvertible < Test::Unit::TestCase assert_equal({}, ret) end assert_match(/invalid byte sequence in UTF-8/, out) - assert_match(/#{@base}/, out) - assert_match(/#{@name}/, out) + assert_match(/#{File.join(@base,@name)}/, out) end end end From cf42c566308c3473f0188e2f28e1250f1bb657a2 Mon Sep 17 00:00:00 2001 From: Eric Theise Date: Thu, 10 Jan 2013 14:07:10 -0800 Subject: [PATCH 34/69] stick to good coding standards --- test/test_convertible.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_convertible.rb b/test/test_convertible.rb index 80353a9d..b5afa86f 100644 --- a/test/test_convertible.rb +++ b/test/test_convertible.rb @@ -26,7 +26,7 @@ class TestConvertible < Test::Unit::TestCase assert_equal({}, ret) end assert_match(/YAML Exception|syntax error/, out) - assert_match(/#{File.join(@base,@name)}/, out) + assert_match(/#{File.join(@base, @name)}/, out) end if RUBY_VERSION >= '1.9.2' @@ -37,7 +37,7 @@ class TestConvertible < Test::Unit::TestCase assert_equal({}, ret) end assert_match(/invalid byte sequence in UTF-8/, out) - assert_match(/#{File.join(@base,@name)}/, out) + assert_match(/#{File.join(@base, @name)}/, out) end end end From a5c908ac3f3d55bf6ed25695831f9e5affbde0c4 Mon Sep 17 00:00:00 2001 From: Eric Theise Date: Thu, 10 Jan 2013 14:13:11 -0800 Subject: [PATCH 35/69] no good reason for making 'name' an instance variable --- test/test_convertible.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_convertible.rb b/test/test_convertible.rb index b5afa86f..82e4d27f 100644 --- a/test/test_convertible.rb +++ b/test/test_convertible.rb @@ -20,24 +20,24 @@ class TestConvertible < Test::Unit::TestCase end should "not parse if there is syntax error in front-matter" do - @name = 'broken_front_matter2.erb' + name = 'broken_front_matter2.erb' out = capture_stdout do - ret = @convertible.read_yaml(@base, @name) + ret = @convertible.read_yaml(@base, name) assert_equal({}, ret) end assert_match(/YAML Exception|syntax error/, out) - assert_match(/#{File.join(@base, @name)}/, out) + assert_match(/#{File.join(@base, name)}/, out) end if RUBY_VERSION >= '1.9.2' should "not parse if there is encoding error in file" do - @name = 'broken_front_matter3.erb' + name = 'broken_front_matter3.erb' out = capture_stdout do - ret = @convertible.read_yaml(@base, @name) + ret = @convertible.read_yaml(@base, name) assert_equal({}, ret) end assert_match(/invalid byte sequence in UTF-8/, out) - assert_match(/#{File.join(@base, @name)}/, out) + assert_match(/#{File.join(@base, name)}/, out) end end end From e210a0603fa76fe4e31dc80729c8ff9f8e61b38e Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Fri, 11 Jan 2013 00:30:17 +0000 Subject: [PATCH 36/69] Revert "Remove migrators" This reverts commit b9da30bc8ffa4a2d0a9280e755d0904609d4fc97. --- Rakefile | 15 ++ lib/jekyll/migrators/csv.rb | 26 +++ lib/jekyll/migrators/drupal.rb | 103 +++++++++ lib/jekyll/migrators/enki.rb | 49 ++++ lib/jekyll/migrators/joomla.rb | 53 +++++ lib/jekyll/migrators/marley.rb | 52 +++++ lib/jekyll/migrators/mephisto.rb | 84 +++++++ lib/jekyll/migrators/mt.rb | 86 +++++++ lib/jekyll/migrators/posterous.rb | 67 ++++++ lib/jekyll/migrators/rss.rb | 47 ++++ lib/jekyll/migrators/textpattern.rb | 58 +++++ lib/jekyll/migrators/tumblr.rb | 195 ++++++++++++++++ lib/jekyll/migrators/typo.rb | 51 ++++ lib/jekyll/migrators/wordpress.rb | 294 ++++++++++++++++++++++++ lib/jekyll/migrators/wordpressdotcom.rb | 70 ++++++ 15 files changed, 1250 insertions(+) create mode 100644 lib/jekyll/migrators/csv.rb create mode 100644 lib/jekyll/migrators/drupal.rb create mode 100644 lib/jekyll/migrators/enki.rb create mode 100644 lib/jekyll/migrators/joomla.rb create mode 100644 lib/jekyll/migrators/marley.rb create mode 100644 lib/jekyll/migrators/mephisto.rb create mode 100644 lib/jekyll/migrators/mt.rb create mode 100644 lib/jekyll/migrators/posterous.rb create mode 100644 lib/jekyll/migrators/rss.rb create mode 100644 lib/jekyll/migrators/textpattern.rb create mode 100644 lib/jekyll/migrators/tumblr.rb create mode 100644 lib/jekyll/migrators/typo.rb create mode 100644 lib/jekyll/migrators/wordpress.rb create mode 100644 lib/jekyll/migrators/wordpressdotcom.rb diff --git a/Rakefile b/Rakefile index d1917c5a..ff52d677 100644 --- a/Rakefile +++ b/Rakefile @@ -82,6 +82,21 @@ end # ############################################################################# +namespace :migrate do + desc "Migrate from mephisto in the current directory" + task :mephisto do + sh %q(ruby -r './lib/jekyll/migrators/mephisto' -e 'Jekyll::Mephisto.postgres(:database => "#{ENV["DB"]}")') + end + desc "Migrate from Movable Type in the current directory" + task :mt do + sh %q(ruby -r './lib/jekyll/migrators/mt' -e 'Jekyll::MT.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")') + end + desc "Migrate from Typo in the current directory" + task :typo do + sh %q(ruby -r './lib/jekyll/migrators/typo' -e 'Jekyll::Typo.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")') + end +end + begin require 'cucumber/rake/task' Cucumber::Rake::Task.new(:features) do |t| diff --git a/lib/jekyll/migrators/csv.rb b/lib/jekyll/migrators/csv.rb new file mode 100644 index 00000000..ce5203b7 --- /dev/null +++ b/lib/jekyll/migrators/csv.rb @@ -0,0 +1,26 @@ +module Jekyll + module CSV + # Reads a csv with title, permalink, body, published_at, and filter. + # It creates a post file for each row in the csv + def self.process(file = "posts.csv") + FileUtils.mkdir_p "_posts" + posts = 0 + FasterCSV.foreach(file) do |row| + next if row[0] == "title" + posts += 1 + name = row[3].split(" ")[0]+"-"+row[1]+(row[4] =~ /markdown/ ? ".markdown" : ".textile") + File.open("_posts/#{name}", "w") do |f| + f.puts <<-HEADER +--- +layout: post +title: #{row[0]} +--- + + HEADER + f.puts row[2] + end + end + "Created #{posts} posts!" + end + end +end diff --git a/lib/jekyll/migrators/drupal.rb b/lib/jekyll/migrators/drupal.rb new file mode 100644 index 00000000..7fd16aef --- /dev/null +++ b/lib/jekyll/migrators/drupal.rb @@ -0,0 +1,103 @@ +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module Drupal + # Reads a MySQL database via Sequel and creates a post file for each post + # in wp_posts that has post_status = 'publish'. This restriction is made + # because 'draft' posts are not guaranteed to have valid dates. + QUERY = "SELECT n.nid, \ + n.title, \ + nr.body, \ + n.created, \ + n.status \ + FROM node AS n, \ + node_revisions AS nr \ + WHERE (n.type = 'blog' OR n.type = 'story') \ + AND n.vid = nr.vid" + + def self.process(dbname, user, pass, host = 'localhost', prefix = '') + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + + if prefix != '' + QUERY[" node "] = " " + prefix + "node " + QUERY[" node_revisions "] = " " + prefix + "node_revisions " + end + + FileUtils.mkdir_p "_posts" + FileUtils.mkdir_p "_drafts" + + # Create the refresh layout + # Change the refresh url if you customized your permalink config + File.open("_layouts/refresh.html", "w") do |f| + f.puts < + + + + + + +EOF + end + + db[QUERY].each do |post| + # Get required fields and construct Jekyll compatible name + node_id = post[:nid] + title = post[:title] + content = post[:body] + created = post[:created] + time = Time.at(created) + is_published = post[:status] == 1 + dir = is_published ? "_posts" : "_drafts" + slug = title.strip.downcase.gsub(/(&|&)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '') + name = time.strftime("%Y-%m-%d-") + slug + '.md' + + # Get the relevant fields as a hash, delete empty fields and convert + # to YAML for the header + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'created' => created, + }.delete_if { |k,v| v.nil? || v == ''}.to_yaml + + # Write out the data and content to file + File.open("#{dir}/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + + # Make a file to redirect from the old Drupal URL + if is_published + aliases = db["SELECT dst FROM #{prefix}url_alias WHERE src = ?", "node/#{node_id}"].all + + aliases.push(:dst => "node/#{node_id}") + + aliases.each do |url_alias| + FileUtils.mkdir_p url_alias[:dst] + File.open("#{url_alias[:dst]}/index.md", "w") do |f| + f.puts "---" + f.puts "layout: refresh" + f.puts "refresh_to_post_id: /#{time.strftime("%Y/%m/%d/") + slug}" + f.puts "---" + end + end + end + end + + # TODO: Make dirs & files for nodes of type 'page' + # Make refresh pages for these as well + + # TODO: Make refresh dirs & files according to entries in url_alias table + end + end +end diff --git a/lib/jekyll/migrators/enki.rb b/lib/jekyll/migrators/enki.rb new file mode 100644 index 00000000..61cb2562 --- /dev/null +++ b/lib/jekyll/migrators/enki.rb @@ -0,0 +1,49 @@ +# Adapted by Rodrigo Pinto +# Based on typo.rb by Toby DiPasquale + +require 'fileutils' +require 'rubygems' +require 'sequel' + +module Jekyll + module Enki + SQL = <<-EOS + SELECT p.id, + p.title, + p.slug, + p.body, + p.published_at as date, + p.cached_tag_list as tags + FROM posts p + EOS + + # Just working with postgres, but can be easily adapted + # to work with both mysql and postgres. + def self.process(dbname, user, pass, host = 'localhost') + FileUtils.mkdir_p('_posts') + db = Sequel.postgres(:database => dbname, + :user => user, + :password => pass, + :host => host, + :encoding => 'utf8') + + db[SQL].each do |post| + name = [ sprintf("%.04d", post[:date].year), + sprintf("%.02d", post[:date].month), + sprintf("%.02d", post[:date].day), + post[:slug].strip ].join('-') + name += '.textile' + + File.open("_posts/#{name}", 'w') do |f| + f.puts({ 'layout' => 'post', + 'title' => post[:title].to_s, + 'enki_id' => post[:id], + 'categories' => post[:tags] + }.delete_if { |k, v| v.nil? || v == '' }.to_yaml) + f.puts '---' + f.puts post[:body].delete("\r") + end + end + end + end +end diff --git a/lib/jekyll/migrators/joomla.rb b/lib/jekyll/migrators/joomla.rb new file mode 100644 index 00000000..87f1e105 --- /dev/null +++ b/lib/jekyll/migrators/joomla.rb @@ -0,0 +1,53 @@ +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This migrator is made for Joomla 1.5 databases. +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module Joomla + def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'jos_', section = '1') + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + + FileUtils.mkdir_p("_posts") + + # Reads a MySQL database via Sequel and creates a post file for each + # post in wp_posts that has post_status = 'publish'. This restriction is + # made because 'draft' posts are not guaranteed to have valid dates. + query = "SELECT `title`, `alias`, CONCAT(`introtext`,`fulltext`) as content, `created`, `id` FROM #{table_prefix}content WHERE state = '0' OR state = '1' AND sectionid = '#{section}'" + + db[query].each do |post| + # Get required fields and construct Jekyll compatible name. + title = post[:title] + slug = post[:alias] + date = post[:created] + content = post[:content] + name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day, + slug] + + # Get the relevant fields as a hash, delete empty fields and convert + # to YAML for the header. + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'joomla_id' => post[:id], + 'joomla_url' => post[:alias], + 'date' => date + }.delete_if { |k,v| v.nil? || v == '' }.to_yaml + + # Write out the data and content to file + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + end + end +end diff --git a/lib/jekyll/migrators/marley.rb b/lib/jekyll/migrators/marley.rb new file mode 100644 index 00000000..21bcead5 --- /dev/null +++ b/lib/jekyll/migrators/marley.rb @@ -0,0 +1,52 @@ +require 'yaml' +require 'fileutils' + +module Jekyll + module Marley + def self.regexp + { :id => /^\d{0,4}-{0,1}(.*)$/, + :title => /^#\s*(.*)\s+$/, + :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/, + :published_on => /.*\s+\(([0-9\/]+)\)$/, + :perex => /^([^\#\n]+\n)$/, + :meta => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp + } + end + + def self.process(marley_data_dir) + raise ArgumentError, "marley dir #{marley_data_dir} not found" unless File.directory?(marley_data_dir) + + FileUtils.mkdir_p "_posts" + + posts = 0 + Dir["#{marley_data_dir}/**/*.txt"].each do |f| + next unless File.exists?(f) + + #copied over from marley's app/lib/post.rb + file_content = File.read(f) + meta_content = file_content.slice!( self.regexp[:meta] ) + body = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip + + title = file_content.scan( self.regexp[:title] ).first.to_s.strip + prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip + published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) ) + meta = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {} + meta['title'] = title + meta['layout'] = 'post' + + formatted_date = published_on.strftime('%Y-%m-%d') + post_name = File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '') + + name = "#{formatted_date}-#{post_name}" + File.open("_posts/#{name}.markdown", "w") do |f| + f.puts meta.to_yaml + f.puts "---\n" + f.puts "\n#{prerex}\n\n" if prerex + f.puts body + end + posts += 1 + end + "Created #{posts} posts!" + end + end +end diff --git a/lib/jekyll/migrators/mephisto.rb b/lib/jekyll/migrators/mephisto.rb new file mode 100644 index 00000000..7622c722 --- /dev/null +++ b/lib/jekyll/migrators/mephisto.rb @@ -0,0 +1,84 @@ +# Quickly hacked together my Michael Ivey +# Based on mt.rb by Nick Gerakines, open source and publically +# available under the MIT license. Use this module at your own risk. + +require 'rubygems' +require 'sequel' +require 'fastercsv' +require 'fileutils' +require File.join(File.dirname(__FILE__),"csv.rb") + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module Mephisto + #Accepts a hash with database config variables, exports mephisto posts into a csv + #export PGPASSWORD if you must + def self.postgres(c) + sql = <<-SQL + BEGIN; + CREATE TEMP TABLE jekyll AS + SELECT title, permalink, body, published_at, filter FROM contents + WHERE user_id = 1 AND type = 'Article' ORDER BY published_at; + COPY jekyll TO STDOUT WITH CSV HEADER; + ROLLBACK; + SQL + command = %Q(psql -h #{c[:host] || "localhost"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"}) + puts command + `#{command}` + CSV.process + end + + # This query will pull blog posts from all entries across all blogs. If + # you've got unpublished, deleted or otherwise hidden posts please sift + # through the created posts to make sure nothing is accidently published. + QUERY = "SELECT id, \ + permalink, \ + body, \ + published_at, \ + title \ + FROM contents \ + WHERE user_id = 1 AND \ + type = 'Article' AND \ + published_at IS NOT NULL \ + ORDER BY published_at" + + def self.process(dbname, user, pass, host = 'localhost') + db = Sequel.mysql(dbname, :user => user, + :password => pass, + :host => host, + :encoding => 'utf8') + + FileUtils.mkdir_p "_posts" + + db[QUERY].each do |post| + title = post[:title] + slug = post[:permalink] + date = post[:published_at] + content = post[:body] + + # Ideally, this script would determine the post format (markdown, + # html, etc) and create files with proper extensions. At this point + # it just assumes that markdown will be acceptable. + name = [date.year, date.month, date.day, slug].join('-') + ".markdown" + + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'mt_id' => post[:entry_id], + }.delete_if { |k,v| v.nil? || v == ''}.to_yaml + + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + + end + end +end diff --git a/lib/jekyll/migrators/mt.rb b/lib/jekyll/migrators/mt.rb new file mode 100644 index 00000000..048c84db --- /dev/null +++ b/lib/jekyll/migrators/mt.rb @@ -0,0 +1,86 @@ +# Created by Nick Gerakines, open source and publically available under the +# MIT license. Use this module at your own risk. +# I'm an Erlang/Perl/C++ guy so please forgive my dirty ruby. + +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module MT + # This query will pull blog posts from all entries across all blogs. If + # you've got unpublished, deleted or otherwise hidden posts please sift + # through the created posts to make sure nothing is accidently published. + QUERY = "SELECT entry_id, \ + entry_basename, \ + entry_text, \ + entry_text_more, \ + entry_authored_on, \ + entry_title, \ + entry_convert_breaks \ + FROM mt_entry" + + def self.process(dbname, user, pass, host = 'localhost') + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + + FileUtils.mkdir_p "_posts" + + db[QUERY].each do |post| + title = post[:entry_title] + slug = post[:entry_basename].gsub(/_/, '-') + date = post[:entry_authored_on] + content = post[:entry_text] + more_content = post[:entry_text_more] + entry_convert_breaks = post[:entry_convert_breaks] + + # Be sure to include the body and extended body. + if more_content != nil + content = content + " \n" + more_content + end + + # Ideally, this script would determine the post format (markdown, + # html, etc) and create files with proper extensions. At this point + # it just assumes that markdown will be acceptable. + name = [date.year, date.month, date.day, slug].join('-') + '.' + + self.suffix(entry_convert_breaks) + + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'mt_id' => post[:entry_id], + 'date' => date + }.delete_if { |k,v| v.nil? || v == '' }.to_yaml + + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + end + + def self.suffix(entry_type) + if entry_type.nil? || entry_type.include?("markdown") + # The markdown plugin I have saves this as + # "markdown_with_smarty_pants", so I just look for "markdown". + "markdown" + elsif entry_type.include?("textile") + # This is saved as "textile_2" on my installation of MT 5.1. + "textile" + elsif entry_type == "0" || entry_type.include?("richtext") + # Richtext looks to me like it's saved as HTML, so I include it here. + "html" + else + # Other values might need custom work. + entry_type + end + end + end +end diff --git a/lib/jekyll/migrators/posterous.rb b/lib/jekyll/migrators/posterous.rb new file mode 100644 index 00000000..0a2280f2 --- /dev/null +++ b/lib/jekyll/migrators/posterous.rb @@ -0,0 +1,67 @@ +require 'rubygems' +require 'jekyll' +require 'fileutils' +require 'net/http' +require 'uri' +require "json" + +# ruby -r './lib/jekyll/migrators/posterous.rb' -e 'Jekyll::Posterous.process(email, pass, api_key, blog)' + +module Jekyll + module Posterous + def self.fetch(uri_str, limit = 10) + # You should choose better exception. + raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0 + + response = nil + Net::HTTP.start('posterous.com') do |http| + req = Net::HTTP::Get.new(uri_str) + req.basic_auth @email, @pass + response = http.request(req) + end + + case response + when Net::HTTPSuccess then response + when Net::HTTPRedirection then fetch(response['location'], limit - 1) + else response.error! + end + end + + def self.process(email, pass, api_token, blog = 'primary') + @email, @pass, @api_token = email, pass, api_token + FileUtils.mkdir_p "_posts" + + posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}").body) + page = 1 + + while posts.any? + posts.each do |post| + title = post["title"] + slug = title.gsub(/[^[:alnum:]]+/, '-').downcase + date = Date.parse(post["display_date"]) + content = post["body_html"] + published = !post["is_private"] + name = "%02d-%02d-%02d-%s.html" % [date.year, date.month, date.day, slug] + + # Get the relevant fields as a hash, delete empty fields and convert + # to YAML for the header + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'published' => published + }.delete_if { |k,v| v.nil? || v == ''}.to_yaml + + # Write out the data and content to file + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + + page += 1 + posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}&page=#{page}").body) + end + end + end +end diff --git a/lib/jekyll/migrators/rss.rb b/lib/jekyll/migrators/rss.rb new file mode 100644 index 00000000..461abd35 --- /dev/null +++ b/lib/jekyll/migrators/rss.rb @@ -0,0 +1,47 @@ +# Created by Kendall Buchanan (https://github.com/kendagriff) on 2011-12-22. +# Use at your own risk. The end. +# +# Usage: +# (URL) +# ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('http://yourdomain.com/your-favorite-feed.xml')" +# +# (Local file) +# ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('./somefile/on/your/computer.xml')" + +require 'rubygems' +require 'rss/1.0' +require 'rss/2.0' +require 'open-uri' +require 'fileutils' +require 'yaml' + +module Jekyll + module MigrateRSS + + # The `source` argument may be a URL or a local file. + def self.process(source) + content = "" + open(source) { |s| content = s.read } + rss = RSS::Parser.parse(content, false) + + raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss + + rss.items.each do |item| + formatted_date = item.date.strftime('%Y-%m-%d') + post_name = item.title.split(%r{ |!|/|:|&|-|$|,}).map { |i| i.downcase if i != '' }.compact.join('-') + name = "#{formatted_date}-#{post_name}" + + header = { + 'layout' => 'post', + 'title' => item.title + } + + File.open("_posts/#{name}.html", "w") do |f| + f.puts header.to_yaml + f.puts "---\n" + f.puts item.description + end + end + end + end +end \ No newline at end of file diff --git a/lib/jekyll/migrators/textpattern.rb b/lib/jekyll/migrators/textpattern.rb new file mode 100644 index 00000000..3b370ed9 --- /dev/null +++ b/lib/jekyll/migrators/textpattern.rb @@ -0,0 +1,58 @@ +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module TextPattern + # Reads a MySQL database via Sequel and creates a post file for each post. + # The only posts selected are those with a status of 4 or 5, which means + # "live" and "sticky" respectively. + # Other statuses are 1 => draft, 2 => hidden and 3 => pending. + QUERY = "SELECT Title, \ + url_title, \ + Posted, \ + Body, \ + Keywords \ + FROM textpattern \ + WHERE Status = '4' OR \ + Status = '5'" + + def self.process(dbname, user, pass, host = 'localhost') + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + + FileUtils.mkdir_p "_posts" + + db[QUERY].each do |post| + # Get required fields and construct Jekyll compatible name. + title = post[:Title] + slug = post[:url_title] + date = post[:Posted] + content = post[:Body] + + name = [date.strftime("%Y-%m-%d"), slug].join('-') + ".textile" + + # Get the relevant fields as a hash, delete empty fields and convert + # to YAML for the header. + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'tags' => post[:Keywords].split(',') + }.delete_if { |k,v| v.nil? || v == ''}.to_yaml + + # Write out the data and content to file. + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + end + end +end diff --git a/lib/jekyll/migrators/tumblr.rb b/lib/jekyll/migrators/tumblr.rb new file mode 100644 index 00000000..367a83c9 --- /dev/null +++ b/lib/jekyll/migrators/tumblr.rb @@ -0,0 +1,195 @@ +require 'rubygems' +require 'open-uri' +require 'fileutils' +require 'nokogiri' +require 'date' +require 'json' +require 'uri' +require 'jekyll' + +module Jekyll + module Tumblr + def self.process(url, format = "html", grab_images = false, + add_highlights = false, rewrite_urls = true) + @grab_images = grab_images + FileUtils.mkdir_p "_posts/tumblr" + url += "/api/read/json/" + per_page = 50 + posts = [] + # Two passes are required so that we can rewrite URLs. + # First pass builds up an array of each post as a hash. + begin + current_page = (current_page || -1) + 1 + feed = open(url + "?num=#{per_page}&start=#{current_page * per_page}") + json = feed.readlines.join("\n")[21...-2] # Strip Tumblr's JSONP chars. + blog = JSON.parse(json) + puts "Page: #{current_page + 1} - Posts: #{blog["posts"].size}" + posts += blog["posts"].map { |post| post_to_hash(post, format) } + end until blog["posts"].size < per_page + # Rewrite URLs and create redirects. + posts = rewrite_urls_and_redirects posts if rewrite_urls + # Second pass for writing post files. + posts.each do |post| + if format == "md" + post[:content] = html_to_markdown post[:content] + post[:content] = add_syntax_highlights post[:content] if add_highlights + end + File.open("_posts/tumblr/#{post[:name]}", "w") do |f| + f.puts post[:header].to_yaml + "---\n" + post[:content] + end + end + end + + private + + # Converts each type of Tumblr post to a hash with all required + # data for Jekyll. + def self.post_to_hash(post, format) + case post['type'] + when "regular" + title = post["regular-title"] + content = post["regular-body"] + when "link" + title = post["link-text"] || post["link-url"] + content = "#{title}" + unless post["link-description"].nil? + content << "
" + post["link-description"] + end + when "photo" + title = post["photo-caption"] + max_size = post.keys.map{ |k| k.gsub("photo-url-", "").to_i }.max + url = post["photo-url"] || post["photo-url-#{max_size}"] + ext = "." + post[post.keys.select { |k| + k =~ /^photo-url-/ && post[k].split("/").last =~ /\./ + }.first].split(".").last + content = "" + unless post["photo-link-url"].nil? + content = "#{content}" + end + when "audio" + if !post["id3-title"].nil? + title = post["id3-title"] + content = post.at["audio-player"] + "
" + post["audio-caption"] + else + title = post["audio-caption"] + content = post.at["audio-player"] + end + when "quote" + title = post["quote-text"] + content = "
#{post["quote-text"]}
" + unless post["quote-source"].nil? + content << "—" + post["quote-source"] + end + when "conversation" + title = post["conversation-title"] + content = "
" + post["conversation"]["line"].each do |line| + content << "
#{line['label']}
#{line}
" + end + content << "
" + when "video" + title = post["video-title"] + content = post["video-player"] + unless post["video-caption"].nil? + content << "
" + post["video-caption"] + end + end + date = Date.parse(post['date']).to_s + title = Nokogiri::HTML(title).text + slug = title.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '') + { + :name => "#{date}-#{slug}.#{format}", + :header => { + "layout" => "post", + "title" => title, + "tags" => post["tags"], + }, + :content => content, + :url => post["url"], + :slug => post["url-with-slug"], + } + end + + # Create a Hash of old urls => new urls, for rewriting and + # redirects, and replace urls in each post. Instantiate Jekyll + # site/posts to get the correct permalink format. + def self.rewrite_urls_and_redirects(posts) + site = Jekyll::Site.new(Jekyll.configuration({})) + dir = File.join(File.dirname(__FILE__), "..") + urls = Hash[posts.map { |post| + # Create an initial empty file for the post so that + # we can instantiate a post object. + File.open("_posts/tumblr/#{post[:name]}", "w") + tumblr_url = URI.parse(post[:slug]).path + jekyll_url = Jekyll::Post.new(site, dir, "", "tumblr/" + post[:name]).url + redirect_dir = tumblr_url.sub(/\//, "") + "/" + FileUtils.mkdir_p redirect_dir + File.open(redirect_dir + "index.html", "w") do |f| + f.puts "" + end + [tumblr_url, jekyll_url] + }] + posts.map { |post| + urls.each do |tumblr_url, jekyll_url| + post[:content].gsub!(/#{tumblr_url}/i, jekyll_url) + end + post + } + end + + # Uses Python's html2text to convert a post's content to + # markdown. Preserve HTML tables as per the markdown docs. + def self.html_to_markdown(content) + preserve = ["table", "tr", "th", "td"] + preserve.each do |tag| + content.gsub!(/<#{tag}/i, "$$" + tag) + content.gsub!(/<\/#{tag}/i, "||" + tag) + end + content = %x[echo '#{content.gsub("'", "''")}' | html2text] + preserve.each do |tag| + content.gsub!("$$" + tag, "<" + tag) + content.gsub!("||" + tag, " +require 'fileutils' +require 'rubygems' +require 'sequel' +require 'yaml' + +module Jekyll + module Typo + # This SQL *should* work for both MySQL and PostgreSQL, but I haven't + # tested PostgreSQL yet (as of 2008-12-16). + SQL = <<-EOS + SELECT c.id id, + c.title title, + c.permalink slug, + c.body body, + c.published_at date, + c.state state, + COALESCE(tf.name, 'html') filter + FROM contents c + LEFT OUTER JOIN text_filters tf + ON c.text_filter_id = tf.id + EOS + + def self.process dbname, user, pass, host='localhost' + FileUtils.mkdir_p '_posts' + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + db[SQL].each do |post| + next unless post[:state] =~ /published/ + + name = [ sprintf("%.04d", post[:date].year), + sprintf("%.02d", post[:date].month), + sprintf("%.02d", post[:date].day), + post[:slug].strip ].join('-') + + # Can have more than one text filter in this field, but we just want + # the first one for this. + name += '.' + post[:filter].split(' ')[0] + + File.open("_posts/#{name}", 'w') do |f| + f.puts({ 'layout' => 'post', + 'title' => post[:title].to_s, + 'typo_id' => post[:id] + }.delete_if { |k, v| v.nil? || v == '' }.to_yaml) + f.puts '---' + f.puts post[:body].delete("\r") + end + end + end + + end +end diff --git a/lib/jekyll/migrators/wordpress.rb b/lib/jekyll/migrators/wordpress.rb new file mode 100644 index 00000000..d2d19039 --- /dev/null +++ b/lib/jekyll/migrators/wordpress.rb @@ -0,0 +1,294 @@ +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module WordPress + + # Main migrator function. Call this to perform the migration. + # + # dbname:: The name of the database + # user:: The database user name + # pass:: The database user's password + # host:: The address of the MySQL database host. Default: 'localhost' + # options:: A hash table of configuration options. + # + # Supported options are: + # + # :table_prefix:: Prefix of database tables used by WordPress. + # Default: 'wp_' + # :clean_entities:: If true, convert non-ASCII characters to HTML + # entities in the posts, comments, titles, and + # names. Requires the 'htmlentities' gem to + # work. Default: true. + # :comments:: If true, migrate post comments too. Comments + # are saved in the post's YAML front matter. + # Default: true. + # :categories:: If true, save the post's categories in its + # YAML front matter. + # :tags:: If true, save the post's tags in its + # YAML front matter. + # :more_excerpt:: If true, when a post has no excerpt but + # does have a tag, use the + # preceding post content as the excerpt. + # Default: true. + # :more_anchor:: If true, convert a tag into + # two HTML anchors with ids "more" and + # "more-NNN" (where NNN is the post number). + # Default: true. + # :status:: Array of allowed post statuses. Only + # posts with matching status will be migrated. + # Known statuses are :publish, :draft, :private, + # and :revision. If this is nil or an empty + # array, all posts are migrated regardless of + # status. Default: [:publish]. + # + def self.process(dbname, user, pass, host='localhost', options={}) + options = { + :table_prefix => 'wp_', + :clean_entities => true, + :comments => true, + :categories => true, + :tags => true, + :more_excerpt => true, + :more_anchor => true, + :status => [:publish] # :draft, :private, :revision + }.merge(options) + + if options[:clean_entities] + begin + require 'htmlentities' + rescue LoadError + STDERR.puts "Could not require 'htmlentities', so the " + + ":clean_entities option is now disabled." + options[:clean_entities] = false + end + end + + FileUtils.mkdir_p("_posts") + + db = Sequel.mysql(dbname, :user => user, :password => pass, + :host => host, :encoding => 'utf8') + + px = options[:table_prefix] + + posts_query = " + SELECT + posts.ID AS `id`, + posts.guid AS `guid`, + posts.post_type AS `type`, + posts.post_status AS `status`, + posts.post_title AS `title`, + posts.post_name AS `slug`, + posts.post_date AS `date`, + posts.post_content AS `content`, + posts.post_excerpt AS `excerpt`, + posts.comment_count AS `comment_count`, + users.display_name AS `author`, + users.user_login AS `author_login`, + users.user_email AS `author_email`, + users.user_url AS `author_url` + FROM #{px}posts AS `posts` + LEFT JOIN #{px}users AS `users` + ON posts.post_author = users.ID" + + if options[:status] and not options[:status].empty? + status = options[:status][0] + posts_query << " + WHERE posts.post_status = '#{status.to_s}'" + options[:status][1..-1].each do |status| + posts_query << " OR + posts.post_status = '#{status.to_s}'" + end + end + + db[posts_query].each do |post| + process_post(post, db, options) + end + end + + + def self.process_post(post, db, options) + px = options[:table_prefix] + + title = post[:title] + if options[:clean_entities] + title = clean_entities(title) + end + + slug = post[:slug] + if !slug or slug.empty? + slug = sluggify(title) + end + + date = post[:date] || Time.now + name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, + date.day, slug] + content = post[:content].to_s + if options[:clean_entities] + content = clean_entities(content) + end + + excerpt = post[:excerpt].to_s + + more_index = content.index(//) + more_anchor = nil + if more_index + if options[:more_excerpt] and + (post[:excerpt].nil? or post[:excerpt].empty?) + excerpt = content[0...more_index] + end + if options[:more_anchor] + more_link = "more" + content.sub!(//, + "" + + "") + end + end + + categories = [] + tags = [] + + if options[:categories] or options[:tags] + + cquery = + "SELECT + terms.name AS `name`, + ttax.taxonomy AS `type` + FROM + #{px}terms AS `terms`, + #{px}term_relationships AS `trels`, + #{px}term_taxonomy AS `ttax` + WHERE + trels.object_id = '#{post[:id]}' AND + trels.term_taxonomy_id = ttax.term_taxonomy_id AND + terms.term_id = ttax.term_id" + + db[cquery].each do |term| + if options[:categories] and term[:type] == "category" + if options[:clean_entities] + categories << clean_entities(term[:name]) + else + categories << term[:name] + end + elsif options[:tags] and term[:type] == "post_tag" + if options[:clean_entities] + tags << clean_entities(term[:name]) + else + tags << term[:name] + end + end + end + end + + comments = [] + + if options[:comments] and post[:comment_count].to_i > 0 + cquery = + "SELECT + comment_ID AS `id`, + comment_author AS `author`, + comment_author_email AS `author_email`, + comment_author_url AS `author_url`, + comment_date AS `date`, + comment_date_gmt AS `date_gmt`, + comment_content AS `content` + FROM #{px}comments + WHERE + comment_post_ID = '#{post[:id]}' AND + comment_approved != 'spam'" + + + db[cquery].each do |comment| + + comcontent = comment[:content].to_s + if comcontent.respond_to?(:force_encoding) + comcontent.force_encoding("UTF-8") + end + if options[:clean_entities] + comcontent = clean_entities(comcontent) + end + comauthor = comment[:author].to_s + if options[:clean_entities] + comauthor = clean_entities(comauthor) + end + + comments << { + 'id' => comment[:id].to_i, + 'author' => comauthor, + 'author_email' => comment[:author_email].to_s, + 'author_url' => comment[:author_url].to_s, + 'date' => comment[:date].to_s, + 'date_gmt' => comment[:date_gmt].to_s, + 'content' => comcontent, + } + end + + comments.sort!{ |a,b| a['id'] <=> b['id'] } + end + + # Get the relevant fields as a hash, delete empty fields and + # convert to YAML for the header. + data = { + 'layout' => post[:type].to_s, + 'status' => post[:status].to_s, + 'published' => (post[:status].to_s == "publish"), + 'title' => title.to_s, + 'author' => post[:author].to_s, + 'author_login' => post[:author_login].to_s, + 'author_email' => post[:author_email].to_s, + 'author_url' => post[:author_url].to_s, + 'excerpt' => excerpt, + 'more_anchor' => more_anchor, + 'wordpress_id' => post[:id], + 'wordpress_url' => post[:guid].to_s, + 'date' => date, + 'categories' => options[:categories] ? categories : nil, + 'tags' => options[:tags] ? tags : nil, + 'comments' => options[:comments] ? comments : nil, + }.delete_if { |k,v| v.nil? || v == '' }.to_yaml + + # Write out the data and content to file + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + + + def self.clean_entities( text ) + if text.respond_to?(:force_encoding) + text.force_encoding("UTF-8") + end + text = HTMLEntities.new.encode(text, :named) + # We don't want to convert these, it would break all + # HTML tags in the post and comments. + text.gsub!("&", "&") + text.gsub!("<", "<") + text.gsub!(">", ">") + text.gsub!(""", '"') + text.gsub!("'", "'") + text + end + + + def self.sluggify( title ) + begin + require 'unidecode' + title = title.to_ascii + rescue LoadError + STDERR.puts "Could not require 'unidecode'. If your post titles have non-ASCII characters, you could get nicer permalinks by installing unidecode." + end + title.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-") + end + + end +end diff --git a/lib/jekyll/migrators/wordpressdotcom.rb b/lib/jekyll/migrators/wordpressdotcom.rb new file mode 100644 index 00000000..701c2af4 --- /dev/null +++ b/lib/jekyll/migrators/wordpressdotcom.rb @@ -0,0 +1,70 @@ +# coding: utf-8 + +require 'rubygems' +require 'hpricot' +require 'fileutils' +require 'yaml' +require 'time' + +module Jekyll + # This importer takes a wordpress.xml file, which can be exported from your + # wordpress.com blog (/wp-admin/export.php). + module WordpressDotCom + def self.process(filename = "wordpress.xml") + import_count = Hash.new(0) + doc = Hpricot::XML(File.read(filename)) + + (doc/:channel/:item).each do |item| + title = item.at(:title).inner_text.strip + permalink_title = item.at('wp:post_name').inner_text + # Fallback to "prettified" title if post_name is empty (can happen) + if permalink_title == "" + permalink_title = title.downcase.split.join('-') + end + + date = Time.parse(item.at('wp:post_date').inner_text) + status = item.at('wp:status').inner_text + + if status == "publish" + published = true + else + published = false + end + + type = item.at('wp:post_type').inner_text + tags = (item/:category).map{|c| c.inner_text}.reject{|c| c == 'Uncategorized'}.uniq + + metas = Hash.new + item.search("wp:postmeta").each do |meta| + key = meta.at('wp:meta_key').inner_text + value = meta.at('wp:meta_value').inner_text + metas[key] = value; + end + + name = "#{date.strftime('%Y-%m-%d')}-#{permalink_title}.html" + header = { + 'layout' => type, + 'title' => title, + 'tags' => tags, + 'status' => status, + 'type' => type, + 'published' => published, + 'meta' => metas + } + + FileUtils.mkdir_p "_#{type}s" + File.open("_#{type}s/#{name}", "w") do |f| + f.puts header.to_yaml + f.puts '---' + f.puts item.at('content:encoded').inner_text + end + + import_count[type] += 1 + end + + import_count.each do |key, value| + puts "Imported #{value} #{key}s" + end + end + end +end From 9c517c6d7d9432b73c125c33c9425362ca8df4f2 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Fri, 11 Jan 2013 00:32:39 +0000 Subject: [PATCH 37/69] Rename migrate command to import --- bin/jekyll | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/jekyll b/bin/jekyll index 3df10e4f..309ffc91 100755 --- a/bin/jekyll +++ b/bin/jekyll @@ -5,14 +5,12 @@ $:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib }) require 'commander/import' require 'jekyll' -# Details about Jekyll program :name, 'jekyll' program :version, Jekyll::VERSION program :description, 'Jekyll is a blog-aware, static site generator in Ruby' default_command :help -# Global options available to every command global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' global_option '--safe', 'Safe mode (defaults to false)' @@ -56,9 +54,9 @@ command :serve do |c| end end -command :migrate do |c| - c.syntax = 'jekyll migrate [options]' - c.description = 'Migrate your own blog to Jekyll' +command :import do |c| + c.syntax = 'jekyll import [options]' + c.description = 'Import your old blog to Jekyll' c.option '--file', 'File to migrate from' c.option '--dbname', 'Database name to migrate from' From 7782e0b1a10a4719721df92ce413143e47353735 Mon Sep 17 00:00:00 2001 From: Matt Rogers Date: Thu, 10 Jan 2013 21:44:08 -0600 Subject: [PATCH 38/69] Add a CONTRIBUTING file This uses the content of the 'Contribute' wiki page. --- CONTRIBUTING.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..fa6fb807 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,45 @@ +Contribute +========== + +So you've got an awesome idea to throw into Jekyll. Great! Please keep the following in mind: + +* **Contributions will not be accepted without tests.** +* If you're creating a small fix or patch to an existing feature, just a simple test will do. Please stay in the confines of the current test suite and use [Shoulda](http://github.com/thoughtbot/shoulda/tree/master) and [RR](http://github.com/btakita/rr/tree/master). +* If it's a brand new feature, make sure to create a new [Cucumber](https://github.com/cucumber/cucumber/) feature and reuse steps where appropriate. Also, whipping up some documentation in your fork's wiki would be appreciated, and once merged it will be transferred over to the main wiki. + +Test Dependencies +----------------- + +To run the test suite and build the gem you'll need to install Jekyll's dependencies. Jekyll uses Bundler, so a quick run of the bundle command and you're all set! + + $ bundle + +Before you start, run the tests and make sure that they pass (to confirm your environment is configured properly): + + $ rake test + $ rake features + +Workflow +-------- + +Here's the most direct way to get your work merged into the project: +* Fork the project +* Clone down your fork ( `git clone git://github.com//jekyll.git` ) +* Create a topic branch to contain your change ( `git checkout -b my_awesome_feature` ) +* Hack away, add tests. Not necessarily in that order. +* Make sure everything still passes by running `rake` +* If necessary, rebase your commits into logical chunks, without errors +* Push the branch up ( `git push origin my_awesome_feature` ) +* Create an issue with a description and link to your branch + +Gotchas +------- + +* If you want to bump the gem version, please put that in a separate commit. This way, the maintainers can control when the gem gets released. +* Try to keep your patch(es) based from the latest commit on mojombo/jekyll. The easier it is to apply your work, the less work the maintainers have to do, which is always a good thing. +* Please don't tag your GitHub issue with [fix], [feature], etc. The maintainers actively read the issues and will label it once they come across it. + +Finally... +---------- + +Thanks! Hacking on Jekyll should be fun, and if for some reason it's a pain to do let us know so we can fix it. From 6253f79de2ebebb5bfc8bf70f05ed92b570b0417 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Thu, 10 Jan 2013 22:11:13 -0800 Subject: [PATCH 39/69] Added space between arguments in Convertible errors --- lib/jekyll/convertible.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb index 26f7a0c9..e71fe1bf 100644 --- a/lib/jekyll/convertible.rb +++ b/lib/jekyll/convertible.rb @@ -33,9 +33,9 @@ module Jekyll self.data = YAML.load($1) end rescue => e - puts "Error reading file #{File.join(base,name)}: #{e.message}" + puts "Error reading file #{File.join(base, name)}: #{e.message}" rescue SyntaxError => e - puts "YAML Exception reading #{File.join(base,name)}: #{e.message}" + puts "YAML Exception reading #{File.join(base, name)}: #{e.message}" end self.data ||= {} From 3cf29eeee2148cd72c648988c56517627fad3309 Mon Sep 17 00:00:00 2001 From: mccxj Date: Fri, 11 Jan 2013 15:33:11 +0800 Subject: [PATCH 40/69] add Enumerable#glob_include? --- lib/jekyll/core_ext.rb | 8 ++++++++ lib/jekyll/site.rb | 8 ++------ test/test_core_ext.rb | 22 ++++++++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/lib/jekyll/core_ext.rb b/lib/jekyll/core_ext.rb index fc40cb38..dfc5bbf7 100644 --- a/lib/jekyll/core_ext.rb +++ b/lib/jekyll/core_ext.rb @@ -50,3 +50,11 @@ class Date strftime("%Y-%m-%dT%H:%M:%S%Z") end if RUBY_VERSION < '1.9' end + +module Enumerable + # Returns true if path matches against any glob pattern. + # Look for more detail about glob pattern in method File::fnmatch. + def glob_include?(e) + any? { |exp| File.fnmatch?(exp, e) } + end +end diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index 8c28ed89..eb5f9f0d 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -318,19 +318,15 @@ module Jekyll # Returns the Array of filtered entries. def filter_entries(entries) entries.reject do |e| - unless glob_include?(self.include, e) + unless self.include.glob_include?(e) ['.', '_', '#'].include?(e[0..0]) || e[-1..-1] == '~' || - glob_include?(self.exclude, e) || + self.exclude.glob_include?(e) || File.symlink?(e) end end end - def glob_include?(exps, e) - exps.any? { |exp| File.fnmatch?(exp, e) } - end - # Get the implementation class for the given Converter. # # klass - The Class of the Converter to fetch. diff --git a/test/test_core_ext.rb b/test/test_core_ext.rb index c3e62cb5..0b79f33f 100644 --- a/test/test_core_ext.rb +++ b/test/test_core_ext.rb @@ -63,4 +63,26 @@ class TestCoreExt < Test::Unit::TestCase end end + + context "enumerable" do + context "glob_include?" do + should "return false with no glob patterns" do + assert ![].glob_include?("a.txt") + end + + should "return false with all not match path" do + data = ["a*", "b?"] + assert !data.glob_include?("ca.txt") + assert !data.glob_include?("ba.txt") + end + + should "return true with match path" do + data = ["a*", "b?", "**/a*"] + assert data.glob_include?("a.txt") + assert data.glob_include?("ba") + assert data.glob_include?("c/a/a.txt") + assert data.glob_include?("c/a/b/a.txt") + end + end + end end From 68333cd221ad7f1c0138aa21481d76005f09842d Mon Sep 17 00:00:00 2001 From: Jashank Jeremy Date: Fri, 11 Jan 2013 20:02:31 +1100 Subject: [PATCH 41/69] Slight stylistic tweak to LSI initialisation. Recommended-by: parkr --- lib/jekyll/post.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jekyll/post.rb b/lib/jekyll/post.rb index d028a290..91eacc6d 100644 --- a/lib/jekyll/post.rb +++ b/lib/jekyll/post.rb @@ -163,7 +163,7 @@ module Jekyll if self.site.lsi self.class.lsi ||= begin puts "Starting the classifier..." - lsi = Classifier::LSI.new :auto_rebuild => false + lsi = Classifier::LSI.new(:auto_rebuild => false) $stdout.print(" Populating LSI... ");$stdout.flush posts.each { |x| $stdout.print(".");$stdout.flush;lsi.add_item(x) } $stdout.print("\n Rebuilding LSI index... ") From d2e750cc744a81398f56d1efd8840e38dac21a15 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 15:54:57 -0800 Subject: [PATCH 42/69] removed extraneous whitespace in test_site.rb --- test/test_site.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_site.rb b/test/test_site.rb index f71a9599..5462fd6e 100644 --- a/test/test_site.rb +++ b/test/test_site.rb @@ -240,7 +240,6 @@ class TestSite < Test::Unit::TestCase assert File.exist?(dest_dir('.svn')) assert File.exist?(dest_dir('.svn/HEAD')) end - end context 'with an invalid markdown processor in the configuration' do From 4336b6dd4ead43760adca1202c6ea2ddc6243376 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 15:55:14 -0800 Subject: [PATCH 43/69] Added Travis CI badge to README --- README.textile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.textile b/README.textile index 529e1e7d..80ca6220 100644 --- a/README.textile +++ b/README.textile @@ -1,5 +1,7 @@ h1. Jekyll +!https://travis-ci.org/mojombo/jekyll.png?branch=master!:https://travis-ci.org/mojombo/jekyll + By Tom Preston-Werner, Nick Quaranto, and many awesome contributors! Jekyll is a simple, blog aware, static site generator. It takes a template directory (representing the raw form of a website), runs it through Textile or Markdown and Liquid converters, and spits out a complete, static website suitable for serving with Apache or your favorite web server. This is also the engine behind "GitHub Pages":http://pages.github.com, which you can use to host your project's page or blog right here from GitHub. From 36ba434115dedc5d1fc791c8e5dc460afc0a9f8b Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 16:17:03 -0800 Subject: [PATCH 44/69] Updated history to reflected merged PRs, #745 and #685. --- History.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/History.txt b/History.txt index d65aad87..260c2661 100644 --- a/History.txt +++ b/History.txt @@ -1,5 +1,7 @@ == HEAD * Minor Enhancements + * "Keep files" feature (#685) + * Output full path & name for files that don't parse (#745) * Add source and destination directory protection (#535) * Better YAML error message (#718) * Bug Fixes From a8d07dd5f572e384f687bdf77a36d5c2fbff6979 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 16:21:31 -0800 Subject: [PATCH 45/69] Updating travis configuration to explicitly run 'rake' behind bundle exec. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 416e8f3a..b6b39b7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ +language: ruby rvm: - 1.9.3 - 1.9.2 - 1.8.7 +script: bundle exec rake From e4517c42af355f31745954c8d133a0461b3e2565 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 16:26:06 -0800 Subject: [PATCH 46/69] Specific patch version of 1.9.3: 362. --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6b39b7d..152517e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: ruby rvm: - - 1.9.3 + - 1.9.3-p362 - 1.9.2 - 1.8.7 -script: bundle exec rake From 8b9baeee0e2fc8c7806e242448e7564ba8349375 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 16:45:17 -0800 Subject: [PATCH 47/69] Revert "Specific patch version of 1.9.3: 362." This reverts commit e4517c42af355f31745954c8d133a0461b3e2565. --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 152517e5..b6b39b7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ language: ruby rvm: - - 1.9.3-p362 + - 1.9.3 - 1.9.2 - 1.8.7 +script: bundle exec rake From 9fc89482b3dc0b8b8b7af9b81601f959e065237d Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 23:50:13 -0800 Subject: [PATCH 48/69] Using build and serve sub-commands in docs --- site/_posts/2012-07-01-deployment-methods.md | 2 +- site/_posts/2012-07-01-extras.md | 2 +- site/_posts/2012-07-01-home.md | 2 +- site/_posts/2012-07-01-troubleshooting.md | 2 +- site/_posts/2012-07-01-usage.md | 19 +++++++++++-------- site/index.html | 2 +- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/site/_posts/2012-07-01-deployment-methods.md b/site/_posts/2012-07-01-deployment-methods.md index 7716e5f8..80b55ae3 100644 --- a/site/_posts/2012-07-01-deployment-methods.md +++ b/site/_posts/2012-07-01-deployment-methods.md @@ -53,7 +53,7 @@ TMP_GIT_CLONE=$HOME/tmp/myrepo PUBLIC_WWW=/var/www/myrepo git clone $GIT_REPO $TMP_GIT_CLONE -jekyll --no-auto $TMP_GIT_CLONE $PUBLIC_WWW +jekyll build $TMP_GIT_CLONE $PUBLIC_WWW rm -Rf $TMP_GIT_CLONE exit {% endhighlight %} diff --git a/site/_posts/2012-07-01-extras.md b/site/_posts/2012-07-01-extras.md index 1af24d10..7bbff0b9 100644 --- a/site/_posts/2012-07-01-extras.md +++ b/site/_posts/2012-07-01-extras.md @@ -91,7 +91,7 @@ sudo gem install rdiscount And then run Jekyll with the following option: {% highlight bash %} -jekyll --rdiscount +jekyll build --markdown rdiscount {% endhighlight %} Or, specify RDiscount as the markdown engine in your `_config.yml` file to have Jekyll run with that option by default (so you don’t have to specify the flag every time). diff --git a/site/_posts/2012-07-01-home.md b/site/_posts/2012-07-01-home.md index 9c27eeef..34de7bb1 100644 --- a/site/_posts/2012-07-01-home.md +++ b/site/_posts/2012-07-01-home.md @@ -19,7 +19,7 @@ For the impatient, here's how to get Jekyll up and running. ~ $ mkdir -p my/new/site ~ $ cd my/new/site ~ $ vim index.html -~/my/new/site $ jekyll --server +~/my/new/site $ jekyll serve # => Now browse to http://localhost:4000 {% endhighlight %} diff --git a/site/_posts/2012-07-01-troubleshooting.md b/site/_posts/2012-07-01-troubleshooting.md index e6a592f2..08dac8e8 100644 --- a/site/_posts/2012-07-01-troubleshooting.md +++ b/site/_posts/2012-07-01-troubleshooting.md @@ -50,7 +50,7 @@ On Debian or Ubuntu, you may need to add /var/lib/gems/1.8/bin/ to your path in ## Base-URL Problems -If you are using base-url option like `jekyll --server --base-url '/blog'` then make sure that you access the site at `http://localhost:4000/blog/index.html`. Just accessing `http://localhost:4000/blog` will not work. +If you are using base-url option like `jekyll serve --baseurl '/blog'` then make sure that you access the site at `http://localhost:4000/blog/index.html`. Just accessing `http://localhost:4000/blog` will not work. ## Configuration problems diff --git a/site/_posts/2012-07-01-usage.md b/site/_posts/2012-07-01-usage.md index c1cfa5b1..ae1b07e1 100644 --- a/site/_posts/2012-07-01-usage.md +++ b/site/_posts/2012-07-01-usage.md @@ -8,28 +8,31 @@ next_section: structure The Jekyll gem makes a `jekyll` executable available to you in your Terminal window. You can use this command in a number of ways: {% highlight bash %} -jekyll +jekyll build #=> The current folder will get generated into ./_site -jekyll +jekyll build --destination #=> The current folder will get generated into -jekyll +jekyll build --source --destination #=> The folder will get generated into +jekyll build --watch +#=> The current folder will get generated into ./_site, +# and watch for changes and regenerate automatically. {% endhighlight %} Jekyll also comes with a built-in development server that will allow you to preview what the generated site will look like in your browser locally. {% highlight bash %} -jekyll --server +jekyll serve #=> A development server will run at http://localhost:4000/ -jekyll --server --auto +jekyll serve --watch #=> As above, but watch for changes and regenerate automatically too. {% endhighlight %} -These are just some of the many [configuration options](../configuration) available. All configuration options can either be specified as flags on the command line, or alternatively (and more commonly) they can be specified in a `_config.yml` file at the root of the source directory. Jekyll will automatically configuration options from this file when run, so placing the following two lines in the configuration file will mean that running `jekyll` would be equivalent to running `jekyll --server --auto`: +These are just some of the many [configuration options](../configuration) available. All configuration options can either be specified as flags on the command line, or alternatively (and more commonly) they can be specified in a `_config.yml` file at the root of the source directory. Jekyll will automatically configuration options from this file when run, so placing the following one line in the configuration file will mean that running `jekyll build` or `jekyll serve` would be equivalent to running `jekyll [build|serve] --source _source --destination _deploy`: {% highlight yaml %} -auto: true -server: true +source: _source +destination: _deploy {% endhighlight %} For more about the possible configuration options, see the [configuration](../configuration) page. diff --git a/site/index.html b/site/index.html index 86711cd8..ee30456d 100644 --- a/site/index.html +++ b/site/index.html @@ -54,7 +54,7 @@ overview: true

~/my/awesome/site $ - jekyll --server + jekyll serve

# => Now browse to http://localhost:4000 From 39b159c258a890e5b52931c33d3971381cbced04 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 23:50:36 -0800 Subject: [PATCH 49/69] Return of the "sites" page - for learning purposes --- site/_includes/docs_contents.html | 3 +++ site/_posts/2012-07-01-sites.md | 25 +++++++++++++++++++++++++ site/_posts/2012-07-01-structure.md | 2 +- 3 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 site/_posts/2012-07-01-sites.md diff --git a/site/_includes/docs_contents.html b/site/_includes/docs_contents.html index ad3f3e16..7e7637f3 100644 --- a/site/_includes/docs_contents.html +++ b/site/_includes/docs_contents.html @@ -71,6 +71,9 @@

  • Troubleshooting
    • +
  • + Sites using Jekyll +
  • Resources
    • diff --git a/site/_posts/2012-07-01-sites.md b/site/_posts/2012-07-01-sites.md new file mode 100644 index 00000000..b24aba41 --- /dev/null +++ b/site/_posts/2012-07-01-sites.md @@ -0,0 +1,25 @@ +--- +layout: docs +title: Sites using Jekyll +prev_section: troubleshooting +next_section: resources +--- + +It’s interesting to see what designs and features others have come up +with. Below are three Jekyll-powered blogs which were hand-picked for +learning purposes. + +- [Tom Preston-Werner](http://tom.preston-werner.com/) + ([source](http://github.com/mojombo/mojombo.github.com)) +- [Nick Quaranto](http://quaran.to/) + ([source](https://github.com/qrush/qrush.github.com)) +- [Parker Moore](http://www.parkermoore.de) + ([source](https://github.com/parkr/blogsource)) +- [Roger Chapman](http://rogchap.com/) + ([source](https://github.com/rogchap/rogchap.github.com)) +- [GitHub Official Teaching Materials](http://teach.github.com) + ([source](https://github.com/github/teach.github.com)) +- [Rasmus Andersson](http://rsms.me/) + ([source](https://github.com/rsms/rsms.github.com)) +- [Scott Chacon](http://schacon.github.com) + ([source](https://github.com/schacon/schacon.github.com)) diff --git a/site/_posts/2012-07-01-structure.md b/site/_posts/2012-07-01-structure.md index 78212f03..87362b44 100644 --- a/site/_posts/2012-07-01-structure.md +++ b/site/_posts/2012-07-01-structure.md @@ -88,7 +88,7 @@ An overview of what each of these does:

    Other Files/Folders

    -

    Every other directory and file except for those listed above—such as css and images folders, favicon.ico files, and so forth—will be transferred over verbatim to the generated site. There's plenty of sites already using Jekyll if you're curious as to how they're laid out.

    +

    Every other directory and file except for those listed above—such as css and images folders, favicon.ico files, and so forth—will be transferred over verbatim to the generated site. There's plenty of sites already using Jekyll if you're curious as to how they're laid out.

    From 64cbaa1d85f87c4feec22c2c7c129974e0495fed Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 23:51:06 -0800 Subject: [PATCH 50/69] Configuration docs updated with new structure --- site/_posts/2012-07-01-configuration.md | 301 +++++++++++++----------- 1 file changed, 164 insertions(+), 137 deletions(-) diff --git a/site/_posts/2012-07-01-configuration.md b/site/_posts/2012-07-01-configuration.md index a942914b..6a66e14c 100644 --- a/site/_posts/2012-07-01-configuration.md +++ b/site/_posts/2012-07-01-configuration.md @@ -9,6 +9,8 @@ Jekyll allows you to concoct your sites in any way you can dream up, and it’s ## Configuration Settings +### Global Configuration + The table below lists the available settings for Jekyll, and the various options (specifed in the configuration file) and flags (specified on the command-line) that control them. @@ -19,6 +21,26 @@ The table below lists the available settings for Jekyll, and the various + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +
    +

    Site Source

    +

    Changes the directory where Jekyll will look to transform files

    +     		+

    source: [string]

    +

    --source [source]

    +
    +

    Site Destination

    +

    Changes the directory where Jekyll will write files to

    +     		+

    destination: [string]

    +

    --destination

    +

    Safe

    @@ -29,140 +51,6 @@ The table below lists the available settings for Jekyll, and the various --safe
    -

    Regeneration

    -

    Enables or disables Jekyll from recreating the site when files are modified.

    -     		-

    auto: [boolean]

    -

    --auto

    -

    --no-auto

    -
    -

    Local Server

    -

    Fires up a server that will host your _site directory

    -     		-

    server: [boolean]

    -

    --server

    -
    -

    Local Server Port

    -

    Changes the port that the Jekyll server will run on

    -     		-

    server_port: [integer]

    -

    --server [port]

    -
    -

    Base URL

    -

    Serve website from a given base URL

    -     		-

    baseurl: [BASE_URL]

    -

    --base-url [url]

    -
    -

    URL

    -

    Sets site.url, useful for environment switching

    -     		-

    url: [URL]

    -

    --url [URL]

    -
    -

    Site Destination

    -

    Changes the directory where Jekyll will write files to

    -     		-

    destination: [dir]

    -

    jekyll [dest]

    -
    -

    Site Source

    -

    Changes the directory where Jekyll will look to transform files

    -     		-

    source: [dir]

    -

    jekyll [source] [dest]

    -
    -

    Markdown

    -

    Uses RDiscount or [engine] instead of Maruku.

    -     		-

    markdown: [engine]

    -

    --rdiscount

    -

    --kramdown

    -

    --redcarpet

    -
    -

    Pygments

    -

    Enables highlight tag with Pygments.

    -     		-

    pygments: [boolean]

    -

    --pygments

    -
    -

    Future

    -

    Publishes posts with a future date

    -     		-

    future: [boolean]

    -

    --no-future

    -

    --future

    -
    -

    LSI

    -

    Produces an index for related posts.

    -     		-

    lsi: [boolean]

    -

    --lsi

    -
    -

    Permalink

    -

    Controls the URLs that posts are generated with. Please refer to the Permalinks page for more info.

    -     		-

    permalink: [style]

    -

    --permalink=[style]

    -
    -

    Pagination

    -

    Splits your posts up over multiple subdirectories called "page2", "page3", ... "pageN"

    -     		-

    paginate: [per_page]

    -

    --paginate [per_page]

    -

    Exclude

    @@ -181,6 +69,98 @@ The table below lists the available settings for Jekyll, and the various include: [dir1, file1, dir2]
    + +### Configuration for "build" Sub-Command + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    SettingOptions and Flags
    +

    Regeneration

    +

    Enables auto-regeneration of the site when files are modified. Off by default.

    +     		+

    --watch

    +
    +

    URL

    +

    Sets site.url, useful for environment switching

    +     		+

    url: [URL]

    +

    --url [URL]

    +
    +

    Markdown

    +

    Uses RDiscount or [engine] instead of Maruku.

    +     		+

    markdown: [engine]

    +

    --markdown [rdiscount|kramdown|redcarpet]

    +
    +

    Pygments

    +

    Enables highlight tag with Pygments.

    +     		+

    pygments: [boolean]

    +

    --pygments

    +
    +

    Future

    +

    Publishes posts with a future date

    +     		+

    future: [boolean]

    +

    --future

    +
    +

    LSI

    +

    Produces an index for related posts.

    +     		+

    lsi: [boolean]

    +

    --lsi

    +
    +

    Permalink

    +

    Controls the URLs that posts are generated with. Please refer to the Permalinks page for more info.

    +     		+

    permalink: [style]

    +

    --permalink [style]

    +
    +

    Pagination

    +

    Splits your posts up over multiple subdirectories called "page2", "page3", ... "pageN"

    +     		+

    paginate: [per_page]

    +

    --paginate [per_page]

    +

    Limit Posts

    @@ -188,10 +168,56 @@ The table below lists the available settings for Jekyll, and the various

    limit_posts: [max_posts]

    -

    --limit_posts=[max_posts]

    +

    --limit_posts [max_posts]
    +### Configuration for "serve" Sub-Command + +In addition to the options below, the `serve` sub-command can accept any of the options +for the `build` sub-command, which are then applied to the site build which occurs right +before your site is served. + + + + + + + + + + + + + + + + + + + + +
    SettingOptions and Flags
    +

    Local Server Port

    +

    Changes the port that the Jekyll server will run on

    +     		+

    port: [integer]

    +

    --port [port]

    +
    +

    Local Server Hostname

    +

    Changes the hostname that the Jekyll server will run on

    +     		+

    host: [string]

    +

    --host [hostname]

    +
    +

    Base URL

    +

    Serve website from a given base URL

    +     		+

    baseurl: [BASE_URL]

    +

    --baseurl [url]

    +
    @@ -206,9 +232,10 @@ Jekyll runs with the following configuration options by default. Unless alternat {% highlight yaml %} safe: false -auto: false +watch: false server: false -server_port: 4000 +host: 0.0.0.0 +port: 4000 baseurl: / url: http://localhost:4000 From 5db1bba6fd33490faac6c2907062d7da996fd3cf Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 23:51:22 -0800 Subject: [PATCH 51/69] auto => watch --- site/_posts/2012-07-01-variables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/_posts/2012-07-01-variables.md b/site/_posts/2012-07-01-variables.md index b04c14f5..01091474 100644 --- a/site/_posts/2012-07-01-variables.md +++ b/site/_posts/2012-07-01-variables.md @@ -68,7 +68,7 @@ Jekyll traverses your site looking for files to process. Any files with [YAML Fr

    site.[CONFIGURATION_DATA]

    -

    All variables set in your _config.yml are available through the site variable. For example, if you have url: http://mysite.com in your configuration file, then in your posts and pages it can be accessed using {{ "{{ site.url " }}}}. Jekyll does not parse changes to _config.yml in auto mode, you have to restart Jekyll to see changes to variables.

    +

    All variables set in your _config.yml are available through the site variable. For example, if you have url: http://mysite.com in your configuration file, then in your posts and pages it can be accessed using {{ "{{ site.url " }}}}. Jekyll does not parse changes to _config.yml in watch mode, you have to restart Jekyll to see changes to variables.

    From 42fa1046db418143772147edf79a6a0875c5a69b Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 23:51:32 -0800 Subject: [PATCH 52/69] Using internal links for homepage --- site/index.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/index.html b/site/index.html index ee30456d..c0a35828 100644 --- a/site/index.html +++ b/site/index.html @@ -16,19 +16,19 @@ overview: true

    No more databases, comment moderation, or pesky updates to install—just your content.

    - How Jekyll works → + How Jekyll works →

    Static

    Markdown (or Textile), Liquid, HTML & CSS go in. Static sites come out ready for deployment.

    - Jekyll template guide → + Jekyll template guide →

    Blog-aware

    Permalinks, categories, pages, posts, and custom layouts are all first-class citizens here.

    - Migrate your blog → + Migrate your blog →
    From 3d093f00358f341802482feb4306638e37fd0304 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Fri, 11 Jan 2013 23:59:21 -0800 Subject: [PATCH 53/69] Using new command structure in Rakefile --- Rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rakefile b/Rakefile index ff52d677..257da617 100644 --- a/Rakefile +++ b/Rakefile @@ -131,7 +131,7 @@ namespace :site do # Generate the site in server mode. puts "Running Jekyll..." Dir.chdir("site") do - sh "#{File.expand_path('bin/jekyll', File.dirname(__FILE__))} --server" + sh "#{File.expand_path('bin/jekyll', File.dirname(__FILE__))} serve --watch" end end From e2affb22574a9fa96075b5cc350ff1001a15ccd1 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 00:14:00 -0800 Subject: [PATCH 54/69] Added passing tests for #616 --- test/test_tags.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_tags.rb b/test/test_tags.rb index 943d980e..816fd809 100644 --- a/test/test_tags.rb +++ b/test/test_tags.rb @@ -32,6 +32,7 @@ title: This is a test This document results in a markdown error with maruku {% highlight text %}#{code}{% endhighlight %} +{% highlight text linenos %}#{code}{% endhighlight %} CONTENT create_post(content, override) end @@ -80,9 +81,13 @@ CONTENT assert_no_match /markdown\-html\-error/, @result end - should "render markdown with pygments line handling" do + should "render markdown with pygments" do assert_match %{
    test\n
    }, @result end + + should "render markdown with pygments with line numbers" do + assert_match %{
    1 test\n
    }, @result + end end context "post content has highlight with file reference" do From d9bc6b57d5a0a1579e86684dec6fcf92928dcb43 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 13:19:29 -0800 Subject: [PATCH 55/69] Updated gemspec to include new command and fixture files in manifest --- jekyll.gemspec | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jekyll.gemspec b/jekyll.gemspec index 433fd78d..86df546b 100644 --- a/jekyll.gemspec +++ b/jekyll.gemspec @@ -44,6 +44,8 @@ Gem::Specification.new do |s| # = MANIFEST = s.files = %w[ + .travis.yml + CONTRIBUTING.md Gemfile History.txt LICENSE @@ -63,6 +65,10 @@ Gem::Specification.new do |s| features/support/env.rb jekyll.gemspec lib/jekyll.rb + lib/jekyll/command.rb + lib/jekyll/commands/build.rb + lib/jekyll/commands/migrate.rb + lib/jekyll/commands/serve.rb lib/jekyll/converter.rb lib/jekyll/converters/identity.rb lib/jekyll/converters/markdown.rb @@ -97,6 +103,8 @@ Gem::Specification.new do |s| lib/jekyll/tags/include.rb lib/jekyll/tags/post_url.rb test/fixtures/broken_front_matter1.erb + test/fixtures/broken_front_matter2.erb + test/fixtures/broken_front_matter3.erb test/fixtures/front_matter.erb test/helper.rb test/source/.htaccess From e8c74fe7a89c3b53c15e7ec10e80d0abbe2158d1 Mon Sep 17 00:00:00 2001 From: 7rans Date: Sat, 12 Jan 2013 13:48:44 -0800 Subject: [PATCH 56/69] Layouts default to page or post depending upon type. --- lib/jekyll/page.rb | 12 ++++++++++++ lib/jekyll/post.rb | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/lib/jekyll/page.rb b/lib/jekyll/page.rb index 82b82048..d3461181 100644 --- a/lib/jekyll/page.rb +++ b/lib/jekyll/page.rb @@ -24,6 +24,18 @@ module Jekyll self.read_yaml(File.join(base, dir), name) end + # Read the YAML frontmatter. + # + # base - The String path to the dir containing the file. + # name - The String filename of the file. + # + # Returns nothing. + def read_yaml(base, name) + super(base, name) + self.data['layout'] ||= 'page' + self.data + end + # The generated directory into which the page will be placed # upon generation. This is derived from the permalink or, if # permalink is absent, we be '/' diff --git a/lib/jekyll/post.rb b/lib/jekyll/post.rb index a360cb8d..6156b5be 100644 --- a/lib/jekyll/post.rb +++ b/lib/jekyll/post.rb @@ -66,6 +66,18 @@ module Jekyll end end + # Read the YAML frontmatter. + # + # base - The String path to the dir containing the file. + # name - The String filename of the file. + # + # Returns nothing. + def read_yaml(base, name) + super(base, name) + self.data['layout'] ||= 'post' + self.data + end + # Compares Post objects. First compares the Post date. If the dates are # equal, it compares the Post slugs. # From 11ca1dfa7805024568c16eb7349756203ef82cc4 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 15:15:32 -0800 Subject: [PATCH 57/69] Updated history to reflect merging of #580. --- History.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/History.txt b/History.txt index 260c2661..5ef30148 100644 --- a/History.txt +++ b/History.txt @@ -1,5 +1,6 @@ == HEAD * Minor Enhancements + * Layout of Page or Post defaults to 'page' or 'post', respectively (#580) * "Keep files" feature (#685) * Output full path & name for files that don't parse (#745) * Add source and destination directory protection (#535) From 5a3e05dbff48394746b519f2015fa1153e2ae17f Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 15:31:14 -0800 Subject: [PATCH 58/69] Ensuring that layout is only set to the default value if layout is not specified --- lib/jekyll/page.rb | 2 +- lib/jekyll/post.rb | 2 +- .../_posts/2013-01-12-nil-layout.textile | 6 +++++ .../_posts/2013-01-12-no-layout.textile | 5 ++++ test/test_generated_site.rb | 2 +- test/test_page.rb | 20 ++++++++++++++ test/test_post.rb | 26 ++++++++++++++++++- 7 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 test/source/_posts/2013-01-12-nil-layout.textile create mode 100644 test/source/_posts/2013-01-12-no-layout.textile diff --git a/lib/jekyll/page.rb b/lib/jekyll/page.rb index d3461181..a821bb63 100644 --- a/lib/jekyll/page.rb +++ b/lib/jekyll/page.rb @@ -32,7 +32,7 @@ module Jekyll # Returns nothing. def read_yaml(base, name) super(base, name) - self.data['layout'] ||= 'page' + self.data['layout'] = 'page' unless self.data.has_key?('layout') self.data end diff --git a/lib/jekyll/post.rb b/lib/jekyll/post.rb index 6156b5be..58721020 100644 --- a/lib/jekyll/post.rb +++ b/lib/jekyll/post.rb @@ -74,7 +74,7 @@ module Jekyll # Returns nothing. def read_yaml(base, name) super(base, name) - self.data['layout'] ||= 'post' + self.data['layout'] = 'post' unless self.data.has_key?('layout') self.data end diff --git a/test/source/_posts/2013-01-12-nil-layout.textile b/test/source/_posts/2013-01-12-nil-layout.textile new file mode 100644 index 00000000..1e12fa9f --- /dev/null +++ b/test/source/_posts/2013-01-12-nil-layout.textile @@ -0,0 +1,6 @@ +--- +layout: nil +title: No layout +--- + +This post has no layout. diff --git a/test/source/_posts/2013-01-12-no-layout.textile b/test/source/_posts/2013-01-12-no-layout.textile new file mode 100644 index 00000000..1c745ae0 --- /dev/null +++ b/test/source/_posts/2013-01-12-no-layout.textile @@ -0,0 +1,5 @@ +--- +title: I have no layout +--- + +This post will be rendered with the "post" layout. diff --git a/test/test_generated_site.rb b/test/test_generated_site.rb index 9fb7a409..552e2417 100644 --- a/test/test_generated_site.rb +++ b/test/test_generated_site.rb @@ -14,7 +14,7 @@ class TestGeneratedSite < Test::Unit::TestCase end should "ensure post count is as expected" do - assert_equal 28, @site.posts.size + assert_equal 30, @site.posts.size end should "insert site.posts into the index" do diff --git a/test/test_page.rb b/test/test_page.rb index 61278997..3c7c9fed 100644 --- a/test/test_page.rb +++ b/test/test_page.rb @@ -66,6 +66,26 @@ class TestPage < Test::Unit::TestCase assert_equal "/about/", @page.dir end end + + context "with unspecified layout" do + setup do + @page = setup_page('contacts.html') + end + + should "default to 'post' layout" do + assert_equal "page", @page.data["layout"] + end + end + + context "with specified layout of nil" do + setup do + @page = setup_page('sitemap.xml') + end + + should "layout of nil is respected" do + assert_equal "nil", @page.data["layout"] + end + end context "rendering" do setup do diff --git a/test/test_post.rb b/test/test_post.rb index 245c30b4..16aff4f1 100644 --- a/test/test_post.rb +++ b/test/test_post.rb @@ -95,7 +95,7 @@ class TestPost < Test::Unit::TestCase should "consume the embedded dashes" do @post.read_yaml(@source, @real_file) - assert_equal({"title" => "Foo --- Bar"}, @post.data) + assert_equal({"title" => "Foo --- Bar", "layout" => "post"}, @post.data) assert_equal "Triple the fun!", @post.content end end @@ -128,6 +128,30 @@ class TestPost < Test::Unit::TestCase end end + context "with unspecified layout" do + setup do + file = '2013-01-12-no-layout.textile' + @post = setup_post(file) + @post.process(file) + end + + should "default to 'post' layout" do + assert_equal "post", @post.data["layout"] + end + end + + context "with specified layout of nil" do + setup do + file = '2013-01-12-nil-layout.textile' + @post = setup_post(file) + @post.process(file) + end + + should "layout of nil is respected" do + assert_equal "nil", @post.data["layout"] + end + end + context "with unspecified (date) style and categories" do setup do @post.categories << "food" From 4bb5158b094879b929c064fa667962da468202d2 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 15:34:05 -0800 Subject: [PATCH 59/69] New sub-titles for sub-command options table --- site/_posts/2012-07-01-configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/_posts/2012-07-01-configuration.md b/site/_posts/2012-07-01-configuration.md index 6a66e14c..7fcb34fe 100644 --- a/site/_posts/2012-07-01-configuration.md +++ b/site/_posts/2012-07-01-configuration.md @@ -72,7 +72,7 @@ The table below lists the available settings for Jekyll, and the various -### Configuration for "build" Sub-Command +### Build Command Options @@ -174,7 +174,7 @@ The table below lists the available settings for Jekyll, and the various
    -### Configuration for "serve" Sub-Command +### Serve Command Options In addition to the options below, the `serve` sub-command can accept any of the options for the `build` sub-command, which are then applied to the site build which occurs right From 078972de4b27ad6541834b29f152dc84a18f44ff Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 15:35:21 -0800 Subject: [PATCH 60/69] Remove Octopress site from list of Jekyll-powered Sites --- site/_posts/2012-07-01-sites.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/site/_posts/2012-07-01-sites.md b/site/_posts/2012-07-01-sites.md index b24aba41..715cea1d 100644 --- a/site/_posts/2012-07-01-sites.md +++ b/site/_posts/2012-07-01-sites.md @@ -13,8 +13,6 @@ learning purposes. ([source](http://github.com/mojombo/mojombo.github.com)) - [Nick Quaranto](http://quaran.to/) ([source](https://github.com/qrush/qrush.github.com)) -- [Parker Moore](http://www.parkermoore.de) - ([source](https://github.com/parkr/blogsource)) - [Roger Chapman](http://rogchap.com/) ([source](https://github.com/rogchap/rogchap.github.com)) - [GitHub Official Teaching Materials](http://teach.github.com) From 45f10a22da8c2d2631c5669a2dcd4314a517b8c6 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 15:36:11 -0800 Subject: [PATCH 61/69] Update number of Jekyll-powered blogs on Sites page --- site/_posts/2012-07-01-sites.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/_posts/2012-07-01-sites.md b/site/_posts/2012-07-01-sites.md index 715cea1d..1cd971c4 100644 --- a/site/_posts/2012-07-01-sites.md +++ b/site/_posts/2012-07-01-sites.md @@ -6,7 +6,7 @@ next_section: resources --- It’s interesting to see what designs and features others have come up -with. Below are three Jekyll-powered blogs which were hand-picked for +with. Below are six Jekyll-powered blogs which were hand-picked for learning purposes. - [Tom Preston-Werner](http://tom.preston-werner.com/) From fbda85210a9521a5f8e0141f980b289aaea32aad Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 21:55:59 -0800 Subject: [PATCH 62/69] Added html_report profile to cucumber --- cucumber.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cucumber.yml b/cucumber.yml index 9692830c..ff1d0580 100644 --- a/cucumber.yml +++ b/cucumber.yml @@ -1 +1,2 @@ -default: --format progress \ No newline at end of file +default: --format progress +html_report: --format progress --format html --out=features_report.html From 6a7a030eee32cd89b891377030b9f9cbf6d44a8c Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 22:25:55 -0800 Subject: [PATCH 63/69] Fixed failing Cucumber test --- features/site_configuration.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/site_configuration.feature b/features/site_configuration.feature index 5a83ee80..b9a082fa 100644 --- a/features/site_configuration.feature +++ b/features/site_configuration.feature @@ -22,7 +22,7 @@ Feature: Site configuration Given I have an "Rakefile" file that contains "I want to be excluded" And I have an "README" file that contains "I want to be excluded" And I have an "index.html" file that contains "I want to be included" - And I have a configuration file with "exclude" set to "Rakefile", "README" + And I have a configuration file with "exclude" set to "['Rakefile', 'README']" When I run jekyll Then I should see "I want to be included" in "_site/index.html" And the "_site/Rakefile" file should not exist From 282a3bce734cbce113d679043544f54542e46356 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Sat, 12 Jan 2013 22:26:05 -0800 Subject: [PATCH 64/69] Updated history --- History.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/History.txt b/History.txt index 5ef30148..1588111b 100644 --- a/History.txt +++ b/History.txt @@ -1,5 +1,6 @@ == HEAD * Minor Enhancements + * Add glob support to include, exclude option (#743) * Layout of Page or Post defaults to 'page' or 'post', respectively (#580) * "Keep files" feature (#685) * Output full path & name for files that don't parse (#745) From 070f5111e1802186ac7b3ec6bded884dbb3fa5a2 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Tue, 15 Jan 2013 14:44:06 -0800 Subject: [PATCH 65/69] A link to the sites page on the wiki (in addition to this curated list) --- site/_posts/2012-07-01-sites.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/_posts/2012-07-01-sites.md b/site/_posts/2012-07-01-sites.md index 1cd971c4..82142aba 100644 --- a/site/_posts/2012-07-01-sites.md +++ b/site/_posts/2012-07-01-sites.md @@ -21,3 +21,8 @@ learning purposes. ([source](https://github.com/rsms/rsms.github.com)) - [Scott Chacon](http://schacon.github.com) ([source](https://github.com/schacon/schacon.github.com)) + +If you would like to explore more examples, you can find a list of sites +and their sources on the ["Sites" page in the Jekyll wiki][jekyll-sites]. + +[jekyll-sites]: https://github.com/mojombo/jekyll/wiki/Sites From c0a6e1e5a25b3e05a51b339b9ae2ad8e39e5bed1 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Tue, 15 Jan 2013 14:56:04 -0800 Subject: [PATCH 66/69] Changed 'six' to 'some' on Sites page --- site/_posts/2012-07-01-sites.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/_posts/2012-07-01-sites.md b/site/_posts/2012-07-01-sites.md index 82142aba..f28c0280 100644 --- a/site/_posts/2012-07-01-sites.md +++ b/site/_posts/2012-07-01-sites.md @@ -6,7 +6,7 @@ next_section: resources --- It’s interesting to see what designs and features others have come up -with. Below are six Jekyll-powered blogs which were hand-picked for +with. Below are some Jekyll-powered blogs which were hand-picked for learning purposes. - [Tom Preston-Werner](http://tom.preston-werner.com/) From a017f36d4b669e2f09399f44a14eb43fb847aad2 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Tue, 15 Jan 2013 15:03:52 -0800 Subject: [PATCH 67/69] Added description to History file --- History.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/History.txt b/History.txt index 260c2661..094676f5 100644 --- a/History.txt +++ b/History.txt @@ -1,4 +1,6 @@ == HEAD + * Major Enhancements + * Refactored jekyll commands into subcommands: build, serve, and migrate. (#690) * Minor Enhancements * "Keep files" feature (#685) * Output full path & name for files that don't parse (#745) From cbbf8b496d2a3cd6a717390212b7b88def4c5ef6 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Wed, 16 Jan 2013 21:35:12 -0500 Subject: [PATCH 68/69] Update history to reflect merge of 496 --- History.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/History.txt b/History.txt index b03fefc0..d199fea4 100644 --- a/History.txt +++ b/History.txt @@ -2,6 +2,7 @@ * Major Enhancements * Refactored jekyll commands into subcommands: build, serve, and migrate. (#690) * Minor Enhancements + * Truncate post slugs when importing from Tumblr (#496) * Add glob support to include, exclude option (#743) * Layout of Page or Post defaults to 'page' or 'post', respectively (#580) * "Keep files" feature (#685) From 8b83feac06992e253e4ebd9b9bc4cbc2ee576012 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Wed, 16 Jan 2013 21:36:48 -0500 Subject: [PATCH 69/69] Update history to reflect merge of #664 --- History.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/History.txt b/History.txt index d199fea4..adb70078 100644 --- a/History.txt +++ b/History.txt @@ -2,6 +2,7 @@ * Major Enhancements * Refactored jekyll commands into subcommands: build, serve, and migrate. (#690) * Minor Enhancements + * Massively accelerate LSI performance (#664) * Truncate post slugs when importing from Tumblr (#496) * Add glob support to include, exclude option (#743) * Layout of Page or Post defaults to 'page' or 'post', respectively (#580)