From 6c0c5b61876d1cfba3b046b0d5b831cffffff934 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Sun, 16 Dec 2012 21:53:44 +0000 Subject: [PATCH 01/20] Add commander dependency --- jekyll.gemspec | 1 + 1 file changed, 1 insertion(+) diff --git a/jekyll.gemspec b/jekyll.gemspec index cd395683..49b8d008 100644 --- a/jekyll.gemspec +++ b/jekyll.gemspec @@ -29,6 +29,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency('maruku', "~> 0.5") s.add_runtime_dependency('kramdown', "~> 0.13.4") s.add_runtime_dependency('pygments.rb', "~> 0.3.2") + s.add_runtime_dependency('commander', "~> 4.1.3") s.add_development_dependency('rake', "~> 0.9") s.add_development_dependency('rdoc', "~> 3.11") From 14766497c8f17cf67270574178b9912a2da75ce9 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 20:34:09 +0000 Subject: [PATCH 02/20] Add bin/jekyll2 and initial BuildCommand The `BuildCommand` class is responsible for handling the building of the site. It can also optionally watch for changes to files and regenerate the site if needed. The `Command` class holds any methods which are used by any command implementation. --- bin/jekyll2 | 31 +++++++++++++++ lib/jekyll.rb | 3 ++ lib/jekyll/command.rb | 14 +++++++ lib/jekyll/commands/build.rb | 77 ++++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 bin/jekyll2 create mode 100644 lib/jekyll/command.rb create mode 100644 lib/jekyll/commands/build.rb diff --git a/bin/jekyll2 b/bin/jekyll2 new file mode 100755 index 00000000..ce85f13b --- /dev/null +++ b/bin/jekyll2 @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby + +$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib }) + +require 'commander/import' +require 'jekyll' + +# Details about Jekyll +program :name, 'jekyll' +program :version, Jekyll::VERSION +program :description, 'Jekyll is a blog-aware, static site generator in Ruby' + +# Global options available to every command +global_option '-s', '--source DIR', String, 'Source directory' +global_option '-d', '--destination DIR', String, 'Destination directory' + +# Build command +# +# Args: +# --source +# --destination +# --watch +command :build do |c| + c.syntax = 'jekyll build [options]' + c.description = 'Build...' + c.option '-w', '--watch', 'Watch for changes and rebuild' + c.action do |args, options| + options.default :watch => false + Jekyll::BuildCommand.process(options) + end +end diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 87984ecf..058fd762 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -41,6 +41,9 @@ require 'jekyll/errors' require 'jekyll/plugin' require 'jekyll/converter' require 'jekyll/generator' +require 'jekyll/command' + +require_all 'jekyll/commands' require_all 'jekyll/converters' require_all 'jekyll/generators' require_all 'jekyll/tags' diff --git a/lib/jekyll/command.rb b/lib/jekyll/command.rb new file mode 100644 index 00000000..340d457d --- /dev/null +++ b/lib/jekyll/command.rb @@ -0,0 +1,14 @@ +module Jekyll + + class Command + def self.globs(source) + Dir.chdir(source) do + dirs = Dir['*'].select { |x| File.directory?(x) } + dirs -= ['_site'] + dirs = dirs.map { |x| "#{x}/**/*" } + dirs += ['*'] + end + end + end + +end diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb new file mode 100644 index 00000000..074f5d49 --- /dev/null +++ b/lib/jekyll/commands/build.rb @@ -0,0 +1,77 @@ +module Jekyll + + class BuildCommand < Command + def self.process(options) + opts = {} + options.__hash__.map do |k,v| + opts[k.to_s] = v + end + + opts = Jekyll.configuration(opts) + site = Jekyll::Site.new(opts) + + source = opts['source'] + destination = opts['destination'] + + if opts['watch'] + self.watch(site, source, destination) + else + self.build(site, source, destination) + end + end + + # Private: Build the site from source into destination. + # + # site - A Jekyll::Site instance + # source - A String of the source path + # destination - A String of the destination path + # + # Returns nothing. + def self.build(site, source, destination) + puts "Building site: #{source} -> #{destination}" + begin + site.process + rescue Jekyll::FatalException => e + puts + puts "ERROR: YOUR SITE COULD NOT BE BUILT:" + puts "------------------------------------" + puts e.message + exit(1) + end + puts "Successfully generated site: #{source} -> #{destination}" + end + + # Private: Watch for file changes and rebuild the site. + # + # site - A Jekyll::Site instance + # source - A String of the source path + # destination - A String of the destination path + # + # Returns nothing. + def self.watch(site, source, destination) + require 'directory_watcher' + + puts "Auto-Regenerating enabled: #{source} -> #{destination}" + + dw = DirectoryWatcher.new(source) + dw.interval = 1 + dw.glob = self.globs(source) + + dw.add_observer do |*args| + t = Time.now.strftime("%Y-%m-%d %H:%M:%S") + puts "[#{t} regeneration: #{args.size} files changed" + site.process + end + + dw.start + + trap("SIGINT") do + puts "Stopping auto-regeneration..." + exit 0 + end + + loop { sleep 1000 } + end + end + +end From 3b4feb41f0a2e767b18d26f01d1a3d3a0fc28713 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:00:24 +0000 Subject: [PATCH 03/20] Add initial serve command The `ServeCommand` will let you serve your site locally for development. You can specify `--port`, `--host` and `--baseurl` options if you wish to change the defaults. Additionally the `BuildCommand` will be called before the processing of the serve command, this makes sure that the site is actually built. This means you are able to pass the `--watch` option to auto-regenerate your site, even while serving it locally. --- bin/jekyll2 | 29 +++++++++++++++++++++++++++++ lib/jekyll/commands/build.rb | 31 ++++++++++++++++++------------- lib/jekyll/commands/serve.rb | 28 ++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 13 deletions(-) create mode 100644 lib/jekyll/commands/serve.rb diff --git a/bin/jekyll2 b/bin/jekyll2 index ce85f13b..d99ba866 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -29,3 +29,32 @@ command :build do |c| Jekyll::BuildCommand.process(options) end end + +# Serve command +# +# Args: +# --source +# --destination +# --watch +# +# --port +# --host +# --baseurl +command :serve do |c| + c.syntax = 'jekyll serve [options]' + c.description = 'Serve...' + c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '-p', '--port [PORT]', 'Port to listen on' + c.option '-h', '--host [HOST]', 'Host to bind to' + c.option '-b', '--baseurl [URL]', 'Base URL' + c.action do |args, options| + options.default :watch => false, + :port => '4000', + :host => '0.0.0.0', + :baseurl => '/', + :serving => true + + Jekyll::BuildCommand.process(options) + Jekyll::ServeCommand.process(options) + end +end diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index 074f5d49..2ff8aa80 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -14,20 +14,21 @@ module Jekyll destination = opts['destination'] if opts['watch'] - self.watch(site, source, destination) + self.watch(site, opts) else - self.build(site, source, destination) + self.build(site, opts) end end # Private: Build the site from source into destination. # # site - A Jekyll::Site instance - # source - A String of the source path - # destination - A String of the destination path + # options - A Hash of options passed to the command # # Returns nothing. - def self.build(site, source, destination) + def self.build(site, options) + source = options['source'] + destination = options['destination'] puts "Building site: #{source} -> #{destination}" begin site.process @@ -44,13 +45,15 @@ module Jekyll # Private: Watch for file changes and rebuild the site. # # site - A Jekyll::Site instance - # source - A String of the source path - # destination - A String of the destination path + # options - A Hash of options passed to the command # # Returns nothing. - def self.watch(site, source, destination) + def self.watch(site, options) require 'directory_watcher' + source = options['source'] + destination = options['destination'] + puts "Auto-Regenerating enabled: #{source} -> #{destination}" dw = DirectoryWatcher.new(source) @@ -65,12 +68,14 @@ module Jekyll dw.start - trap("SIGINT") do - puts "Stopping auto-regeneration..." - exit 0 - end + unless options['serving'] + loop { sleep 1000 } - loop { sleep 1000 } + trap("INT") do + puts "Stopping auto-regeneration..." + exit 0 + end + end end end diff --git a/lib/jekyll/commands/serve.rb b/lib/jekyll/commands/serve.rb new file mode 100644 index 00000000..665a26cf --- /dev/null +++ b/lib/jekyll/commands/serve.rb @@ -0,0 +1,28 @@ +module Jekyll + + class ServeCommand < Command + def self.process(options) + require 'webrick' + include WEBrick + + destination = options.destination + + FileUtils.mkdir_p(destination) + + mime_types = WEBrick::HTTPUtils::DefaultMimeTypes + mime_types.store 'js', 'application/javascript' + + s = HTTPServer.new( + :Port => options.port, + :BindAddress => options.host, + :MimeTypes => mime_types + ) + + s.mount(options.baseurl, HTTPServlet::FileHandler, destination) + t = Thread.new { s.start } + trap("INT") { s.shutdown } + t.join() + end + end + +end From 72f99eb8a7c39509753f8e9bec45c42c151a59e4 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:18:05 +0000 Subject: [PATCH 04/20] Update global options and command descriptions --- bin/jekyll2 | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index d99ba866..eba186dd 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -11,8 +11,11 @@ program :version, Jekyll::VERSION program :description, 'Jekyll is a blog-aware, static site generator in Ruby' # Global options available to every command -global_option '-s', '--source DIR', String, 'Source directory' -global_option '-d', '--destination DIR', String, 'Destination directory' +global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' +global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' +global_option '--safe', 'Safe mode (defaults to false)' +global_option '--plugins', 'Plugins directory (defaults to ./_plugins)' +global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' # Build command # @@ -22,10 +25,9 @@ global_option '-d', '--destination DIR', String, 'Destination directory' # --watch command :build do |c| c.syntax = 'jekyll build [options]' - c.description = 'Build...' + c.description = 'Build your site with the option of auto-renegeration' c.option '-w', '--watch', 'Watch for changes and rebuild' c.action do |args, options| - options.default :watch => false Jekyll::BuildCommand.process(options) end end @@ -42,14 +44,13 @@ end # --baseurl command :serve do |c| c.syntax = 'jekyll serve [options]' - c.description = 'Serve...' + c.description = 'Serve your site locally with the option of auto-regeneration' c.option '-w', '--watch', 'Watch for changes and rebuild' c.option '-p', '--port [PORT]', 'Port to listen on' c.option '-h', '--host [HOST]', 'Host to bind to' c.option '-b', '--baseurl [URL]', 'Base URL' c.action do |args, options| - options.default :watch => false, - :port => '4000', + options.default :port => '4000', :host => '0.0.0.0', :baseurl => '/', :serving => true From d8f328b87ce171f210614509c339644b4529aa01 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:41:18 +0000 Subject: [PATCH 05/20] Update loop position so trap handler is used --- lib/jekyll/commands/build.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index 2ff8aa80..2a7348a7 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -69,12 +69,12 @@ module Jekyll dw.start unless options['serving'] - loop { sleep 1000 } - trap("INT") do puts "Stopping auto-regeneration..." exit 0 end + + loop { sleep 1000 } end end end From 5b2e95b44392fa6b6ed94e1f04dbed9bd7bef8c9 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:54:01 +0000 Subject: [PATCH 06/20] Add missing ] in self.watch string --- lib/jekyll/commands/build.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index 2a7348a7..d5a0e570 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -62,7 +62,7 @@ module Jekyll dw.add_observer do |*args| t = Time.now.strftime("%Y-%m-%d %H:%M:%S") - puts "[#{t} regeneration: #{args.size} files changed" + puts "[#{t}] regeneration: #{args.size} files changed" site.process end From bd1c8fe760d31a67ee9081c3d2767cb60781d909 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 21:55:00 +0000 Subject: [PATCH 07/20] Update Jekyll.configuration to convert symbol keys Because Commander uses symbol keys in the options hash and I don't want to go back backport every hash string key to symbols in Jekyll. :star: --- bin/jekyll2 | 6 +++--- lib/jekyll.rb | 3 +++ lib/jekyll/commands/build.rb | 19 +++++++------------ lib/jekyll/commands/serve.rb | 8 ++++---- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index eba186dd..34d040aa 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -28,7 +28,7 @@ command :build do |c| c.description = 'Build your site with the option of auto-renegeration' c.option '-w', '--watch', 'Watch for changes and rebuild' c.action do |args, options| - Jekyll::BuildCommand.process(options) + Jekyll::BuildCommand.process(options.__hash__) end end @@ -55,7 +55,7 @@ command :serve do |c| :baseurl => '/', :serving => true - Jekyll::BuildCommand.process(options) - Jekyll::ServeCommand.process(options) + Jekyll::BuildCommand.process(options.__hash__) + Jekyll::ServeCommand.process(options.__hash__) end end diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 058fd762..98f99402 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -123,6 +123,9 @@ module Jekyll # # Returns the final configuration Hash. def self.configuration(override) + # Convert any symbol keys to strings and remove the old key/values + override.keys.each { |k| override[k.to_s] = override.delete(k) } + # _config.yml may override default source location, but until # then, we need to know where to look for _config.yml source = override['source'] || Jekyll::DEFAULTS['source'] diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index d5a0e570..6c8a1254 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -2,21 +2,16 @@ module Jekyll class BuildCommand < Command def self.process(options) - opts = {} - options.__hash__.map do |k,v| - opts[k.to_s] = v - end + options = Jekyll.configuration(options) + site = Jekyll::Site.new(options) - opts = Jekyll.configuration(opts) - site = Jekyll::Site.new(opts) + source = options['source'] + destination = options['destination'] - source = opts['source'] - destination = opts['destination'] - - if opts['watch'] - self.watch(site, opts) + if options['watch'] + self.watch(site, options) else - self.build(site, opts) + self.build(site, options) end end diff --git a/lib/jekyll/commands/serve.rb b/lib/jekyll/commands/serve.rb index 665a26cf..df717bea 100644 --- a/lib/jekyll/commands/serve.rb +++ b/lib/jekyll/commands/serve.rb @@ -5,7 +5,7 @@ module Jekyll require 'webrick' include WEBrick - destination = options.destination + destination = options['destination'] FileUtils.mkdir_p(destination) @@ -13,12 +13,12 @@ module Jekyll mime_types.store 'js', 'application/javascript' s = HTTPServer.new( - :Port => options.port, - :BindAddress => options.host, + :Port => options['port'], + :BindAddress => options['host'], :MimeTypes => mime_types ) - s.mount(options.baseurl, HTTPServlet::FileHandler, destination) + s.mount(options['baseurl'], HTTPServlet::FileHandler, destination) t = Thread.new { s.start } trap("INT") { s.shutdown } t.join() From dc139e2ac97197182b2d036e8794cc1e45306b40 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Tue, 18 Dec 2012 22:13:17 +0000 Subject: [PATCH 08/20] Update the sym->str key conversion --- bin/jekyll2 | 9 ++++++--- lib/jekyll.rb | 2 +- lib/jekyll/commands/build.rb | 1 - 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index 34d040aa..f46f2dc7 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -28,7 +28,9 @@ command :build do |c| c.description = 'Build your site with the option of auto-renegeration' c.option '-w', '--watch', 'Watch for changes and rebuild' c.action do |args, options| - Jekyll::BuildCommand.process(options.__hash__) + options.defaults :serving => false + options = Jekyll.configuration(options.__hash__) + Jekyll::BuildCommand.process(options) end end @@ -55,7 +57,8 @@ command :serve do |c| :baseurl => '/', :serving => true - Jekyll::BuildCommand.process(options.__hash__) - Jekyll::ServeCommand.process(options.__hash__) + options = Jekyll.configuration(options.__hash__) + Jekyll::BuildCommand.process(options) + Jekyll::ServeCommand.process(options) end end diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 98f99402..2b008f88 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -124,7 +124,7 @@ module Jekyll # Returns the final configuration Hash. def self.configuration(override) # Convert any symbol keys to strings and remove the old key/values - override.keys.each { |k| override[k.to_s] = override.delete(k) } + override = override.reduce({}) { |hsh,(k,v)| hsh.merge(k.to_s => v) } # _config.yml may override default source location, but until # then, we need to know where to look for _config.yml diff --git a/lib/jekyll/commands/build.rb b/lib/jekyll/commands/build.rb index 6c8a1254..27461101 100644 --- a/lib/jekyll/commands/build.rb +++ b/lib/jekyll/commands/build.rb @@ -2,7 +2,6 @@ module Jekyll class BuildCommand < Command def self.process(options) - options = Jekyll.configuration(options) site = Jekyll::Site.new(options) source = options['source'] From be20a03bb9ccebe2f74fa5ec4fd3bafebd19dcc7 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 14:09:02 +0000 Subject: [PATCH 09/20] Update to make the help command default --- bin/jekyll2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/jekyll2 b/bin/jekyll2 index f46f2dc7..c8dfde16 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -10,6 +10,8 @@ program :name, 'jekyll' program :version, Jekyll::VERSION program :description, 'Jekyll is a blog-aware, static site generator in Ruby' +default_command :help + # Global options available to every command global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' From 053c2eb0a72c5fc6719cccc76f3d48bdf2e1a1b1 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 14:13:32 +0000 Subject: [PATCH 10/20] Rename Args to Options in command docs --- bin/jekyll2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index c8dfde16..7c99f953 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -21,7 +21,7 @@ global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' # Build command # -# Args: +# Options: # --source # --destination # --watch @@ -38,7 +38,7 @@ end # Serve command # -# Args: +# Options: # --source # --destination # --watch From 4578c15a24c976860b1437a216ff1bc7748b6602 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 16:34:03 +0000 Subject: [PATCH 11/20] Add TODOs for current switches --- bin/jekyll | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/bin/jekyll b/bin/jekyll index 74cb99b0..8c0d2dad 100755 --- a/bin/jekyll +++ b/bin/jekyll @@ -25,88 +25,108 @@ options = {} opts = OptionParser.new do |opts| opts.banner = help + # TODO: delete, migrator related opts.on("--file [PATH]", "File to import from") do |import_file| options['file'] = import_file end + # TODO: delete, migrator related opts.on("--dbname [TEXT]", "DB to import from") do |import_dbname| options['dbname'] = import_dbname end + # TODO: delete, migrator related opts.on("--user [TEXT]", "Username to use when importing") do |import_user| options['user'] = import_user end + # TODO: delete, migrator related opts.on("--pass [TEXT]", "Password to use when importing") do |import_pass| options['pass'] = import_pass end + # TODO: delete, migrator related opts.on("--host [HOST ADDRESS]", "Host to import from") do |import_host| options['host'] = import_host end + # TODO: delete, migrator related opts.on("--site [SITE NAME]", "Site to import from") do |import_site| options['site'] = import_site end - + # TODO: global option opts.on("--[no-]safe", "Safe mode (default unsafe)") do |safe| options['safe'] = safe end + # TODO: option for build and serve command opts.on("--[no-]auto", "Auto-regenerate") do |auto| options['auto'] = auto end + # TODO: new serve command opts.on("--server [PORT]", "Start web server (default port 4000)") do |port| options['server'] = true options['server_port'] = port unless port.nil? end + # TODO: remove opts.on("--no-server", "Do not start a web server") do |part| options['server'] = false end + # TODO: option for serve command opts.on("--base-url [BASE_URL]", "Serve website from a given base URL (default '/'") do |baseurl| options['baseurl'] = baseurl end + # TODO: does anyone actually use this? opts.on("--default-mimetype [MT]", "Mimetype to use when no file extension (if --server)") do |mt| options['default-mimetype'] = mt end + # TODO: option for build and serve command opts.on("--[no-]lsi", "Use LSI for better related posts") do |lsi| options['lsi'] = lsi end + # TODO: remove and just enable all the time? opts.on("--[no-]pygments", "Use pygments to highlight code") do |pygments| options['pygments'] = pygments end + # TODO: read from config opts.on("--rdiscount", "Use rdiscount gem for Markdown") do options['markdown'] = 'rdiscount' end + # TODO: read from config opts.on("--redcarpet", "Use redcarpet gem for Markdown") do options['markdown'] = 'redcarpet' end + # TODO: read from config opts.on("--kramdown", "Use kramdown gem for Markdown") do options['markdown'] = 'kramdown' end + # TODO: remove and just generate the site as is? opts.on("--time [TIME]", "Time to generate the site for") do |time| options['time'] = Time.parse(time) end + # TODO: remove and just render all posts which aren't 'unpublished'? opts.on("--[no-]future", "Render future dated posts") do |future| options['future'] = future end + # TODO: read from config opts.on("--permalink [TYPE]", "Use 'date' (default) for YYYY/MM/DD") do |style| options['permalink'] = style unless style.nil? end + # TODO: read from config opts.on("--paginate [POSTS_PER_PAGE]", "Paginate a blog's posts") do |per_page| begin options['paginate'] = per_page.to_i @@ -117,6 +137,7 @@ opts = OptionParser.new do |opts| end end + # TODO: read from config opts.on("--paginate_path [PAGINATED_URL_FORMAT]", "Leave blank for /page") do |paginate_path| begin options['paginate_path'] = paginate_path @@ -127,6 +148,7 @@ opts = OptionParser.new do |opts| end end + # TODO: read from config opts.on("--limit_posts [MAX_POSTS]", "Limit the number of posts to publish") do |limit_posts| begin options['limit_posts'] = limit_posts.to_i @@ -137,6 +159,7 @@ opts = OptionParser.new do |opts| end end + # TODO: read from config opts.on("--url [URL]", "Set custom site.url") do |url| options['url'] = url end @@ -206,6 +229,7 @@ end # Get source and destination from command line +# TODO: source and destination are now global options case ARGV.size when 0 when 1 From d298e2bd5759514343a4ac64665a8fa853fec0b2 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 16:44:03 +0000 Subject: [PATCH 12/20] Add LSI option for build and serve commands --- bin/jekyll2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/jekyll2 b/bin/jekyll2 index 7c99f953..76e4c94d 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -29,6 +29,7 @@ command :build do |c| c.syntax = 'jekyll build [options]' c.description = 'Build your site with the option of auto-renegeration' c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '--lsi', 'Use LSI for improved related posts' c.action do |args, options| options.defaults :serving => false options = Jekyll.configuration(options.__hash__) @@ -50,6 +51,7 @@ command :serve do |c| c.syntax = 'jekyll serve [options]' c.description = 'Serve your site locally with the option of auto-regeneration' c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '--lsi', 'Use LSI for improved related posts' c.option '-p', '--port [PORT]', 'Port to listen on' c.option '-h', '--host [HOST]', 'Host to bind to' c.option '-b', '--baseurl [URL]', 'Base URL' From 9c65ceb22dea9fa507ea6ece5132a0fabd0915bc Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 16:56:08 +0000 Subject: [PATCH 13/20] Remove comments --- bin/jekyll2 | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/bin/jekyll2 b/bin/jekyll2 index 76e4c94d..29030c47 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -19,17 +19,13 @@ global_option '--safe', 'Safe mode (defaults to false)' global_option '--plugins', 'Plugins directory (defaults to ./_plugins)' global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' -# Build command -# -# Options: -# --source -# --destination -# --watch command :build do |c| c.syntax = 'jekyll build [options]' c.description = 'Build your site with the option of auto-renegeration' + c.option '-w', '--watch', 'Watch for changes and rebuild' c.option '--lsi', 'Use LSI for improved related posts' + c.action do |args, options| options.defaults :serving => false options = Jekyll.configuration(options.__hash__) @@ -37,21 +33,13 @@ command :build do |c| end end -# Serve command -# -# Options: -# --source -# --destination -# --watch -# -# --port -# --host -# --baseurl command :serve do |c| c.syntax = 'jekyll serve [options]' c.description = 'Serve your site locally with the option of auto-regeneration' + c.option '-w', '--watch', 'Watch for changes and rebuild' c.option '--lsi', 'Use LSI for improved related posts' + c.option '-p', '--port [PORT]', 'Port to listen on' c.option '-h', '--host [HOST]', 'Host to bind to' c.option '-b', '--baseurl [URL]', 'Base URL' From b7944c527496a107e4afecb92735604d5cbbe162 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 17:50:21 +0000 Subject: [PATCH 14/20] Add initial MigrateCommand Not all migrators can actually be calld from the comand line. Some require options which are not passed in and have to be called by other means. --- bin/jekyll2 | 16 ++++++++++++ lib/jekyll/commands/migrate.rb | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 lib/jekyll/commands/migrate.rb diff --git a/bin/jekyll2 b/bin/jekyll2 index 29030c47..3df10e4f 100755 --- a/bin/jekyll2 +++ b/bin/jekyll2 @@ -43,6 +43,7 @@ command :serve do |c| c.option '-p', '--port [PORT]', 'Port to listen on' c.option '-h', '--host [HOST]', 'Host to bind to' c.option '-b', '--baseurl [URL]', 'Base URL' + c.action do |args, options| options.default :port => '4000', :host => '0.0.0.0', @@ -54,3 +55,18 @@ command :serve do |c| Jekyll::ServeCommand.process(options) end end + +command :migrate do |c| + c.syntax = 'jekyll migrate [options]' + c.description = 'Migrate your own blog to Jekyll' + + c.option '--file', 'File to migrate from' + c.option '--dbname', 'Database name to migrate from' + c.option '--user', 'Username to use when migrating' + c.option '--pass', 'Password to use when migrating' + c.option '--host', 'Host address to use when migrating' + + c.action do |args, options| + Jekyll::MigrateCommand.process(args.first, options) + end +end diff --git a/lib/jekyll/commands/migrate.rb b/lib/jekyll/commands/migrate.rb new file mode 100644 index 00000000..26937357 --- /dev/null +++ b/lib/jekyll/commands/migrate.rb @@ -0,0 +1,47 @@ +module Jekyll + + class MigrateCommand < Command + MIGRATORS = { + :csv => 'CSV', + :drupal => 'Drupal', + :enki => 'Enki', + :mephisto => 'Mephisto', + :mt => 'MT', + :posterous => 'Posterous', + :textpattern => 'TextPattern', + :tumblr => 'Tumblr', + :typo => 'Typo', + :wordpressdotcom => 'WordpressDotCom', + :wordpress => 'WordPress' + } + + def self.process(migrator, options) + abort 'missing argument. Please specify a migrator' if migrator.nil? + migrator = migrator.downcase + + cmd_options = [] + [ :file, :dbname, :user, :pass, :host, :site ].each do |p| + cmd_options << "\"#{options[p]}\"" unless options[p].nil? + end + + + if MIGRATORS.keys.include?(migrator) + app_root = File.expand_path( + File.join(File.dirname(__FILE__), '..', '..', '..') + ) + + require "#{app_root}/lib/jekyll/migrators/#{migrator}" + + if Jekyll.const_defiend?(MIGRATORS[migrator.to_sym]) + puts 'Importing...' + migrator_class = Jekyll.const_get(MIGRATORS[migrator.to_sym]) + migrator_class.process(*cmd_options) + exit 0 + end + end + + abort 'invalid migrator. Please specify a valid migrator' + end + end + +end From e24bb02576e9ac727176d383632dad8b899d9aa4 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 17:54:58 +0000 Subject: [PATCH 15/20] Update the Jekyll command for features testing :star: :metal: :star2: --- features/support/env.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/features/support/env.rb b/features/support/env.rb index 3166ce9c..912b735c 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -7,10 +7,11 @@ World do end TEST_DIR = File.join('/', 'tmp', 'jekyll') -JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll') +JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll2') def run_jekyll(opts = {}) command = JEKYLL_PATH + command << " build" command << " >> /dev/null 2>&1" if opts[:debug].nil? system command end From a151a16f09c6b23f331966d1a737c530454f5720 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 19 Dec 2012 18:23:34 +0000 Subject: [PATCH 16/20] Remove command options from default config Removing command line options from the config is a path towards cleaning up the configuration file and not including options which don't really belong there. --- lib/jekyll.rb | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/jekyll.rb b/lib/jekyll.rb index 2b008f88..d7f011d3 100644 --- a/lib/jekyll.rb +++ b/lib/jekyll.rb @@ -51,26 +51,22 @@ require_all 'jekyll/tags' module Jekyll VERSION = '0.11.2' - # Default options. Overriden by values in _config.yml or command-line opts. + # Default options. Overriden by values in _config.yml. # Strings rather than symbols are used for compatability with YAML. DEFAULTS = { - 'safe' => false, - 'auto' => false, - 'server' => false, - 'server_port' => 4000, - 'source' => Dir.pwd, 'destination' => File.join(Dir.pwd, '_site'), + 'plugins' => File.join(Dir.pwd, '_plugins'), 'layouts' => '_layouts', - 'future' => true, - 'lsi' => false, - 'pygments' => false, - 'markdown' => 'maruku', - 'permalink' => 'date', - 'include' => ['.htaccess'], - 'paginate_path' => 'page:num', + 'future' => true, # remove and make true just default + 'pygments' => false, # remove and make true just default + + 'markdown' => 'maruku', # no longer a command option + 'permalink' => 'date', # no longer a command option + 'include' => ['.htaccess'], # no longer a command option + 'paginate_path' => 'page:num', # no longer a command option 'markdown_ext' => 'markdown,mkd,mkdn,md', 'textile_ext' => 'textile', From b9da30bc8ffa4a2d0a9280e755d0904609d4fc97 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Sat, 22 Dec 2012 17:49:33 +0000 Subject: [PATCH 17/20] Remove migrators --- Rakefile | 15 -- lib/jekyll/migrators/csv.rb | 26 --- lib/jekyll/migrators/drupal.rb | 103 --------- lib/jekyll/migrators/enki.rb | 49 ---- lib/jekyll/migrators/joomla.rb | 53 ----- lib/jekyll/migrators/marley.rb | 52 ----- lib/jekyll/migrators/mephisto.rb | 84 ------- lib/jekyll/migrators/mt.rb | 86 ------- lib/jekyll/migrators/posterous.rb | 67 ------ lib/jekyll/migrators/rss.rb | 47 ---- lib/jekyll/migrators/textpattern.rb | 58 ----- lib/jekyll/migrators/tumblr.rb | 195 ---------------- lib/jekyll/migrators/typo.rb | 51 ---- lib/jekyll/migrators/wordpress.rb | 294 ------------------------ lib/jekyll/migrators/wordpressdotcom.rb | 70 ------ 15 files changed, 1250 deletions(-) delete mode 100644 lib/jekyll/migrators/csv.rb delete mode 100644 lib/jekyll/migrators/drupal.rb delete mode 100644 lib/jekyll/migrators/enki.rb delete mode 100644 lib/jekyll/migrators/joomla.rb delete mode 100644 lib/jekyll/migrators/marley.rb delete mode 100644 lib/jekyll/migrators/mephisto.rb delete mode 100644 lib/jekyll/migrators/mt.rb delete mode 100644 lib/jekyll/migrators/posterous.rb delete mode 100644 lib/jekyll/migrators/rss.rb delete mode 100644 lib/jekyll/migrators/textpattern.rb delete mode 100644 lib/jekyll/migrators/tumblr.rb delete mode 100644 lib/jekyll/migrators/typo.rb delete mode 100644 lib/jekyll/migrators/wordpress.rb delete mode 100644 lib/jekyll/migrators/wordpressdotcom.rb diff --git a/Rakefile b/Rakefile index 8eb1360d..a267c50d 100644 --- a/Rakefile +++ b/Rakefile @@ -82,21 +82,6 @@ end # ############################################################################# -namespace :migrate do - desc "Migrate from mephisto in the current directory" - task :mephisto do - sh %q(ruby -r './lib/jekyll/migrators/mephisto' -e 'Jekyll::Mephisto.postgres(:database => "#{ENV["DB"]}")') - end - desc "Migrate from Movable Type in the current directory" - task :mt do - sh %q(ruby -r './lib/jekyll/migrators/mt' -e 'Jekyll::MT.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")') - end - desc "Migrate from Typo in the current directory" - task :typo do - sh %q(ruby -r './lib/jekyll/migrators/typo' -e 'Jekyll::Typo.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")') - end -end - begin require 'cucumber/rake/task' Cucumber::Rake::Task.new(:features) do |t| diff --git a/lib/jekyll/migrators/csv.rb b/lib/jekyll/migrators/csv.rb deleted file mode 100644 index ce5203b7..00000000 --- a/lib/jekyll/migrators/csv.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Jekyll - module CSV - # Reads a csv with title, permalink, body, published_at, and filter. - # It creates a post file for each row in the csv - def self.process(file = "posts.csv") - FileUtils.mkdir_p "_posts" - posts = 0 - FasterCSV.foreach(file) do |row| - next if row[0] == "title" - posts += 1 - name = row[3].split(" ")[0]+"-"+row[1]+(row[4] =~ /markdown/ ? ".markdown" : ".textile") - File.open("_posts/#{name}", "w") do |f| - f.puts <<-HEADER ---- -layout: post -title: #{row[0]} ---- - - HEADER - f.puts row[2] - end - end - "Created #{posts} posts!" - end - end -end diff --git a/lib/jekyll/migrators/drupal.rb b/lib/jekyll/migrators/drupal.rb deleted file mode 100644 index 7fd16aef..00000000 --- a/lib/jekyll/migrators/drupal.rb +++ /dev/null @@ -1,103 +0,0 @@ -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module Drupal - # Reads a MySQL database via Sequel and creates a post file for each post - # in wp_posts that has post_status = 'publish'. This restriction is made - # because 'draft' posts are not guaranteed to have valid dates. - QUERY = "SELECT n.nid, \ - n.title, \ - nr.body, \ - n.created, \ - n.status \ - FROM node AS n, \ - node_revisions AS nr \ - WHERE (n.type = 'blog' OR n.type = 'story') \ - AND n.vid = nr.vid" - - def self.process(dbname, user, pass, host = 'localhost', prefix = '') - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - - if prefix != '' - QUERY[" node "] = " " + prefix + "node " - QUERY[" node_revisions "] = " " + prefix + "node_revisions " - end - - FileUtils.mkdir_p "_posts" - FileUtils.mkdir_p "_drafts" - - # Create the refresh layout - # Change the refresh url if you customized your permalink config - File.open("_layouts/refresh.html", "w") do |f| - f.puts < - - - - - - -EOF - end - - db[QUERY].each do |post| - # Get required fields and construct Jekyll compatible name - node_id = post[:nid] - title = post[:title] - content = post[:body] - created = post[:created] - time = Time.at(created) - is_published = post[:status] == 1 - dir = is_published ? "_posts" : "_drafts" - slug = title.strip.downcase.gsub(/(&|&)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '') - name = time.strftime("%Y-%m-%d-") + slug + '.md' - - # Get the relevant fields as a hash, delete empty fields and convert - # to YAML for the header - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'created' => created, - }.delete_if { |k,v| v.nil? || v == ''}.to_yaml - - # Write out the data and content to file - File.open("#{dir}/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - - # Make a file to redirect from the old Drupal URL - if is_published - aliases = db["SELECT dst FROM #{prefix}url_alias WHERE src = ?", "node/#{node_id}"].all - - aliases.push(:dst => "node/#{node_id}") - - aliases.each do |url_alias| - FileUtils.mkdir_p url_alias[:dst] - File.open("#{url_alias[:dst]}/index.md", "w") do |f| - f.puts "---" - f.puts "layout: refresh" - f.puts "refresh_to_post_id: /#{time.strftime("%Y/%m/%d/") + slug}" - f.puts "---" - end - end - end - end - - # TODO: Make dirs & files for nodes of type 'page' - # Make refresh pages for these as well - - # TODO: Make refresh dirs & files according to entries in url_alias table - end - end -end diff --git a/lib/jekyll/migrators/enki.rb b/lib/jekyll/migrators/enki.rb deleted file mode 100644 index 61cb2562..00000000 --- a/lib/jekyll/migrators/enki.rb +++ /dev/null @@ -1,49 +0,0 @@ -# Adapted by Rodrigo Pinto -# Based on typo.rb by Toby DiPasquale - -require 'fileutils' -require 'rubygems' -require 'sequel' - -module Jekyll - module Enki - SQL = <<-EOS - SELECT p.id, - p.title, - p.slug, - p.body, - p.published_at as date, - p.cached_tag_list as tags - FROM posts p - EOS - - # Just working with postgres, but can be easily adapted - # to work with both mysql and postgres. - def self.process(dbname, user, pass, host = 'localhost') - FileUtils.mkdir_p('_posts') - db = Sequel.postgres(:database => dbname, - :user => user, - :password => pass, - :host => host, - :encoding => 'utf8') - - db[SQL].each do |post| - name = [ sprintf("%.04d", post[:date].year), - sprintf("%.02d", post[:date].month), - sprintf("%.02d", post[:date].day), - post[:slug].strip ].join('-') - name += '.textile' - - File.open("_posts/#{name}", 'w') do |f| - f.puts({ 'layout' => 'post', - 'title' => post[:title].to_s, - 'enki_id' => post[:id], - 'categories' => post[:tags] - }.delete_if { |k, v| v.nil? || v == '' }.to_yaml) - f.puts '---' - f.puts post[:body].delete("\r") - end - end - end - end -end diff --git a/lib/jekyll/migrators/joomla.rb b/lib/jekyll/migrators/joomla.rb deleted file mode 100644 index 87f1e105..00000000 --- a/lib/jekyll/migrators/joomla.rb +++ /dev/null @@ -1,53 +0,0 @@ -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This migrator is made for Joomla 1.5 databases. -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module Joomla - def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'jos_', section = '1') - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - - FileUtils.mkdir_p("_posts") - - # Reads a MySQL database via Sequel and creates a post file for each - # post in wp_posts that has post_status = 'publish'. This restriction is - # made because 'draft' posts are not guaranteed to have valid dates. - query = "SELECT `title`, `alias`, CONCAT(`introtext`,`fulltext`) as content, `created`, `id` FROM #{table_prefix}content WHERE state = '0' OR state = '1' AND sectionid = '#{section}'" - - db[query].each do |post| - # Get required fields and construct Jekyll compatible name. - title = post[:title] - slug = post[:alias] - date = post[:created] - content = post[:content] - name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day, - slug] - - # Get the relevant fields as a hash, delete empty fields and convert - # to YAML for the header. - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'joomla_id' => post[:id], - 'joomla_url' => post[:alias], - 'date' => date - }.delete_if { |k,v| v.nil? || v == '' }.to_yaml - - # Write out the data and content to file - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - end - end -end diff --git a/lib/jekyll/migrators/marley.rb b/lib/jekyll/migrators/marley.rb deleted file mode 100644 index 21bcead5..00000000 --- a/lib/jekyll/migrators/marley.rb +++ /dev/null @@ -1,52 +0,0 @@ -require 'yaml' -require 'fileutils' - -module Jekyll - module Marley - def self.regexp - { :id => /^\d{0,4}-{0,1}(.*)$/, - :title => /^#\s*(.*)\s+$/, - :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/, - :published_on => /.*\s+\(([0-9\/]+)\)$/, - :perex => /^([^\#\n]+\n)$/, - :meta => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp - } - end - - def self.process(marley_data_dir) - raise ArgumentError, "marley dir #{marley_data_dir} not found" unless File.directory?(marley_data_dir) - - FileUtils.mkdir_p "_posts" - - posts = 0 - Dir["#{marley_data_dir}/**/*.txt"].each do |f| - next unless File.exists?(f) - - #copied over from marley's app/lib/post.rb - file_content = File.read(f) - meta_content = file_content.slice!( self.regexp[:meta] ) - body = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip - - title = file_content.scan( self.regexp[:title] ).first.to_s.strip - prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip - published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) ) - meta = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {} - meta['title'] = title - meta['layout'] = 'post' - - formatted_date = published_on.strftime('%Y-%m-%d') - post_name = File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '') - - name = "#{formatted_date}-#{post_name}" - File.open("_posts/#{name}.markdown", "w") do |f| - f.puts meta.to_yaml - f.puts "---\n" - f.puts "\n#{prerex}\n\n" if prerex - f.puts body - end - posts += 1 - end - "Created #{posts} posts!" - end - end -end diff --git a/lib/jekyll/migrators/mephisto.rb b/lib/jekyll/migrators/mephisto.rb deleted file mode 100644 index 7622c722..00000000 --- a/lib/jekyll/migrators/mephisto.rb +++ /dev/null @@ -1,84 +0,0 @@ -# Quickly hacked together my Michael Ivey -# Based on mt.rb by Nick Gerakines, open source and publically -# available under the MIT license. Use this module at your own risk. - -require 'rubygems' -require 'sequel' -require 'fastercsv' -require 'fileutils' -require File.join(File.dirname(__FILE__),"csv.rb") - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module Mephisto - #Accepts a hash with database config variables, exports mephisto posts into a csv - #export PGPASSWORD if you must - def self.postgres(c) - sql = <<-SQL - BEGIN; - CREATE TEMP TABLE jekyll AS - SELECT title, permalink, body, published_at, filter FROM contents - WHERE user_id = 1 AND type = 'Article' ORDER BY published_at; - COPY jekyll TO STDOUT WITH CSV HEADER; - ROLLBACK; - SQL - command = %Q(psql -h #{c[:host] || "localhost"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"}) - puts command - `#{command}` - CSV.process - end - - # This query will pull blog posts from all entries across all blogs. If - # you've got unpublished, deleted or otherwise hidden posts please sift - # through the created posts to make sure nothing is accidently published. - QUERY = "SELECT id, \ - permalink, \ - body, \ - published_at, \ - title \ - FROM contents \ - WHERE user_id = 1 AND \ - type = 'Article' AND \ - published_at IS NOT NULL \ - ORDER BY published_at" - - def self.process(dbname, user, pass, host = 'localhost') - db = Sequel.mysql(dbname, :user => user, - :password => pass, - :host => host, - :encoding => 'utf8') - - FileUtils.mkdir_p "_posts" - - db[QUERY].each do |post| - title = post[:title] - slug = post[:permalink] - date = post[:published_at] - content = post[:body] - - # Ideally, this script would determine the post format (markdown, - # html, etc) and create files with proper extensions. At this point - # it just assumes that markdown will be acceptable. - name = [date.year, date.month, date.day, slug].join('-') + ".markdown" - - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'mt_id' => post[:entry_id], - }.delete_if { |k,v| v.nil? || v == ''}.to_yaml - - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - - end - end -end diff --git a/lib/jekyll/migrators/mt.rb b/lib/jekyll/migrators/mt.rb deleted file mode 100644 index 048c84db..00000000 --- a/lib/jekyll/migrators/mt.rb +++ /dev/null @@ -1,86 +0,0 @@ -# Created by Nick Gerakines, open source and publically available under the -# MIT license. Use this module at your own risk. -# I'm an Erlang/Perl/C++ guy so please forgive my dirty ruby. - -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module MT - # This query will pull blog posts from all entries across all blogs. If - # you've got unpublished, deleted or otherwise hidden posts please sift - # through the created posts to make sure nothing is accidently published. - QUERY = "SELECT entry_id, \ - entry_basename, \ - entry_text, \ - entry_text_more, \ - entry_authored_on, \ - entry_title, \ - entry_convert_breaks \ - FROM mt_entry" - - def self.process(dbname, user, pass, host = 'localhost') - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - - FileUtils.mkdir_p "_posts" - - db[QUERY].each do |post| - title = post[:entry_title] - slug = post[:entry_basename].gsub(/_/, '-') - date = post[:entry_authored_on] - content = post[:entry_text] - more_content = post[:entry_text_more] - entry_convert_breaks = post[:entry_convert_breaks] - - # Be sure to include the body and extended body. - if more_content != nil - content = content + " \n" + more_content - end - - # Ideally, this script would determine the post format (markdown, - # html, etc) and create files with proper extensions. At this point - # it just assumes that markdown will be acceptable. - name = [date.year, date.month, date.day, slug].join('-') + '.' + - self.suffix(entry_convert_breaks) - - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'mt_id' => post[:entry_id], - 'date' => date - }.delete_if { |k,v| v.nil? || v == '' }.to_yaml - - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - end - - def self.suffix(entry_type) - if entry_type.nil? || entry_type.include?("markdown") - # The markdown plugin I have saves this as - # "markdown_with_smarty_pants", so I just look for "markdown". - "markdown" - elsif entry_type.include?("textile") - # This is saved as "textile_2" on my installation of MT 5.1. - "textile" - elsif entry_type == "0" || entry_type.include?("richtext") - # Richtext looks to me like it's saved as HTML, so I include it here. - "html" - else - # Other values might need custom work. - entry_type - end - end - end -end diff --git a/lib/jekyll/migrators/posterous.rb b/lib/jekyll/migrators/posterous.rb deleted file mode 100644 index 0a2280f2..00000000 --- a/lib/jekyll/migrators/posterous.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'rubygems' -require 'jekyll' -require 'fileutils' -require 'net/http' -require 'uri' -require "json" - -# ruby -r './lib/jekyll/migrators/posterous.rb' -e 'Jekyll::Posterous.process(email, pass, api_key, blog)' - -module Jekyll - module Posterous - def self.fetch(uri_str, limit = 10) - # You should choose better exception. - raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0 - - response = nil - Net::HTTP.start('posterous.com') do |http| - req = Net::HTTP::Get.new(uri_str) - req.basic_auth @email, @pass - response = http.request(req) - end - - case response - when Net::HTTPSuccess then response - when Net::HTTPRedirection then fetch(response['location'], limit - 1) - else response.error! - end - end - - def self.process(email, pass, api_token, blog = 'primary') - @email, @pass, @api_token = email, pass, api_token - FileUtils.mkdir_p "_posts" - - posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}").body) - page = 1 - - while posts.any? - posts.each do |post| - title = post["title"] - slug = title.gsub(/[^[:alnum:]]+/, '-').downcase - date = Date.parse(post["display_date"]) - content = post["body_html"] - published = !post["is_private"] - name = "%02d-%02d-%02d-%s.html" % [date.year, date.month, date.day, slug] - - # Get the relevant fields as a hash, delete empty fields and convert - # to YAML for the header - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'published' => published - }.delete_if { |k,v| v.nil? || v == ''}.to_yaml - - # Write out the data and content to file - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - - page += 1 - posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}&page=#{page}").body) - end - end - end -end diff --git a/lib/jekyll/migrators/rss.rb b/lib/jekyll/migrators/rss.rb deleted file mode 100644 index 461abd35..00000000 --- a/lib/jekyll/migrators/rss.rb +++ /dev/null @@ -1,47 +0,0 @@ -# Created by Kendall Buchanan (https://github.com/kendagriff) on 2011-12-22. -# Use at your own risk. The end. -# -# Usage: -# (URL) -# ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('http://yourdomain.com/your-favorite-feed.xml')" -# -# (Local file) -# ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('./somefile/on/your/computer.xml')" - -require 'rubygems' -require 'rss/1.0' -require 'rss/2.0' -require 'open-uri' -require 'fileutils' -require 'yaml' - -module Jekyll - module MigrateRSS - - # The `source` argument may be a URL or a local file. - def self.process(source) - content = "" - open(source) { |s| content = s.read } - rss = RSS::Parser.parse(content, false) - - raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss - - rss.items.each do |item| - formatted_date = item.date.strftime('%Y-%m-%d') - post_name = item.title.split(%r{ |!|/|:|&|-|$|,}).map { |i| i.downcase if i != '' }.compact.join('-') - name = "#{formatted_date}-#{post_name}" - - header = { - 'layout' => 'post', - 'title' => item.title - } - - File.open("_posts/#{name}.html", "w") do |f| - f.puts header.to_yaml - f.puts "---\n" - f.puts item.description - end - end - end - end -end \ No newline at end of file diff --git a/lib/jekyll/migrators/textpattern.rb b/lib/jekyll/migrators/textpattern.rb deleted file mode 100644 index 3b370ed9..00000000 --- a/lib/jekyll/migrators/textpattern.rb +++ /dev/null @@ -1,58 +0,0 @@ -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module TextPattern - # Reads a MySQL database via Sequel and creates a post file for each post. - # The only posts selected are those with a status of 4 or 5, which means - # "live" and "sticky" respectively. - # Other statuses are 1 => draft, 2 => hidden and 3 => pending. - QUERY = "SELECT Title, \ - url_title, \ - Posted, \ - Body, \ - Keywords \ - FROM textpattern \ - WHERE Status = '4' OR \ - Status = '5'" - - def self.process(dbname, user, pass, host = 'localhost') - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - - FileUtils.mkdir_p "_posts" - - db[QUERY].each do |post| - # Get required fields and construct Jekyll compatible name. - title = post[:Title] - slug = post[:url_title] - date = post[:Posted] - content = post[:Body] - - name = [date.strftime("%Y-%m-%d"), slug].join('-') + ".textile" - - # Get the relevant fields as a hash, delete empty fields and convert - # to YAML for the header. - data = { - 'layout' => 'post', - 'title' => title.to_s, - 'tags' => post[:Keywords].split(',') - }.delete_if { |k,v| v.nil? || v == ''}.to_yaml - - # Write out the data and content to file. - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - end - end -end diff --git a/lib/jekyll/migrators/tumblr.rb b/lib/jekyll/migrators/tumblr.rb deleted file mode 100644 index 367a83c9..00000000 --- a/lib/jekyll/migrators/tumblr.rb +++ /dev/null @@ -1,195 +0,0 @@ -require 'rubygems' -require 'open-uri' -require 'fileutils' -require 'nokogiri' -require 'date' -require 'json' -require 'uri' -require 'jekyll' - -module Jekyll - module Tumblr - def self.process(url, format = "html", grab_images = false, - add_highlights = false, rewrite_urls = true) - @grab_images = grab_images - FileUtils.mkdir_p "_posts/tumblr" - url += "/api/read/json/" - per_page = 50 - posts = [] - # Two passes are required so that we can rewrite URLs. - # First pass builds up an array of each post as a hash. - begin - current_page = (current_page || -1) + 1 - feed = open(url + "?num=#{per_page}&start=#{current_page * per_page}") - json = feed.readlines.join("\n")[21...-2] # Strip Tumblr's JSONP chars. - blog = JSON.parse(json) - puts "Page: #{current_page + 1} - Posts: #{blog["posts"].size}" - posts += blog["posts"].map { |post| post_to_hash(post, format) } - end until blog["posts"].size < per_page - # Rewrite URLs and create redirects. - posts = rewrite_urls_and_redirects posts if rewrite_urls - # Second pass for writing post files. - posts.each do |post| - if format == "md" - post[:content] = html_to_markdown post[:content] - post[:content] = add_syntax_highlights post[:content] if add_highlights - end - File.open("_posts/tumblr/#{post[:name]}", "w") do |f| - f.puts post[:header].to_yaml + "---\n" + post[:content] - end - end - end - - private - - # Converts each type of Tumblr post to a hash with all required - # data for Jekyll. - def self.post_to_hash(post, format) - case post['type'] - when "regular" - title = post["regular-title"] - content = post["regular-body"] - when "link" - title = post["link-text"] || post["link-url"] - content = "#{title}" - unless post["link-description"].nil? - content << "
" + post["link-description"] - end - when "photo" - title = post["photo-caption"] - max_size = post.keys.map{ |k| k.gsub("photo-url-", "").to_i }.max - url = post["photo-url"] || post["photo-url-#{max_size}"] - ext = "." + post[post.keys.select { |k| - k =~ /^photo-url-/ && post[k].split("/").last =~ /\./ - }.first].split(".").last - content = "" - unless post["photo-link-url"].nil? - content = "#{content}" - end - when "audio" - if !post["id3-title"].nil? - title = post["id3-title"] - content = post.at["audio-player"] + "
" + post["audio-caption"] - else - title = post["audio-caption"] - content = post.at["audio-player"] - end - when "quote" - title = post["quote-text"] - content = "
#{post["quote-text"]}
" - unless post["quote-source"].nil? - content << "—" + post["quote-source"] - end - when "conversation" - title = post["conversation-title"] - content = "
" - post["conversation"]["line"].each do |line| - content << "
#{line['label']}
#{line}
" - end - content << "
" - when "video" - title = post["video-title"] - content = post["video-player"] - unless post["video-caption"].nil? - content << "
" + post["video-caption"] - end - end - date = Date.parse(post['date']).to_s - title = Nokogiri::HTML(title).text - slug = title.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '') - { - :name => "#{date}-#{slug}.#{format}", - :header => { - "layout" => "post", - "title" => title, - "tags" => post["tags"], - }, - :content => content, - :url => post["url"], - :slug => post["url-with-slug"], - } - end - - # Create a Hash of old urls => new urls, for rewriting and - # redirects, and replace urls in each post. Instantiate Jekyll - # site/posts to get the correct permalink format. - def self.rewrite_urls_and_redirects(posts) - site = Jekyll::Site.new(Jekyll.configuration({})) - dir = File.join(File.dirname(__FILE__), "..") - urls = Hash[posts.map { |post| - # Create an initial empty file for the post so that - # we can instantiate a post object. - File.open("_posts/tumblr/#{post[:name]}", "w") - tumblr_url = URI.parse(post[:slug]).path - jekyll_url = Jekyll::Post.new(site, dir, "", "tumblr/" + post[:name]).url - redirect_dir = tumblr_url.sub(/\//, "") + "/" - FileUtils.mkdir_p redirect_dir - File.open(redirect_dir + "index.html", "w") do |f| - f.puts "" - end - [tumblr_url, jekyll_url] - }] - posts.map { |post| - urls.each do |tumblr_url, jekyll_url| - post[:content].gsub!(/#{tumblr_url}/i, jekyll_url) - end - post - } - end - - # Uses Python's html2text to convert a post's content to - # markdown. Preserve HTML tables as per the markdown docs. - def self.html_to_markdown(content) - preserve = ["table", "tr", "th", "td"] - preserve.each do |tag| - content.gsub!(/<#{tag}/i, "$$" + tag) - content.gsub!(/<\/#{tag}/i, "||" + tag) - end - content = %x[echo '#{content.gsub("'", "''")}' | html2text] - preserve.each do |tag| - content.gsub!("$$" + tag, "<" + tag) - content.gsub!("||" + tag, " -require 'fileutils' -require 'rubygems' -require 'sequel' -require 'yaml' - -module Jekyll - module Typo - # This SQL *should* work for both MySQL and PostgreSQL, but I haven't - # tested PostgreSQL yet (as of 2008-12-16). - SQL = <<-EOS - SELECT c.id id, - c.title title, - c.permalink slug, - c.body body, - c.published_at date, - c.state state, - COALESCE(tf.name, 'html') filter - FROM contents c - LEFT OUTER JOIN text_filters tf - ON c.text_filter_id = tf.id - EOS - - def self.process dbname, user, pass, host='localhost' - FileUtils.mkdir_p '_posts' - db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') - db[SQL].each do |post| - next unless post[:state] =~ /published/ - - name = [ sprintf("%.04d", post[:date].year), - sprintf("%.02d", post[:date].month), - sprintf("%.02d", post[:date].day), - post[:slug].strip ].join('-') - - # Can have more than one text filter in this field, but we just want - # the first one for this. - name += '.' + post[:filter].split(' ')[0] - - File.open("_posts/#{name}", 'w') do |f| - f.puts({ 'layout' => 'post', - 'title' => post[:title].to_s, - 'typo_id' => post[:id] - }.delete_if { |k, v| v.nil? || v == '' }.to_yaml) - f.puts '---' - f.puts post[:body].delete("\r") - end - end - end - - end -end diff --git a/lib/jekyll/migrators/wordpress.rb b/lib/jekyll/migrators/wordpress.rb deleted file mode 100644 index d2d19039..00000000 --- a/lib/jekyll/migrators/wordpress.rb +++ /dev/null @@ -1,294 +0,0 @@ -require 'rubygems' -require 'sequel' -require 'fileutils' -require 'yaml' - -# NOTE: This converter requires Sequel and the MySQL gems. -# The MySQL gem can be difficult to install on OS X. Once you have MySQL -# installed, running the following commands should work: -# $ sudo gem install sequel -# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config - -module Jekyll - module WordPress - - # Main migrator function. Call this to perform the migration. - # - # dbname:: The name of the database - # user:: The database user name - # pass:: The database user's password - # host:: The address of the MySQL database host. Default: 'localhost' - # options:: A hash table of configuration options. - # - # Supported options are: - # - # :table_prefix:: Prefix of database tables used by WordPress. - # Default: 'wp_' - # :clean_entities:: If true, convert non-ASCII characters to HTML - # entities in the posts, comments, titles, and - # names. Requires the 'htmlentities' gem to - # work. Default: true. - # :comments:: If true, migrate post comments too. Comments - # are saved in the post's YAML front matter. - # Default: true. - # :categories:: If true, save the post's categories in its - # YAML front matter. - # :tags:: If true, save the post's tags in its - # YAML front matter. - # :more_excerpt:: If true, when a post has no excerpt but - # does have a tag, use the - # preceding post content as the excerpt. - # Default: true. - # :more_anchor:: If true, convert a tag into - # two HTML anchors with ids "more" and - # "more-NNN" (where NNN is the post number). - # Default: true. - # :status:: Array of allowed post statuses. Only - # posts with matching status will be migrated. - # Known statuses are :publish, :draft, :private, - # and :revision. If this is nil or an empty - # array, all posts are migrated regardless of - # status. Default: [:publish]. - # - def self.process(dbname, user, pass, host='localhost', options={}) - options = { - :table_prefix => 'wp_', - :clean_entities => true, - :comments => true, - :categories => true, - :tags => true, - :more_excerpt => true, - :more_anchor => true, - :status => [:publish] # :draft, :private, :revision - }.merge(options) - - if options[:clean_entities] - begin - require 'htmlentities' - rescue LoadError - STDERR.puts "Could not require 'htmlentities', so the " + - ":clean_entities option is now disabled." - options[:clean_entities] = false - end - end - - FileUtils.mkdir_p("_posts") - - db = Sequel.mysql(dbname, :user => user, :password => pass, - :host => host, :encoding => 'utf8') - - px = options[:table_prefix] - - posts_query = " - SELECT - posts.ID AS `id`, - posts.guid AS `guid`, - posts.post_type AS `type`, - posts.post_status AS `status`, - posts.post_title AS `title`, - posts.post_name AS `slug`, - posts.post_date AS `date`, - posts.post_content AS `content`, - posts.post_excerpt AS `excerpt`, - posts.comment_count AS `comment_count`, - users.display_name AS `author`, - users.user_login AS `author_login`, - users.user_email AS `author_email`, - users.user_url AS `author_url` - FROM #{px}posts AS `posts` - LEFT JOIN #{px}users AS `users` - ON posts.post_author = users.ID" - - if options[:status] and not options[:status].empty? - status = options[:status][0] - posts_query << " - WHERE posts.post_status = '#{status.to_s}'" - options[:status][1..-1].each do |status| - posts_query << " OR - posts.post_status = '#{status.to_s}'" - end - end - - db[posts_query].each do |post| - process_post(post, db, options) - end - end - - - def self.process_post(post, db, options) - px = options[:table_prefix] - - title = post[:title] - if options[:clean_entities] - title = clean_entities(title) - end - - slug = post[:slug] - if !slug or slug.empty? - slug = sluggify(title) - end - - date = post[:date] || Time.now - name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, - date.day, slug] - content = post[:content].to_s - if options[:clean_entities] - content = clean_entities(content) - end - - excerpt = post[:excerpt].to_s - - more_index = content.index(//) - more_anchor = nil - if more_index - if options[:more_excerpt] and - (post[:excerpt].nil? or post[:excerpt].empty?) - excerpt = content[0...more_index] - end - if options[:more_anchor] - more_link = "more" - content.sub!(//, - "" + - "") - end - end - - categories = [] - tags = [] - - if options[:categories] or options[:tags] - - cquery = - "SELECT - terms.name AS `name`, - ttax.taxonomy AS `type` - FROM - #{px}terms AS `terms`, - #{px}term_relationships AS `trels`, - #{px}term_taxonomy AS `ttax` - WHERE - trels.object_id = '#{post[:id]}' AND - trels.term_taxonomy_id = ttax.term_taxonomy_id AND - terms.term_id = ttax.term_id" - - db[cquery].each do |term| - if options[:categories] and term[:type] == "category" - if options[:clean_entities] - categories << clean_entities(term[:name]) - else - categories << term[:name] - end - elsif options[:tags] and term[:type] == "post_tag" - if options[:clean_entities] - tags << clean_entities(term[:name]) - else - tags << term[:name] - end - end - end - end - - comments = [] - - if options[:comments] and post[:comment_count].to_i > 0 - cquery = - "SELECT - comment_ID AS `id`, - comment_author AS `author`, - comment_author_email AS `author_email`, - comment_author_url AS `author_url`, - comment_date AS `date`, - comment_date_gmt AS `date_gmt`, - comment_content AS `content` - FROM #{px}comments - WHERE - comment_post_ID = '#{post[:id]}' AND - comment_approved != 'spam'" - - - db[cquery].each do |comment| - - comcontent = comment[:content].to_s - if comcontent.respond_to?(:force_encoding) - comcontent.force_encoding("UTF-8") - end - if options[:clean_entities] - comcontent = clean_entities(comcontent) - end - comauthor = comment[:author].to_s - if options[:clean_entities] - comauthor = clean_entities(comauthor) - end - - comments << { - 'id' => comment[:id].to_i, - 'author' => comauthor, - 'author_email' => comment[:author_email].to_s, - 'author_url' => comment[:author_url].to_s, - 'date' => comment[:date].to_s, - 'date_gmt' => comment[:date_gmt].to_s, - 'content' => comcontent, - } - end - - comments.sort!{ |a,b| a['id'] <=> b['id'] } - end - - # Get the relevant fields as a hash, delete empty fields and - # convert to YAML for the header. - data = { - 'layout' => post[:type].to_s, - 'status' => post[:status].to_s, - 'published' => (post[:status].to_s == "publish"), - 'title' => title.to_s, - 'author' => post[:author].to_s, - 'author_login' => post[:author_login].to_s, - 'author_email' => post[:author_email].to_s, - 'author_url' => post[:author_url].to_s, - 'excerpt' => excerpt, - 'more_anchor' => more_anchor, - 'wordpress_id' => post[:id], - 'wordpress_url' => post[:guid].to_s, - 'date' => date, - 'categories' => options[:categories] ? categories : nil, - 'tags' => options[:tags] ? tags : nil, - 'comments' => options[:comments] ? comments : nil, - }.delete_if { |k,v| v.nil? || v == '' }.to_yaml - - # Write out the data and content to file - File.open("_posts/#{name}", "w") do |f| - f.puts data - f.puts "---" - f.puts content - end - end - - - def self.clean_entities( text ) - if text.respond_to?(:force_encoding) - text.force_encoding("UTF-8") - end - text = HTMLEntities.new.encode(text, :named) - # We don't want to convert these, it would break all - # HTML tags in the post and comments. - text.gsub!("&", "&") - text.gsub!("<", "<") - text.gsub!(">", ">") - text.gsub!(""", '"') - text.gsub!("'", "'") - text - end - - - def self.sluggify( title ) - begin - require 'unidecode' - title = title.to_ascii - rescue LoadError - STDERR.puts "Could not require 'unidecode'. If your post titles have non-ASCII characters, you could get nicer permalinks by installing unidecode." - end - title.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-") - end - - end -end diff --git a/lib/jekyll/migrators/wordpressdotcom.rb b/lib/jekyll/migrators/wordpressdotcom.rb deleted file mode 100644 index 701c2af4..00000000 --- a/lib/jekyll/migrators/wordpressdotcom.rb +++ /dev/null @@ -1,70 +0,0 @@ -# coding: utf-8 - -require 'rubygems' -require 'hpricot' -require 'fileutils' -require 'yaml' -require 'time' - -module Jekyll - # This importer takes a wordpress.xml file, which can be exported from your - # wordpress.com blog (/wp-admin/export.php). - module WordpressDotCom - def self.process(filename = "wordpress.xml") - import_count = Hash.new(0) - doc = Hpricot::XML(File.read(filename)) - - (doc/:channel/:item).each do |item| - title = item.at(:title).inner_text.strip - permalink_title = item.at('wp:post_name').inner_text - # Fallback to "prettified" title if post_name is empty (can happen) - if permalink_title == "" - permalink_title = title.downcase.split.join('-') - end - - date = Time.parse(item.at('wp:post_date').inner_text) - status = item.at('wp:status').inner_text - - if status == "publish" - published = true - else - published = false - end - - type = item.at('wp:post_type').inner_text - tags = (item/:category).map{|c| c.inner_text}.reject{|c| c == 'Uncategorized'}.uniq - - metas = Hash.new - item.search("wp:postmeta").each do |meta| - key = meta.at('wp:meta_key').inner_text - value = meta.at('wp:meta_value').inner_text - metas[key] = value; - end - - name = "#{date.strftime('%Y-%m-%d')}-#{permalink_title}.html" - header = { - 'layout' => type, - 'title' => title, - 'tags' => tags, - 'status' => status, - 'type' => type, - 'published' => published, - 'meta' => metas - } - - FileUtils.mkdir_p "_#{type}s" - File.open("_#{type}s/#{name}", "w") do |f| - f.puts header.to_yaml - f.puts '---' - f.puts item.at('content:encoded').inner_text - end - - import_count[type] += 1 - end - - import_count.each do |key, value| - puts "Imported #{value} #{key}s" - end - end - end -end From e3bd250e696c38b54f5fe196e618183b2909d07d Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Wed, 9 Jan 2013 23:42:57 +0000 Subject: [PATCH 18/20] Remove old jekyll command --- bin/jekyll | 347 ++++++---------------------------------- bin/jekyll2 | 72 --------- features/support/env.rb | 2 +- 3 files changed, 49 insertions(+), 372 deletions(-) delete mode 100755 bin/jekyll2 diff --git a/bin/jekyll b/bin/jekyll index f94420e8..3df10e4f 100755 --- a/bin/jekyll +++ b/bin/jekyll @@ -1,323 +1,72 @@ #!/usr/bin/env ruby -$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib]) +$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib }) -help = < ./_site - jekyll # . -> - jekyll # -> - jekyll import # imports posts using named import script - - Configuration is read from '/_config.yml' but can be overriden - using the following options: - -HELP - -require 'optparse' +require 'commander/import' require 'jekyll' +# Details about Jekyll +program :name, 'jekyll' +program :version, Jekyll::VERSION +program :description, 'Jekyll is a blog-aware, static site generator in Ruby' -exec = {} -options = {} -opts = OptionParser.new do |opts| - opts.banner = help +default_command :help - # TODO: delete, migrator related - opts.on("--file [PATH]", "File to import from") do |import_file| - options['file'] = import_file - end - - # TODO: delete, migrator related - opts.on("--dbname [TEXT]", "DB to import from") do |import_dbname| - options['dbname'] = import_dbname - end - - # TODO: delete, migrator related - opts.on("--user [TEXT]", "Username to use when importing") do |import_user| - options['user'] = import_user - end - - # TODO: delete, migrator related - opts.on("--pass [TEXT]", "Password to use when importing") do |import_pass| - options['pass'] = import_pass - end - - # TODO: delete, migrator related - opts.on("--host [HOST ADDRESS]", "Host to import from") do |import_host| - options['host'] = import_host - end - - # TODO: delete, migrator related - opts.on("--site [SITE NAME]", "Site to import from") do |import_site| - options['site'] = import_site - end - - # TODO: global option - opts.on("--[no-]safe", "Safe mode (default unsafe)") do |safe| - options['safe'] = safe - end +# Global options available to every command +global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' +global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' +global_option '--safe', 'Safe mode (defaults to false)' +global_option '--plugins', 'Plugins directory (defaults to ./_plugins)' +global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' - # TODO: option for build and serve command - opts.on("--[no-]auto", "Auto-regenerate") do |auto| - options['auto'] = auto - end +command :build do |c| + c.syntax = 'jekyll build [options]' + c.description = 'Build your site with the option of auto-renegeration' - # TODO: new serve command - opts.on("--server [PORT]", "Start web server (default port 4000)") do |port| - options['server'] = true - options['server_port'] = port unless port.nil? - end + c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '--lsi', 'Use LSI for improved related posts' - # TODO: remove - opts.on("--no-server", "Do not start a web server") do |part| - options['server'] = false - end - - # TODO: option for serve command - opts.on("--base-url [BASE_URL]", "Serve website from a given base URL (default '/'") do |baseurl| - options['baseurl'] = baseurl - end - - # TODO: does anyone actually use this? - opts.on("--default-mimetype [MT]", "Mimetype to use when no file extension (if --server)") do |mt| - options['default-mimetype'] = mt - end - - # TODO: option for build and serve command - opts.on("--[no-]lsi", "Use LSI for better related posts") do |lsi| - options['lsi'] = lsi - end - - # TODO: remove and just enable all the time? - opts.on("--[no-]pygments", "Use pygments to highlight code") do |pygments| - options['pygments'] = pygments - end - - # TODO: read from config - opts.on("--rdiscount", "Use rdiscount gem for Markdown") do - options['markdown'] = 'rdiscount' - end - - # TODO: read from config - opts.on("--redcarpet", "Use redcarpet gem for Markdown") do - options['markdown'] = 'redcarpet' - end - - # TODO: read from config - opts.on("--kramdown", "Use kramdown gem for Markdown") do - options['markdown'] = 'kramdown' - end - - # TODO: remove and just generate the site as is? - opts.on("--time [TIME]", "Time to generate the site for") do |time| - options['time'] = Time.parse(time) - end - - # TODO: remove and just render all posts which aren't 'unpublished'? - opts.on("--[no-]future", "Render future dated posts") do |future| - options['future'] = future - end - - # TODO: read from config - opts.on("--permalink [TYPE]", "Use 'date' (default) for YYYY/MM/DD") do |style| - options['permalink'] = style unless style.nil? - end - - # TODO: read from config - opts.on("--paginate [POSTS_PER_PAGE]", "Paginate a blog's posts") do |per_page| - begin - options['paginate'] = per_page.to_i - raise ArgumentError if options['paginate'] == 0 - rescue - puts 'you must specify a number of posts by page bigger than 0' - exit 0 - end - end - - # TODO: read from config - opts.on("--paginate_path [PAGINATED_URL_FORMAT]", "Leave blank for /page") do |paginate_path| - begin - options['paginate_path'] = paginate_path - raise ArgumentError if options['paginate_path'].nil? - rescue - puts 'You must specify a pagination url format' - exit 0 - end - end - - # TODO: read from config - opts.on("--limit_posts [MAX_POSTS]", "Limit the number of posts to publish") do |limit_posts| - begin - options['limit_posts'] = limit_posts.to_i - raise ArgumentError if options['limit_posts'] < 1 - rescue - puts 'you must specify a number of posts by page bigger than 0' - exit 0 - end - end - - # TODO: read from config - opts.on("--url [URL]", "Set custom site.url") do |url| - options['url'] = url - end - - opts.on("--version", "Display current version") do - puts "Jekyll " + Jekyll::VERSION - exit 0 + c.action do |args, options| + options.defaults :serving => false + options = Jekyll.configuration(options.__hash__) + Jekyll::BuildCommand.process(options) end end -# Read command line options into `options` hash -opts.parse! +command :serve do |c| + c.syntax = 'jekyll serve [options]' + c.description = 'Serve your site locally with the option of auto-regeneration' + c.option '-w', '--watch', 'Watch for changes and rebuild' + c.option '--lsi', 'Use LSI for improved related posts' -# Check for import stuff -if ARGV.size > 0 - if ARGV[0] == 'import' - migrator = ARGV[1] + c.option '-p', '--port [PORT]', 'Port to listen on' + c.option '-h', '--host [HOST]', 'Host to bind to' + c.option '-b', '--baseurl [URL]', 'Base URL' - if migrator.nil? - puts "Invalid options. Run `jekyll --help` for assistance." - exit(1) - else - migrator = migrator.downcase - end + c.action do |args, options| + options.default :port => '4000', + :host => '0.0.0.0', + :baseurl => '/', + :serving => true - cmd_options = [] - ['file', 'dbname', 'user', 'pass', 'host', 'site'].each do |p| - cmd_options << "\"#{options[p]}\"" unless options[p].nil? - end - - # It's import time - puts "Importing..." - - # Ideally, this shouldn't be necessary. Maybe parse the actual - # src files for the migrator name? - migrators = { - :posterous => 'Posterous', - :wordpressdotcom => 'WordpressDotCom', - :wordpress => 'WordPress', - :csv => 'CSV', - :drupal => 'Drupal', - :enki => 'Enki', - :mephisto => 'Mephisto', - :mt => 'MT', - :textpattern => 'TextPattern', - :tumblr => 'Tumblr', - :typo => 'Typo' - } - - app_root = File.join(File.dirname(__FILE__), '..') - - require "#{app_root}/lib/jekyll/migrators/#{migrator}" - - if Jekyll.const_defined?(migrators[migrator.to_sym]) - migrator_class = Jekyll.const_get(migrators[migrator.to_sym]) - migrator_class.process(*cmd_options) - else - puts "Invalid migrator. Run `jekyll --help` for assistance." - exit(1) - end - - exit(0) + options = Jekyll.configuration(options.__hash__) + Jekyll::BuildCommand.process(options) + Jekyll::ServeCommand.process(options) end end +command :migrate do |c| + c.syntax = 'jekyll migrate [options]' + c.description = 'Migrate your own blog to Jekyll' + c.option '--file', 'File to migrate from' + c.option '--dbname', 'Database name to migrate from' + c.option '--user', 'Username to use when migrating' + c.option '--pass', 'Password to use when migrating' + c.option '--host', 'Host address to use when migrating' -# Get source and destination from command line -# TODO: source and destination are now global options -case ARGV.size - when 0 - when 1 - options['destination'] = ARGV[0] - when 2 - options['source'] = ARGV[0] - options['destination'] = ARGV[1] - else - puts "Invalid options. Run `jekyll --help` for assistance." - exit(1) -end - -options = Jekyll.configuration(options) - -# Get source and destination directories (possibly set by config file) -source = options['source'] -destination = options['destination'] - -# Files to watch -def globs(source, destination) - Dir.chdir(source) do - dirs = Dir['*'].select { |x| File.directory?(x) } - dirs -= [destination] - dirs = dirs.map { |x| "#{x}/**/*" } - dirs += ['*'] + c.action do |args, options| + Jekyll::MigrateCommand.process(args.first, options) end end - -# Create the Site -site = Jekyll::Site.new(options) - -# Run the directory watcher for auto-generation, if required -if options['auto'] - require 'directory_watcher' - - puts "Auto-regenerating enabled: #{source} -> #{destination}" - - dw = DirectoryWatcher.new(source) - dw.interval = 1 - dw.glob = globs(source, destination) - - dw.add_observer do |*args| - t = Time.now.strftime("%Y-%m-%d %H:%M:%S") - puts "[#{t}] regeneration: #{args.size} files changed" - site.process - end - - dw.start - - unless options['server'] - loop { sleep 1000 } - end -else - puts "Building site: #{source} -> #{destination}" - begin - site.process - rescue Jekyll::FatalException => e - puts - puts "ERROR: YOUR SITE COULD NOT BE BUILT:" - puts "------------------------------------" - puts e.message - exit(1) - end - puts "Successfully generated site: #{source} -> #{destination}" -end - -# Run the server on the specified port, if required -if options['server'] - require 'webrick' - include WEBrick - - FileUtils.mkdir_p(destination) - - mime_types = WEBrick::HTTPUtils::DefaultMimeTypes - mime_types.store 'js', 'application/javascript' - if options['default-mimetype'] - mime_types.store(nil, options['default-mimetype']) - end - - s = HTTPServer.new( - :Port => options['server_port'], - :MimeTypes => mime_types - ) - s.mount(options['baseurl'], HTTPServlet::FileHandler, destination) - t = Thread.new { - s.start - } - - trap("INT") { s.shutdown } - t.join() -end diff --git a/bin/jekyll2 b/bin/jekyll2 deleted file mode 100755 index 3df10e4f..00000000 --- a/bin/jekyll2 +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env ruby - -$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib }) - -require 'commander/import' -require 'jekyll' - -# Details about Jekyll -program :name, 'jekyll' -program :version, Jekyll::VERSION -program :description, 'Jekyll is a blog-aware, static site generator in Ruby' - -default_command :help - -# Global options available to every command -global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' -global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' -global_option '--safe', 'Safe mode (defaults to false)' -global_option '--plugins', 'Plugins directory (defaults to ./_plugins)' -global_option '--layouts', 'Layouts directory (defaults to ./_layouts)' - -command :build do |c| - c.syntax = 'jekyll build [options]' - c.description = 'Build your site with the option of auto-renegeration' - - c.option '-w', '--watch', 'Watch for changes and rebuild' - c.option '--lsi', 'Use LSI for improved related posts' - - c.action do |args, options| - options.defaults :serving => false - options = Jekyll.configuration(options.__hash__) - Jekyll::BuildCommand.process(options) - end -end - -command :serve do |c| - c.syntax = 'jekyll serve [options]' - c.description = 'Serve your site locally with the option of auto-regeneration' - - c.option '-w', '--watch', 'Watch for changes and rebuild' - c.option '--lsi', 'Use LSI for improved related posts' - - c.option '-p', '--port [PORT]', 'Port to listen on' - c.option '-h', '--host [HOST]', 'Host to bind to' - c.option '-b', '--baseurl [URL]', 'Base URL' - - c.action do |args, options| - options.default :port => '4000', - :host => '0.0.0.0', - :baseurl => '/', - :serving => true - - options = Jekyll.configuration(options.__hash__) - Jekyll::BuildCommand.process(options) - Jekyll::ServeCommand.process(options) - end -end - -command :migrate do |c| - c.syntax = 'jekyll migrate [options]' - c.description = 'Migrate your own blog to Jekyll' - - c.option '--file', 'File to migrate from' - c.option '--dbname', 'Database name to migrate from' - c.option '--user', 'Username to use when migrating' - c.option '--pass', 'Password to use when migrating' - c.option '--host', 'Host address to use when migrating' - - c.action do |args, options| - Jekyll::MigrateCommand.process(args.first, options) - end -end diff --git a/features/support/env.rb b/features/support/env.rb index 912b735c..1ed330a1 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -7,7 +7,7 @@ World do end TEST_DIR = File.join('/', 'tmp', 'jekyll') -JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll2') +JEKYLL_PATH = File.join(ENV['PWD'], 'bin', 'jekyll') def run_jekyll(opts = {}) command = JEKYLL_PATH From e210a0603fa76fe4e31dc80729c8ff9f8e61b38e Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Fri, 11 Jan 2013 00:30:17 +0000 Subject: [PATCH 19/20] Revert "Remove migrators" This reverts commit b9da30bc8ffa4a2d0a9280e755d0904609d4fc97. --- Rakefile | 15 ++ lib/jekyll/migrators/csv.rb | 26 +++ lib/jekyll/migrators/drupal.rb | 103 +++++++++ lib/jekyll/migrators/enki.rb | 49 ++++ lib/jekyll/migrators/joomla.rb | 53 +++++ lib/jekyll/migrators/marley.rb | 52 +++++ lib/jekyll/migrators/mephisto.rb | 84 +++++++ lib/jekyll/migrators/mt.rb | 86 +++++++ lib/jekyll/migrators/posterous.rb | 67 ++++++ lib/jekyll/migrators/rss.rb | 47 ++++ lib/jekyll/migrators/textpattern.rb | 58 +++++ lib/jekyll/migrators/tumblr.rb | 195 ++++++++++++++++ lib/jekyll/migrators/typo.rb | 51 ++++ lib/jekyll/migrators/wordpress.rb | 294 ++++++++++++++++++++++++ lib/jekyll/migrators/wordpressdotcom.rb | 70 ++++++ 15 files changed, 1250 insertions(+) create mode 100644 lib/jekyll/migrators/csv.rb create mode 100644 lib/jekyll/migrators/drupal.rb create mode 100644 lib/jekyll/migrators/enki.rb create mode 100644 lib/jekyll/migrators/joomla.rb create mode 100644 lib/jekyll/migrators/marley.rb create mode 100644 lib/jekyll/migrators/mephisto.rb create mode 100644 lib/jekyll/migrators/mt.rb create mode 100644 lib/jekyll/migrators/posterous.rb create mode 100644 lib/jekyll/migrators/rss.rb create mode 100644 lib/jekyll/migrators/textpattern.rb create mode 100644 lib/jekyll/migrators/tumblr.rb create mode 100644 lib/jekyll/migrators/typo.rb create mode 100644 lib/jekyll/migrators/wordpress.rb create mode 100644 lib/jekyll/migrators/wordpressdotcom.rb diff --git a/Rakefile b/Rakefile index d1917c5a..ff52d677 100644 --- a/Rakefile +++ b/Rakefile @@ -82,6 +82,21 @@ end # ############################################################################# +namespace :migrate do + desc "Migrate from mephisto in the current directory" + task :mephisto do + sh %q(ruby -r './lib/jekyll/migrators/mephisto' -e 'Jekyll::Mephisto.postgres(:database => "#{ENV["DB"]}")') + end + desc "Migrate from Movable Type in the current directory" + task :mt do + sh %q(ruby -r './lib/jekyll/migrators/mt' -e 'Jekyll::MT.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")') + end + desc "Migrate from Typo in the current directory" + task :typo do + sh %q(ruby -r './lib/jekyll/migrators/typo' -e 'Jekyll::Typo.process("#{ENV["DB"]}", "#{ENV["USER"]}", "#{ENV["PASS"]}")') + end +end + begin require 'cucumber/rake/task' Cucumber::Rake::Task.new(:features) do |t| diff --git a/lib/jekyll/migrators/csv.rb b/lib/jekyll/migrators/csv.rb new file mode 100644 index 00000000..ce5203b7 --- /dev/null +++ b/lib/jekyll/migrators/csv.rb @@ -0,0 +1,26 @@ +module Jekyll + module CSV + # Reads a csv with title, permalink, body, published_at, and filter. + # It creates a post file for each row in the csv + def self.process(file = "posts.csv") + FileUtils.mkdir_p "_posts" + posts = 0 + FasterCSV.foreach(file) do |row| + next if row[0] == "title" + posts += 1 + name = row[3].split(" ")[0]+"-"+row[1]+(row[4] =~ /markdown/ ? ".markdown" : ".textile") + File.open("_posts/#{name}", "w") do |f| + f.puts <<-HEADER +--- +layout: post +title: #{row[0]} +--- + + HEADER + f.puts row[2] + end + end + "Created #{posts} posts!" + end + end +end diff --git a/lib/jekyll/migrators/drupal.rb b/lib/jekyll/migrators/drupal.rb new file mode 100644 index 00000000..7fd16aef --- /dev/null +++ b/lib/jekyll/migrators/drupal.rb @@ -0,0 +1,103 @@ +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module Drupal + # Reads a MySQL database via Sequel and creates a post file for each post + # in wp_posts that has post_status = 'publish'. This restriction is made + # because 'draft' posts are not guaranteed to have valid dates. + QUERY = "SELECT n.nid, \ + n.title, \ + nr.body, \ + n.created, \ + n.status \ + FROM node AS n, \ + node_revisions AS nr \ + WHERE (n.type = 'blog' OR n.type = 'story') \ + AND n.vid = nr.vid" + + def self.process(dbname, user, pass, host = 'localhost', prefix = '') + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + + if prefix != '' + QUERY[" node "] = " " + prefix + "node " + QUERY[" node_revisions "] = " " + prefix + "node_revisions " + end + + FileUtils.mkdir_p "_posts" + FileUtils.mkdir_p "_drafts" + + # Create the refresh layout + # Change the refresh url if you customized your permalink config + File.open("_layouts/refresh.html", "w") do |f| + f.puts < + + + + + + +EOF + end + + db[QUERY].each do |post| + # Get required fields and construct Jekyll compatible name + node_id = post[:nid] + title = post[:title] + content = post[:body] + created = post[:created] + time = Time.at(created) + is_published = post[:status] == 1 + dir = is_published ? "_posts" : "_drafts" + slug = title.strip.downcase.gsub(/(&|&)/, ' and ').gsub(/[\s\.\/\\]/, '-').gsub(/[^\w-]/, '').gsub(/[-_]{2,}/, '-').gsub(/^[-_]/, '').gsub(/[-_]$/, '') + name = time.strftime("%Y-%m-%d-") + slug + '.md' + + # Get the relevant fields as a hash, delete empty fields and convert + # to YAML for the header + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'created' => created, + }.delete_if { |k,v| v.nil? || v == ''}.to_yaml + + # Write out the data and content to file + File.open("#{dir}/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + + # Make a file to redirect from the old Drupal URL + if is_published + aliases = db["SELECT dst FROM #{prefix}url_alias WHERE src = ?", "node/#{node_id}"].all + + aliases.push(:dst => "node/#{node_id}") + + aliases.each do |url_alias| + FileUtils.mkdir_p url_alias[:dst] + File.open("#{url_alias[:dst]}/index.md", "w") do |f| + f.puts "---" + f.puts "layout: refresh" + f.puts "refresh_to_post_id: /#{time.strftime("%Y/%m/%d/") + slug}" + f.puts "---" + end + end + end + end + + # TODO: Make dirs & files for nodes of type 'page' + # Make refresh pages for these as well + + # TODO: Make refresh dirs & files according to entries in url_alias table + end + end +end diff --git a/lib/jekyll/migrators/enki.rb b/lib/jekyll/migrators/enki.rb new file mode 100644 index 00000000..61cb2562 --- /dev/null +++ b/lib/jekyll/migrators/enki.rb @@ -0,0 +1,49 @@ +# Adapted by Rodrigo Pinto +# Based on typo.rb by Toby DiPasquale + +require 'fileutils' +require 'rubygems' +require 'sequel' + +module Jekyll + module Enki + SQL = <<-EOS + SELECT p.id, + p.title, + p.slug, + p.body, + p.published_at as date, + p.cached_tag_list as tags + FROM posts p + EOS + + # Just working with postgres, but can be easily adapted + # to work with both mysql and postgres. + def self.process(dbname, user, pass, host = 'localhost') + FileUtils.mkdir_p('_posts') + db = Sequel.postgres(:database => dbname, + :user => user, + :password => pass, + :host => host, + :encoding => 'utf8') + + db[SQL].each do |post| + name = [ sprintf("%.04d", post[:date].year), + sprintf("%.02d", post[:date].month), + sprintf("%.02d", post[:date].day), + post[:slug].strip ].join('-') + name += '.textile' + + File.open("_posts/#{name}", 'w') do |f| + f.puts({ 'layout' => 'post', + 'title' => post[:title].to_s, + 'enki_id' => post[:id], + 'categories' => post[:tags] + }.delete_if { |k, v| v.nil? || v == '' }.to_yaml) + f.puts '---' + f.puts post[:body].delete("\r") + end + end + end + end +end diff --git a/lib/jekyll/migrators/joomla.rb b/lib/jekyll/migrators/joomla.rb new file mode 100644 index 00000000..87f1e105 --- /dev/null +++ b/lib/jekyll/migrators/joomla.rb @@ -0,0 +1,53 @@ +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This migrator is made for Joomla 1.5 databases. +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module Joomla + def self.process(dbname, user, pass, host = 'localhost', table_prefix = 'jos_', section = '1') + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + + FileUtils.mkdir_p("_posts") + + # Reads a MySQL database via Sequel and creates a post file for each + # post in wp_posts that has post_status = 'publish'. This restriction is + # made because 'draft' posts are not guaranteed to have valid dates. + query = "SELECT `title`, `alias`, CONCAT(`introtext`,`fulltext`) as content, `created`, `id` FROM #{table_prefix}content WHERE state = '0' OR state = '1' AND sectionid = '#{section}'" + + db[query].each do |post| + # Get required fields and construct Jekyll compatible name. + title = post[:title] + slug = post[:alias] + date = post[:created] + content = post[:content] + name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, date.day, + slug] + + # Get the relevant fields as a hash, delete empty fields and convert + # to YAML for the header. + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'joomla_id' => post[:id], + 'joomla_url' => post[:alias], + 'date' => date + }.delete_if { |k,v| v.nil? || v == '' }.to_yaml + + # Write out the data and content to file + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + end + end +end diff --git a/lib/jekyll/migrators/marley.rb b/lib/jekyll/migrators/marley.rb new file mode 100644 index 00000000..21bcead5 --- /dev/null +++ b/lib/jekyll/migrators/marley.rb @@ -0,0 +1,52 @@ +require 'yaml' +require 'fileutils' + +module Jekyll + module Marley + def self.regexp + { :id => /^\d{0,4}-{0,1}(.*)$/, + :title => /^#\s*(.*)\s+$/, + :title_with_date => /^#\s*(.*)\s+\(([0-9\/]+)\)$/, + :published_on => /.*\s+\(([0-9\/]+)\)$/, + :perex => /^([^\#\n]+\n)$/, + :meta => /^\{\{\n(.*)\}\}\n$/mi # Multiline Regexp + } + end + + def self.process(marley_data_dir) + raise ArgumentError, "marley dir #{marley_data_dir} not found" unless File.directory?(marley_data_dir) + + FileUtils.mkdir_p "_posts" + + posts = 0 + Dir["#{marley_data_dir}/**/*.txt"].each do |f| + next unless File.exists?(f) + + #copied over from marley's app/lib/post.rb + file_content = File.read(f) + meta_content = file_content.slice!( self.regexp[:meta] ) + body = file_content.sub( self.regexp[:title], '').sub( self.regexp[:perex], '').strip + + title = file_content.scan( self.regexp[:title] ).first.to_s.strip + prerex = file_content.scan( self.regexp[:perex] ).first.to_s.strip + published_on = DateTime.parse( post[:published_on] ) rescue File.mtime( File.dirname(f) ) + meta = ( meta_content ) ? YAML::load( meta_content.scan( self.regexp[:meta]).to_s ) : {} + meta['title'] = title + meta['layout'] = 'post' + + formatted_date = published_on.strftime('%Y-%m-%d') + post_name = File.dirname(f).split(%r{/}).last.gsub(/\A\d+-/, '') + + name = "#{formatted_date}-#{post_name}" + File.open("_posts/#{name}.markdown", "w") do |f| + f.puts meta.to_yaml + f.puts "---\n" + f.puts "\n#{prerex}\n\n" if prerex + f.puts body + end + posts += 1 + end + "Created #{posts} posts!" + end + end +end diff --git a/lib/jekyll/migrators/mephisto.rb b/lib/jekyll/migrators/mephisto.rb new file mode 100644 index 00000000..7622c722 --- /dev/null +++ b/lib/jekyll/migrators/mephisto.rb @@ -0,0 +1,84 @@ +# Quickly hacked together my Michael Ivey +# Based on mt.rb by Nick Gerakines, open source and publically +# available under the MIT license. Use this module at your own risk. + +require 'rubygems' +require 'sequel' +require 'fastercsv' +require 'fileutils' +require File.join(File.dirname(__FILE__),"csv.rb") + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module Mephisto + #Accepts a hash with database config variables, exports mephisto posts into a csv + #export PGPASSWORD if you must + def self.postgres(c) + sql = <<-SQL + BEGIN; + CREATE TEMP TABLE jekyll AS + SELECT title, permalink, body, published_at, filter FROM contents + WHERE user_id = 1 AND type = 'Article' ORDER BY published_at; + COPY jekyll TO STDOUT WITH CSV HEADER; + ROLLBACK; + SQL + command = %Q(psql -h #{c[:host] || "localhost"} -c "#{sql.strip}" #{c[:database]} #{c[:username]} -o #{c[:filename] || "posts.csv"}) + puts command + `#{command}` + CSV.process + end + + # This query will pull blog posts from all entries across all blogs. If + # you've got unpublished, deleted or otherwise hidden posts please sift + # through the created posts to make sure nothing is accidently published. + QUERY = "SELECT id, \ + permalink, \ + body, \ + published_at, \ + title \ + FROM contents \ + WHERE user_id = 1 AND \ + type = 'Article' AND \ + published_at IS NOT NULL \ + ORDER BY published_at" + + def self.process(dbname, user, pass, host = 'localhost') + db = Sequel.mysql(dbname, :user => user, + :password => pass, + :host => host, + :encoding => 'utf8') + + FileUtils.mkdir_p "_posts" + + db[QUERY].each do |post| + title = post[:title] + slug = post[:permalink] + date = post[:published_at] + content = post[:body] + + # Ideally, this script would determine the post format (markdown, + # html, etc) and create files with proper extensions. At this point + # it just assumes that markdown will be acceptable. + name = [date.year, date.month, date.day, slug].join('-') + ".markdown" + + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'mt_id' => post[:entry_id], + }.delete_if { |k,v| v.nil? || v == ''}.to_yaml + + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + + end + end +end diff --git a/lib/jekyll/migrators/mt.rb b/lib/jekyll/migrators/mt.rb new file mode 100644 index 00000000..048c84db --- /dev/null +++ b/lib/jekyll/migrators/mt.rb @@ -0,0 +1,86 @@ +# Created by Nick Gerakines, open source and publically available under the +# MIT license. Use this module at your own risk. +# I'm an Erlang/Perl/C++ guy so please forgive my dirty ruby. + +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module MT + # This query will pull blog posts from all entries across all blogs. If + # you've got unpublished, deleted or otherwise hidden posts please sift + # through the created posts to make sure nothing is accidently published. + QUERY = "SELECT entry_id, \ + entry_basename, \ + entry_text, \ + entry_text_more, \ + entry_authored_on, \ + entry_title, \ + entry_convert_breaks \ + FROM mt_entry" + + def self.process(dbname, user, pass, host = 'localhost') + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + + FileUtils.mkdir_p "_posts" + + db[QUERY].each do |post| + title = post[:entry_title] + slug = post[:entry_basename].gsub(/_/, '-') + date = post[:entry_authored_on] + content = post[:entry_text] + more_content = post[:entry_text_more] + entry_convert_breaks = post[:entry_convert_breaks] + + # Be sure to include the body and extended body. + if more_content != nil + content = content + " \n" + more_content + end + + # Ideally, this script would determine the post format (markdown, + # html, etc) and create files with proper extensions. At this point + # it just assumes that markdown will be acceptable. + name = [date.year, date.month, date.day, slug].join('-') + '.' + + self.suffix(entry_convert_breaks) + + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'mt_id' => post[:entry_id], + 'date' => date + }.delete_if { |k,v| v.nil? || v == '' }.to_yaml + + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + end + + def self.suffix(entry_type) + if entry_type.nil? || entry_type.include?("markdown") + # The markdown plugin I have saves this as + # "markdown_with_smarty_pants", so I just look for "markdown". + "markdown" + elsif entry_type.include?("textile") + # This is saved as "textile_2" on my installation of MT 5.1. + "textile" + elsif entry_type == "0" || entry_type.include?("richtext") + # Richtext looks to me like it's saved as HTML, so I include it here. + "html" + else + # Other values might need custom work. + entry_type + end + end + end +end diff --git a/lib/jekyll/migrators/posterous.rb b/lib/jekyll/migrators/posterous.rb new file mode 100644 index 00000000..0a2280f2 --- /dev/null +++ b/lib/jekyll/migrators/posterous.rb @@ -0,0 +1,67 @@ +require 'rubygems' +require 'jekyll' +require 'fileutils' +require 'net/http' +require 'uri' +require "json" + +# ruby -r './lib/jekyll/migrators/posterous.rb' -e 'Jekyll::Posterous.process(email, pass, api_key, blog)' + +module Jekyll + module Posterous + def self.fetch(uri_str, limit = 10) + # You should choose better exception. + raise ArgumentError, 'Stuck in a redirect loop. Please double check your email and password' if limit == 0 + + response = nil + Net::HTTP.start('posterous.com') do |http| + req = Net::HTTP::Get.new(uri_str) + req.basic_auth @email, @pass + response = http.request(req) + end + + case response + when Net::HTTPSuccess then response + when Net::HTTPRedirection then fetch(response['location'], limit - 1) + else response.error! + end + end + + def self.process(email, pass, api_token, blog = 'primary') + @email, @pass, @api_token = email, pass, api_token + FileUtils.mkdir_p "_posts" + + posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}").body) + page = 1 + + while posts.any? + posts.each do |post| + title = post["title"] + slug = title.gsub(/[^[:alnum:]]+/, '-').downcase + date = Date.parse(post["display_date"]) + content = post["body_html"] + published = !post["is_private"] + name = "%02d-%02d-%02d-%s.html" % [date.year, date.month, date.day, slug] + + # Get the relevant fields as a hash, delete empty fields and convert + # to YAML for the header + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'published' => published + }.delete_if { |k,v| v.nil? || v == ''}.to_yaml + + # Write out the data and content to file + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + + page += 1 + posts = JSON.parse(self.fetch("/api/v2/users/me/sites/#{blog}/posts?api_token=#{@api_token}&page=#{page}").body) + end + end + end +end diff --git a/lib/jekyll/migrators/rss.rb b/lib/jekyll/migrators/rss.rb new file mode 100644 index 00000000..461abd35 --- /dev/null +++ b/lib/jekyll/migrators/rss.rb @@ -0,0 +1,47 @@ +# Created by Kendall Buchanan (https://github.com/kendagriff) on 2011-12-22. +# Use at your own risk. The end. +# +# Usage: +# (URL) +# ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('http://yourdomain.com/your-favorite-feed.xml')" +# +# (Local file) +# ruby -r '_import/rss.rb' -e "Jekyll::MigrateRSS.process('./somefile/on/your/computer.xml')" + +require 'rubygems' +require 'rss/1.0' +require 'rss/2.0' +require 'open-uri' +require 'fileutils' +require 'yaml' + +module Jekyll + module MigrateRSS + + # The `source` argument may be a URL or a local file. + def self.process(source) + content = "" + open(source) { |s| content = s.read } + rss = RSS::Parser.parse(content, false) + + raise "There doesn't appear to be any RSS items at the source (#{source}) provided." unless rss + + rss.items.each do |item| + formatted_date = item.date.strftime('%Y-%m-%d') + post_name = item.title.split(%r{ |!|/|:|&|-|$|,}).map { |i| i.downcase if i != '' }.compact.join('-') + name = "#{formatted_date}-#{post_name}" + + header = { + 'layout' => 'post', + 'title' => item.title + } + + File.open("_posts/#{name}.html", "w") do |f| + f.puts header.to_yaml + f.puts "---\n" + f.puts item.description + end + end + end + end +end \ No newline at end of file diff --git a/lib/jekyll/migrators/textpattern.rb b/lib/jekyll/migrators/textpattern.rb new file mode 100644 index 00000000..3b370ed9 --- /dev/null +++ b/lib/jekyll/migrators/textpattern.rb @@ -0,0 +1,58 @@ +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module TextPattern + # Reads a MySQL database via Sequel and creates a post file for each post. + # The only posts selected are those with a status of 4 or 5, which means + # "live" and "sticky" respectively. + # Other statuses are 1 => draft, 2 => hidden and 3 => pending. + QUERY = "SELECT Title, \ + url_title, \ + Posted, \ + Body, \ + Keywords \ + FROM textpattern \ + WHERE Status = '4' OR \ + Status = '5'" + + def self.process(dbname, user, pass, host = 'localhost') + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + + FileUtils.mkdir_p "_posts" + + db[QUERY].each do |post| + # Get required fields and construct Jekyll compatible name. + title = post[:Title] + slug = post[:url_title] + date = post[:Posted] + content = post[:Body] + + name = [date.strftime("%Y-%m-%d"), slug].join('-') + ".textile" + + # Get the relevant fields as a hash, delete empty fields and convert + # to YAML for the header. + data = { + 'layout' => 'post', + 'title' => title.to_s, + 'tags' => post[:Keywords].split(',') + }.delete_if { |k,v| v.nil? || v == ''}.to_yaml + + # Write out the data and content to file. + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + end + end +end diff --git a/lib/jekyll/migrators/tumblr.rb b/lib/jekyll/migrators/tumblr.rb new file mode 100644 index 00000000..367a83c9 --- /dev/null +++ b/lib/jekyll/migrators/tumblr.rb @@ -0,0 +1,195 @@ +require 'rubygems' +require 'open-uri' +require 'fileutils' +require 'nokogiri' +require 'date' +require 'json' +require 'uri' +require 'jekyll' + +module Jekyll + module Tumblr + def self.process(url, format = "html", grab_images = false, + add_highlights = false, rewrite_urls = true) + @grab_images = grab_images + FileUtils.mkdir_p "_posts/tumblr" + url += "/api/read/json/" + per_page = 50 + posts = [] + # Two passes are required so that we can rewrite URLs. + # First pass builds up an array of each post as a hash. + begin + current_page = (current_page || -1) + 1 + feed = open(url + "?num=#{per_page}&start=#{current_page * per_page}") + json = feed.readlines.join("\n")[21...-2] # Strip Tumblr's JSONP chars. + blog = JSON.parse(json) + puts "Page: #{current_page + 1} - Posts: #{blog["posts"].size}" + posts += blog["posts"].map { |post| post_to_hash(post, format) } + end until blog["posts"].size < per_page + # Rewrite URLs and create redirects. + posts = rewrite_urls_and_redirects posts if rewrite_urls + # Second pass for writing post files. + posts.each do |post| + if format == "md" + post[:content] = html_to_markdown post[:content] + post[:content] = add_syntax_highlights post[:content] if add_highlights + end + File.open("_posts/tumblr/#{post[:name]}", "w") do |f| + f.puts post[:header].to_yaml + "---\n" + post[:content] + end + end + end + + private + + # Converts each type of Tumblr post to a hash with all required + # data for Jekyll. + def self.post_to_hash(post, format) + case post['type'] + when "regular" + title = post["regular-title"] + content = post["regular-body"] + when "link" + title = post["link-text"] || post["link-url"] + content = "#{title}" + unless post["link-description"].nil? + content << "
" + post["link-description"] + end + when "photo" + title = post["photo-caption"] + max_size = post.keys.map{ |k| k.gsub("photo-url-", "").to_i }.max + url = post["photo-url"] || post["photo-url-#{max_size}"] + ext = "." + post[post.keys.select { |k| + k =~ /^photo-url-/ && post[k].split("/").last =~ /\./ + }.first].split(".").last + content = "" + unless post["photo-link-url"].nil? + content = "#{content}" + end + when "audio" + if !post["id3-title"].nil? + title = post["id3-title"] + content = post.at["audio-player"] + "
" + post["audio-caption"] + else + title = post["audio-caption"] + content = post.at["audio-player"] + end + when "quote" + title = post["quote-text"] + content = "
#{post["quote-text"]}
" + unless post["quote-source"].nil? + content << "—" + post["quote-source"] + end + when "conversation" + title = post["conversation-title"] + content = "
" + post["conversation"]["line"].each do |line| + content << "
#{line['label']}
#{line}
" + end + content << "
" + when "video" + title = post["video-title"] + content = post["video-player"] + unless post["video-caption"].nil? + content << "
" + post["video-caption"] + end + end + date = Date.parse(post['date']).to_s + title = Nokogiri::HTML(title).text + slug = title.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '') + { + :name => "#{date}-#{slug}.#{format}", + :header => { + "layout" => "post", + "title" => title, + "tags" => post["tags"], + }, + :content => content, + :url => post["url"], + :slug => post["url-with-slug"], + } + end + + # Create a Hash of old urls => new urls, for rewriting and + # redirects, and replace urls in each post. Instantiate Jekyll + # site/posts to get the correct permalink format. + def self.rewrite_urls_and_redirects(posts) + site = Jekyll::Site.new(Jekyll.configuration({})) + dir = File.join(File.dirname(__FILE__), "..") + urls = Hash[posts.map { |post| + # Create an initial empty file for the post so that + # we can instantiate a post object. + File.open("_posts/tumblr/#{post[:name]}", "w") + tumblr_url = URI.parse(post[:slug]).path + jekyll_url = Jekyll::Post.new(site, dir, "", "tumblr/" + post[:name]).url + redirect_dir = tumblr_url.sub(/\//, "") + "/" + FileUtils.mkdir_p redirect_dir + File.open(redirect_dir + "index.html", "w") do |f| + f.puts "" + end + [tumblr_url, jekyll_url] + }] + posts.map { |post| + urls.each do |tumblr_url, jekyll_url| + post[:content].gsub!(/#{tumblr_url}/i, jekyll_url) + end + post + } + end + + # Uses Python's html2text to convert a post's content to + # markdown. Preserve HTML tables as per the markdown docs. + def self.html_to_markdown(content) + preserve = ["table", "tr", "th", "td"] + preserve.each do |tag| + content.gsub!(/<#{tag}/i, "$$" + tag) + content.gsub!(/<\/#{tag}/i, "||" + tag) + end + content = %x[echo '#{content.gsub("'", "''")}' | html2text] + preserve.each do |tag| + content.gsub!("$$" + tag, "<" + tag) + content.gsub!("||" + tag, " +require 'fileutils' +require 'rubygems' +require 'sequel' +require 'yaml' + +module Jekyll + module Typo + # This SQL *should* work for both MySQL and PostgreSQL, but I haven't + # tested PostgreSQL yet (as of 2008-12-16). + SQL = <<-EOS + SELECT c.id id, + c.title title, + c.permalink slug, + c.body body, + c.published_at date, + c.state state, + COALESCE(tf.name, 'html') filter + FROM contents c + LEFT OUTER JOIN text_filters tf + ON c.text_filter_id = tf.id + EOS + + def self.process dbname, user, pass, host='localhost' + FileUtils.mkdir_p '_posts' + db = Sequel.mysql(dbname, :user => user, :password => pass, :host => host, :encoding => 'utf8') + db[SQL].each do |post| + next unless post[:state] =~ /published/ + + name = [ sprintf("%.04d", post[:date].year), + sprintf("%.02d", post[:date].month), + sprintf("%.02d", post[:date].day), + post[:slug].strip ].join('-') + + # Can have more than one text filter in this field, but we just want + # the first one for this. + name += '.' + post[:filter].split(' ')[0] + + File.open("_posts/#{name}", 'w') do |f| + f.puts({ 'layout' => 'post', + 'title' => post[:title].to_s, + 'typo_id' => post[:id] + }.delete_if { |k, v| v.nil? || v == '' }.to_yaml) + f.puts '---' + f.puts post[:body].delete("\r") + end + end + end + + end +end diff --git a/lib/jekyll/migrators/wordpress.rb b/lib/jekyll/migrators/wordpress.rb new file mode 100644 index 00000000..d2d19039 --- /dev/null +++ b/lib/jekyll/migrators/wordpress.rb @@ -0,0 +1,294 @@ +require 'rubygems' +require 'sequel' +require 'fileutils' +require 'yaml' + +# NOTE: This converter requires Sequel and the MySQL gems. +# The MySQL gem can be difficult to install on OS X. Once you have MySQL +# installed, running the following commands should work: +# $ sudo gem install sequel +# $ sudo gem install mysql -- --with-mysql-config=/usr/local/mysql/bin/mysql_config + +module Jekyll + module WordPress + + # Main migrator function. Call this to perform the migration. + # + # dbname:: The name of the database + # user:: The database user name + # pass:: The database user's password + # host:: The address of the MySQL database host. Default: 'localhost' + # options:: A hash table of configuration options. + # + # Supported options are: + # + # :table_prefix:: Prefix of database tables used by WordPress. + # Default: 'wp_' + # :clean_entities:: If true, convert non-ASCII characters to HTML + # entities in the posts, comments, titles, and + # names. Requires the 'htmlentities' gem to + # work. Default: true. + # :comments:: If true, migrate post comments too. Comments + # are saved in the post's YAML front matter. + # Default: true. + # :categories:: If true, save the post's categories in its + # YAML front matter. + # :tags:: If true, save the post's tags in its + # YAML front matter. + # :more_excerpt:: If true, when a post has no excerpt but + # does have a tag, use the + # preceding post content as the excerpt. + # Default: true. + # :more_anchor:: If true, convert a tag into + # two HTML anchors with ids "more" and + # "more-NNN" (where NNN is the post number). + # Default: true. + # :status:: Array of allowed post statuses. Only + # posts with matching status will be migrated. + # Known statuses are :publish, :draft, :private, + # and :revision. If this is nil or an empty + # array, all posts are migrated regardless of + # status. Default: [:publish]. + # + def self.process(dbname, user, pass, host='localhost', options={}) + options = { + :table_prefix => 'wp_', + :clean_entities => true, + :comments => true, + :categories => true, + :tags => true, + :more_excerpt => true, + :more_anchor => true, + :status => [:publish] # :draft, :private, :revision + }.merge(options) + + if options[:clean_entities] + begin + require 'htmlentities' + rescue LoadError + STDERR.puts "Could not require 'htmlentities', so the " + + ":clean_entities option is now disabled." + options[:clean_entities] = false + end + end + + FileUtils.mkdir_p("_posts") + + db = Sequel.mysql(dbname, :user => user, :password => pass, + :host => host, :encoding => 'utf8') + + px = options[:table_prefix] + + posts_query = " + SELECT + posts.ID AS `id`, + posts.guid AS `guid`, + posts.post_type AS `type`, + posts.post_status AS `status`, + posts.post_title AS `title`, + posts.post_name AS `slug`, + posts.post_date AS `date`, + posts.post_content AS `content`, + posts.post_excerpt AS `excerpt`, + posts.comment_count AS `comment_count`, + users.display_name AS `author`, + users.user_login AS `author_login`, + users.user_email AS `author_email`, + users.user_url AS `author_url` + FROM #{px}posts AS `posts` + LEFT JOIN #{px}users AS `users` + ON posts.post_author = users.ID" + + if options[:status] and not options[:status].empty? + status = options[:status][0] + posts_query << " + WHERE posts.post_status = '#{status.to_s}'" + options[:status][1..-1].each do |status| + posts_query << " OR + posts.post_status = '#{status.to_s}'" + end + end + + db[posts_query].each do |post| + process_post(post, db, options) + end + end + + + def self.process_post(post, db, options) + px = options[:table_prefix] + + title = post[:title] + if options[:clean_entities] + title = clean_entities(title) + end + + slug = post[:slug] + if !slug or slug.empty? + slug = sluggify(title) + end + + date = post[:date] || Time.now + name = "%02d-%02d-%02d-%s.markdown" % [date.year, date.month, + date.day, slug] + content = post[:content].to_s + if options[:clean_entities] + content = clean_entities(content) + end + + excerpt = post[:excerpt].to_s + + more_index = content.index(//) + more_anchor = nil + if more_index + if options[:more_excerpt] and + (post[:excerpt].nil? or post[:excerpt].empty?) + excerpt = content[0...more_index] + end + if options[:more_anchor] + more_link = "more" + content.sub!(//, + "" + + "") + end + end + + categories = [] + tags = [] + + if options[:categories] or options[:tags] + + cquery = + "SELECT + terms.name AS `name`, + ttax.taxonomy AS `type` + FROM + #{px}terms AS `terms`, + #{px}term_relationships AS `trels`, + #{px}term_taxonomy AS `ttax` + WHERE + trels.object_id = '#{post[:id]}' AND + trels.term_taxonomy_id = ttax.term_taxonomy_id AND + terms.term_id = ttax.term_id" + + db[cquery].each do |term| + if options[:categories] and term[:type] == "category" + if options[:clean_entities] + categories << clean_entities(term[:name]) + else + categories << term[:name] + end + elsif options[:tags] and term[:type] == "post_tag" + if options[:clean_entities] + tags << clean_entities(term[:name]) + else + tags << term[:name] + end + end + end + end + + comments = [] + + if options[:comments] and post[:comment_count].to_i > 0 + cquery = + "SELECT + comment_ID AS `id`, + comment_author AS `author`, + comment_author_email AS `author_email`, + comment_author_url AS `author_url`, + comment_date AS `date`, + comment_date_gmt AS `date_gmt`, + comment_content AS `content` + FROM #{px}comments + WHERE + comment_post_ID = '#{post[:id]}' AND + comment_approved != 'spam'" + + + db[cquery].each do |comment| + + comcontent = comment[:content].to_s + if comcontent.respond_to?(:force_encoding) + comcontent.force_encoding("UTF-8") + end + if options[:clean_entities] + comcontent = clean_entities(comcontent) + end + comauthor = comment[:author].to_s + if options[:clean_entities] + comauthor = clean_entities(comauthor) + end + + comments << { + 'id' => comment[:id].to_i, + 'author' => comauthor, + 'author_email' => comment[:author_email].to_s, + 'author_url' => comment[:author_url].to_s, + 'date' => comment[:date].to_s, + 'date_gmt' => comment[:date_gmt].to_s, + 'content' => comcontent, + } + end + + comments.sort!{ |a,b| a['id'] <=> b['id'] } + end + + # Get the relevant fields as a hash, delete empty fields and + # convert to YAML for the header. + data = { + 'layout' => post[:type].to_s, + 'status' => post[:status].to_s, + 'published' => (post[:status].to_s == "publish"), + 'title' => title.to_s, + 'author' => post[:author].to_s, + 'author_login' => post[:author_login].to_s, + 'author_email' => post[:author_email].to_s, + 'author_url' => post[:author_url].to_s, + 'excerpt' => excerpt, + 'more_anchor' => more_anchor, + 'wordpress_id' => post[:id], + 'wordpress_url' => post[:guid].to_s, + 'date' => date, + 'categories' => options[:categories] ? categories : nil, + 'tags' => options[:tags] ? tags : nil, + 'comments' => options[:comments] ? comments : nil, + }.delete_if { |k,v| v.nil? || v == '' }.to_yaml + + # Write out the data and content to file + File.open("_posts/#{name}", "w") do |f| + f.puts data + f.puts "---" + f.puts content + end + end + + + def self.clean_entities( text ) + if text.respond_to?(:force_encoding) + text.force_encoding("UTF-8") + end + text = HTMLEntities.new.encode(text, :named) + # We don't want to convert these, it would break all + # HTML tags in the post and comments. + text.gsub!("&", "&") + text.gsub!("<", "<") + text.gsub!(">", ">") + text.gsub!(""", '"') + text.gsub!("'", "'") + text + end + + + def self.sluggify( title ) + begin + require 'unidecode' + title = title.to_ascii + rescue LoadError + STDERR.puts "Could not require 'unidecode'. If your post titles have non-ASCII characters, you could get nicer permalinks by installing unidecode." + end + title.downcase.gsub(/[^0-9A-Za-z]+/, " ").strip.gsub(" ", "-") + end + + end +end diff --git a/lib/jekyll/migrators/wordpressdotcom.rb b/lib/jekyll/migrators/wordpressdotcom.rb new file mode 100644 index 00000000..701c2af4 --- /dev/null +++ b/lib/jekyll/migrators/wordpressdotcom.rb @@ -0,0 +1,70 @@ +# coding: utf-8 + +require 'rubygems' +require 'hpricot' +require 'fileutils' +require 'yaml' +require 'time' + +module Jekyll + # This importer takes a wordpress.xml file, which can be exported from your + # wordpress.com blog (/wp-admin/export.php). + module WordpressDotCom + def self.process(filename = "wordpress.xml") + import_count = Hash.new(0) + doc = Hpricot::XML(File.read(filename)) + + (doc/:channel/:item).each do |item| + title = item.at(:title).inner_text.strip + permalink_title = item.at('wp:post_name').inner_text + # Fallback to "prettified" title if post_name is empty (can happen) + if permalink_title == "" + permalink_title = title.downcase.split.join('-') + end + + date = Time.parse(item.at('wp:post_date').inner_text) + status = item.at('wp:status').inner_text + + if status == "publish" + published = true + else + published = false + end + + type = item.at('wp:post_type').inner_text + tags = (item/:category).map{|c| c.inner_text}.reject{|c| c == 'Uncategorized'}.uniq + + metas = Hash.new + item.search("wp:postmeta").each do |meta| + key = meta.at('wp:meta_key').inner_text + value = meta.at('wp:meta_value').inner_text + metas[key] = value; + end + + name = "#{date.strftime('%Y-%m-%d')}-#{permalink_title}.html" + header = { + 'layout' => type, + 'title' => title, + 'tags' => tags, + 'status' => status, + 'type' => type, + 'published' => published, + 'meta' => metas + } + + FileUtils.mkdir_p "_#{type}s" + File.open("_#{type}s/#{name}", "w") do |f| + f.puts header.to_yaml + f.puts '---' + f.puts item.at('content:encoded').inner_text + end + + import_count[type] += 1 + end + + import_count.each do |key, value| + puts "Imported #{value} #{key}s" + end + end + end +end From 9c517c6d7d9432b73c125c33c9425362ca8df4f2 Mon Sep 17 00:00:00 2001 From: Tom Bell Date: Fri, 11 Jan 2013 00:32:39 +0000 Subject: [PATCH 20/20] Rename migrate command to import --- bin/jekyll | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/jekyll b/bin/jekyll index 3df10e4f..309ffc91 100755 --- a/bin/jekyll +++ b/bin/jekyll @@ -5,14 +5,12 @@ $:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib }) require 'commander/import' require 'jekyll' -# Details about Jekyll program :name, 'jekyll' program :version, Jekyll::VERSION program :description, 'Jekyll is a blog-aware, static site generator in Ruby' default_command :help -# Global options available to every command global_option '-s', '--source [DIR]', 'Source directory (defaults to ./)' global_option '-d', '--destination [DIR]', 'Destination directory (defaults to ./_site)' global_option '--safe', 'Safe mode (defaults to false)' @@ -56,9 +54,9 @@ command :serve do |c| end end -command :migrate do |c| - c.syntax = 'jekyll migrate [options]' - c.description = 'Migrate your own blog to Jekyll' +command :import do |c| + c.syntax = 'jekyll import [options]' + c.description = 'Import your old blog to Jekyll' c.option '--file', 'File to migrate from' c.option '--dbname', 'Database name to migrate from'