From 8a28d806909b4ac15491faf81f29de4b05dfef52 Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Tue, 30 Jul 2013 22:21:09 +0900 Subject: [PATCH 1/8] add encoding for configuration --- lib/jekyll/configuration.rb | 1 + lib/jekyll/convertible.rb | 2 +- lib/jekyll/site.rb | 7 ++++++- lib/jekyll/tags/include.rb | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/jekyll/configuration.rb b/lib/jekyll/configuration.rb index 05a097c7..75230bb8 100644 --- a/lib/jekyll/configuration.rb +++ b/lib/jekyll/configuration.rb @@ -11,6 +11,7 @@ module Jekyll 'plugins' => '_plugins', 'layouts' => '_layouts', 'keep_files' => ['.git','.svn'], + 'encoding' => nil, 'timezone' => nil, # use the local timezone diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb index 5f493800..74125e7b 100644 --- a/lib/jekyll/convertible.rb +++ b/lib/jekyll/convertible.rb @@ -28,7 +28,7 @@ module Jekyll # Returns nothing. def read_yaml(base, name) begin - self.content = File.read(File.join(base, name)) + self.content = File.read(File.join(base, name), self.site.file_read_opts) if self.content =~ /\A(---\s*\n.*?\n?)^(---\s*$\n?)/m self.content = $POSTMATCH diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index ef7be1fc..6aec1c19 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -3,7 +3,7 @@ module Jekyll attr_accessor :config, :layouts, :posts, :pages, :static_files, :categories, :exclude, :include, :source, :dest, :lsi, :pygments, :permalink_style, :tags, :time, :future, :safe, :plugins, :limit_posts, - :show_drafts, :keep_files, :baseurl + :show_drafts, :keep_files, :baseurl, :file_read_opts attr_accessor :converters, :generators @@ -22,6 +22,11 @@ module Jekyll self.plugins = plugins_path self.permalink_style = config['permalink'].to_sym + self.file_read_opts = {} + if encoding = config['encoding'] + self.file_read_opts[:encoding] = Encoding.find(encoding) + end + self.reset self.setup end diff --git a/lib/jekyll/tags/include.rb b/lib/jekyll/tags/include.rb index 2b0d82b4..b8b54353 100644 --- a/lib/jekyll/tags/include.rb +++ b/lib/jekyll/tags/include.rb @@ -109,7 +109,7 @@ eos # This method allows to modify the file content by inheriting from the class. def source(file) - File.read(file) + File.read(file, context.registers[:site].file_read_opts) end end end From f4c2383d3c266314b5a8481eb3b03b38e21be7b2 Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Tue, 30 Jul 2013 22:28:30 +0900 Subject: [PATCH 2/8] write document for config.encoding --- lib/jekyll/configuration.rb | 3 ++- site/docs/configuration.md | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/jekyll/configuration.rb b/lib/jekyll/configuration.rb index 75230bb8..43244b47 100644 --- a/lib/jekyll/configuration.rb +++ b/lib/jekyll/configuration.rb @@ -11,10 +11,11 @@ module Jekyll 'plugins' => '_plugins', 'layouts' => '_layouts', 'keep_files' => ['.git','.svn'], - 'encoding' => nil, 'timezone' => nil, # use the local timezone + 'encoding' => nil, # use the system encoding + 'safe' => false, 'detach' => false, # default to not detaching the server 'show_drafts' => nil, diff --git a/site/docs/configuration.md b/site/docs/configuration.md index fa92c6d1..79d4a3cf 100644 --- a/site/docs/configuration.md +++ b/site/docs/configuration.md @@ -99,6 +99,18 @@ class="flag">flags (specified on the command-line) that control them.

timezone: TIMEZONE

+ + +

Encoding

+

+ Set the encoding of files. The default is the system encoding, + which determined by LANG environment variable. +

+ + +

encoding: ENCODING

+ + @@ -266,6 +278,7 @@ include: ['.htaccess'] exclude: [] keep_files: ['.git','.svn'] timezone: nil +encoding: nil future: true show_drafts: nil From 4cb24f4b877e1e0d1ead5f2def09f18fc9eb9408 Mon Sep 17 00:00:00 2001 From: MURAOKA Taro Date: Tue, 30 Jul 2013 23:19:26 +0900 Subject: [PATCH 3/8] consider a case of "site" is unavailable in Convertible --- lib/jekyll/convertible.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb index 74125e7b..da8858ee 100644 --- a/lib/jekyll/convertible.rb +++ b/lib/jekyll/convertible.rb @@ -28,7 +28,8 @@ module Jekyll # Returns nothing. def read_yaml(base, name) begin - self.content = File.read(File.join(base, name), self.site.file_read_opts) + opts = self.site ? self.site.file_read_opts : {} + self.content = File.read(File.join(base, name), opts) if self.content =~ /\A(---\s*\n.*?\n?)^(---\s*$\n?)/m self.content = $POSTMATCH From 8b892ed735bfcaaeecfcf8d97db763561ee466a9 Mon Sep 17 00:00:00 2001 From: Shigeya Suzuki Date: Sun, 18 Aug 2013 10:43:02 +0900 Subject: [PATCH 4/8] read_yaml to accept optional parameter to override defaults --- lib/jekyll/convertible.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb index da8858ee..21f950d0 100644 --- a/lib/jekyll/convertible.rb +++ b/lib/jekyll/convertible.rb @@ -24,11 +24,13 @@ module Jekyll # # base - The String path to the dir containing the file. # name - The String filename of the file. + # opts - optional parameter to File.read, default at site configs # # Returns nothing. - def read_yaml(base, name) + def read_yaml(base, name, opts = {}) begin - opts = self.site ? self.site.file_read_opts : {} + opts = (self.site ? self.site.file_read_opts : {}).merge(opts) + self.content = File.read(File.join(base, name), opts) if self.content =~ /\A(---\s*\n.*?\n?)^(---\s*$\n?)/m From fbea3471ddb458838fe352cd3b6f30459dcdec90 Mon Sep 17 00:00:00 2001 From: Shigeya Suzuki Date: Sun, 18 Aug 2013 10:43:25 +0900 Subject: [PATCH 5/8] Make TestConvertible to pass with utf-8 encoding --- test/test_convertible.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_convertible.rb b/test/test_convertible.rb index ee7fc8e3..694c152c 100644 --- a/test/test_convertible.rb +++ b/test/test_convertible.rb @@ -40,7 +40,7 @@ class TestConvertible < Test::Unit::TestCase should "not parse if there is encoding error in file" do name = 'broken_front_matter3.erb' out = capture_stdout do - ret = @convertible.read_yaml(@base, name) + ret = @convertible.read_yaml(@base, name, :encoding => 'utf-8') assert_equal({}, ret) end assert_match(/invalid byte sequence in UTF-8/, out) From 97e052df639df2091502b65a7dfa64db323584cc Mon Sep 17 00:00:00 2001 From: Shigeya Suzuki Date: Tue, 20 Aug 2013 09:23:21 +0900 Subject: [PATCH 6/8] Pass encoding parameter as string, not an object --- lib/jekyll/site.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/jekyll/site.rb b/lib/jekyll/site.rb index 6aec1c19..6084631a 100644 --- a/lib/jekyll/site.rb +++ b/lib/jekyll/site.rb @@ -23,9 +23,7 @@ module Jekyll self.permalink_style = config['permalink'].to_sym self.file_read_opts = {} - if encoding = config['encoding'] - self.file_read_opts[:encoding] = Encoding.find(encoding) - end + self.file_read_opts[:encoding] = config['encoding'] if config['encoding'] self.reset self.setup From c625ddf6cd9481965b3339f119f8b2011520cce4 Mon Sep 17 00:00:00 2001 From: Shigeya Suzuki Date: Sat, 24 Aug 2013 14:40:55 +0900 Subject: [PATCH 7/8] Invoke File.read with or without options depends on Ruby version - Extract option fetch method as a separate method - Added File.read_with_options method to use - With performance fix --- lib/jekyll/convertible.rb | 12 ++++++++---- lib/jekyll/core_ext.rb | 15 +++++++++++++++ lib/jekyll/tags/include.rb | 11 ++++++++--- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/lib/jekyll/convertible.rb b/lib/jekyll/convertible.rb index 21f950d0..44bd8b9a 100644 --- a/lib/jekyll/convertible.rb +++ b/lib/jekyll/convertible.rb @@ -20,6 +20,12 @@ module Jekyll self.content || '' end + # Returns merged optin hash for File.read of self.site (if exists) + # and a given param + def merged_file_read_opts(opts) + (self.site ? self.site.file_read_opts : {}).merge(opts) + end + # Read the YAML frontmatter. # # base - The String path to the dir containing the file. @@ -29,10 +35,8 @@ module Jekyll # Returns nothing. def read_yaml(base, name, opts = {}) begin - opts = (self.site ? self.site.file_read_opts : {}).merge(opts) - - self.content = File.read(File.join(base, name), opts) - + self.content = File.read_with_options(File.join(base, name), + merged_file_read_opts(opts)) if self.content =~ /\A(---\s*\n.*?\n?)^(---\s*$\n?)/m self.content = $POSTMATCH self.data = YAML.safe_load($1) diff --git a/lib/jekyll/core_ext.rb b/lib/jekyll/core_ext.rb index 1a7b1816..54f7c9d5 100644 --- a/lib/jekyll/core_ext.rb +++ b/lib/jekyll/core_ext.rb @@ -69,3 +69,18 @@ module Enumerable any? { |exp| File.fnmatch?(exp, e) } end end + +# Ruby 1.8's File.read don't support option. +# read_with_options ignore optional parameter for 1.8, +# and act as alias for 1.9 or later. +class File + if RUBY_VERSION < '1.9' + def self.read_with_options(path, opts = {}) + self.read(path) + end + else + def self.read_with_options(path, opts = {}) + self.read(path, opts) + end + end +end diff --git a/lib/jekyll/tags/include.rb b/lib/jekyll/tags/include.rb index b8b54353..57095ce7 100644 --- a/lib/jekyll/tags/include.rb +++ b/lib/jekyll/tags/include.rb @@ -70,6 +70,11 @@ eos end end + # Grab file read opts in the context + def file_read_opts(context) + context.registers[:site].file_read_opts + end + def render(context) dir = File.join(context.registers[:site].source, INCLUDES_DIR) if error = validate_dir(dir, context.registers[:site].safe) @@ -81,7 +86,7 @@ eos return error end - partial = Liquid::Template.parse(source(file)) + partial = Liquid::Template.parse(source(file, context)) context.stack do context['include'] = parse_params(context) if @params @@ -108,8 +113,8 @@ eos end # This method allows to modify the file content by inheriting from the class. - def source(file) - File.read(file, context.registers[:site].file_read_opts) + def source(file, context) + File.read_with_options(file, file_read_opts(context)) end end end From a93b1a26eb20cb29eaac48aa42c3c464a950ea3c Mon Sep 17 00:00:00 2001 From: Shigeya Suzuki Date: Thu, 29 Aug 2013 07:38:53 +0900 Subject: [PATCH 8/8] Documentation update for the encoding configuration --- site/docs/configuration.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/site/docs/configuration.md b/site/docs/configuration.md index 79d4a3cf..31bf7048 100644 --- a/site/docs/configuration.md +++ b/site/docs/configuration.md @@ -103,8 +103,12 @@ class="flag">flags (specified on the command-line) that control them.

Encoding

- Set the encoding of files. The default is the system encoding, - which determined by LANG environment variable. + Set the encoding of files by name. Only available for Ruby + 1.9 or later). + The default value is nil, which use Ruby default, + ASCII-8BIT. + Available encoding for the ruby in use, can be shown by + command ruby -e 'puts Encoding::list.join("\n")'