From bf149a0b97187259cccb5323d3396f313b3461b0 Mon Sep 17 00:00:00 2001 From: Parker Moore Date: Mon, 9 Feb 2015 21:42:50 -0800 Subject: [PATCH 1/2] highlight: fix problem with linenos and rouge. Found by @EdMcBane in https://github.com/jekyll/jekyll/pull/3435 The strange regexp we were doing to replace the
bits in the Pygments output were wreaking havoc on Rouge output because Rouge uses 
's to wrap line numbers.

To be consistent, the output from render_* should *not* include
the wrapping 
 and 
 tags. It should just be what was
inside. We can then wrap it in our own custom tags without using
any regular expressions, as God intended. Death to regular
expressions and HTML manipulation!
---
 lib/jekyll/tags/highlight.rb | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/lib/jekyll/tags/highlight.rb b/lib/jekyll/tags/highlight.rb
index 5af5f2bd..5589d38f 100644
--- a/lib/jekyll/tags/highlight.rb
+++ b/lib/jekyll/tags/highlight.rb
@@ -77,8 +77,6 @@ eos
       def render_pygments(code, is_safe)
         Jekyll::External.require_with_graceful_fail('pygments')
 
-        @options[:encoding] = 'utf-8'
-
         highlighted_code = Pygments.highlight(
           code,
           :lexer   => @lang,
@@ -96,26 +94,26 @@ eos
           raise ArgumentError.new("Pygments.rb returned an unacceptable value when attempting to highlight some code.")
         end
 
-        highlighted_code
+        highlighted_code.sub('
', '').sub('
', '')
       end
 
       def render_rouge(code)
         Jekyll::External.require_with_graceful_fail('rouge')
         formatter = Rouge::Formatters::HTML.new(line_numbers: @options[:linenos], wrap: false)
         lexer = Rouge::Lexer.find_fancy(@lang, code) || Rouge::Lexers::PlainText
-        code = formatter.format(lexer.lex(code))
-        "
#{code}
"
+        formatter.format(lexer.lex(code))
       end
 
       def render_codehighlighter(code)
-        "
#{h(code).strip}
"
+        h(code).strip
       end
 
       def add_code_tag(code)
-        # Add nested  tags to code blocks
-        code = code.sub(/
\n*/,'
')
-        code = code.sub(/\n*<\/pre>/,"
")
-        code.strip
+        code_attributes = [
+          "class=\"language-#{@lang.to_s.gsub('+', '-')}\"",
+          "data-lang=\"#{@lang.to_s}\""
+        ].join(" ")
+        "
#{code.chomp}
"
       end
 
     end

From 1f503b24b3e4597716d411fe7710a680880125f8 Mon Sep 17 00:00:00 2001
From: Parker Moore 
Date: Mon, 9 Feb 2015 21:57:43 -0800
Subject: [PATCH 2/2] highlight: duplicate tests for pygments for rouge

Ensure that the output we get for pygments will match
that we get for rouge in all cases except line numbers.
---
 test/test_tags.rb | 223 +++++++++++++++++++++++++++++++++-------------
 1 file changed, 162 insertions(+), 61 deletions(-)

diff --git a/test/test_tags.rb b/test/test_tags.rb
index 9fbd9e39..931c3830 100644
--- a/test/test_tags.rb
+++ b/test/test_tags.rb
@@ -7,7 +7,7 @@ class TestTags < Test::Unit::TestCase
   def create_post(content, override = {}, converter_class = Jekyll::Converters::Markdown)
     stub(Jekyll).configuration do
       site_configuration({
-        "highlighter" => "pygments"
+        "highlighter" => "rouge"
       }.merge(override))
     end
     site = Site.new(Jekyll.configuration)
@@ -136,63 +136,65 @@ CONTENT
     end
   end
 
-  context "post content has highlight tag" do
-    setup do
-      fill_post("test")
+  context "with the pygments highlighter" do
+    context "post content has highlight tag" do
+      setup do
+        fill_post("test", {'highlighter' => 'pygments'})
+      end
+
+      should "not cause a markdown error" do
+        assert_no_match /markdown\-html\-error/, @result
+      end
+
+      should "render markdown with pygments" do
+        assert_match %{
test
}, @result
+      end
+
+      should "render markdown with pygments with line numbers" do
+        assert_match %{
1 test
}, @result
+      end
     end
 
-    should "not cause a markdown error" do
-      assert_no_match /markdown\-html\-error/, @result
+    context "post content has highlight with file reference" do
+      setup do
+        fill_post("./jekyll.gemspec", {'highlighter' => 'pygments'})
+      end
+
+      should "not embed the file" do
+        assert_match %{
./jekyll.gemspec
}, @result
+      end
     end
 
-    should "render markdown with pygments" do
-      assert_match %{
test
}, @result
+    context "post content has highlight tag with UTF character" do
+      setup do
+        fill_post("Æ", {'highlighter' => 'pygments'})
+      end
+
+      should "render markdown with pygments line handling" do
+        assert_match %{
Æ
}, @result
+      end
     end
 
-    should "render markdown with pygments with line numbers" do
-      assert_match %{
1 test
}, @result
-    end
-  end
-
-  context "post content has highlight with file reference" do
-    setup do
-      fill_post("./jekyll.gemspec")
-    end
-
-    should "not embed the file" do
-      assert_match %{
./jekyll.gemspec
}, @result
-    end
-  end
-
-  context "post content has highlight tag with UTF character" do
-    setup do
-      fill_post("Æ")
-    end
-
-    should "render markdown with pygments line handling" do
-      assert_match %{
Æ
}, @result
-    end
-  end
-
-  context "post content has highlight tag with preceding spaces & lines" do
-    setup do
-      fill_post <<-EOS
+    context "post content has highlight tag with preceding spaces & lines" do
+      setup do
+        code = <<-EOS
 
 
      [,1] [,2]
 [1,] FALSE TRUE
 [2,] FALSE TRUE
 EOS
+        fill_post(code, {'highlighter' => 'pygments'})
+      end
+
+      should "only strip the preceding newlines" do
+        assert_match %{
     [,1] [,2]}, @result
+      end
     end
 
-    should "only strip the preceding newlines" do
-      assert_match %{
     [,1] [,2]}, @result
-    end
-  end
-
-  context "post content has highlight tag with preceding spaces & lines in several places" do
-    setup do
-      fill_post <<-EOS
+    context "post content has highlight tag with preceding spaces & lines in several places" do
+      setup do
+        code = <<-EOS
 
 
      [,1] [,2]
@@ -203,34 +205,133 @@ EOS
 
 
 EOS
+        fill_post(code, {'highlighter' => 'pygments'})
+      end
+
+      should "only strip the newlines which precede and succeed the entire block" do
+        assert_match "
     [,1] [,2]\n\n\n[1,] FALSE TRUE\n[2,] FALSE TRUE
", @result
+      end
     end
 
-    should "only strip the newlines which precede and succeed the entire block" do
-      assert_match "
     [,1] [,2]\n\n\n[1,] FALSE TRUE\n[2,] FALSE TRUE
", @result
-    end
-  end
+    context "post content has highlight tag with preceding spaces & Windows-style newlines" do
+      setup do
+        fill_post "\r\n\r\n\r\n     [,1] [,2]", {'highlighter' => 'pygments'}
+      end
 
-  context "post content has highlight tag with preceding spaces & Windows-style newlines" do
-    setup do
-      fill_post "\r\n\r\n\r\n     [,1] [,2]"
+      should "only strip the preceding newlines" do
+        assert_match %{
     [,1] [,2]}, @result
+      end
     end
 
-    should "only strip the preceding newlines" do
-      assert_match %{
     [,1] [,2]}, @result
-    end
-  end
-
-  context "post content has highlight tag with only preceding spaces" do
-    setup do
-      fill_post <<-EOS
+    context "post content has highlight tag with only preceding spaces" do
+      setup do
+        code = <<-EOS
      [,1] [,2]
 [1,] FALSE TRUE
 [2,] FALSE TRUE
 EOS
+        fill_post(code, {'highlighter' => 'pygments'})
+      end
+
+      should "only strip the preceding newlines" do
+        assert_match %{
     [,1] [,2]}, @result
+      end
+    end
+  end
+
+  context "with the rouge highlighter" do
+    context "post content has highlight tag" do
+      setup do
+        fill_post("test")
+      end
+
+      should "render markdown with rouge" do
+        assert_match %{
test
}, @result
+      end
+
+      should "render markdown with rouge with line numbers" do
+        assert_match %{
1
test\n
}, @result
+      end
     end
 
-    should "only strip the preceding newlines" do
-      assert_match %{
     [,1] [,2]}, @result
+    context "post content has highlight with file reference" do
+      setup do
+        fill_post("./jekyll.gemspec")
+      end
+
+      should "not embed the file" do
+        assert_match %{
./jekyll.gemspec
}, @result
+      end
+    end
+
+    context "post content has highlight tag with UTF character" do
+      setup do
+        fill_post("Æ")
+      end
+
+      should "render markdown with pygments line handling" do
+        assert_match %{
Æ
}, @result
+      end
+    end
+
+    context "post content has highlight tag with preceding spaces & lines" do
+      setup do
+        fill_post <<-EOS
+
+
+     [,1] [,2]
+[1,] FALSE TRUE
+[2,] FALSE TRUE
+EOS
+      end
+
+      should "only strip the preceding newlines" do
+        assert_match %{
     [,1] [,2]}, @result
+      end
+    end
+
+    context "post content has highlight tag with preceding spaces & lines in several places" do
+      setup do
+        fill_post <<-EOS
+
+
+     [,1] [,2]
+
+
+[1,] FALSE TRUE
+[2,] FALSE TRUE
+
+
+EOS
+      end
+
+      should "only strip the newlines which precede and succeed the entire block" do
+        assert_match "
     [,1] [,2]\n\n\n[1,] FALSE TRUE\n[2,] FALSE TRUE
", @result
+      end
+    end
+
+    context "post content has highlight tag with preceding spaces & Windows-style newlines" do
+      setup do
+        fill_post "\r\n\r\n\r\n     [,1] [,2]"
+      end
+
+      should "only strip the preceding newlines" do
+        assert_match %{
     [,1] [,2]}, @result
+      end
+    end
+
+    context "post content has highlight tag with only preceding spaces" do
+      setup do
+        fill_post <<-EOS
+     [,1] [,2]
+[1,] FALSE TRUE
+[2,] FALSE TRUE
+EOS
+      end
+
+      should "only strip the preceding newlines" do
+        assert_match %{
     [,1] [,2]}, @result
+      end
     end
   end