Handle empty meta tag attributes during encoding detection

Additionally, handle malformed metatags by assuming no encoding
inside of raising a RuntimeError.

Fixes #311

Author: jeremyevans

lib/net/http/response.rb

@@ -472,40 +472,42 @@ def check_bom(str)
   def scanning_meta(str)
     require 'strscan'
     ss = StringScanner.new(str)
-    if ss.scan_until(/<meta[\t\n\f\r ]*/)
-      attrs = {} # attribute_list
-      got_pragma = false
-      need_pragma = nil
-      charset = nil
-
-      # step: Attributes
-      while attr = get_attribute(ss)
-        name, value = *attr
-        next if attrs[name]
-        attrs[name] = true
-        case name
-        when 'http-equiv'
-          got_pragma = true if value == 'content-type'
-        when 'content'
-          encoding = extracting_encodings_from_meta_elements(value)
-          unless charset
-            charset = encoding
+    catch(:invalid_meta_tag) do
+      if ss.scan_until(/<meta[\t\n\f\r ]*/)
+        attrs = {} # attribute_list
+        got_pragma = false
+        need_pragma = nil
+        charset = nil
+
+        # step: Attributes
+        while attr = get_attribute(ss)
+          name, value = *attr
+          next if attrs[name]
+          attrs[name] = true
+          case name
+          when 'http-equiv'
+            got_pragma = true if value == 'content-type'
+          when 'content'
+            encoding = extracting_encodings_from_meta_elements(value)
+            unless charset
+              charset = encoding
+            end
+            need_pragma = true
+          when 'charset'
+            need_pragma = false
+            charset = value
           end
-          need_pragma = true
-        when 'charset'
-          need_pragma = false
-          charset = value
         end
-      end
 
-      # step: Processing
-      return if need_pragma.nil?
-      return if need_pragma && !got_pragma
+        # step: Processing
+        return if need_pragma.nil?
+        return if need_pragma && !got_pragma
 
-      charset = Encoding.find(charset) rescue nil
-      return unless charset
-      charset = Encoding::UTF_8 if charset == Encoding::UTF_16
-      return charset # tentative
+        charset = Encoding.find(charset) rescue nil
+        return unless charset
+        charset = Encoding::UTF_8 if charset == Encoding::UTF_16
+        return charset # tentative
+      end
     end
     nil
   end
@@ -518,7 +520,7 @@ def get_attribute(ss)
     end
     name = ss.scan(/[^=\t\n\f\r \/>]*/)
     name.downcase!
-    raise if name.empty?
+    throw :invalid_meta_tag if name.empty?
     ss.skip(/[\t\n\f\r ]*/)
     if ss.getch != '='
       value = ''
@@ -528,18 +530,18 @@ def get_attribute(ss)
     case ss.peek(1)
     when '"'
       ss.getch
-      value = ss.scan(/[^"]+/)
+      value = ss.scan(/[^"]*/)
       value.downcase!
       ss.getch
     when "'"
       ss.getch
-      value = ss.scan(/[^']+/)
+      value = ss.scan(/[^']*/)
       value.downcase!
       ss.getch
     when '>'
       value = ''
     else
-      value = ss.scan(/[^\t\n\f\r >]+/)
+      throw :invalid_meta_tag unless value = ss.scan(/[^\t\n\f\r >]+/)
       value.downcase!
     end
     [name, value]

test/net/http/test_httpresponse.rb

@@ -313,6 +313,78 @@ def test_read_body_body_encoding_true_with_iso8859_1_meta_content_charset
     assert_equal Encoding::ISO_8859_1, body.encoding
   end
 
+  def test_read_body_body_encoding_with_empty_attribute
+    res_body = "<meta http-equiv='content-type' empty1='' empty2="" content='text/html; charset=UTF-8'>hello\u1234</html>"
+    io = dummy_io(<<EOS)
+HTTP/1.1 200 OK
+Connection: close
+Content-Length: #{res_body.bytesize}
+Content-Type: text/html
+
+#{res_body}
+EOS
+
+    res = Net::HTTPResponse.read_new(io)
+    res.body_encoding = true
+
+    body = nil
+
+    res.reading_body io, true do
+      body = res.read_body
+    end
+
+    assert_equal res_body, body
+    assert_equal Encoding::UTF_8, body.encoding
+  end
+
+  def test_read_body_body_encoding_unclosed_meta_tag_in_attribute
+    res_body = "<meta http-equiv='content-type'"
+    io = dummy_io(<<EOS)
+HTTP/1.1 200 OK
+Connection: close
+Content-Length: #{res_body.bytesize}
+Content-Type: text/html
+
+#{res_body}
+EOS
+
+    res = Net::HTTPResponse.read_new(io)
+    res.body_encoding = true
+
+    body = nil
+
+    res.reading_body io, true do
+      body = res.read_body
+    end
+
+    assert_equal res_body, body
+    assert_equal Encoding::US_ASCII, body.encoding
+  end
+
+  def test_read_body_body_encoding_unclosed_meta_tag_in_attribute_value
+    res_body = "<meta http-equiv='content-type' bad="
+    io = dummy_io(<<EOS)
+HTTP/1.1 200 OK
+Connection: close
+Content-Length: #{res_body.bytesize}
+Content-Type: text/html
+
+#{res_body}
+EOS
+
+    res = Net::HTTPResponse.read_new(io)
+    res.body_encoding = true
+
+    body = nil
+
+    res.reading_body io, true do
+      body = res.read_body
+    end
+
+    assert_equal res_body, body
+    assert_equal Encoding::US_ASCII, body.encoding
+  end
+
   def test_read_body_block
     io = dummy_io(<<EOS)
 HTTP/1.1 200 OK