report all libxml errors in updater debug output

force utf8 encoding if devforceupdate is on
parser: try to convert non-unicode feeds with specified encoding to utf8
before trying to remove dangling utf8 characters in case of utf8-related
libxml errors because doing so produces garbage content

diff --git a/classes/feedparser.php b/classes/feedparser.php
index 1c97e496b25b09bd4fd69b572f5c47530128bd8c..de6c565428a1a803ea316790da8279b742f389e6 100644 (file)
--- a/classes/feedparser.php
+++ b/classes/feedparser.php
@@ -13,6 +13,16 @@ class FeedParser {
        const FEED_RSS = 1;
        const FEED_ATOM = 2;
 
+       function normalize_encoding($data) {
+               if (preg_match('/^(<\?xml[\t\n\r ].*?encoding[\t\n\r ]*=[\t\n\r ]*["\'])(.+?)(["\'].*?\?>)/s', $data, $matches) === 1) {
+                       $data = mb_convert_encoding($data, 'UTF-8', $matches[2]);
+
+                       $data = preg_replace('/^<\?xml[\t\n\r ].*?\?>/s', $matches[1] . "UTF-8" . $matches[3] , $data);
+               }
+
+               return $data;
+       }
+
        function __construct($data) {
                libxml_use_internal_errors(true);
                libxml_clear_errors();
@@ -25,19 +35,15 @@ class FeedParser {
 
                // libxml compiled without iconv?
                if ($error && $error->code == 32) {
-                       if (preg_match('/^(<\?xml[\t\n\r ].*?encoding[\t\n\r ]*=[\t\n\r ]*["\'])(.+?)(["\'].*?\?>)/s', $data, $matches) === 1) {
-                               $data = mb_convert_encoding($data, 'UTF-8', $matches[2]);
-
-                               $data = preg_replace('/^<\?xml[\t\n\r ].*?\?>/s', $matches[1] . "UTF-8" . $matches[3] , $data);
+                       $data = $this->normalize_encoding($data);
 
-                               if ($data) {
-                                       libxml_clear_errors();
+                       if ($data) {
+                               libxml_clear_errors();
 
-                                       $this->doc = new DOMDocument();
-                                       $this->doc->loadXML($data);
+                               $this->doc = new DOMDocument();
+                               $this->doc->loadXML($data);
 
-                                       $error = libxml_get_last_error();
-                               }
+                               $error = libxml_get_last_error();
                        }
                }
 
@@ -45,6 +51,9 @@ class FeedParser {
                if ($error) {
                        foreach (libxml_get_errors() as $err) {
                                if ($err->code == 9) {
+                                       // if the source feed is not in utf8, next conversion will fail
+                                       $data = $this->normalize_encoding($data);
+
                                        // remove dangling bytes
                                        $data = mb_convert_encoding($data, 'UTF-8', 'UTF-8');
 

diff --git a/classes/feeds.php b/classes/feeds.php
index 7f5fd10af3fa8996856b3de80bcf36d3abc26879..2c17a225788ad454d0c335e4fcfeed806f51d90c 100644 (file)
--- a/classes/feeds.php
+++ b/classes/feeds.php
@@ -148,7 +148,7 @@ class Feeds extends Handler_Protected {
                                        $override_order = false, $include_children = false) {
 
                if (isset($_REQUEST["DevForceUpdate"]))
-                       header("Content-Type: text/plain");
+                       header("Content-Type: text/plain; charset=utf-8");
 
                $disable_cache = false;
 

diff --git a/include/rssfuncs.php b/include/rssfuncs.php
index bfbec0919275c29c4fc722766e27c84ddf4a4364..bc6048217f671869e1bceb249231fc291d05cd7d 100644 (file)
--- a/include/rssfuncs.php
+++ b/include/rssfuncs.php
@@ -1110,7 +1110,13 @@
 
                        $error_msg = db_escape_string(mb_substr($rss->error(), 0, 245));
 
-                       _debug("error fetching feed: $error_msg", $debug_enabled);
+                       _debug("fetch error: $error_msg", $debug_enabled);
+
+                       if (count($rss->errors()) > 1) {
+                               foreach ($rss->errors() as $error) {
+                                       _debug("+ $error");
+                               }
+                       }
 
                        db_query(
                                "UPDATE ttrss_feeds SET last_error = '$error_msg',