improve encoding utf8->ascii for non-printable chars

diff --git a/NEWS b/NEWS
index 3d2f11d..eff5673 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,9 @@
+XML-RPC for PHP version 4.XX - unreleased
+
+* improved: when encoding utf8 text into us-ascii xml, use character entity references for characters number 0-31
+  (ascii non printable characters), as we were already doing when encoding iso-8859-1 text into us-ascii xml
+
+

 XML-RPC for PHP version 4.5.2 - 2021/1/11
 
 * improved: better phpdocs in the the php code generated by the Wrapper class
 XML-RPC for PHP version 4.5.2 - 2021/1/11
 
 * improved: better phpdocs in the the php code generated by the Wrapper class

diff --git a/src/Helper/Charset.php b/src/Helper/Charset.php
index eca3e46..a72016e 100644 (file)
--- a/src/Helper/Charset.php
+++ b/src/Helper/Charset.php
@@ -168,7 +168,14 @@ class Charset
                     $ch = $data[$nn];
                     $ii = ord($ch);
                     // 7 bits: 0bbbbbbb (127)
                     $ch = $data[$nn];
                     $ii = ord($ch);
                     // 7 bits: 0bbbbbbb (127)

-                    if ($ii < 128) {
+                    if ($ii < 32) {
+                        if ($conversion == 'UTF-8_' || $conversion == 'UTF-8_US-ASCII') {
+                            $escapedData .= sprintf('&#%d;', $ii);
+                        } else {
+                            $escapedData .= $ch;
+                        }
+                    }
+                    else if ($ii < 128) {

                         /// @todo shall we replace this with a (supposedly) faster str_replace?
                         switch ($ii) {
                             case 34:
                         /// @todo shall we replace this with a (supposedly) faster str_replace?
                         switch ($ii) {
                             case 34:


@@ -192,35 +199,26 @@ class Charset
                     } // 11 bits: 110bbbbb 10bbbbbb (2047)
                     elseif ($ii >> 5 == 6) {
                         $b1 = ($ii & 31);
                     } // 11 bits: 110bbbbb 10bbbbbb (2047)
                     elseif ($ii >> 5 == 6) {
                         $b1 = ($ii & 31);

-                        $ii = ord($data[$nn + 1]);
-                        $b2 = ($ii & 63);
+                        $b2 = (ord($data[$nn + 1]) & 63);

                         $ii = ($b1 * 64) + $b2;
                         $ii = ($b1 * 64) + $b2;

-                        $ent = sprintf('&#%d;', $ii);
-                        $escapedData .= $ent;
+                        $escapedData .= sprintf('&#%d;', $ii);

                         $nn += 1;
                     } // 16 bits: 1110bbbb 10bbbbbb 10bbbbbb
                     elseif ($ii >> 4 == 14) {
                         $b1 = ($ii & 15);
                         $nn += 1;
                     } // 16 bits: 1110bbbb 10bbbbbb 10bbbbbb
                     elseif ($ii >> 4 == 14) {
                         $b1 = ($ii & 15);

-                        $ii = ord($data[$nn + 1]);
-                        $b2 = ($ii & 63);
-                        $ii = ord($data[$nn + 2]);
-                        $b3 = ($ii & 63);
+                        $b2 = (ord($data[$nn + 1]) & 63);
+                        $b3 = (ord($data[$nn + 2]) & 63);

                         $ii = ((($b1 * 64) + $b2) * 64) + $b3;
                         $ii = ((($b1 * 64) + $b2) * 64) + $b3;

-                        $ent = sprintf('&#%d;', $ii);
-                        $escapedData .= $ent;
+                        $escapedData .= sprintf('&#%d;', $ii);

                         $nn += 2;
                     } // 21 bits: 11110bbb 10bbbbbb 10bbbbbb 10bbbbbb
                     elseif ($ii >> 3 == 30) {
                         $b1 = ($ii & 7);
                         $nn += 2;
                     } // 21 bits: 11110bbb 10bbbbbb 10bbbbbb 10bbbbbb
                     elseif ($ii >> 3 == 30) {
                         $b1 = ($ii & 7);

-                        $ii = ord($data[$nn + 1]);
-                        $b2 = ($ii & 63);
-                        $ii = ord($data[$nn + 2]);
-                        $b3 = ($ii & 63);
-                        $ii = ord($data[$nn + 3]);
-                        $b4 = ($ii & 63);
+                        $b2 = (ord($data[$nn + 1]) & 63);
+                        $b3 = (ord($data[$nn + 2]) & 63);
+                        $b4 = (ord($data[$nn + 3]) & 63);

                         $ii = ((((($b1 * 64) + $b2) * 64) + $b3) * 64) + $b4;
                         $ii = ((((($b1 * 64) + $b2) * 64) + $b3) * 64) + $b4;

-                        $ent = sprintf('&#%d;', $ii);
-                        $escapedData .= $ent;
+                        $escapedData .= sprintf('&#%d;', $ii);

                         $nn += 3;
                     }
                 }
                         $nn += 3;
                     }
                 }

diff --git a/tests/0CharsetTest.php b/tests/0CharsetTest.php
index f94e3f4..549b0f4 100644 (file)
--- a/tests/0CharsetTest.php
+++ b/tests/0CharsetTest.php
@@ -15,8 +15,9 @@ use PhpXmlRpc\Helper\Charset;
  *     chcp 28591 (latin1)
  *     chcp 65001 (utf8)
  *
  *     chcp 28591 (latin1)
  *     chcp 65001 (utf8)
  *

- * @todo add tests for conversion: utf8 -> ascii
+ * @todo add tests for conversion: utf8 -> ascii (incl. chars 0-31)

  * @todo add tests for conversion: latin1 -> utf8
  * @todo add tests for conversion: latin1 -> utf8

+ * @todo add tests for conversion: latin1 -> ascii

  */
 class CharsetTest extends PhpXmlRpc_PolyfillTestCase
 {
  */
 class CharsetTest extends PhpXmlRpc_PolyfillTestCase
 {


@@ -49,6 +50,15 @@ class CharsetTest extends PhpXmlRpc_PolyfillTestCase
         );
     }
 
         );
     }
 

+    protected function utf8ToAscii($data)
+    {
+        return Charset::instance()->encodeEntities(
+            $data,
+            'UTF-8',
+            'US-ASCII'
+        );
+    }
+

     public function testUtf8ToLatin1All()
     {
         /*$this->assertEquals(
     public function testUtf8ToLatin1All()
     {
         /*$this->assertEquals(