LCOV - code coverage report
Current view: top level - lib/util/charset/tests - iconv.c (source / functions) Hit Total Coverage
Test: coverage report for master 6248eab5 Lines: 141 216 65.3 %
Date: 2021-08-25 13:27:56 Functions: 9 11 81.8 %

          Line data    Source code
       1             : /* 
       2             :    Unix SMB/CIFS implementation.
       3             : 
       4             :    local testing of iconv routines. This tests the system iconv code against
       5             :    the built-in iconv code
       6             : 
       7             :    Copyright (C) Andrew Tridgell 2004
       8             :    
       9             :    This program is free software; you can redistribute it and/or modify
      10             :    it under the terms of the GNU General Public License as published by
      11             :    the Free Software Foundation; either version 3 of the License, or
      12             :    (at your option) any later version.
      13             :    
      14             :    This program is distributed in the hope that it will be useful,
      15             :    but WITHOUT ANY WARRANTY; without even the implied warranty of
      16             :    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      17             :    GNU General Public License for more details.
      18             :    
      19             :    You should have received a copy of the GNU General Public License
      20             :    along with this program.  If not, see <http://www.gnu.org/licenses/>.
      21             : */
      22             : 
      23             : #include "includes.h"
      24             : #include "torture/torture.h"
      25             : #include "system/iconv.h"
      26             : #include "system/time.h"
      27             : #include "libcli/raw/libcliraw.h"
      28             : #include "param/param.h"
      29             : #include "torture/util.h"
      30             : #include "torture/local/proto.h"
      31             : #include "talloc.h"
      32             : 
      33             : #ifdef HAVE_NATIVE_ICONV
      34             : 
      35           3 : static bool iconv_untestable(struct torture_context *tctx)
      36             : {
      37             :         iconv_t cd;
      38             : 
      39           3 :         cd = iconv_open("UTF-16LE", "UCS-4LE");
      40           3 :         if (cd == (iconv_t)-1)
      41           0 :                 torture_skip(tctx, "unable to test - system iconv library does not support UTF-16LE -> UCS-4LE");
      42           3 :         iconv_close(cd);
      43             : 
      44           3 :         cd = iconv_open("UTF-16LE", "CP850");
      45           3 :         if (cd == (iconv_t)-1)
      46           0 :                 torture_skip(tctx, "unable to test - system iconv library does not support UTF-16LE -> CP850\n");
      47           3 :         iconv_close(cd);
      48             : 
      49           3 :         return false;
      50             : }
      51             : 
      52             : /*
      53             :   generate a UTF-16LE buffer for a given unicode codepoint
      54             : */
      55     1048576 : static int gen_codepoint_utf16(unsigned int codepoint,
      56             :                                char *buf, size_t *size)
      57             : {
      58             :         static iconv_t cd;
      59             :         uint8_t in[4];
      60             :         char *ptr_in;
      61             :         size_t size_in, size_out, ret;
      62     1048576 :         if (!cd) {
      63           1 :                 cd = iconv_open("UTF-16LE", "UCS-4LE");
      64           1 :                 if (cd == (iconv_t)-1) {
      65           0 :                         cd = NULL;
      66           0 :                         return -1;
      67             :                 }
      68             :         }
      69             : 
      70     1048576 :         in[0] = codepoint & 0xFF;
      71     1048576 :         in[1] = (codepoint>>8) & 0xFF;
      72     1048576 :         in[2] = (codepoint>>16) & 0xFF;
      73     1048576 :         in[3] = (codepoint>>24) & 0xFF;
      74             : 
      75     1048576 :         ptr_in = (char *)in;
      76     1048576 :         size_in = 4;
      77     1048576 :         size_out = 8;
      78             : 
      79     1048576 :         ret = iconv(cd, &ptr_in, &size_in, &buf, &size_out);
      80             : 
      81     1048576 :         *size = 8 - size_out;
      82             : 
      83     1048576 :         return ret;
      84             : }
      85             : 
      86             : 
      87             : /*
      88             :   work out the unicode codepoint of the first UTF-8 character in the buffer
      89             : */
      90           0 : static unsigned int get_codepoint(char *buf, size_t size, const char *charset)
      91             : {
      92             :         iconv_t cd;
      93             :         uint8_t out[4];
      94             :         char *ptr_out;
      95             :         size_t size_out, size_in, ret;
      96             : 
      97           0 :         cd = iconv_open("UCS-4LE", charset);
      98             : 
      99           0 :         size_in = size;
     100           0 :         ptr_out = (char *)out;
     101           0 :         size_out = sizeof(out);
     102           0 :         memset(out, 0, sizeof(out));
     103             : 
     104           0 :         ret = iconv(cd, &buf, &size_in, &ptr_out, &size_out);
     105           0 :         iconv_close(cd);
     106           0 :         if (ret == (size_t) -1) {
     107           0 :                 return (unsigned int)-1;
     108             :         }
     109             : 
     110           0 :         return out[0] | (out[1]<<8) | (out[2]<<16) | (out[3]<<24);
     111             : }
     112             : 
     113             : /*
     114             :   display a buffer with name prefix
     115             : */
     116           0 : static void show_buf(const char *name, uint8_t *buf, size_t size)
     117             : {
     118             :         int i;
     119           0 :         printf("%s ", name);
     120           0 :         for (i=0;i<size;i++) {
     121           0 :                 printf("%02x ", buf[i]);
     122             :         }
     123           0 :         printf("\n");
     124           0 : }
     125             : 
     126             : /*
     127             :   given a UTF-16LE buffer, test the system and built-in iconv code to
     128             :   make sure they do exactly the same thing in converting the buffer to
     129             :   "charset", then convert it back again and ensure we get the same
     130             :   buffer back
     131             : */
     132     2046528 : static bool test_buffer(struct torture_context *test, 
     133             :                         uint8_t *inbuf, size_t size, const char *charset)
     134             : {
     135             :         uint8_t buf1[1000], buf2[1000], buf3[1000];
     136             :         size_t outsize1, outsize2, outsize3;
     137             :         const char *ptr_in1;
     138             :         char *ptr_in2;
     139             :         char *ptr_out;
     140             :         size_t size_in1, size_in2, size_in3;
     141             :         size_t ret1, ret2, ret3, len1, len2;
     142             :         int errno1, errno2;
     143             :         static iconv_t cd;
     144             :         static smb_iconv_t cd2, cd3;
     145             :         static const char *last_charset;
     146             : 
     147     2046528 :         if (cd && last_charset) {
     148     2046527 :                 iconv_close(cd);
     149     2046527 :                 smb_iconv_close(cd2);
     150     2046527 :                 smb_iconv_close(cd3);
     151     2046527 :                 cd = NULL;
     152             :         }
     153             : 
     154     2046528 :         if (!cd) {
     155     2046528 :                 cd = iconv_open(charset, "UTF-16LE");
     156     2046528 :                 if (cd == (iconv_t)-1) {
     157           0 :                         torture_fail(test, 
     158             :                                      talloc_asprintf(test, 
     159             :                                                      "failed to open %s to UTF-16LE",
     160             :                                                      charset));
     161             :                 }
     162     2046528 :                 cd2 = smb_iconv_open_ex(test, charset, "UTF-16LE", lpcfg_parm_bool(test->lp_ctx, NULL, "iconv", "use_builtin_handlers", true));
     163     2046528 :                 if (cd2 == (iconv_t)-1) {
     164           0 :                         torture_fail(test, 
     165             :                                      talloc_asprintf(test, 
     166             :                                                      "failed to open %s to UTF-16LE via smb_iconv_open_ex",
     167             :                                                      charset));
     168             :                 }
     169     2046528 :                 cd3 = smb_iconv_open_ex(test, "UTF-16LE", charset, lpcfg_parm_bool(test->lp_ctx, NULL, "iconv", "use_builtin_handlers", true));
     170     2046528 :                 if (cd3 == (iconv_t)-1) {
     171           0 :                         torture_fail(test, 
     172             :                                      talloc_asprintf(test, 
     173             :                                                      "failed to open UTF-16LE to %s via smb_iconv_open_ex",
     174             :                                                      charset));
     175             :                 }
     176     2046528 :                 last_charset = charset;
     177             :         }
     178             : 
     179             :         /* internal convert to charset - placing result in buf1 */
     180     2046528 :         ptr_in1 = (const char *)inbuf;
     181     2046528 :         ptr_out = (char *)buf1;
     182     2046528 :         size_in1 = size;
     183     2046528 :         outsize1 = sizeof(buf1);
     184             : 
     185     4093056 :         memset(ptr_out, 0, outsize1);
     186     2046528 :         errno = 0;
     187     2046528 :         ret1 = smb_iconv(cd2, &ptr_in1, &size_in1, &ptr_out, &outsize1);
     188     2046528 :         errno1 = errno;
     189             : 
     190             :         /* system convert to charset - placing result in buf2 */
     191     2046528 :         ptr_in2 = (char *)inbuf;
     192     2046528 :         ptr_out = (char *)buf2;
     193     2046528 :         size_in2 = size;
     194     2046528 :         outsize2 = sizeof(buf2);
     195             :         
     196     4093056 :         memset(ptr_out, 0, outsize2);
     197     2046528 :         errno = 0;
     198     2046528 :         ret2 = iconv(cd, &ptr_in2, &size_in2, &ptr_out, &outsize2);
     199     2046528 :         errno2 = errno;
     200             : 
     201     2046528 :         len1 = sizeof(buf1) - outsize1;
     202     2046528 :         len2 = sizeof(buf2) - outsize2;
     203             : 
     204             :         /* codepoints above 1M are not interesting for now */
     205     2046528 :         if (len2 > len1 && 
     206           0 :             memcmp(buf1, buf2, len1) == 0 && 
     207           0 :             get_codepoint((char *)(buf2+len1), len2-len1, charset) >= (1<<20)) {
     208           0 :                 return true;
     209             :         }
     210     2046528 :         if (len1 > len2 && 
     211           0 :             memcmp(buf1, buf2, len2) == 0 && 
     212           0 :             get_codepoint((char *)(buf1+len2), len1-len2, charset) >= (1<<20)) {
     213           0 :                 return true;
     214             :         }
     215             : 
     216     2046528 :         torture_assert_int_equal(test, ret1, ret2, "ret mismatch");
     217             : 
     218     2046528 :         if (errno1 != errno2) {
     219           0 :                 show_buf(" rem1:", inbuf+(size-size_in1), size_in1);
     220           0 :                 show_buf(" rem2:", inbuf+(size-size_in2), size_in2);
     221           0 :                 torture_fail(test, talloc_asprintf(test, 
     222             :                                                    "errno mismatch with %s internal=%d/%s system=%d/%s", 
     223             :                                                    charset, 
     224             :                                                    errno1, strerror(errno1), 
     225             :                                                    errno2, strerror(errno2)));
     226             :         }
     227             :         
     228     2046528 :         torture_assert_int_equal(test, outsize1, outsize2, "outsize mismatch");
     229             :         
     230     2046528 :         torture_assert_int_equal(test, size_in1, size_in2, "size_in mismatch");
     231             : 
     232     4093056 :         if (len1 != len2 ||
     233     2046528 :             memcmp(buf1, buf2, len1) != 0) {
     234           0 :                 torture_comment(test, "size=%d ret1=%d ret2=%d", (int)size, (int)ret1, (int)ret2);
     235           0 :                 show_buf(" IN1:", inbuf, size-size_in1);
     236           0 :                 show_buf(" IN2:", inbuf, size-size_in2);
     237           0 :                 show_buf("OUT1:", buf1, len1);
     238           0 :                 show_buf("OUT2:", buf2, len2);
     239           0 :                 if (len2 > len1 && memcmp(buf1, buf2, len1) == 0) {
     240           0 :                         torture_comment(test, "next codepoint is %u", 
     241             :                                get_codepoint((char *)(buf2+len1), len2-len1, charset));
     242             :                 }
     243           0 :                 if (len1 > len2 && memcmp(buf1, buf2, len2) == 0) {
     244           0 :                         torture_comment(test, "next codepoint is %u", 
     245             :                                get_codepoint((char *)(buf1+len2),len1-len2, charset));
     246             :                 }
     247             : 
     248           0 :                 torture_fail(test, "failed");
     249             :         }
     250             : 
     251             :         /* convert back to UTF-16, putting result in buf3 */
     252     2046528 :         size = size - size_in1;
     253     2046528 :         ptr_in1 = (const char *)buf1;
     254     2046528 :         ptr_out = (char *)buf3;
     255     2046528 :         size_in3 = len1;
     256     2046528 :         outsize3 = sizeof(buf3);
     257             : 
     258     4093056 :         memset(ptr_out, 0, outsize3);
     259     2046528 :         ret3 = smb_iconv(cd3, &ptr_in1, &size_in3, &ptr_out, &outsize3);
     260             : 
     261             :         /* we only internally support the first 1M codepoints */
     262     2046528 :         if (outsize3 != sizeof(buf3) - size &&
     263           0 :             get_codepoint((char *)(inbuf+sizeof(buf3) - outsize3), 
     264           0 :                           size - (sizeof(buf3) - outsize3),
     265             :                           "UTF-16LE") >= (1<<20)) {
     266           0 :                 return true;
     267             :         }
     268             : 
     269     2046528 :         torture_assert_int_equal(test, ret3, 0, talloc_asprintf(test, 
     270             :                                                                 "pull failed - %s", strerror(errno)));
     271             : 
     272     2046528 :         if (strncmp(charset, "UTF", 3) != 0) {
     273             :                 /* don't expect perfect mappings for non UTF charsets */
     274           0 :                 return true;
     275             :         }
     276             : 
     277             : 
     278     1546528 :         torture_assert_int_equal(test, outsize3, sizeof(buf3) - size, 
     279             :                 "wrong outsize3");
     280             :         
     281     1546528 :         if (memcmp(buf3, inbuf, size) != 0) {
     282           0 :                 torture_comment(test, "pull bytes mismatch:");
     283           0 :                 show_buf("inbuf", inbuf, size);
     284           0 :                 show_buf(" buf3", buf3, sizeof(buf3) - outsize3);
     285           0 :                 torture_comment(test, "next codepoint is %u\n", 
     286           0 :                        get_codepoint((char *)(inbuf+sizeof(buf3) - outsize3), 
     287           0 :                                      size - (sizeof(buf3) - outsize3),
     288             :                                      "UTF-16LE"));
     289           0 :                 torture_fail(test, "");
     290             :         }
     291             : 
     292           0 :         return true;
     293             : }
     294             : 
     295             : 
     296             : /*
     297             :   test the push_codepoint() and next_codepoint() functions for a given
     298             :   codepoint
     299             : */
     300     1048576 : static bool test_codepoint(struct torture_context *tctx, unsigned int codepoint)
     301             : {
     302             :         uint8_t buf[10];
     303             :         size_t size, size2;
     304             :         codepoint_t c;
     305             : 
     306     1048576 :         size = push_codepoint_handle(lpcfg_iconv_handle(tctx->lp_ctx), (char *)buf, codepoint);
     307     1048576 :         torture_assert(tctx, size != -1 || (codepoint >= 0xd800 && codepoint <= 0x10000), 
     308             :                        "Invalid Codepoint range");
     309             : 
     310     1048576 :         if (size == -1) return true;
     311             : 
     312     1046528 :         buf[size] = random();
     313     1046528 :         buf[size+1] = random();
     314     1046528 :         buf[size+2] = random();
     315     1046528 :         buf[size+3] = random();
     316             : 
     317     1046528 :         c = next_codepoint_handle(lpcfg_iconv_handle(tctx->lp_ctx), (char *)buf, &size2);
     318             : 
     319     1046528 :         torture_assert(tctx, c == codepoint, 
     320             :                        talloc_asprintf(tctx, 
     321             :                                        "next_codepoint(%u) failed - gave %u", codepoint, c));
     322             : 
     323     1046528 :         torture_assert(tctx, size2 == size, 
     324             :                         talloc_asprintf(tctx, "next_codepoint(%u) gave wrong size %d (should be %d)\n", 
     325             :                        codepoint, (int)size2, (int)size));
     326             : 
     327           0 :         return true;
     328             : }
     329             : 
     330           1 : static bool test_next_codepoint(struct torture_context *tctx)
     331             : {
     332             :         unsigned int codepoint;
     333           1 :         if (iconv_untestable(tctx))
     334           0 :                 return true;
     335             : 
     336     1048576 :         for (codepoint=0;codepoint<(1<<20);codepoint++) {
     337     1048576 :                 if (!test_codepoint(tctx, codepoint))
     338           0 :                         return false;
     339             :         }
     340           0 :         return true;
     341             : }
     342             : 
     343           1 : static bool test_first_1m(struct torture_context *tctx)
     344             : {
     345             :         unsigned int codepoint;
     346             :         size_t size;
     347             :         unsigned char inbuf[1000];
     348             : 
     349           1 :         if (iconv_untestable(tctx))
     350           0 :                 return true;
     351             : 
     352     1048576 :         for (codepoint=0;codepoint<(1<<20);codepoint++) {
     353     1048576 :                 if (gen_codepoint_utf16(codepoint, (char *)inbuf, &size) != 0) {
     354        2048 :                         continue;
     355             :                 }
     356             : 
     357     1046528 :                 if (codepoint % 1000 == 0) {
     358        1047 :                         if (torture_setting_bool(tctx, "progress", true)) {
     359           0 :                                 torture_comment(tctx, "codepoint=%u   \r", codepoint);
     360           0 :                                 fflush(stdout);
     361             :                         }
     362             :                 }
     363             : 
     364     1046528 :                 if (!test_buffer(tctx, inbuf, size, "UTF-8"))
     365           0 :                         return false;
     366             :         }
     367           0 :         return true;
     368             : }
     369             : 
     370           1 : static bool test_random_5m(struct torture_context *tctx)
     371             : {
     372             :         unsigned char inbuf[1000];
     373             :         unsigned int i;
     374             : 
     375           1 :         if (iconv_untestable(tctx))
     376           0 :                 return true;
     377             : 
     378      500000 :         for (i=0;i<500000;i++) {
     379             :                 size_t size;
     380             :                 unsigned int c;
     381             : 
     382      500000 :                 if (i % 1000 == 0) {
     383         500 :                         if (torture_setting_bool(tctx, "progress", true)) {
     384           0 :                                 torture_comment(tctx, "i=%u              \r", i);
     385           0 :                                 fflush(stdout);
     386             :                         }
     387             :                 }
     388             : 
     389      500000 :                 size = random() % 100;
     390    25238277 :                 for (c=0;c<size;c++) {
     391    24738277 :                         if (random() % 100 < 80) {
     392    19788847 :                                 inbuf[c] = random() % 128;
     393             :                         } else {
     394     4949430 :                                 inbuf[c] = random();
     395             :                         }
     396    24738277 :                         if (random() % 10 == 0) {
     397     2474412 :                                 inbuf[c] |= 0xd8;
     398             :                         }
     399    24738277 :                         if (random() % 10 == 0) {
     400     2472313 :                                 inbuf[c] |= 0xdc;
     401             :                         }
     402             :                 }
     403      500000 :                 if (!test_buffer(tctx, inbuf, size, "UTF-8")) {
     404           0 :                         printf("i=%d failed UTF-8\n", i);
     405           0 :                         return false;
     406             :                 }
     407             : 
     408      500000 :                 if (!test_buffer(tctx, inbuf, size, "CP850")) {
     409           0 :                         printf("i=%d failed CP850\n", i);
     410           0 :                         return false;
     411             :                 }
     412             :         }
     413           0 :         return true;
     414             : }
     415             : 
     416             : 
     417           2 : static bool test_string2key(struct torture_context *tctx)
     418             : {
     419             :         uint16_t *buf;
     420           2 :         char *dest = NULL;
     421           2 :         TALLOC_CTX *mem_ctx = talloc_new(tctx);
     422           2 :         size_t len = (random()%1000)+1;
     423           2 :         const uint16_t in1[10] = { 'a', 0xd805, 'b', 0xdcf0, 'c', 0, 'd', 'e', 'f', 'g' };
     424             :         uint8_t le1[20];
     425             :         uint8_t *munged1;
     426             :         uint8_t *out1;
     427             :         size_t ret;
     428             :         int i;
     429           2 :         const char *correct = "a\357\277\275b\357\277\275c\001defg";
     430             : 
     431           2 :         buf = talloc_size(mem_ctx, len*2);
     432           2 :         generate_random_buffer((uint8_t *)buf, len*2);
     433             : 
     434           2 :         torture_comment(tctx, "converting random buffer\n");
     435             : 
     436           2 :         if (!convert_string_talloc(mem_ctx, CH_UTF16MUNGED, CH_UTF8, (void *)buf, len*2, (void**)&dest, &ret)) {
     437           0 :                 torture_fail(tctx, "Failed to convert random buffer\n");
     438             :         }
     439             : 
     440          20 :         for (i=0;i<10;i++) {
     441          20 :                 SSVAL(&le1[2*i], 0, in1[i]);
     442             :         }
     443             : 
     444           2 :         torture_comment(tctx, "converting fixed buffer to UTF16\n");
     445             : 
     446           2 :         if (!convert_string_talloc(mem_ctx, CH_UTF16MUNGED, CH_UTF16, (void *)le1, 20, (void**)&munged1, &ret)) {
     447           0 :                 torture_fail(tctx, "Failed to convert fixed buffer to UTF16_MUNGED\n");
     448             :         }
     449             : 
     450           2 :         torture_assert(tctx, ret == 20, "conversion should give 20 bytes\n");
     451             : 
     452           2 :         torture_comment(tctx, "converting fixed buffer to UTF8\n");
     453             : 
     454           2 :         if (!convert_string_talloc(mem_ctx, CH_UTF16MUNGED, CH_UTF8, (void *)le1, 20, (void**)&out1, &ret)) {
     455           0 :                 torture_fail(tctx, "Failed to convert fixed buffer to UTF8\n");
     456             :         }
     457             : 
     458           2 :         torture_assert(tctx, strcmp(correct, (const char *) out1) == 0,
     459             :                 "conversion gave incorrect result\n");
     460             : 
     461           2 :         talloc_free(mem_ctx);
     462             : 
     463           2 :         return true;
     464             : }
     465             : 
     466        2353 : struct torture_suite *torture_local_iconv(TALLOC_CTX *mem_ctx)
     467             : {
     468        2353 :         struct torture_suite *suite = torture_suite_create(mem_ctx, "iconv");
     469             : 
     470        2353 :         torture_suite_add_simple_test(suite, "string2key",
     471             :                                       test_string2key);
     472             : 
     473        2353 :         torture_suite_add_simple_test(suite, "next_codepoint()",
     474             :                                       test_next_codepoint);
     475             : 
     476        2353 :         torture_suite_add_simple_test(suite, "first 1M codepoints",
     477             :                                       test_first_1m);
     478             : 
     479        2353 :         torture_suite_add_simple_test(suite, "5M random UTF-16LE sequences",
     480             :                                       test_random_5m);
     481             : 
     482        2353 :         torture_suite_add_simple_test(suite, "string2key",
     483             :                                       test_string2key);
     484        2353 :         return suite;
     485             : }
     486             : 
     487             : #else
     488             : 
     489             : struct torture_suite *torture_local_iconv(TALLOC_CTX *mem_ctx) 
     490             : {
     491             :         printf("No native iconv library - can't run iconv test\n");
     492             :         return NULL;
     493             : }
     494             : 
     495             : #endif

Generated by: LCOV version 1.13