1/* Convert using charmaps and possibly iconv().
2 Copyright (C) 2001-2020 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4 Contributed by Ulrich Drepper <drepper@redhat.com>, 2001.
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published
8 by the Free Software Foundation; version 2 of the License, or
9 (at your option) any later version.
10
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <https://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <error.h>
22#include <fcntl.h>
23#include <iconv.h>
24#include <libintl.h>
25#include <stdio.h>
26#include <stdlib.h>
27#include <unistd.h>
28#include <stdint.h>
29#include <sys/mman.h>
30#include <sys/stat.h>
31
32#include "iconv_prog.h"
33
34
35/* Prototypes for a few program-wide used functions. */
36#include <programs/xmalloc.h>
37
38
39struct convtable
40{
41 int term[256 / 8];
42 union
43 {
44 struct convtable *sub;
45 struct charseq *out;
46 } val[256];
47};
48
49
50static inline struct convtable *
51allocate_table (void)
52{
53 return (struct convtable *) xcalloc (1, sizeof (struct convtable));
54}
55
56
57static inline int
58is_term (struct convtable *tbl, unsigned int idx)
59{
60 return tbl->term[idx / 8] & (1 << (idx % 8));
61}
62
63
64static inline void
65clear_term (struct convtable *tbl, unsigned int idx)
66{
67 tbl->term[idx / 8] &= ~(1 << (idx % 8));
68}
69
70
71static inline void
72set_term (struct convtable *tbl, unsigned int idx)
73{
74 tbl->term[idx / 8] |= 1 << (idx % 8);
75}
76
77
78/* Generate the conversion table. */
79static struct convtable *use_from_charmap (struct charmap_t *from_charmap,
80 const char *to_code);
81static struct convtable *use_to_charmap (const char *from_code,
82 struct charmap_t *to_charmap);
83static struct convtable *use_both_charmaps (struct charmap_t *from_charmap,
84 struct charmap_t *to_charmap);
85
86/* Prototypes for the functions doing the actual work. */
87static int process_block (struct convtable *tbl, char *addr, size_t len,
88 FILE *output);
89static int process_fd (struct convtable *tbl, int fd, FILE *output);
90static int process_file (struct convtable *tbl, FILE *input, FILE *output);
91
92
93int
94charmap_conversion (const char *from_code, struct charmap_t *from_charmap,
95 const char *to_code, struct charmap_t *to_charmap,
96 int argc, int remaining, char *argv[],
97 const char *output_file)
98{
99 struct convtable *cvtbl;
100 int status = EXIT_SUCCESS;
101
102 /* We have three different cases to handle:
103
104 - both, from_charmap and to_charmap, are available. This means we
105 can assume that the symbolic names match and use them to create
106 the mapping.
107
108 - only from_charmap is available. In this case we can only hope that
109 the symbolic names used are of the <Uxxxx> form in which case we
110 can use a UCS4->"to_code" iconv() conversion for the second step.
111
112 - only to_charmap is available. This is similar, only that we would
113 use iconv() for the "to_code"->UCS4 conversion.
114
115 We first create a table which maps input bytes into output bytes.
116 Once this is done we can handle all three of the cases above
117 equally. */
118 if (from_charmap != NULL)
119 {
120 if (to_charmap == NULL)
121 cvtbl = use_from_charmap (from_charmap, to_code);
122 else
123 cvtbl = use_both_charmaps (from_charmap, to_charmap);
124 }
125 else
126 {
127 assert (to_charmap != NULL);
128 cvtbl = use_to_charmap (from_code, to_charmap);
129 }
130
131 /* If we couldn't generate a table stop now. */
132 if (cvtbl == NULL)
133 return EXIT_FAILURE;
134
135 /* Determine output file. */
136 FILE *output;
137 if (output_file != NULL && strcmp (output_file, "-") != 0)
138 {
139 output = fopen (output_file, "w");
140 if (output == NULL)
141 error (EXIT_FAILURE, errno, _("cannot open output file"));
142 }
143 else
144 output = stdout;
145
146 /* We can now start the conversion. */
147 if (remaining == argc)
148 {
149 if (process_file (cvtbl, stdin, output) != 0)
150 status = EXIT_FAILURE;
151 }
152 else
153 do
154 {
155 int fd;
156
157 if (verbose)
158 printf ("%s:\n", argv[remaining]);
159 if (strcmp (argv[remaining], "-") == 0)
160 fd = 0;
161 else
162 {
163 fd = open (argv[remaining], O_RDONLY);
164
165 if (fd == -1)
166 {
167 error (0, errno, _("cannot open input file `%s'"),
168 argv[remaining]);
169 status = EXIT_FAILURE;
170 continue;
171 }
172 }
173
174#ifdef _POSIX_MAPPED_FILES
175 struct stat64 st;
176 char *addr;
177 /* We have possibilities for reading the input file. First try
178 to mmap() it since this will provide the fastest solution. */
179 if (fstat64 (fd, &st) == 0
180 && ((addr = mmap (NULL, st.st_size, PROT_READ, MAP_PRIVATE,
181 fd, 0)) != MAP_FAILED))
182 {
183 /* Yes, we can use mmap(). The descriptor is not needed
184 anymore. */
185 if (close (fd) != 0)
186 error (EXIT_FAILURE, errno,
187 _("error while closing input `%s'"), argv[remaining]);
188
189 if (process_block (cvtbl, addr, st.st_size, output) < 0)
190 {
191 /* Something went wrong. */
192 status = EXIT_FAILURE;
193
194 /* We don't need the input data anymore. */
195 munmap ((void *) addr, st.st_size);
196
197 /* We cannot go on with producing output since it might
198 lead to problem because the last output might leave
199 the output stream in an undefined state. */
200 break;
201 }
202
203 /* We don't need the input data anymore. */
204 munmap ((void *) addr, st.st_size);
205 }
206 else
207#endif /* _POSIX_MAPPED_FILES */
208 {
209 /* Read the file in pieces. */
210 if (process_fd (cvtbl, fd, output) != 0)
211 {
212 /* Something went wrong. */
213 status = EXIT_FAILURE;
214
215 /* We don't need the input file anymore. */
216 close (fd);
217
218 /* We cannot go on with producing output since it might
219 lead to problem because the last output might leave
220 the output stream in an undefined state. */
221 break;
222 }
223
224 /* Now close the file. */
225 close (fd);
226 }
227 }
228 while (++remaining < argc);
229
230 /* All done. */
231 return status;
232}
233
234
235/* Add the IN->OUT mapping to TBL. OUT is potentially stored in the table.
236 IN is used only here, so it need not be kept live afterwards. */
237static void
238add_bytes (struct convtable *tbl, const struct charseq *in, struct charseq *out)
239{
240 int n = 0;
241 unsigned int byte;
242
243 assert (in->nbytes > 0);
244
245 byte = ((unsigned char *) in->bytes)[n];
246 while (n + 1 < in->nbytes)
247 {
248 if (is_term (tbl, byte) || tbl->val[byte].sub == NULL)
249 {
250 /* Note that we simply ignore a definition for a byte sequence
251 which is also the prefix for a longer one. */
252 clear_term (tbl, byte);
253 tbl->val[byte].sub =
254 (struct convtable *) xcalloc (1, sizeof (struct convtable));
255 }
256
257 tbl = tbl->val[byte].sub;
258
259 byte = ((unsigned char *) in->bytes)[++n];
260 }
261
262 /* Only add the new sequence if there is none yet and the byte sequence
263 is not part of an even longer one. */
264 if (! is_term (tbl, byte) && tbl->val[byte].sub == NULL)
265 {
266 set_term (tbl, byte);
267 tbl->val[byte].out = out;
268 }
269}
270
271/* Try to convert SEQ from WCHAR_T format using CD.
272 Returns a malloc'd struct or NULL. */
273static struct charseq *
274convert_charseq (iconv_t cd, const struct charseq *seq)
275{
276 struct charseq *result = NULL;
277
278 if (seq->ucs4 != UNINITIALIZED_CHAR_VALUE)
279 {
280 /* There is a chance. Try the iconv module. */
281 wchar_t inbuf[1] = { seq->ucs4 };
282 unsigned char outbuf[64];
283 char *inptr = (char *) inbuf;
284 size_t inlen = sizeof (inbuf);
285 char *outptr = (char *) outbuf;
286 size_t outlen = sizeof (outbuf);
287
288 (void) iconv (cd, &inptr, &inlen, &outptr, &outlen);
289
290 if (outptr != (char *) outbuf)
291 {
292 /* We got some output. Good, use it. */
293 outlen = sizeof (outbuf) - outlen;
294 assert ((char *) outbuf + outlen == outptr);
295
296 result = xmalloc (sizeof (struct charseq) + outlen);
297 result->name = seq->name;
298 result->ucs4 = seq->ucs4;
299 result->nbytes = outlen;
300 memcpy (result->bytes, outbuf, outlen);
301 }
302
303 /* Clear any possible state left behind. */
304 (void) iconv (cd, NULL, NULL, NULL, NULL);
305 }
306
307 return result;
308}
309
310
311static struct convtable *
312use_from_charmap (struct charmap_t *from_charmap, const char *to_code)
313{
314 /* We iterate over all entries in the from_charmap and for those which
315 have a known UCS4 representation we use an iconv() call to determine
316 the mapping to the to_code charset. */
317 struct convtable *rettbl;
318 iconv_t cd;
319 void *ptr = NULL;
320 const void *key;
321 size_t keylen;
322 void *data;
323
324 cd = iconv_open (to_code, "WCHAR_T");
325 if (cd == (iconv_t) -1)
326 /* We cannot do anything. */
327 return NULL;
328
329 rettbl = allocate_table ();
330
331 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
332 >= 0)
333 {
334 struct charseq *in = data;
335 struct charseq *newp = convert_charseq (cd, in);
336 if (newp != NULL)
337 add_bytes (rettbl, in, newp);
338 }
339
340 iconv_close (cd);
341
342 return rettbl;
343}
344
345
346static struct convtable *
347use_to_charmap (const char *from_code, struct charmap_t *to_charmap)
348{
349 /* We iterate over all entries in the to_charmap and for those which
350 have a known UCS4 representation we use an iconv() call to determine
351 the mapping to the from_code charset. */
352 struct convtable *rettbl;
353 iconv_t cd;
354 void *ptr = NULL;
355 const void *key;
356 size_t keylen;
357 void *data;
358
359 /* Note that the conversion we use here is the reverse direction. Without
360 exhaustive search we cannot figure out which input yields the UCS4
361 character we are looking for. Therefore we determine it the other
362 way round. */
363 cd = iconv_open (from_code, "WCHAR_T");
364 if (cd == (iconv_t) -1)
365 /* We cannot do anything. */
366 return NULL;
367
368 rettbl = allocate_table ();
369
370 while (iterate_table (&to_charmap->char_table, &ptr, &key, &keylen, &data)
371 >= 0)
372 {
373 struct charseq *out = data;
374 struct charseq *newp = convert_charseq (cd, out);
375 if (newp != NULL)
376 {
377 add_bytes (rettbl, newp, out);
378 free (newp);
379 }
380 }
381
382 iconv_close (cd);
383
384 return rettbl;
385}
386
387
388static struct convtable *
389use_both_charmaps (struct charmap_t *from_charmap,
390 struct charmap_t *to_charmap)
391{
392 /* In this case we iterate over all the entries in the from_charmap,
393 determine the internal name, and find an appropriate entry in the
394 to_charmap (if it exists). */
395 struct convtable *rettbl = allocate_table ();
396 void *ptr = NULL;
397 const void *key;
398 size_t keylen;
399 void *data;
400
401 while (iterate_table (&from_charmap->char_table, &ptr, &key, &keylen, &data)
402 >= 0)
403 {
404 struct charseq *in = (struct charseq *) data;
405 struct charseq *out = charmap_find_value (to_charmap, key, keylen);
406
407 if (out != NULL)
408 add_bytes (rettbl, in, out);
409 }
410
411 return rettbl;
412}
413
414
415static int
416process_block (struct convtable *tbl, char *addr, size_t len, FILE *output)
417{
418 size_t n = 0;
419
420 while (n < len)
421 {
422 struct convtable *cur = tbl;
423 unsigned char *curp = (unsigned char *) addr;
424 unsigned int byte = *curp;
425 int cnt;
426 struct charseq *out;
427
428 while (! is_term (cur, byte))
429 if (cur->val[byte].sub == NULL)
430 {
431 /* This is an invalid sequence. Skip the first byte if we are
432 ignoring errors. Otherwise punt. */
433 if (! omit_invalid)
434 {
435 error (0, 0, _("illegal input sequence at position %Zd"), n);
436 return -1;
437 }
438
439 n -= curp - (unsigned char *) addr;
440
441 byte = *(curp = (unsigned char *) ++addr);
442 if (++n >= len)
443 /* All converted. */
444 return 0;
445
446 cur = tbl;
447 }
448 else
449 {
450 cur = cur->val[byte].sub;
451
452 if (++n >= len)
453 {
454 error (0, 0, _("\
455incomplete character or shift sequence at end of buffer"));
456 return -1;
457 }
458
459 byte = *++curp;
460 }
461
462 /* We found a final byte. Write the output bytes. */
463 out = cur->val[byte].out;
464 for (cnt = 0; cnt < out->nbytes; ++cnt)
465 fputc_unlocked (out->bytes[cnt], output);
466
467 addr = (char *) curp + 1;
468 ++n;
469 }
470
471 return 0;
472}
473
474
475static int
476process_fd (struct convtable *tbl, int fd, FILE *output)
477{
478 /* We have a problem with reading from a descriptor since we must not
479 provide the iconv() function an incomplete character or shift
480 sequence at the end of the buffer. Since we have to deal with
481 arbitrary encodings we must read the whole text in a buffer and
482 process it in one step. */
483 static char *inbuf = NULL;
484 static size_t maxlen = 0;
485 char *inptr = inbuf;
486 size_t actlen = 0;
487
488 while (actlen < maxlen)
489 {
490 ssize_t n = read (fd, inptr, maxlen - actlen);
491
492 if (n == 0)
493 /* No more text to read. */
494 break;
495
496 if (n == -1)
497 {
498 /* Error while reading. */
499 error (0, errno, _("error while reading the input"));
500 return -1;
501 }
502
503 inptr += n;
504 actlen += n;
505 }
506
507 if (actlen == maxlen)
508 while (1)
509 {
510 ssize_t n;
511 char *new_inbuf;
512
513 /* Increase the buffer. */
514 new_inbuf = (char *) realloc (inbuf, maxlen + 32768);
515 if (new_inbuf == NULL)
516 {
517 error (0, errno, _("unable to allocate buffer for input"));
518 return -1;
519 }
520 inbuf = new_inbuf;
521 maxlen += 32768;
522 inptr = inbuf + actlen;
523
524 do
525 {
526 n = read (fd, inptr, maxlen - actlen);
527
528 if (n == 0)
529 /* No more text to read. */
530 break;
531
532 if (n == -1)
533 {
534 /* Error while reading. */
535 error (0, errno, _("error while reading the input"));
536 return -1;
537 }
538
539 inptr += n;
540 actlen += n;
541 }
542 while (actlen < maxlen);
543
544 if (n == 0)
545 /* Break again so we leave both loops. */
546 break;
547 }
548
549 /* Now we have all the input in the buffer. Process it in one run. */
550 return process_block (tbl, inbuf, actlen, output);
551}
552
553
554static int
555process_file (struct convtable *tbl, FILE *input, FILE *output)
556{
557 /* This should be safe since we use this function only for `stdin' and
558 we haven't read anything so far. */
559 return process_fd (tbl, fileno (input), output);
560}
561