§1  The thaiconv program  THAICONV.C 1

1. thaiconv.c .

/*
  This is the file thaiconv.c of the CJK macro package ver. 4.3.0
  (20-Jun-1999).
  Thai preprocessor
  based on the program cttex ver 1.15 written by
  Vuthichai Ampornaramveth <vuthi@ctrl.titech.ac.jp>
  It uses a dictionary to find break points between words.
  This version is for special use with cjk-enc.el (and MULEenc.sty) of
  the CJK package for LaTeX 2e:
    each line will be started and ended with special commands which
    start and end a Thai environment.
    each processed character (and breakpoints between Thai words) will be
    treated similarly.
*/
/* Maximum length of input line */

#define 1000

/* Maximum number of WORDS in one line */

#define 60

/* Maximum number of words to LOOKBACK */

#define 3

/* Characters to be skipped */

#define (x) (((x) < 128) ∨ (((x) ≤ ^{#}F9) ∧ ((x) ≥ ^{#}F0)))

/* HIGH Chars */

#define (x) (((x) ≥ 128))

/* Check level of a character */

#define (x) ((x) < ^{#}D0 ? 0 : (levtable[(x) −^{#}D0] ≠ 0))

/* the internal word break character */

#define 254

2.

#include #include #include

/* the dictionary */

2  THAICONV.C  The thaiconv program §2

#include void dooneline(unsigned char ∗, unsigned char ∗); void savestatus (int ∗, int ∗, int ∗, int ∗, int ∗, int ∗, unsigned char ∗, int); int mystrncmp(unsigned char ∗, unsigned char ∗, int); int findword (unsigned char ∗, int ∗); int countmatch(unsigned char ∗in, unsigned char ∗out ); void adj (unsigned char ∗); void filter (unsigned char ∗); int moveleft (int);

3.

/* Table Look-Up for level of a character */

int levtable[ ] = { 0, 2, 0, 0, 2, 2, 2, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 3, 3, 3, 3, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0 };

int lefttab[ ] = {

136, 131,  /∗ Meaning : change 136 to 131, ... ∗/
137, 132,  /∗ Up Level Mai Ek, To, Ti ... ∗/

138, 133, 139, 134, 140, 135,

#ED,^{#}8F,  /∗ Circle ∗/
#E8,^{#}98,  /∗ Top Level Mai Ek, To, Ti, ... ∗/
#E9,^{#}99,
#EA,^{#}9A,
#EB,^{#}9B,
#EC,^{#}9C,
#D4,^{#}94,  /∗ Sara I, EE, ... ∗/
#D5,^{#}95,
#D6,^{#}96,
#D7,^{#}97,
#D1,^{#}92,
#E7,^{#}93

};

/* the character code of a visible representation of a wordbreak */

int cutcode;

/* to avoid zillions of signedness warnings */

unsigned char ∗∗wordptr = (unsigned char ∗∗) Wordptr ;

4.

int main(int argc, char ∗argv [ ]) { FILE ∗fp, ∗fopen( ); unsigned char str [ ], out [ ]; int i, c;

cutcode = ;

if (argc > 1)

sscanf (argv [1], , &cutcode);

fp = stdin;

while (! feof (fp)) { (void) fgets ((char ∗) str , − 1, fp);

if (! feof (fp))

{ dooneline(str , out );

if (argc > 1)

      printf (  , out );  /∗ for testing purposes ∗/

else {

      adj (out );  /∗ Choose appropriate WANNAYUK ∗/
    /* Mule command #57 (begin Thai) */

printf ( );

i = 0;

while ((c = out [i^{++}])) if (c ≡ )

        /* Mule command #58 (end Thai) */

printf ( ); else if (! (c))

putchar (c); else if (c ≡ )

        /* Mule command #61 (Thai word break) */

printf ( ); else if ( (c))

          /* Mule command #60 (print Thai vowel/tone) */

printf ( , c); else

          /* Mule command #62 (print Thai consonant with glue) */

printf ( , c); }

} }

return 0; }

5.

void dooneline(unsigned char ∗in, unsigned char ∗out ) { int i, j, k, l, old i ; int wlist [ ], poslist [ ], jlist [ ], windex , pos , fence, backmode; int prev error = 0;

i = old i = j = l = 0; windex = 0; fence = 0; backmode = 0; prev error = 0;

while (in[i]) {

/* old_i is the value of i before looking back mode */
/* i > old_i means looking back was successful, cancel it */

if (i > old i )

backmode = 0;

if ( (in[i]))

   {  /∗ Chars to be skipped ? ∗/

if (prev error )

{

    /* Mark words not in dict */

out [j^{++}] = cutcode + 1;

prev error = 0;

}

backmode = fence = windex = 0; /∗ Begin new word list ∗/ while ( (in[i]) ∧ in[i]) /∗ Skip non-Thai chars ∗/ out [j^{++}] = in[i^{++}]; /∗ and Thai numbers ∗/

}

if (in[i]) /∗ Still not EOL ? ∗/ do { if ((k = findword (in + i, &pos )))

       {  /∗ Found a word in dict ∗/

if (prev error )

{ out [j^{++}] = cutcode + 1; prev error = 0; }

wlist [windex ] = i; poslist [windex ] = pos ; jlist [windex ] = j; windex^{++};

      /* For speed, limit the number of words to LOOK back */
      /* by creating a fence */
      /* Fence may only INCREASE */

if (windex − > fence) fence = windex − ; for (l = 0; l < k; l^{++}) /∗ Copy word ∗/

out [j^{++}] = in[i^{++}];

/* Mai Ya Mok & Pai Yan Noi */
/* I was using this code before adding these two characters
into the dict. Now they are in the dict and I no longer need
these two lines.
*/

while ((in[i] ≡ ^{#}E6) ∨ (in[i] ≡ ^{#}CF))

out [j^{++}] = in[i^{++}];

if (!  (in[i]))  /∗ Make sure it's not the last Thai word ∗/

out [j^{++}] = cutcode; /∗ Insert word sep symbol ∗/ } else

{  /∗ Not in Dict ∗/
/* Shortening the prev words may help */
/* Try to Look Back */

while ((windex > fence) ∧! k) {

/* Save status before looking back */

if (! backmode) { backmode = 1; savestatus (&windex , wlist , poslist , jlist , &i, &j, out , 1); old i = i; }

pos = poslist [windex − 1] − 1; /∗ Skip back one word ∗/ while ((pos ≥ 0) ∧ (l = countmatch(wordptr [pos ] + 1, in + wlist [windex − 1]))) { if ((l ≡ wordptr [pos ][0]) ∧ ! (in[wlist [windex − 1] + l])) { k = 1; break; } pos^{−}; }

/* A shorter version of prev word found */

if (k) { out [j = jlist [windex − 1] + l] = cutcode; poslist [windex − 1] = pos ; j^{++}; i = wlist [windex − 1] + l; } else { if (backmode ∧ (windex ≡ fence + 1)) {

 /* Search-Back method can't help, restore prev status */

savestatus (&windex , wlist , poslist , jlist , &i, &j, out , 0); break; } windex^{−}; } }

      /* Sure that word is not in dictionary */

if (k ≡ 0)

{

          prev error = 1;  /∗ Begin unknown word area ∗/

out [j^{++}] = in[i^{++}]; /∗ Copy it ∗/ backmode = fence = windex = 0; /∗ Clear Word List ∗/ } }

} while ((k ≡ 0) ∧ (! (in[i])));

} out [j] = 0;

/* Filter out words not in dict */

filter (out ); }

6.

#if 0

/* Sequential version */

int findword (unsigned char ∗in) { int i;

for (i = − 1; i ≥ 0; i^{−}) { if (mystrncmp(in, wordptr [i] + 1, wordptr [i][0]) ≡ 0) { printf ( ␣ ␣ , wordptr [i] + 1, wordptr [i][0]); return wordptr [i][0]; }

} return 0; }

#endif

7.

/* Calling: Index to a string
Return : Length of recognized word, and position of that word in
        dictionary
Binary search method.
*/

int findword (unsigned char ∗in, int ∗pos ) { int up, low , mid , a, l;

up =  − 1;  /∗ Upper bound ∗/
low = 0;  /∗ Lower bound ∗/
∗pos = −1;  /∗ If word not found ∗/
/* Found word at the boundaries ? */

if (mystrncmp(in, wordptr [up] + 1, wordptr [up][0]) ≡ 0)

 {  /∗ last (longest) match found ∗/

∗pos = up;

return wordptr [up][0]; }

if (mystrncmp(in, wordptr [low ] + 1, wordptr [low ][0]) ≡ 0)

  mid = low ;  /∗ first (shortest) match found ∗/

else

 {  /∗ Begin binary search ∗/

do

{

mid = (up + low )/2;

a = mystrncmp(in, wordptr [mid ] + 1, wordptr [mid ][0]);

#if 0

printf ( ␣ ␣ ␣ , low , mid , up, wordptr [mid ] + 1);

#endif

if (a ≠ 0)

{ if (a > 0)

low = mid ; else

up = mid ; }

} while ((a ≠ 0) ∧ (up − low > 1));

if (a ≠ 0)

   {  /∗ Word not found ∗/
    mid^{−};  /∗ Go back in dictionary ∗/
  /* Can we find a shorter word ? */

if (! countmatch(wordptr [mid ] + 1, in))

      return 0;  /∗ No character matches ∗/

while (mid ∧ (l = countmatch(wordptr [mid ] + 1, in)))

{ if ((l ≡ wordptr [mid ][0]) ∧! (in[l])) { ∗pos = mid ; return l; } mid^{−}; }

if (a)

      return 0;  /∗ really no match ∗/

} }

up = mid ; if (up < )

do

   {  /∗ Find the longest match ∗/

up^{++};

a = mystrncmp(in, wordptr [up] + 1, wordptr [up][0]); if (a ≡ 0) mid = up;

} while ((a ≥ 0) ∧ (up < − 1));

#if 0

printf ( ␣ ␣ ␣ , wordptr [mid ] + 1, wordptr [mid ][0]); #endif

∗pos = mid ; return wordptr [mid ][0]; }

8.

int countmatch(unsigned char ∗in, unsigned char ∗out ) { int i;

i = 0; while (in[i] ≡ out [i])

i^{++}; return i; }

9.

void savestatus (int ∗windex , int ∗wlist , int ∗poslist , int ∗jlist , int ∗oi , int ∗j, unsigned char ∗out , int mode) { static int lwindex , lwlist [ ], lposlist [ ], ljlist [ ], li , lj ; int i; static unsigned char lout [ ];

#if 0

printf (  ␣    ␣  , mode);

#endif

if (mode) { /∗ Save ∗/ lwindex = ∗windex ; for (i = 0; i < lwindex ; i^{++}) { lwlist [i] = wlist [i]; lposlist [i] = poslist [i]; ljlist [i] = jlist [i]; }

for (i = 0; i < ∗j; i^{++})

lout [i] = out [i];

li = ∗oi ; lj = ∗j; } else

{  /∗ Restore ∗/

∗windex = lwindex ; for (i = 0; i < lwindex ; i^{++}) { wlist [i] = lwlist [i]; poslist [i] = lposlist [i]; jlist [i] = ljlist [i]; }

for (i = 0; i < lj ; i^{++})

out [i] = lout [i];

∗oi = li ; ∗j = lj ; }

}

10.

/*
Thai version of strncmp: b must be the word from dictionary.
*/

int mystrncmp(unsigned char ∗a, unsigned char ∗b, int l) { int i;

i = strncmp((char ∗) a, (char ∗) b, l); if (i)

return i; else

return (a[l]); }

11.

/* What to do with words outside dictionary */

void filter (unsigned char ∗line) { int i, j, c, a, found ; unsigned char str [ ];

strcpy ((char ∗) str , (char ∗) line); found = i = 0; a = −1; while ((c = str [i])) { if (c ≡ cutcode)

a = i; else if (c ≡ cutcode + 1) { found = 1; if (! (str [i + 1])) str [i] = cutcode;

if (a ≥ 0)

{ str [a] = cutcode + 1; a = −1; }

} else if ( (c))

a = −1;

i^{++}; }

if (found ) { i = j = 0; while ((c = str [i^{++}]))

if (c ≠ cutcode + 1) line[j^{++}] = c;

line[j] = 0; }

}

12.

void adj (unsigned char ∗line) { unsigned char top[ ]; unsigned char up[ ]; unsigned char middle[ ]; unsigned char low [ ];

int i, k, c;

/* Split string into 4 levels */
/* Clear Buffer */

for (i = 0; i < ; i^{++}) top[i] = up[i] = middle[i] = low [i] = 0;

i = 0; k = −1;

while ((c = line[i^{++}])) { if (k ≥ 0) { switch ((c > ^{#}D0) ? levtable[c −^{#}D0] : 0) { case 0: /∗ Middle ∗/

/* Special Case for Sara-Am */

if (c ≡ ^{#}D3) { up[k] = ^{#}ED; middle[^{++}k] = ^{#}D2; /∗ Put Sara-Ar ∗/ } else

middle[^{++}k] = c; break;

case 1: /∗ Low ∗/ low [k] = c; break;

case 2: /∗ Up ∗/ up[k] = c; break;

case 3: /∗ Top ∗/ top[k] = c; break;

} } else

middle[^{++}k] = c; }

/* Beauty Part Begins */

for (i = 0; i ≤ k; i^{++}) {

/* Move down from Top -> Up */

if ((top[i]) ∧ (up[i] ≡ 0)) { up[i] = top[i] − 96; top[i] = 0; }

/* Avoid characters with long tail */

if (middle[i] ≡ ^{#}BB ∨ /∗ Por Pla ∗/ middle[i] ≡ ^{#}BD ∨ /∗ For Far ∗/ middle[i] ≡ ^{#}BF) /∗ For Fun ∗/ { if (up[i])

up[i] = moveleft (up[i]); if (top[i])

top[i] = moveleft (top[i]); }

/* Remove lower part of TorSanTan and YorPhuYing if necessary */

if (middle[i] ≡ ^{#}B0 ∧ low [i]) /∗ TorSanTan ∗/

middle[i] = ^{#}9F; if (middle[i] ≡ ^{#}AD ∧ low [i]) /∗ YorPhuYing ∗/

middle[i] = ^{#}90;

/* Move lower sara down , for DorChaDa, TorPaTak */

if (middle[i] ≡ ^{#}AE ∨ middle[i] ≡ ^{#}AF) { if (low [i])

low [i] = low [i] + 36; }

}

/* Pack Back To A Line */

i = 0; k = 0;

while (middle[i]) { line[k^{++}] = middle[i]; if (low [i])

line[k^{++}] = low [i]; if (up[i])

line[k^{++}] = up[i]; if (top[i])

line[k^{++}] = top[i]; i^{++}; }

/* Number of bytes might change */

line[k] = 0; }

13.

int moveleft (int c) { int i; for (i = 0; i < 34; i += 2) { if (lefttab[i] ≡ c)

return lefttab[i + 1]; } return c; }

14.

/* end of thaiconv.c */

14  INDEX  The thaiconv program §15

15. Index.

a: \underline{7}, \underline{10}, \underline{11}. adj ( ): \underline{2}, 4, \underline{12}. argc: \underline{4}. argv : \underline{4}. b: \underline{10}. : \underline{1}, 5. backmode: \underline{5}. c: \underline{4}, \underline{11}, \underline{12}, \underline{13}. countmatch( ): \underline{2}, 5, 7, \underline{8}. cutcode: \underline{3}, 4, 5, 11. : \underline{1}, 4. dooneline( ): \underline{2}, 4, \underline{5}. fence: \underline{5}. feof ( ): 4. fgets ( ): 4. filter ( ): \underline{2}, 5, \underline{11}. findword ( ): \underline{2}, 5, \underline{6}, \underline{7}. fopen( ): \underline{4}. found : \underline{11}. fp: \underline{4}. ( ): \underline{1}, 4. i: \underline{4}, \underline{5}, \underline{6}, \underline{8}, \underline{9}, \underline{10}, \underline{11}, \underline{12}, \underline{13}. in: \underline{2}, \underline{5}, \underline{6}, \underline{7}, \underline{8}. j: \underline{5}, \underline{9}, \underline{11}. jlist : \underline{5}, \underline{9}. k: \underline{5}, \underline{12}. l: \underline{5}, \underline{7}, \underline{10}. lefttab: \underline{3}, 13. levtable: 1, \underline{3}, 12. li : \underline{9}. line: \underline{11}, \underline{12}. lj : \underline{9}. ljlist : \underline{9}. lout : \underline{9}. low : \underline{7}, \underline{12}. lposlist : \underline{9}. lwindex : \underline{9}. lwlist : \underline{9}. main( ): \underline{4}. : \underline{1}, 4, 9, 11, 12. mid : \underline{7}. middle: \underline{12}. mode: \underline{9}. moveleft ( ): \underline{2}, 12, \underline{13}. : \underline{1}, 5, 9. mystrncmp( ): \underline{2}, 6, 7, \underline{10}. ( ): \underline{1}, 4, 5, 7, 10. : 6, 7. oi : \underline{9}. old i : \underline{5}. out : \underline{2}, \underline{4}, \underline{5}, \underline{8}, \underline{9}. pos : \underline{5}, \underline{7}. poslist : \underline{5}, \underline{9}. prev error : \underline{5}. printf ( ): 4, 6, 7, 9. putchar ( ): 4. savestatus ( ): \underline{2}, 5, \underline{9}. ( ): \underline{1}, 5, 11. sscanf ( ): 4. stdin: 4. str : \underline{4}, \underline{11}. strcpy ( ): 11. strncmp( ): 10. top: \underline{12}. up: \underline{7}, \underline{12}. windex : \underline{5}, \underline{9}. wlist : \underline{5}, \underline{9}. wordptr : \underline{3}, 5, 6, 7. Wordptr : 3.

The thaiconv program

Section Page

thaiconv.c. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 1 1 Index . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 15 14

This document was produced with the program