path: root/pico/lib/picosa.c

/*
 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * @file picosa.c
 *
 * sentence analysis - POS disambiguation
 *
 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
 * All rights reserved.
 *
 * History:
 * - 2009-04-20 -- initial version
 *
 */

#include "picoos.h"
#include "picodbg.h"
#include "picobase.h"
#include "picokdt.h"
#include "picoklex.h"
#include "picoktab.h"
#include "picokfst.h"
#include "picotrns.h"
#include "picodata.h"
#include "picosa.h"

#ifdef __cplusplus
extern "C" {
#endif
#if 0
}
#endif


/* PU saStep states */
#define SA_STEPSTATE_COLLECT       0
#define SA_STEPSTATE_PROCESS_POSD 10
#define SA_STEPSTATE_PROCESS_WPHO 11
#define SA_STEPSTATE_PROCESS_TRNS_PARSE 12
#define SA_STEPSTATE_PROCESS_TRNS_FST 13
#define SA_STEPSTATE_FEED          2

#define SA_MAX_ALTDESC_SIZE (30*(PICOTRNS_MAX_NUM_POSSYM + 2))

#define SA_MSGSTR_SIZE 32

/*  subobject    : SentAnaUnit
 *  shortcut     : sa
 *  context size : one phrase, max. 30 non-PUNC items, for non-processed items
 *                 one item if internal input empty
 */

/** @addtogroup picosa

  internal buffers:

  - headx: array for extended item heads of fixed size (head plus
    index for content, plus two fields for boundary strength/type)

  - cbuf1, cbuf2: buffers for item contents (referenced by index in
    headx). Future: replace these two buffers by a single double-sided
    buffer (double shrink-grow type)

  0. bottom up filling of items in headx and cbuf1

  1. POS disambiguation (right-to-left, top-to-bottom):
  - number and sequence of items unchanged
  - item content can only get smaller (reducing nr of results in WORDINDEX)
  -> info stays in "headx, cbuf1" and changed in place                      \n
     WORDGRAPH(POSes,NA)graph             -> WORDGRAPH(POS,NA)graph         \n
     WORDINDEX(POSes,NA)POS1ind1...POSNindN  -> WORDINDEX(POS,NA)POS|ind    \n

  2. lex-index lookup and G2P (both directions possible, left-to-right done):
  - number and sequence of items unchanged, item head info and content
    changes
  -> headx changed in place; cbuf1 to cbuf2                                 \n
     WORDGRAPH(POS,NA)graph    -> WORDPHON(POS,NA)phon                      \n
     WORDINDEX(POS,NA)POS|ind  -> WORDPHON(POS,NA)phon                      \n

  3. phrasing (right-to-left):

     Previous (before introducing SBEG)\n
     ----------------------------------
                                           1|          2|             3|    4|    \n
     e.g. from      WP WP WP       WP WP PUNC  WP WP PUNC  WP WP WP PUNC FLUSH    \n
     e.g. to  BINIT WP WP WP BPHR3 WP WP BPHR1 WP WP BSEND WP WP WP BSEND BTERM   \n
              |1                         |2          |3             |4            \n

     3-level bound state: to keep track of bound strength from end of
     previous punc-phrase, then BOUND item output as first item
     (strength from prev punc-phrase and type from current
     punc-phrase).

     trailing PUNC item       bound states
                              INIT         SEND         PHR1
       PUNC(SENTEND, T)       B(I,T)>SEND  B(S,T)>SEND  B(P1,T)>SEND
       PUNC(SENTEND, Q)       B(I,Q)>SEND  B(S,Q)>SEND  B(P1,Q)>SEND
       PUNC(SENTEND, E)       B(I,E)>SEND  B(S,E)>SEND  B(P1,E)>SEND
       PUNC(PHRASEEND, P)     B(I,P)>PHR1  B(S,P)>PHR1  B(P1,P)>PHR1
       PUNC(PHRASEEND, FORC)  B(I,P)>PHR1  B(S,P)>PHR1  B(P1,P)>PHR1
       PUNC(FLUSH, T)         B(I,T)..     B(S,T)..     B(P1,T)..
                                B(T,NA)      B(T,NA)      B(T,NA)
                                >INIT        >INIT        >INIT

     PHR2/3 case:
     trailing PUNC item       bound states
                          INIT              SEND              PHR1
       PUNC(SENTEND, T)   B(I,P)B(P,T)>SEND B(S,P)B(P,T)>SEND B(P1,P)B(P,T)>SEND
       PUNC(SENTEND, Q)   B(I,P)B(P,Q)>SEND B(S,P)B(P,Q)>SEND B(P1,P)B(P,Q)>SEND
       PUNC(SENTEND, E)   B(I,P)B(P,E)>SEND B(S,P)B(P,E)>SEND B(P1,P)B(P,E)>SEND
       PUNC(PHRASEEND, P) B(I,P)B(P,P)>PHR1 B(S,P)B(P,P)>PHR1 B(P1,P)B(P,P)>PHR1
       PUNC(PHREND, FORC) B(I,P)B(P,P)>PHR1 B(S,P)B(P,P)>PHR1 B(P1,P)B(P,P)>PHR1
       PUNC(FLUSH, T)     B(I,P)B(P,T)..    B(S,T)B(P,T)..    B(P1,T)B(P,T)..
                            B(T,NA)             B(T,NA)             B(T,NA)
                            >INIT               >INIT               >INIT

     Current
     --------
     e.g. from      WP WP WP       WP WP PUNC  WP WP PUNC        WP WP WP PUNC  FLUSH
     e.g. to  BSBEG WP WP WP BPHR3 WP WP BPHR1 WP WP BSEND BSBEG WP WP WP BSEND BTERM
              |1                         |2                |3                   |4

     2-level bound state: The internal buffer contains one primary phrase (sometimes forced, if buffer
     allmost full), with the trailing PUNCT item included (last item).
     If the trailing PUNC is a a primary phrase separator, the
       item is not output, but instead, the bound state is set to PPHR, so that the correct BOUND can
       be output at the start of the next primary phrase.
     Otherwise,
       the item is converted to the corresponding BOUND and output. the bound state is set to SSEP,
       so that a BOUND of type SBEG is output at the start of the next primary phrase.

     trailing PUNC item       bound states
                              SSEP           PPHR
       PUNC(SENTEND, X)       B(B,X)>SSEP    B(P1,X)>SSEP  (X = T | Q | E)
       PUNC(FLUSH, T)         B(B,T)>SSEP*    B(P1,T)>SSEP
       PUNC(PHRASEEND, P)     B(B,P)>PPHR    B(P1,P)>PPHR
       PUNC(PHRASEEND, FORC)  B(B,P)>PPHR    B(P1,P)>PPHR

*    If more than one sentence separators follow each other (e.g. SEND-FLUSH, SEND-SEND) then
     all but the first will be treated as an (empty) phrase containing just this item.
     If this (single) item is a flush, creation of SBEG is suppressed.


  - dtphr phrasing tree (rather subphrasing tree it should be called)
    determines
      BOUND_PHR2
      BOUND_PHR3
  - boundary strenghts are determined for every word (except the
    first one) from right-to-left. The boundary types mark the phrase
    type of the phrase following the boundary.
  - number of items actually changed (new BOUND items added): because
    of fixed size without content, two fields are contained in headx
    to indicate if a BOUND needs to be added to the LEFT of the item.
    -> headx further extended with boundary strength and type info to
    indicate that to the left of the headx ele a BOUND needs to be
    inserted when outputting.

  4. accentuation:
  - number of items unchanged, content unchanged, only head info changes
  -> changed in place in headx
*/


typedef struct {
    picodata_itemhead_t head;
    picoos_uint16 cind;
} picosa_headx_t;


typedef struct sa_subobj {
    picoos_uint8 procState; /* for next processing step decision */

    picoos_uint8 inspaceok;      /* flag: headx/cbuf1 has space for an item */
    picoos_uint8 needsmoreitems; /* flag: need more items */
    picoos_uint8 phonesTransduced; /* flag: */

    picoos_uint8 tmpbuf[PICODATA_MAX_ITEMSIZE];  /* tmp. location for an item */

    picosa_headx_t headx[PICOSA_MAXNR_HEADX];
    picoos_uint16 headxBottom; /* bottom */
    picoos_uint16 headxLen;    /* length, 0 if empty */

    picoos_uint8 cbuf1[PICOSA_MAXSIZE_CBUF];
    picoos_uint16 cbuf1BufSize; /* actually allocated size */
    picoos_uint16 cbuf1Len;     /* length, 0 if empty */

    picoos_uint8 cbuf2[PICOSA_MAXSIZE_CBUF];
    picoos_uint16 cbuf2BufSize; /* actually allocated size */
    picoos_uint16 cbuf2Len;     /* length, 0 if empty */

    picotrns_possym_t phonBufA[PICOTRNS_MAX_NUM_POSSYM+1];
    picotrns_possym_t phonBufB[PICOTRNS_MAX_NUM_POSSYM+1];
    picotrns_possym_t * phonBuf;
    picotrns_possym_t * phonBufOut;
    picoos_uint16 phonReadPos, phonWritePos; /* next pos to read from phonBufIn, next pos to write to phonBufIn */
    picoos_uint16 nextReadPos; /* position of (potential) next item to read from */


    /* buffer for internal calculation of transducer */
    picotrns_AltDesc altDescBuf;
    /* the number of AltDesc in the buffer */
    picoos_uint16 maxAltDescLen;

    /* tab knowledge base */
    picoktab_Graphs tabgraphs;
    picoktab_Phones tabphones;
    picoktab_Pos tabpos;
    picoktab_FixedIds fixedIds;

    /* dtposd knowledge base */
    picokdt_DtPosD dtposd;

    /* dtg2p knowledge base */
    picokdt_DtG2P dtg2p;

    /* lex knowledge base */
    picoklex_Lex lex;

    /* ulex knowledge bases */
    picoos_uint8 numUlex;
    picoklex_Lex ulex[PICOKNOW_MAX_NUM_ULEX];

    /* fst knowledge bases */
    picoos_uint8 numFsts;
    picokfst_FST fst[PICOKNOW_MAX_NUM_WPHO_FSTS];
    picoos_uint8 curFst; /* the fst to be applied next */


} sa_subobj_t;


static pico_status_t saInitialize(register picodata_ProcessingUnit this, picoos_int32 resetMode) {
    sa_subobj_t * sa;
    picoos_uint16 i;
    picokfst_FST fst;
    picoknow_kb_id_t fstKbIds[PICOKNOW_MAX_NUM_WPHO_FSTS] = PICOKNOW_KBID_WPHO_ARRAY;
    picoklex_Lex ulex;
    picoknow_kb_id_t ulexKbIds[PICOKNOW_MAX_NUM_ULEX] = PICOKNOW_KBID_ULEX_ARRAY;

    PICODBG_DEBUG(("calling"));

    if (NULL == this || NULL == this->subObj) {
        return picoos_emRaiseException(this->common->em,
                                       PICO_ERR_NULLPTR_ACCESS, NULL, NULL);
    }
    sa = (sa_subobj_t *) this->subObj;

    /*  sa->common = this->common; */

    sa->procState = SA_STEPSTATE_COLLECT;

    sa->inspaceok = TRUE;
    sa->needsmoreitems = TRUE;

    sa->headxBottom = 0;
    sa->headxLen = 0;
    sa->cbuf1BufSize = PICOSA_MAXSIZE_CBUF;
    sa->cbuf2BufSize = PICOSA_MAXSIZE_CBUF;
    sa->cbuf1Len = 0;
    sa->cbuf2Len = 0;

    /* init headx, cbuf1, cbuf2 */
    for (i = 0; i < PICOSA_MAXNR_HEADX; i++){
        sa->headx[i].head.type = 0;
        sa->headx[i].head.info1 = PICODATA_ITEMINFO1_NA;
        sa->headx[i].head.info2 = PICODATA_ITEMINFO2_NA;
        sa->headx[i].head.len = 0;
        sa->headx[i].cind = 0;
    }
    for (i = 0; i < PICOSA_MAXSIZE_CBUF; i++) {
        sa->cbuf1[i] = 0;
        sa->cbuf2[i] = 0;
    }


    /* possym buffer */
    sa->phonesTransduced = FALSE;
    sa->phonBuf = sa->phonBufA;
    sa->phonBufOut = sa->phonBufB;
    sa->phonReadPos = 0;
    sa->phonWritePos = 0;
    sa->nextReadPos = 0;

    if (resetMode == PICO_RESET_SOFT) {
        /*following initializations needed only at startup or after a full reset*/
        return PICO_OK;
    }

    /* kb fst[] */
    sa->numFsts = 0;
    for (i = 0; i<PICOKNOW_MAX_NUM_WPHO_FSTS; i++) {
        fst = picokfst_getFST(this->voice->kbArray[fstKbIds[i]]);
        if (NULL != fst) {
            sa->fst[sa->numFsts++] = fst;
        }
    }
    sa->curFst = 0;
    PICODBG_DEBUG(("got %i fsts", sa->numFsts));
    /* kb fixedIds */
    sa->fixedIds = picoktab_getFixedIds(this->voice->kbArray[PICOKNOW_KBID_FIXED_IDS]);

    /* kb tabgraphs */
    sa->tabgraphs =
        picoktab_getGraphs(this->voice->kbArray[PICOKNOW_KBID_TAB_GRAPHS]);
    if (sa->tabgraphs == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got tabgraphs"));

    /* kb tabphones */
    sa->tabphones =
        picoktab_getPhones(this->voice->kbArray[PICOKNOW_KBID_TAB_PHONES]);
    if (sa->tabphones == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got tabphones"));

#ifdef PICO_DEBU
    {
        picoos_uint16 itmp;
        for (itmp = 0; itmp < 256; itmp++) {
            if (picoktab_hasVowelProp(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones hasVowel: %d", itmp));
            }
            if (picoktab_hasDiphthProp(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones hasDiphth: %d", itmp));
            }
            if (picoktab_hasGlottProp(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones hasGlott: %d", itmp));
            }
            if (picoktab_hasNonsyllvowelProp(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones hasNonsyllvowel: %d", itmp));
            }
            if (picoktab_hasSyllconsProp(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones hasSyllcons: %d", itmp));
            }
            if (picoktab_isPrimstress(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones isPrimstress: %d", itmp));
            }
            if (picoktab_isSecstress(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones isSecstress: %d", itmp));
            }
            if (picoktab_isSyllbound(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones isSyllbound: %d", itmp));
            }
            if (picoktab_isPause(sa->tabphones, itmp)) {
                PICODBG_DEBUG(("tabphones isPause: %d", itmp));
            }
        }

        PICODBG_DEBUG(("tabphones primstressID: %d",
                       picoktab_getPrimstressID(sa->tabphones)));
        PICODBG_DEBUG(("tabphones secstressID: %d",
                       picoktab_getSecstressID(sa->tabphones)));
        PICODBG_DEBUG(("tabphones syllboundID: %d",
                       picoktab_getSyllboundID(sa->tabphones)));
        PICODBG_DEBUG(("tabphones pauseID: %d",
                       picoktab_getPauseID(sa->tabphones)));
    }
#endif

    /* kb tabpos */
    sa->tabpos =
        picoktab_getPos(this->voice->kbArray[PICOKNOW_KBID_TAB_POS]);
    if (sa->tabpos == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got tabpos"));

    /* kb dtposd */
    sa->dtposd = picokdt_getDtPosD(this->voice->kbArray[PICOKNOW_KBID_DT_POSD]);
    if (sa->dtposd == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got dtposd"));

    /* kb dtg2p */
    sa->dtg2p = picokdt_getDtG2P(this->voice->kbArray[PICOKNOW_KBID_DT_G2P]);
    if (sa->dtg2p == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got dtg2p"));

    /* kb lex */
    sa->lex = picoklex_getLex(this->voice->kbArray[PICOKNOW_KBID_LEX_MAIN]);
    if (sa->lex == NULL) {
        return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING,
                                       NULL, NULL);
    }
    PICODBG_DEBUG(("got lex"));

    /* kb ulex[] */
    sa->numUlex = 0;
    for (i = 0; i<PICOKNOW_MAX_NUM_ULEX; i++) {
        ulex = picoklex_getLex(this->voice->kbArray[ulexKbIds[i]]);
        if (NULL != ulex) {
            sa->ulex[sa->numUlex++] = ulex;
        }
    }
    PICODBG_DEBUG(("got %i user lexica", sa->numUlex));

    return PICO_OK;
}

static picodata_step_result_t saStep(register picodata_ProcessingUnit this,
                                     picoos_int16 mode,
                                     picoos_uint16 *numBytesOutput);

static pico_status_t saTerminate(register picodata_ProcessingUnit this) {
    return PICO_OK;
}

static pico_status_t saSubObjDeallocate(register picodata_ProcessingUnit this,
                                        picoos_MemoryManager mm) {
    sa_subobj_t * sa;
    if (NULL != this) {
        sa = (sa_subobj_t *) this->subObj;
        picotrns_deallocate_alt_desc_buf(mm,&sa->altDescBuf);
        picoos_deallocate(mm, (void *) &this->subObj);
    }
    return PICO_OK;
}


picodata_ProcessingUnit picosa_newSentAnaUnit(picoos_MemoryManager mm,
                                              picoos_Common common,
                                              picodata_CharBuffer cbIn,
                                              picodata_CharBuffer cbOut,
                                              picorsrc_Voice voice) {
    picodata_ProcessingUnit this;
    sa_subobj_t * sa;
    this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice);
    if (this == NULL) {
        return NULL;
    }

    this->initialize = saInitialize;
    PICODBG_DEBUG(("set this->step to saStep"));
    this->step = saStep;
    this->terminate = saTerminate;
    this->subDeallocate = saSubObjDeallocate;

    this->subObj = picoos_allocate(mm, sizeof(sa_subobj_t));
    if (this->subObj == NULL) {
        picoos_deallocate(mm, (void *)&this);
        picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM, NULL, NULL);
        return NULL;
    }

    sa = (sa_subobj_t *) this->subObj;

    sa->altDescBuf = picotrns_allocate_alt_desc_buf(mm, SA_MAX_ALTDESC_SIZE, &sa->maxAltDescLen);
    if (NULL == sa->altDescBuf) {
        picotrns_deallocate_alt_desc_buf(mm,&sa->altDescBuf);
        picoos_deallocate(mm, (void *)&sa);
        picoos_deallocate(mm, (void *)&this);
        picoos_emRaiseException(common->em,PICO_EXC_OUT_OF_MEM, NULL, NULL);
    }


    saInitialize(this, PICO_RESET_FULL);
    return this;
}


/* ***********************************************************************/
/* PROCESS_POSD disambiguation functions */
/* ***********************************************************************/

/* find next POS to the right of 'ind' and return its POS and index */
static picoos_uint8 saPosDItemSeqGetPosRight(register picodata_ProcessingUnit this,
                                            register sa_subobj_t *sa,
                                            const picoos_uint16 ind,
                                            const picoos_uint16 top,
                                            picoos_uint16 *rightind) {
    picoos_uint8 val;
    picoos_int32 i;

    val = PICOKDT_EPSILON;
    for (i = ind + 1; ((val == PICOKDT_EPSILON) && (i < top)); i++) {
        if ((sa->headx[i].head.type == PICODATA_ITEM_WORDGRAPH) ||
                (sa->headx[i].head.type == PICODATA_ITEM_WORDINDEX)  ||
                (sa->headx[i].head.type == PICODATA_ITEM_WORDPHON) ) {
            val = sa->headx[i].head.info1;
        }
    }
    *rightind = i - 1;
    return val;
}


/* left-to-right, for each WORDGRAPH/WORDINDEX/WORDPHON do posd */
static pico_status_t saDisambPos(register picodata_ProcessingUnit this,
                                 register sa_subobj_t *sa) {
    picokdt_classify_result_t dtres;
    picoos_uint8 half_nratt_posd = PICOKDT_NRATT_POSD >> 1;
    picoos_uint16 valbuf[PICOKDT_NRATT_POSD]; /* only [0..half_nratt_posd] can be >2^8 */
    picoos_uint16 prevout;   /* direct dt output (hist.) or POS of prev word */
    picoos_uint16 lastprev3; /* last index of POS(es) found to the left */
    picoos_uint16 curPOS;     /* POS(es) of current word */
    picoos_int32 first;    /* index of first item with POS(es) */
    picoos_int32 ci;
    picoos_uint8 okay;       /* two uses: processing okay and lexind resovled */
    picoos_uint8 i;
    picoos_uint16 inval;
    picoos_uint16 fallback;

    /* set initial values */
    okay = TRUE;
    prevout = PICOKDT_HISTORY_ZERO;
    curPOS = PICODATA_ITEMINFO1_ERR;
    first = 0;

    while ((first < sa->headxLen) &&
           (sa->headx[first].head.type != PICODATA_ITEM_WORDGRAPH) &&
           (sa->headx[first].head.type != PICODATA_ITEM_WORDINDEX) &&
           (sa->headx[first].head.type != PICODATA_ITEM_WORDPHON)) {
        first++;
    }
    if (first >= sa->headxLen) {
        /* phrase not containing an item with POSes info, e.g. single flush */
        PICODBG_DEBUG(("no item with POSes found"));
        return PICO_OK;
    }

    lastprev3 = first;

    for (i = 0; i <= half_nratt_posd; i++) {
        valbuf[i] = PICOKDT_HISTORY_ZERO;
    }
    /* set POS(es) of current word, will be shifted afterwards */
    valbuf[half_nratt_posd+1] = sa->headx[first].head.info1;
    for (i = half_nratt_posd+2; i < PICOKDT_NRATT_POSD; i++) {
    /* find next POS to the right and set valbuf[i] */
        valbuf[i] = saPosDItemSeqGetPosRight(this, sa, lastprev3, sa->headxLen, &lastprev3);
    }

    PICODBG_TRACE(("headxLen: %d", sa->headxLen));

    /* process from left to right all items in headx */
    for (ci = first; ci < sa->headxLen; ci++) {
        okay = TRUE;

        PICODBG_TRACE(("iter: %d, type: %c", ci, sa->headx[ci].head.type));

        /* if not (WORDGRAPH or WORDINDEX) */
        if ((sa->headx[ci].head.type != PICODATA_ITEM_WORDGRAPH) &&
                (sa->headx[ci].head.type != PICODATA_ITEM_WORDINDEX)  &&
                (sa->headx[ci].head.type != PICODATA_ITEM_WORDPHON)) {
            continue;
        }

        PICODBG_TRACE(("iter: %d, curPOS: %d", ci, sa->headx[ci].head.info1));

        /* no continue so far => at [ci] we have a WORDGRAPH / WORDINDEX item */
        /* shift all elements one position to the left */
        /* shift predicted values (history) */
        for (i=1; i<half_nratt_posd; i++) {
            valbuf[i-1] = valbuf[i];
        }
        /* insert previously predicted value (now history) */
        valbuf[half_nratt_posd-1] = prevout;
        /* shift not yet predicted values */
        for (i=half_nratt_posd+1; i<PICOKDT_NRATT_POSD; i++) {
            valbuf[i-1] = valbuf[i];
        }
        /* find next POS to the right and set valbuf[PICOKDT_NRATT_POSD-1] */
        valbuf[PICOKDT_NRATT_POSD-1] = saPosDItemSeqGetPosRight(this, sa, lastprev3, sa->headxLen, &lastprev3);

        /* just to be on the safe side; the following should never happen */
        if (sa->headx[ci].head.info1 != valbuf[half_nratt_posd]) {
            PICODBG_WARN(("syncing POS"));
            picoos_emRaiseWarning(this->common->em, PICO_WARN_INVECTOR,
                                  NULL, NULL);
            valbuf[half_nratt_posd] = sa->headx[ci].head.info1;
        }

        curPOS = valbuf[half_nratt_posd];

        /* Check if POS disambiguation not needed */
        if (picoktab_isUniquePos(sa->tabpos, (picoos_uint8) curPOS)) {
            /* not needed */
            inval = 0;
            fallback = 0;
            if (!picokdt_dtPosDreverseMapOutFixed(sa->dtposd, curPOS,
                                       &prevout, &fallback)) {
                if (fallback) {
                    prevout = fallback;

                } else {
                    PICODBG_ERROR(("problem doing reverse output mapping"));
                    prevout = curPOS;
                }
            }
            PICODBG_DEBUG(("keeping: %d", sa->headx[ci].head.info1));
            continue;
        }

        /* assuming PICOKDT_NRATT_POSD == 7 */
        PICODBG_DEBUG(("%d: [%d %d %d %d %d %d %d]",
                       ci, valbuf[0], valbuf[1], valbuf[2],
                       valbuf[3], valbuf[4], valbuf[5], valbuf[6]));

        /* no continue so far => POS disambiguation needed */
        /* construct input vector, which is set in dtposd */
        if (!picokdt_dtPosDconstructInVec(sa->dtposd, valbuf)) {
            /* error constructing invec */
            PICODBG_WARN(("problem with invec"));
            picoos_emRaiseWarning(this->common->em, PICO_WARN_INVECTOR,
                                  NULL, NULL);
            okay = FALSE;
        }
        /* classify */
        if (okay && (!picokdt_dtPosDclassify(sa->dtposd, &prevout))) {
            /* error doing classification */
            PICODBG_WARN(("problem classifying"));
            picoos_emRaiseWarning(this->common->em, PICO_WARN_CLASSIFICATION,
                                  NULL, NULL);
            okay = FALSE;
        }
        /* decompose */
        if (okay && (!picokdt_dtPosDdecomposeOutClass(sa->dtposd, &dtres))) {
            /* error decomposing */
            PICODBG_WARN(("problem decomposing"));
            picoos_emRaiseWarning(this->common->em, PICO_WARN_OUTVECTOR,
                                  NULL, NULL);
            okay = FALSE;
        }
        if (okay && dtres.set) {
            PICODBG_DEBUG(("in: %d, out: %d", valbuf[3], dtres.class));
        } else {
            PICODBG_WARN(("problem disambiguating POS"));
            dtres.class = PICODATA_ITEMINFO1_ERR;
        }

        if (dtres.class > 255) {
            PICODBG_WARN(("dt result outside valid range, setting pos to ERR"));
            dtres.class = PICODATA_ITEMINFO1_ERR;
        }

        sa->headx[ci].head.info1 = (picoos_uint8)dtres.class;
        if (sa->headx[ci].head.type == PICODATA_ITEM_WORDINDEX) {
            /* find pos/ind entry in cbuf matching unique,
               disambiguated POS, adapt current headx cind/len
               accordingly */
            PICODBG_DEBUG(("select phon based on POS disambiguation"));
            okay = FALSE;
            for (i = 0; i < sa->headx[ci].head.len; i += PICOKLEX_POSIND_SIZE) {
                PICODBG_DEBUG(("comparing POS at cind + %d", i));
                if (picoktab_isPartOfPosGroup(sa->tabpos,
                            (picoos_uint8)dtres.class,
                            sa->cbuf1[sa->headx[ci].cind + i])) {
                    PICODBG_DEBUG(("found match for entry %d",
                                   i/PICOKLEX_POSIND_SIZE + 1));
                    sa->headx[ci].cind += i;
                    okay = TRUE;
                    break;
                }
            }
            /* not finding a match is possible if posd predicts a POS that
               is not part of any of the input POSes -> no warning */
#if defined(PICO_DEBUG)
            if (!okay) {
                PICODBG_DEBUG(("no match found, selecting 1st entry"));
            }
#endif
            sa->headx[ci].head.len = PICOKLEX_POSIND_SIZE;
        }
    }
    return PICO_OK;
}


/* ***********************************************************************/
/* PROCESS_WPHO functions, copy, lexindex, and g2p */
/* ***********************************************************************/

/* ************** copy ***************/

static pico_status_t saCopyItemContent1to2(register picodata_ProcessingUnit this,
                                           register sa_subobj_t *sa,
                                           picoos_uint16 ind) {
    picoos_uint16 i;
    picoos_uint16 cind1;

    /* set headx.cind, and copy content, head unchanged */
    cind1 = sa->headx[ind].cind;
    sa->headx[ind].cind = sa->cbuf2Len;

    /* check cbufLen */
    if (sa->headx[ind].head.len > (sa->cbuf2BufSize - sa->cbuf2Len)) {
        sa->headx[ind].head.len = sa->cbuf2BufSize - sa->cbuf2Len;
        PICODBG_WARN(("phones skipped"));
        picoos_emRaiseWarning(this->common->em,
                              PICO_WARN_INCOMPLETE, NULL, NULL);
        if (sa->headx[ind].head.len == 0) {
            sa->headx[ind].cind = 0;
        }
    }

    for (i = 0; i < sa->headx[ind].head.len; i++) {
        sa->cbuf2[sa->cbuf2Len] = sa->cbuf1[cind1 + i];
        sa->cbuf2Len++;
    }

    PICODBG_DEBUG(("%c item, len: %d",
                   sa->headx[ind].head.type, sa->headx[ind].head.len));

    return PICO_OK;
}


/* ************** lexindex ***************/

static pico_status_t saLexIndLookup(register picodata_ProcessingUnit this,
                                    register sa_subobj_t *sa,
                                    picoklex_Lex lex,
                                    picoos_uint16 ind) {
    picoos_uint8 pos;
    picoos_uint8 *phones;
    picoos_uint8 plen;
    picoos_uint16 i;

    if (picoklex_lexIndLookup(lex, &(sa->cbuf1[sa->headx[ind].cind + 1]),
                              PICOKLEX_IND_SIZE, &pos, &phones, &plen)) {
        sa->headx[ind].cind = sa->cbuf2Len;

        /* check cbufLen */
        if (plen > (sa->cbuf2BufSize - sa->cbuf2Len)) {
            plen = sa->cbuf2BufSize - sa->cbuf2Len;
            PICODBG_WARN(("phones skipped"));
            picoos_emRaiseWarning(this->common->em,
                                  PICO_WARN_INCOMPLETE, NULL, NULL);
            if (plen == 0) {
                sa->headx[ind].cind = 0;
            }
        }

        /* set item head, info1, info2 unchanged */
        sa->headx[ind].head.type = PICODATA_ITEM_WORDPHON;
        sa->headx[ind].head.len = plen;

        for (i = 0; i < plen; i++) {
            sa->cbuf2[sa->cbuf2Len] = phones[i];
            sa->cbuf2Len++;
        }

        PICODBG_DEBUG(("%c item, pos: %d, plen: %d",
                       PICODATA_ITEM_WORDPHON, pos, plen));

    } else {
        PICODBG_WARN(("lexIndLookup problem"));
        picoos_emRaiseWarning(this->common->em, PICO_WARN_PU_IRREG_ITEM,
                              NULL, NULL);
    }
    return PICO_OK;
}



/* ************** g2p ***************/


/* Name    :   saGetNvowel
   Function:   returns vowel info in a word or word seq
   Input   :   sInChar         the grapheme string to be converted in phoneme
               inLen           number of bytes in grapheme buffer
               inPos           start position of current grapheme (0..inLen-1)
   Output  :   nVow            number of vowels in the word
               nVord           vowel order in the word
   Returns :   TRUE: processing successful;  FALSE: errors
*/
static picoos_uint8 saGetNrVowel(register picodata_ProcessingUnit this,
                                 register sa_subobj_t *sa,
                                 const picoos_uint8 *sInChar,
                                 const picoos_uint16 inLen,
                                 const picoos_uint8 inPos,
                                 picoos_uint8 *nVow,
                                 picoos_uint8 *nVord) {
    picoos_uint32 nCount;
    picoos_uint32 pos;
    picoos_uint8 cstr[PICOBASE_UTF8_MAXLEN + 1];

    /*defaults*/
    *nVow = 0;
    *nVord = 0;
    /*1:check wether the current char is a vowel*/
    pos = inPos;
    if (!picobase_get_next_utf8char(sInChar, inLen, &pos, cstr) ||
        !picoktab_hasVowellikeProp(sa->tabgraphs, cstr, PICOBASE_UTF8_MAXLEN)) {
        return FALSE;
    }
    /*2:count number of vowels in current word and find vowel order*/
    for (nCount = 0; nCount < inLen; ) {
      if (!picobase_get_next_utf8char(sInChar, inLen, &nCount, cstr)) {
            return FALSE;
      }
        if (picoktab_hasVowellikeProp(sa->tabgraphs, cstr,
                                      PICOBASE_UTF8_MAXLEN)) {
            (*nVow)++;
            if (nCount == pos) {
                (*nVord) = (*nVow);
        }
        }
    }
    return TRUE;
}


/* do g2p for a full word, right-to-left */
static picoos_uint8 saDoG2P(register picodata_ProcessingUnit this,
                            register sa_subobj_t *sa,
                            const picoos_uint8 *graph,
                            const picoos_uint8 graphlen,
                            const picoos_uint8 pos,
                            picoos_uint8 *phones,
                            const picoos_uint16 phonesmaxlen,
                            picoos_uint16 *plen) {
    picoos_uint16 outNp1Ch; /*last 3 outputs produced*/
    picoos_uint16 outNp2Ch;
    picoos_uint16 outNp3Ch;
    picoos_uint8 nPrimary;
    picoos_uint8 nCount;
    picoos_uint32 utfpos;
    picoos_uint16 nOutVal;
    picoos_uint8 okay;
    picoos_uint16 phonesind;
    picoos_uint8 nrvow;
    picoos_uint8 ordvow;
    picokdt_classify_vecresult_t dtresv;
    picoos_uint16 i;

    *plen = 0;
    okay = TRUE;

    /* use sa->tmpbuf[PICOSA_MAXITEMSIZE] to temporarly store the
       phones which are predicted in reverse order. Once all are
       available put them in phones in usuable order. phonesind is
       used to fille item in reverse order starting at the end of
       tmpbuf. */
    phonesind = PICOSA_MAXITEMSIZE - 1;

    /* prepare the data for loop operations */
    outNp1Ch = PICOKDT_HISTORY_ZERO;
    outNp2Ch = PICOKDT_HISTORY_ZERO;
    outNp3Ch = PICOKDT_HISTORY_ZERO;

    /* inner loop */
    nPrimary = 0;

    /* ************************************************/
    /* go backward grapheme by grapheme, it's utf8... */
    /* ************************************************/

    /* set start nCount to position of start of last utfchar */
    /* ! watch out! somethimes starting at 1, sometimes at 0,
       ! sometimes counting per byte, sometimes per UTF8 char */
    /* nCount is (start position + 1) of utf8 char */
    utfpos = graphlen;
    if (picobase_get_prev_utf8charpos(graph, 0, &utfpos)) {
        nCount = utfpos + 1;
    } else {
        /* should not occurr */
        PICODBG_ERROR(("invalid utf8 string, graphlen: %d", graphlen));
        return FALSE;
    }

    while (nCount > 0) {
        PICODBG_TRACE(("right-to-left g2p, count: %d", nCount));
        okay = TRUE;

        if (!saGetNrVowel(this, sa, graph, graphlen, nCount-1, &nrvow,
                          &ordvow)) {
            nrvow = 0;
            ordvow = 0;
        }

        /* prepare input vector, set inside tree object invec,
         * g2pBuildVector will call the constructInVec tree method */
        if (!picokdt_dtG2PconstructInVec(sa->dtg2p,
                                         graph, /*grapheme start*/
                                         graphlen, /*grapheme length*/
                                         nCount-1, /*grapheme current position*/
                                         pos, /*Word POS*/
                                         nrvow, /*nr vowels if vowel, 0 else */
                                         ordvow, /*ord of vowel if vowel, 0 el*/
                                         &nPrimary,  /*primary stress flag*/
                                         outNp1Ch, /*Right phoneme context +1*/
                                         outNp2Ch, /*Right phoneme context +2*/
                                         outNp3Ch)) { /*Right phon context +3*/
            /*Errors in preparing the input vector : skip processing*/
            PICODBG_WARN(("problem with invec"));
            picoos_emRaiseWarning(this->common->em, PICO_WARN_INVECTOR,
                                  NULL, NULL);
            okay = FALSE;
        }

        /* classify using the invec in the tree object and save the direct
           tree output also in the tree object */
        if (okay && (!picokdt_dtG2Pclassify(sa->dtg2p, &nOutVal))) {
            /* error doing classification */
            PICODBG_WARN(("problem classifying"));
            picoos_emRaiseWarning(this->common->em, PICO_WARN_CLASSIFICATION,
                                  NULL, NULL);
            okay = FALSE;
        }

        /* decompose the invec in the tree object and return result in dtresv */
        if (okay && (!picokdt_dtG2PdecomposeOutClass(sa->dtg2p, &dtresv))) {
            /* error decomposing */
            PICODBG_WARN(("problem decomposing"));
            picoos_emRaiseWarning(this->common->em, PICO_WARN_OUTVECTOR,
                                  NULL, NULL);
            okay = FALSE;
        }

        if (okay) {
            if ((dtresv.nr == 0) || (dtresv.classvec[0] == PICOKDT_EPSILON)) {
                /* no phones to be added */
                PICODBG_TRACE(("epsilon, no phone added %c", graph[nCount-1]));
                ;
            } else {
                /* add decomposed output to tmpbuf, reverse order */
                for (i = dtresv.nr; ((((PICOSA_MAXITEMSIZE - 1) -
                                       phonesind)<phonesmaxlen) &&
                                     (i > 0)); ) {
                    i--;
                    PICODBG_TRACE(("%c %d",graph[nCount-1],dtresv.classvec[i]));
                    if (dtresv.classvec[i] > 255) {
                        PICODBG_WARN(("dt result outside valid range, "
                                      "skipping phone"));
                        continue;
                    }
                    sa->tmpbuf[phonesind--] = (picoos_uint8)dtresv.classvec[i];
                    if (!nPrimary) {
                        if (picoktab_isPrimstress(sa->tabphones,
                          (picoos_uint8)dtresv.classvec[i])) {
                            nPrimary = 1;
            }
                    }
                    (*plen)++;
                }
                if (i > 0) {
                    PICODBG_WARN(("phones skipped"));
                    picoos_emRaiseWarning(this->common->em,
                                          PICO_WARN_INCOMPLETE, NULL, NULL);
                }
            }
        }

        /*shift tree output history and update*/
        outNp3Ch = outNp2Ch;
        outNp2Ch = outNp1Ch;
        outNp1Ch = nOutVal;

        /* go backward one utf8 char */
        /* nCount is in +1 domain */
        if (nCount <= 1) {
            /* end of str */
            nCount = 0;
        } else {
            utfpos = nCount - 1;
            if (!picobase_get_prev_utf8charpos(graph, 0, &utfpos)) {
                /* should not occur */
                PICODBG_ERROR(("invalid utf8 string, utfpos: %d", utfpos));
                return FALSE;
            } else {
                nCount = utfpos + 1;
            }
        }
    }

    /* a must be: (PICOSA_MAXITEMSIZE-1) - phonesind == *plen */
    /* now that we have all phone IDs, copy in correct order to phones */
    /* phonesind point to next free slot in the reverse domainn,
       ie. inc first */
    phonesind++;
    for (i = 0; i < *plen; i++, phonesind++) {
        phones[i] = sa->tmpbuf[phonesind];
    }
    return TRUE;
}


/* item in headx[ind]/cbuf1, out: modified headx and cbuf2 */

static pico_status_t saGraphemeToPhoneme(register picodata_ProcessingUnit this,
                                         register sa_subobj_t *sa,
                                         picoos_uint16 ind) {
    picoos_uint16 plen;

    PICODBG_TRACE(("starting g2p"));

    if (saDoG2P(this, sa, &(sa->cbuf1[sa->headx[ind].cind]),
                sa->headx[ind].head.len, sa->headx[ind].head.info1,
                &(sa->cbuf2[sa->cbuf2Len]), (sa->cbuf2BufSize - sa->cbuf2Len),
                &plen)) {

        /* check of cbuf2Len done in saDoG2P, phones skipped if needed */
        if (plen > 255) {
            PICODBG_WARN(("maximum number of phones exceeded (%d), skipping",
                          plen));
            plen = 255;
        }

        /* set item head, info1, info2 unchanged */
        sa->headx[ind].head.type = PICODATA_ITEM_WORDPHON;
        sa->headx[ind].head.len = (picoos_uint8)plen;
        sa->headx[ind].cind = sa->cbuf2Len;
        sa->cbuf2Len += plen;
        PICODBG_DEBUG(("%c item, plen: %d",
                       PICODATA_ITEM_WORDPHON, plen));
    } else {
        PICODBG_WARN(("problem doing g2p"));
        picoos_emRaiseWarning(this->common->em, PICO_WARN_PU_IRREG_ITEM,
                              NULL, NULL);
    }
    return PICO_OK;
}


/* ***********************************************************************/
/*                          extract phonemes of an item into a phonBuf   */
/* ***********************************************************************/

static pico_status_t saAddPhoneme(register sa_subobj_t *sa, picoos_uint16 pos, picoos_uint16 sym) {
    /* picoos_uint8 plane, unshifted; */

    /* just for debuging */
    /*
    unshifted = picotrns_unplane(sym,&plane);
    PICODBG_DEBUG(("adding %i/%i (%c on plane %i) at phonBuf[%i]",pos,sym,unshifted,plane,sa->phonWritePos));
    */
    if (PICOTRNS_MAX_NUM_POSSYM <= sa->phonWritePos) {
        /* not an error! */
        PICODBG_DEBUG(("couldn't add because phon buffer full"));
        return PICO_EXC_BUF_OVERFLOW;
    } else {
        sa->phonBuf[sa->phonWritePos].pos = pos;
        sa->phonBuf[sa->phonWritePos].sym = sym;
        sa->phonWritePos++;
        return PICO_OK;
    }
}

/*
static pico_status_t saAddStartPhoneme(register sa_subobj_t *sa) {
    return saAddPhoneme(sa, PICOTRNS_POS_IGNORE,
            (PICOKFST_PLANE_INTERN << 8) + sa->fixedIds->phonStartId);
}


static pico_status_t saAddTermPhoneme(register sa_subobj_t *sa) {
    return saAddPhoneme(sa, PICOTRNS_POS_IGNORE,
            (PICOKFST_PLANE_INTERN << 8) + sa->fixedIds->phonTermId);
}

*/

static pico_status_t saExtractPhonemes(register picodata_ProcessingUnit this,
        register sa_subobj_t *sa, picoos_uint16 pos,
        picodata_itemhead_t* head, const picoos_uint8* content)
{
    pico_status_t rv= PICO_OK;
    picoos_uint8 i;
    picoos_int16 fstSymbol;
#if defined(PICO_DEBUG)
    picoos_char msgstr[SA_MSGSTR_SIZE];
#endif

    PICODBG_TRACE(("doing item %s",
                    picodata_head_to_string(head,msgstr,SA_MSGSTR_SIZE)));
    /*
     Items  considered in a transduction are WORDPHON item. its starting offset within the inBuf is given as
     'pos'.
     Elements that go into the transduction receive "their" position in the buffer.
     */
    sa->phonWritePos = 0;
    /* WORDPHON(POS,WACC)phon */
    rv = saAddPhoneme(sa, PICOTRNS_POS_IGNORE,
                (PICOKFST_PLANE_INTERN << 8) + sa->fixedIds->phonStartId);
    for (i = 0; i < head->len; i++) {
        fstSymbol = /* (PICOKFST_PLANE_PHONEMES << 8) + */content[i];
        /*  */
        PICODBG_TRACE(("adding phoneme %c",fstSymbol));
        rv = saAddPhoneme(sa, pos+PICODATA_ITEM_HEADSIZE+i, fstSymbol);
    }
    rv = saAddPhoneme(sa, PICOTRNS_POS_IGNORE,
                (PICOKFST_PLANE_INTERN << 8) + sa->fixedIds->phonTermId);
    sa->nextReadPos = pos + PICODATA_ITEM_HEADSIZE +  head->len;
    return rv;
}


#define SA_POSSYM_OK           0
#define SA_POSSYM_OUT_OF_RANGE 1
#define SA_POSSYM_END          2
#define SA_POSSYM_INVALID     -3
/* *readPos is the next position in phonBuf to be read, and *writePos is the first position not to be read (may be outside
 * buf).
 * 'rangeEnd' is the first possym position outside the desired range.
 * Possible return values:
 * SA_POSSYM_OK            : 'pos' and 'sym' are set to the read possym, *readPos is advanced
 * SA_POSSYM_OUT_OF_RANGE  : pos is out of range. 'pos' is set to that of the read possym, 'sym' is undefined
 * SA_POSSYM_UNDERFLOW     : no more data in buf. 'pos' is set to PICOTRNS_POS_INVALID,    'sym' is undefined
 * SA_POSSYM_INVALID       : "strange" pos.       'pos' is set to PICOTRNS_POS_INVALID,    'sym' is undefined
 */
static pico_status_t getNextPosSym(sa_subobj_t * sa, picoos_int16 * pos, picoos_int16 * sym,
        picoos_int16 rangeEnd) {
    /* skip POS_IGNORE */
    while ((sa->phonReadPos < sa->phonWritePos) && (PICOTRNS_POS_IGNORE == sa->phonBuf[sa->phonReadPos].pos))  {
        PICODBG_DEBUG(("ignoring phone at sa->phonBuf[%i] because it has pos==IGNORE",sa->phonReadPos));
        sa->phonReadPos++;
    }
    if ((sa->phonReadPos < sa->phonWritePos)) {
        *pos = sa->phonBuf[sa->phonReadPos].pos;
        if ((PICOTRNS_POS_INSERT == *pos) || ((0 <= *pos) && (*pos < rangeEnd))) {
            *sym = sa->phonBuf[sa->phonReadPos++].sym;
            return SA_POSSYM_OK;
        } else if (*pos < 0){ /* *pos is "strange" (e.g. POS_INVALID) */
            return SA_POSSYM_INVALID;
        } else {
            return SA_POSSYM_OUT_OF_RANGE;
        }
    } else {
        /* no more possyms to read */
        *pos = PICOTRNS_POS_INVALID;
        return SA_POSSYM_END;
    }
}




/* ***********************************************************************/
/*                          saStep function                              */
/* ***********************************************************************/

/*
complete phrase processed in one step, if not fast enough -> rework

init, collect into internal buffer, process, and then feed to
output buffer

init state: INIT ext           ext
state trans:     in hc1  hc2   out

INIT | putItem   =  0    0    +1      | BUSY  -> COLL (put B-SBEG item,
                                                   set do-init to false)

                                    inspace-ok-hc1
                                  needs-more-items-(phrase-or-flush)
COLL1 |getItems -n +n             0 1 | ATOMIC -> PPOSD     (got items,
                                                      if flush set do-init)
COLL2 |getItems -n +n             1 0 | ATOMIC -> PPOSD (got items, forced)
COLL3 |getItems -n +n             1 1 | IDLE          (got items, need more)
COLL4 |getItems  =  =             1 1 | IDLE             (got no items)

PPOSD | posd     = ~n~n               | BUSY     -> PWP     (posd done)
PWP   | lex/g2p  = ~n-n  0+n          | BUSY     -> PPHR    (lex/g2p done)
PPHR  | phr      = -n 0 +m=n          | BUSY     -> PACC    (phr done, m>=n)
PACC  | acc      =  0 0 ~m=n          | BUSY     -> FEED    (acc done)

                                  doinit-flag
FEED | putItems  0  0 0 -m-n  +m  0   | BUSY -> COLL    (put items)
FEED | putItems  0  0 0 -m-n  +m  1   | BUSY -> INIT    (put items)
FEED | putItems  0  0 0 -d-d  +d      | OUT_FULL        (put some items)
*/

static picodata_step_result_t saStep(register picodata_ProcessingUnit this,
                                     picoos_int16 mode,
                                     picoos_uint16 *numBytesOutput) {
    register sa_subobj_t *sa;
    pico_status_t rv = PICO_OK;
    pico_status_t rvP = PICO_OK;
    picoos_uint16 blen = 0;
    picoos_uint16 clen = 0;
    picoos_uint16 i;
    picoklex_Lex lex;


    if (NULL == this || NULL == this->subObj) {
        return PICODATA_PU_ERROR;
    }
    sa = (sa_subobj_t *) this->subObj;
    mode = mode;        /* avoid warning "var not used in this function"*/
    *numBytesOutput = 0;
    while (1) { /* exit via return */
        PICODBG_DEBUG(("doing state %i, hLen|c1Len|c2Len: %d|%d|%d",
                       sa->procState, sa->headxLen, sa->cbuf1Len,
                       sa->cbuf2Len));

        switch (sa->procState) {

            /* *********************************************************/
            /* collect state: get item(s) from charBuf and store in
             * internal buffers, need a complete punctuation-phrase
             */
            case SA_STEPSTATE_COLLECT:

                while (sa->inspaceok && sa->needsmoreitems
                       && (PICO_OK ==
                           (rv = picodata_cbGetItem(this->cbIn, sa->tmpbuf,
                                            PICOSA_MAXITEMSIZE, &blen)))) {
                    rvP = picodata_get_itemparts(sa->tmpbuf,
                                            PICOSA_MAXITEMSIZE,
                                            &(sa->headx[sa->headxLen].head),
                                            &(sa->cbuf1[sa->cbuf1Len]),
                                            sa->cbuf1BufSize-sa->cbuf1Len,
                                            &clen);
                    if (rvP != PICO_OK) {
                        PICODBG_ERROR(("problem getting item parts"));
                        picoos_emRaiseException(this->common->em, rvP,
                                                NULL, NULL);
                        return PICODATA_PU_ERROR;
                    }

                    /* if CMD(...FLUSH...) -> PUNC(...FLUSH...),
                       construct PUNC-FLUSH item in headx */
                    if ((sa->headx[sa->headxLen].head.type ==
                         PICODATA_ITEM_CMD) &&
                        (sa->headx[sa->headxLen].head.info1 ==
                         PICODATA_ITEMINFO1_CMD_FLUSH)) {
                        sa->headx[sa->headxLen].head.type =
                            PICODATA_ITEM_PUNC;
                        sa->headx[sa->headxLen].head.info1 =
                            PICODATA_ITEMINFO1_PUNC_FLUSH;
                        sa->headx[sa->headxLen].head.info2 =
                            PICODATA_ITEMINFO2_PUNC_SENT_T;
                        sa->headx[sa->headxLen].head.len = 0;
                    }

                    /* convert opening phoneme command to WORDPHON
                     * and assign user-POS XX to it (Bug 432) */
                    sa->headx[sa->headxLen].cind = sa->cbuf1Len;
                    /* maybe overwritten later */
                    if ((sa->headx[sa->headxLen].head.type ==
                        PICODATA_ITEM_CMD) &&
                       (sa->headx[sa->headxLen].head.info1 ==
                        PICODATA_ITEMINFO1_CMD_PHONEME)&&
                        (sa->headx[sa->headxLen].head.info2 ==
                         PICODATA_ITEMINFO2_CMD_START)) {
                        picoos_uint8 i;
                        picoos_uint8 wordsep = picoktab_getWordboundID(sa->tabphones);
                        PICODBG_INFO(("wordsep id is %i",wordsep));
                        sa->headx[sa->headxLen].head.type = PICODATA_ITEM_WORDPHON;
                        sa->headx[sa->headxLen].head.info1 = PICODATA_POS_XX;
                        sa->headx[sa->headxLen].head.info2 = PICODATA_ITEMINFO2_NA;
                        /* cut off additional words */
                        i = 0;
                        while ((i < sa->headx[sa->headxLen].head.len) && (wordsep != sa->cbuf1[sa->headx[sa->headxLen].cind+i])) {
                            PICODBG_INFO(("accepting phoneme %i",sa->cbuf1[sa->headx[sa->headxLen].cind+i]));

                            i++;
                        }
                        if (i < sa->headx[sa->headxLen].head.len) {
                            PICODBG_INFO(("cutting off superfluous phonetic words at %i",i));
                            sa->headx[sa->headxLen].head.len = i;
                        }
                    }

                    /* check/set needsmoreitems */
                    if (sa->headx[sa->headxLen].head.type ==
                        PICODATA_ITEM_PUNC) {
                        sa->needsmoreitems = FALSE;
                    }

                    /* check/set inspaceok, keep spare slot for forcing */
                    if ((sa->headxLen >= (PICOSA_MAXNR_HEADX - 2)) ||
                        ((sa->cbuf1BufSize - sa->cbuf1Len) <
                         PICOSA_MAXITEMSIZE)) {
                        sa->inspaceok = FALSE;
                    }

                    if (clen > 0) {
                        sa->headx[sa->headxLen].cind = sa->cbuf1Len;
                        sa->cbuf1Len += clen;
                    } else {
                        sa->headx[sa->headxLen].cind = 0;
                    }
                    sa->headxLen++;
                }

                if (!sa->needsmoreitems) {
                    /* 1, phrase buffered */
                    sa->procState = SA_STEPSTATE_PROCESS_POSD;
                    return PICODATA_PU_ATOMIC;
                } else if (!sa->inspaceok) {
                    /* 2, forced phrase end */
                    /* at least one slot is still free, use it to
                       force a trailing PUNC item */
                    sa->headx[sa->headxLen].head.type = PICODATA_ITEM_PUNC;
                    sa->headx[sa->headxLen].head.info1 =
                        PICODATA_ITEMINFO1_PUNC_PHRASEEND;
                    sa->headx[sa->headxLen].head.info2 =
                        PICODATA_ITEMINFO2_PUNC_PHRASE_FORCED;
                    sa->headx[sa->headxLen].head.len = 0;
                    sa->needsmoreitems = FALSE; /* not really needed for now */
                    sa->headxLen++;
                    PICODBG_WARN(("forcing phrase end, added PUNC_PHRASEEND"));
                    picoos_emRaiseWarning(this->common->em,
                                          PICO_WARN_FALLBACK, NULL,
                                          (picoos_char *)"forced phrase end");
                    sa->procState = SA_STEPSTATE_PROCESS_POSD;
                    return PICODATA_PU_ATOMIC;
                } else if (rv == PICO_EOF) {
                    /* 3, 4 */
                    return PICODATA_PU_IDLE;
                } else if ((rv == PICO_EXC_BUF_UNDERFLOW) ||
                           (rv == PICO_EXC_BUF_OVERFLOW)) {
                    /* error, no valid item in cb (UNDER) */
                    /*        or tmpbuf not large enough, not possible (OVER) */
                    /* no exception raised, left for ctrl to handle */
                    PICODBG_ERROR(("buffer under/overflow, rv: %d", rv));
                    return PICODATA_PU_ERROR;
                } else {
                    /* error, only possible if cbGetItem implementation
                       changes without this function being adapted*/
                    PICODBG_ERROR(("untreated return value, rv: %d", rv));
                    return PICODATA_PU_ERROR;
                }
                break;


            /* *********************************************************/
            /* process posd state: process items in headx/cbuf1
             * and change in place
             */
            case SA_STEPSTATE_PROCESS_POSD:
                /* ensure there is an item in inBuf */
                if (sa->headxLen > 0) {
                    /* we have a phrase in headx, cbuf1 (can be
                       single PUNC item without POS), do pos disamb */
                    if (PICO_OK != saDisambPos(this, sa)) {
                        picoos_emRaiseException(this->common->em,
                                                PICO_ERR_OTHER, NULL, NULL);
                        return PICODATA_PU_ERROR;
                    }
                    sa->procState = SA_STEPSTATE_PROCESS_WPHO;

                } else if (sa->headxLen == 0) {    /* no items in inBuf */
                    PICODBG_WARN(("no items in inBuf"));
                    sa->procState = SA_STEPSTATE_COLLECT;
                    return PICODATA_PU_BUSY;
                }

#if defined (PICO_DEBUG)
                if (1) {
                    picoos_uint8 i, j, ittype;
                    for (i = 0; i < sa->headxLen; i++) {
                        ittype = sa->headx[i].head.type;
                        PICODBG_INFO_CTX();
                        PICODBG_INFO_MSG(("sa-d: ("));
                        PICODBG_INFO_MSG(("'%c',", ittype));
                        if ((32 <= sa->headx[i].head.info1) &&
                            (sa->headx[i].head.info1 < 127) &&
                            (ittype != PICODATA_ITEM_WORDGRAPH) &&
                            (ittype != PICODATA_ITEM_WORDINDEX)) {
                            PICODBG_INFO_MSG(("'%c',",sa->headx[i].head.info1));
                        } else {
                            PICODBG_INFO_MSG(("%3d,", sa->headx[i].head.info1));
                        }
                        if ((32 <= sa->headx[i].head.info2) &&
                            (sa->headx[i].head.info2 < 127)) {
                            PICODBG_INFO_MSG(("'%c',",sa->headx[i].head.info2));
                        } else {
                            PICODBG_INFO_MSG(("%3d,", sa->headx[i].head.info2));
                        }
                        PICODBG_INFO_MSG(("%3d)", sa->headx[i].head.len));

                        for (j = 0; j < sa->headx[i].head.len; j++) {
                            if ((ittype == PICODATA_ITEM_WORDGRAPH) ||
                                (ittype == PICODATA_ITEM_CMD)) {
                                PICODBG_INFO_MSG(("%c",
                                        sa->cbuf1[sa->headx[i].cind+j]));
                            } else {
                                PICODBG_INFO_MSG(("%4d",
                                        sa->cbuf1[sa->headx[i].cind+j]));
                            }
                        }
                        PICODBG_INFO_MSG(("\n"));
                    }
                }
#endif

                break;


            /* *********************************************************/
            /* process wpho state: process items in headx/cbuf1 and modify
             * headx in place and fill cbuf2
             */
            case SA_STEPSTATE_PROCESS_WPHO:
                /* ensure there is an item in inBuf */
                if (sa->headxLen > 0) {
                    /* we have a phrase in headx, cbuf1 (can be single
                       PUNC item), do lex lookup, g2p, or copy */

                    /* check if cbuf2 is empty as it should be */
                    if (sa->cbuf2Len > 0) {
                        /* enforce emptyness */
                        PICODBG_WARN(("forcing empty cbuf2, discarding buf"));
                        picoos_emRaiseWarning(this->common->em,
                                              PICO_WARN_PU_DISCARD_BUF,
                                              NULL, NULL);
                    }

                    /* cbuf2 overflow avoided in saGrapheme*, saLexInd*,
                       saCopyItem*, phones skipped if needed */
                    for (i = 0; i < sa->headxLen; i++) {
                        switch (sa->headx[i].head.type) {
                            case PICODATA_ITEM_WORDGRAPH:
                                if (PICO_OK != saGraphemeToPhoneme(this, sa,
                                                                   i)) {
                                    /* not possible, phones skipped if needed */
                                    picoos_emRaiseException(this->common->em,
                                                            PICO_ERR_OTHER,
                                                            NULL, NULL);
                                    return PICODATA_PU_ERROR;
                                }
                                break;
                            case PICODATA_ITEM_WORDINDEX:
                                if (0 == sa->headx[i].head.info2) {
                                  lex = sa->lex;
                                } else {
                                    lex = sa->ulex[sa->headx[i].head.info2-1];
                                }
                                if (PICO_OK != saLexIndLookup(this, sa, lex, i)) {
                                    /* not possible, phones skipped if needed */
                                    picoos_emRaiseException(this->common->em,
                                                            PICO_ERR_OTHER,
                                                            NULL, NULL);
                                    return PICODATA_PU_ERROR;
                                }
                                break;
                            default:
                                /* copy item unmodified, ie. headx untouched,
                                   content from cbuf1 to cbuf2 */
                                if (PICO_OK != saCopyItemContent1to2(this, sa,
                                                                     i)) {
                                    /* not possible, phones skipped if needed */
                                    picoos_emRaiseException(this->common->em,
                                                            PICO_ERR_OTHER,
                                                            NULL, NULL);
                                    return PICODATA_PU_ERROR;
                                }
                                break;
                        }
                    }
                    /* set cbuf1 to empty */
                    sa->cbuf1Len = 0;
                    sa->procState = SA_STEPSTATE_PROCESS_TRNS_PARSE;

                } else if (sa->headxLen == 0) {    /* no items in inBuf */
                    PICODBG_WARN(("no items in inBuf"));
                    sa->procState = SA_STEPSTATE_COLLECT;
                    return PICODATA_PU_BUSY;
                }

#if defined (PICO_DEBUG)
                if (1) {
                    picoos_uint8 i, j, ittype;
                    for (i = 0; i < sa->headxLen; i++) {
                        ittype = sa->headx[i].head.type;
                        PICODBG_INFO_CTX();
                        PICODBG_INFO_MSG(("sa-g: ("));
                        PICODBG_INFO_MSG(("'%c',", ittype));
                        if ((32 <= sa->headx[i].head.info1) &&
                            (sa->headx[i].head.info1 < 127) &&
                            (ittype != PICODATA_ITEM_WORDPHON)) {
                            PICODBG_INFO_MSG(("'%c',",sa->headx[i].head.info1));
                        } else {
                            PICODBG_INFO_MSG(("%3d,", sa->headx[i].head.info1));
                        }
                        if ((32 <= sa->headx[i].head.info2) &&
                            (sa->headx[i].head.info2 < 127)) {
                            PICODBG_INFO_MSG(("'%c',",sa->headx[i].head.info2));
                        } else {
                            PICODBG_INFO_MSG(("%3d,", sa->headx[i].head.info2));
                        }
                        PICODBG_INFO_MSG(("%3d)", sa->headx[i].head.len));

                        for (j = 0; j < sa->headx[i].head.len; j++) {
                            if ((ittype == PICODATA_ITEM_CMD)) {
                                PICODBG_INFO_MSG(("%c",
                                        sa->cbuf2[sa->headx[i].cind+j]));
                            } else {
                                PICODBG_INFO_MSG(("%4d",
                                        sa->cbuf2[sa->headx[i].cind+j]));
                            }
                        }
                        PICODBG_INFO_MSG(("\n"));
                    }
                }
#endif

                break;


                /* *********************************************************/
                /* transduction parse state: extract phonemes of item in internal outBuf */
           case SA_STEPSTATE_PROCESS_TRNS_PARSE:

                PICODBG_DEBUG(("transduce item (bot, remain): (%d, %d)",
                                sa->headxBottom, sa->headxLen));

                /* check for termination condition first */
                if (0 == sa->headxLen) {
                    /* reset headx, cbuf2 */
                    sa->headxBottom = 0;
                    sa->cbuf2Len = 0;
                    /* reset collect state support variables */
                    sa->inspaceok = TRUE;
                    sa->needsmoreitems = TRUE;

                    sa->procState = SA_STEPSTATE_COLLECT;
                    return PICODATA_PU_BUSY;
                }

                sa->procState = SA_STEPSTATE_FEED;
                /* copy item unmodified */
                rv = picodata_put_itemparts(
                        &(sa->headx[sa->headxBottom].head),
                        &(sa->cbuf2[sa->headx[sa->headxBottom].cind]),
                        sa->headx[sa->headxBottom].head.len, sa->tmpbuf,
                        PICOSA_MAXITEMSIZE, &blen);

                if (PICODATA_ITEM_WORDPHON == sa->headx[sa->headxBottom].head.type) {
                   PICODBG_DEBUG(("PARSE found WORDPHON"));
                   rv = saExtractPhonemes(this, sa, 0, &(sa->headx[sa->headxBottom].head),
                           &(sa->cbuf2[sa->headx[sa->headxBottom].cind]));
                   if (PICO_OK == rv) {
                       PICODBG_DEBUG(("PARSE successfully returned from phoneme extraction"));
                       sa->procState = SA_STEPSTATE_PROCESS_TRNS_FST;
                   } else {
                       PICODBG_WARN(("PARSE phone extraction returned exception %i, output WORDPHON untransduced",rv));
                   }
               } else {
                   PICODBG_DEBUG(("PARSE found other item, just copying"));
               }
               if (SA_STEPSTATE_FEED == sa->procState) {
                    PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
                            (picoos_uint8 *)"sa-p: ",
                            sa->tmpbuf, PICOSA_MAXITEMSIZE);

                }

                /* consume item */
                sa->headxBottom++;
                sa->headxLen--;

                break;

                /* *********************************************************/
                /* transduce state: copy item in internal outBuf to tmpBuf and transduce */
           case SA_STEPSTATE_PROCESS_TRNS_FST:





               /* if no word-level FSTs: doing trivial syllabification instead */
               if (0 == sa->numFsts) {
                   PICODBG_DEBUG(("doing trivial sylabification with %i phones", sa->phonWritePos));
#if defined(PICO_DEBUG)
                   {
                       PICODBG_INFO_CTX();
                       PICODBG_INFO_MSG(("sa trying to trivially syllabify: "));
                       PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], sa->phonBuf, sa->phonWritePos);
                       PICODBG_INFO_MSG(("\n"));
                   }
#endif

                   picotrns_trivial_syllabify(sa->tabphones, sa->phonBuf,
                           sa->phonWritePos, sa->phonBufOut,
                           &sa->phonWritePos,PICOTRNS_MAX_NUM_POSSYM);
                   PICODBG_DEBUG(("returned from trivial sylabification with %i phones", sa->phonWritePos));
#if defined(PICO_DEBUG)
                   {
                       PICODBG_INFO_CTX();
                       PICODBG_INFO_MSG(("sa returned from syllabification: "));
                       PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], sa->phonBufOut, sa->phonWritePos);
                       PICODBG_INFO_MSG(("\n"));
                   }
#endif

                   /* eliminate deep epsilons */
                   PICODBG_DEBUG(("doing epsilon elimination with %i phones", sa->phonWritePos));
                   picotrns_eliminate_epsilons(sa->phonBufOut,
                           sa->phonWritePos, sa->phonBuf,
                           &sa->phonWritePos,PICOTRNS_MAX_NUM_POSSYM);
                   PICODBG_DEBUG(("returning from epsilon elimination with %i phones", sa->phonWritePos));
                   sa->phonReadPos = 0;
                   sa->phonesTransduced = 1;
                   sa->procState = SA_STEPSTATE_FEED;
                   break;
               }

               /* there are word-level FSTs */
               /* termination condition first */
               if (sa->curFst >= sa->numFsts) {
                   /* reset for next transduction */
                   sa->curFst = 0;
                   sa->phonReadPos = 0;
                   sa->phonesTransduced = 1;
                   sa->procState = SA_STEPSTATE_FEED;
                   break;
               }

               /* transduce from phonBufIn to PhonBufOut */
               {

                   picoos_uint32 nrSteps;
#if defined(PICO_DEBUG)
                   {
                       PICODBG_INFO_CTX();
                       PICODBG_INFO_MSG(("sa trying to transduce: "));
                       PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], sa->phonBuf, sa->phonWritePos);
                       PICODBG_INFO_MSG(("\n"));
                   }
#endif
                   picotrns_transduce(sa->fst[sa->curFst], FALSE,
                           picotrns_printSolution, sa->phonBuf, sa->phonWritePos, sa->phonBufOut,
                           &sa->phonWritePos,
                           PICOTRNS_MAX_NUM_POSSYM, sa->altDescBuf,
                           sa->maxAltDescLen, &nrSteps);
#if defined(PICO_DEBUG)
                   {
                       PICODBG_INFO_CTX();
                       PICODBG_INFO_MSG(("sa returned from transduction: "));
                       PICOTRNS_PRINTSYMSEQ(this->voice->kbArray[PICOKNOW_KBID_DBG], sa->phonBufOut, sa->phonWritePos);
                       PICODBG_INFO_MSG(("\n"));
                   }
#endif
               }



               /*
                The trasduction output will contain equivalent items i.e. (x,y')  for each (x,y) plus inserted deep symbols (-1,d).
                In case of deletions, (x,0) might also be omitted...
                */
               /* eliminate deep epsilons */
               picotrns_eliminate_epsilons(sa->phonBufOut,
                       sa->phonWritePos, sa->phonBuf, &sa->phonWritePos,PICOTRNS_MAX_NUM_POSSYM);
               sa->phonesTransduced = 1;

               sa->curFst++;

               return PICODATA_PU_ATOMIC;
               /* break; */

                /* *********************************************************/
                /* feed state: copy item in internal outBuf to output charBuf */

           case SA_STEPSTATE_FEED:

               PICODBG_DEBUG(("FEED"));

               if (sa->phonesTransduced) {
                   /* replace original phones by transduced */
                   picoos_uint16 phonWritePos = PICODATA_ITEM_HEADSIZE;
                   picoos_uint8 plane;
                   picoos_int16 sym, pos;
                   while (SA_POSSYM_OK == (rv = getNextPosSym(sa,&pos,&sym,sa->nextReadPos))) {
                       PICODBG_TRACE(("FEED inserting phoneme %c into inBuf[%i]",sym,phonWritePos));
                       sym = picotrns_unplane(sym, &plane);
                       PICODBG_ASSERT((PICOKFST_PLANE_PHONEMES == plane));
                       sa->tmpbuf[phonWritePos++] = (picoos_uint8) sym;
                   }
                   PICODBG_DEBUG(("FEED setting item length to %i",phonWritePos - PICODATA_ITEM_HEADSIZE));
                   picodata_set_itemlen(sa->tmpbuf,PICODATA_ITEM_HEADSIZE,phonWritePos - PICODATA_ITEM_HEADSIZE);
                   if (SA_POSSYM_INVALID == rv) {
                       PICODBG_ERROR(("FEED unexpected symbol or unexpected end of phoneme list"));
                       return (picodata_step_result_t)picoos_emRaiseException(this->common->em, PICO_WARN_INCOMPLETE, NULL, NULL);
                   }
                   sa->phonesTransduced = 0;

               } /* if (sa->phonesTransduced) */


                rvP = picodata_cbPutItem(this->cbOut, sa->tmpbuf,
                PICOSA_MAXITEMSIZE, &clen);

                *numBytesOutput += clen;

                PICODBG_DEBUG(("put item, status: %d", rvP));

                if (rvP == PICO_OK) {
                } else if (rvP == PICO_EXC_BUF_OVERFLOW) {
                    /* try again next time */
                    PICODBG_DEBUG(("feeding overflow"));
                    return PICODATA_PU_OUT_FULL;
                } else {
                    /* error, should never happen */
                    PICODBG_ERROR(("untreated return value, rvP: %d", rvP));
                    return PICODATA_PU_ERROR;
                }

                PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
                        (picoos_uint8 *)"sana: ",
                        sa->tmpbuf, PICOSA_MAXITEMSIZE);

                sa->procState = SA_STEPSTATE_PROCESS_TRNS_PARSE;
                /* return PICODATA_PU_BUSY; */
                break;

            default:
                break;
        } /* switch */

    } /* while */

    /* should be never reached */
    PICODBG_ERROR(("reached end of function"));
    picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL);
    return PICODATA_PU_ERROR;
}

#ifdef __cplusplus
}
#endif


/* end */