/* * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /** * @file picowa.c * * word analysis PU - lexicon lookup and POS prediction * * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland * All rights reserved. * * History: * - 2009-04-20 -- initial version * */ #include "picoos.h" #include "picodbg.h" #include "picodata.h" #include "picowa.h" #include "picoklex.h" #include "picokdt.h" #include "picoktab.h" #ifdef __cplusplus extern "C" { #endif #if 0 } #endif /* PU waStep states */ #define WA_STEPSTATE_COLLECT 0 #define WA_STEPSTATE_PROCESS 1 #define WA_STEPSTATE_FEED 2 /* subobject : WordAnaUnit * shortcut : wa * context size : one item */ typedef struct wa_subobj { picoos_uint8 procState; /* for next processing step decision */ /* one item only */ picoos_uint8 inBuf[PICOWA_MAXITEMSIZE]; /* internal input buffer */ picoos_uint16 inBufSize; /* actually allocated size */ picoos_uint16 inLen; /* length of item in inBuf, 0 for empty buf */ picoos_uint8 outBuf[PICOWA_MAXITEMSIZE]; /* internal output buffer */ picoos_uint16 outBufSize; /* actually allocated size */ picoos_uint16 outLen; /* length of item in outBuf, 0 for empty buf */ /* lex knowledge base */ picoklex_Lex lex; /* ulex knowledge bases */ picoos_uint8 numUlex; picoklex_Lex ulex[PICOKNOW_MAX_NUM_ULEX]; /* tab knowledge base */ picoktab_Pos tabpos; /* dtposp knowledge base */ picokdt_DtPosP dtposp; } wa_subobj_t; static pico_status_t waInitialize(register picodata_ProcessingUnit this, picoos_int32 resetMode) { picoos_uint8 i; picoklex_Lex ulex; wa_subobj_t * wa; picoknow_kb_id_t ulexKbIds[PICOKNOW_MAX_NUM_ULEX] = PICOKNOW_KBID_ULEX_ARRAY; PICODBG_DEBUG(("calling")); if (NULL == this || NULL == this->subObj) { return (picodata_step_result_t) picoos_emRaiseException(this->common->em, PICO_ERR_NULLPTR_ACCESS, NULL, NULL); } wa = (wa_subobj_t *) this->subObj; wa->procState = WA_STEPSTATE_COLLECT; wa->inBufSize = PICOWA_MAXITEMSIZE; wa->inLen = 0; wa->outBufSize = PICOWA_MAXITEMSIZE; wa->outLen = 0; if (resetMode == PICO_RESET_SOFT) { /*following initializations needed only at startup or after a full reset*/ return PICO_OK; } /* kb lex */ wa->lex = picoklex_getLex(this->voice->kbArray[PICOKNOW_KBID_LEX_MAIN]); if (wa->lex == NULL) { return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING, NULL, NULL); } PICODBG_DEBUG(("got lex")); /* kb ulex[] */ wa->numUlex = 0; for (i = 0; ivoice->kbArray[ulexKbIds[i]]); if (NULL != ulex) { wa->ulex[wa->numUlex++] = ulex; } } PICODBG_DEBUG(("got %i user lexica", wa->numUlex)); /* kb tabpos */ wa->tabpos = picoktab_getPos(this->voice->kbArray[PICOKNOW_KBID_TAB_POS]); if (wa->tabpos == NULL) { return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING, NULL, NULL); } PICODBG_DEBUG(("got tabpos")); /* kb dtposp */ wa->dtposp = picokdt_getDtPosP(this->voice->kbArray[PICOKNOW_KBID_DT_POSP]); if (wa->dtposp == NULL) { return picoos_emRaiseException(this->common->em, PICO_EXC_KB_MISSING, NULL, NULL); } PICODBG_DEBUG(("got dtposp")); return PICO_OK; } static picodata_step_result_t waStep(register picodata_ProcessingUnit this, picoos_int16 mode, picoos_uint16 *numBytesOutput); static pico_status_t waTerminate(register picodata_ProcessingUnit this) { return PICO_OK; } static pico_status_t waSubObjDeallocate(register picodata_ProcessingUnit this, picoos_MemoryManager mm) { if (NULL != this) { picoos_deallocate(this->common->mm, (void *) &this->subObj); } mm = mm; /* avoid warning "var not used in this function"*/ return PICO_OK; } picodata_ProcessingUnit picowa_newWordAnaUnit(picoos_MemoryManager mm, picoos_Common common, picodata_CharBuffer cbIn, picodata_CharBuffer cbOut, picorsrc_Voice voice) { picodata_ProcessingUnit this; this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice); if (this == NULL) { return NULL; } this->initialize = waInitialize; PICODBG_DEBUG(("set this->step to waStep")); this->step = waStep; this->terminate = waTerminate; this->subDeallocate = waSubObjDeallocate; this->subObj = picoos_allocate(mm, sizeof(wa_subobj_t)); if (this->subObj == NULL) { picoos_deallocate(mm, (void *)&this); picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM, NULL, NULL); return NULL; } waInitialize(this, PICO_RESET_FULL); return this; } /* ***********************************************************************/ /* WORDGRAPH proc functions */ /* ***********************************************************************/ static picoos_uint8 waClassifyPos(register picodata_ProcessingUnit this, register wa_subobj_t *wa, const picoos_uint8 *graph, const picoos_uint16 graphlen) { picokdt_classify_result_t dtres; picoos_uint8 specchar; picoos_uint16 i; PICODBG_DEBUG(("graphlen %d", graphlen)); /* check existence of special char (e.g. hyphen) in graph: for now, check existence of hard-coded ascii hyphen, ie. preproc needs to match all UTF8 hyphens to the ascii hyphen. */ /* @todo : consider specifying special char(s) in lingware. */ specchar = FALSE; i = 0; while ((i < graphlen) && (!specchar)) { if (graph[i++] == '-') { specchar = TRUE; } } /* construct input vector, which is set in dtposp */ if (!picokdt_dtPosPconstructInVec(wa->dtposp, graph, graphlen, specchar)) { /* error constructing invec */ PICODBG_WARN(("problem with invec")); picoos_emRaiseWarning(this->common->em, PICO_WARN_INVECTOR, NULL, NULL); return PICODATA_ITEMINFO1_ERR; } /* classify */ if (!picokdt_dtPosPclassify(wa->dtposp)) { /* error doing classification */ PICODBG_WARN(("problem classifying")); picoos_emRaiseWarning(this->common->em, PICO_WARN_CLASSIFICATION, NULL, NULL); return PICODATA_ITEMINFO1_ERR; } /* decompose */ if (!picokdt_dtPosPdecomposeOutClass(wa->dtposp, &dtres)) { /* error decomposing */ PICODBG_WARN(("problem decomposing")); picoos_emRaiseWarning(this->common->em, PICO_WARN_OUTVECTOR, NULL, NULL); return PICODATA_ITEMINFO1_ERR; } if (dtres.set) { PICODBG_DEBUG(("class %d", dtres.class)); return (picoos_uint8)dtres.class; } else { PICODBG_WARN(("result not set")); picoos_emRaiseWarning(this->common->em, PICO_WARN_CLASSIFICATION, NULL, NULL); return PICODATA_ITEMINFO1_ERR; } } static pico_status_t waProcessWordgraph(register picodata_ProcessingUnit this, register wa_subobj_t *wa /*inout*/, picodata_itemhead_t *head /*inout*/, const picoos_uint8 *content) { pico_status_t status; picoklex_lexl_result_t lexres; picoos_uint8 posbuf[PICOKTAB_MAXNRPOS_IN_COMB]; picoos_uint8 i; picoos_uint8 foundIndex; picoos_bool found; PICODBG_DEBUG(("type %c, len %d", head->type, head->len)); /* do lookup if no entry found: do POS prediction: -> WORDGRAPH(POSes,NA)graph else: if incl-phone: N entries possible -> WORDINDEX(POSes,NA)POS1|ind1...POSN|indN (N in {1,...,PICOKLEX_MAX_NRRES}, now up to 4) else: no phone, one entry -> WORDGRAPH(POS,NA)graph */ found = FALSE; i = 0; while (!found && (i < wa->numUlex)) { found = picoklex_lexLookup(wa->ulex[i], content, head->len, &lexres); i++; } /* note that if found, i will be incremented nevertheless, so i >= 1 */ if (found) { foundIndex = i; } else { foundIndex = 0; } if (!found && !picoklex_lexLookup(wa->lex, content, head->len, &lexres)) { /* no lex entry found, WORDGRAPH(POS,NA)graph */ if (PICO_OK == picodata_copy_item(wa->inBuf, wa->inLen, wa->outBuf, wa->outBufSize, &wa->outLen)) { wa->inLen = 0; /* predict and modify pos in info1 */ if (PICO_OK != picodata_set_iteminfo1(wa->outBuf, wa->outLen, waClassifyPos(this, wa, content, head->len))) { return picoos_emRaiseException(this->common->em, PICO_EXC_BUF_OVERFLOW,NULL,NULL); } } } else { /* at least one entry found */ PICODBG_DEBUG(("at least one entry found in lexicon %i",foundIndex)); if (lexres.phonfound) { /* incl. ind-phone and possibly multi-ent. */ if (lexres.nrres > PICOKLEX_MAX_NRRES) { /* not possible with system lexicon, needs to be ensured for user lex too */ picoos_emRaiseWarning(this->common->em, PICO_WARN_FALLBACK,NULL, (picoos_char *)"using %d lexicon lookup results", PICOKLEX_MAX_NRRES); lexres.nrres = PICOKLEX_MAX_NRRES; } head->type = PICODATA_ITEM_WORDINDEX; if (lexres.nrres == 1) { head->info1 = lexres.posind[0]; } else { /* more than one result, POSgroup info needs to be determined for later POS disambiguation */ for (i = 0; i < lexres.nrres; i++) { posbuf[i] = lexres.posind[i * PICOKLEX_POSIND_SIZE]; } head->info1 = picoktab_getPosGroup(wa->tabpos, posbuf, lexres.nrres); } head->info2 = foundIndex; head->len = lexres.posindlen; if ((status = picodata_put_itemparts(head, lexres.posind, lexres.posindlen, wa->outBuf, wa->outBufSize, &wa->outLen)) == PICO_OK) { wa->inLen = 0; } else { return picoos_emRaiseException(this->common->em, status, NULL, NULL); } } else { /* no phone, :G2P, one entry: WORDGRAPH(POS,NA)graph */ if (PICO_OK == picodata_copy_item(wa->inBuf, wa->inLen, wa->outBuf, wa->outBufSize, &wa->outLen)) { wa->inLen = 0; /* set lex pos in info1 */ if (PICO_OK != picodata_set_iteminfo1(wa->outBuf, wa->outLen, lexres.posind[0])) { return picoos_emRaiseException(this->common->em, PICO_EXC_BUF_OVERFLOW, NULL, NULL); } } } } return PICO_OK; } /* ***********************************************************************/ /* waStep function */ /* ***********************************************************************/ /* collect into internal buffer, process, and then feed to output buffer init state: COLLECT ext ext state transitions: in IN OUTout COLLECT | getOneItem ->-1 +1 0 0 | (ATOMIC) -> PROCESS (got item) COLLECT | getOneItem -> 0 0 0 0 | IDLE (got no item) PROCESS | procOneItem -> 0 -1 +1 0 | (ATOMIC) -> FEED (proc'ed item) PROCESS | procOneItem -> 0 -1 0 0 | BUSY -> COLLECT (item skipped) FEED | putOneItem -> 0 0 -1 +1 | BUSY -> COLLECT (put item) FEED | putOneItem -> 0 0 1 0 | OUT_FULL (put no item) */ static picodata_step_result_t waStep(register picodata_ProcessingUnit this, picoos_int16 mode, picoos_uint16 * numBytesOutput) { register wa_subobj_t *wa; pico_status_t rv = PICO_OK; if (NULL == this || NULL == this->subObj) { return PICODATA_PU_ERROR; } wa = (wa_subobj_t *) this->subObj; mode = mode; /* avoid warning "var not used in this function"*/ *numBytesOutput = 0; while (1) { /* exit via return */ PICODBG_DEBUG(("doing state %i, inLen: %d, outLen: %d", wa->procState, wa->inLen, wa->outLen)); switch (wa->procState) { /* collect state: get item from charBuf and store in * internal inBuf */ case WA_STEPSTATE_COLLECT: if (wa->inLen == 0) { /* is input buffer empty? */ picoos_uint16 blen; /* try to get one item */ rv = picodata_cbGetItem(this->cbIn, wa->inBuf, wa->inBufSize, &blen); PICODBG_DEBUG(("after getting item, status: %d", rv)); if (PICO_OK == rv) { /* we now have one item */ wa->inLen = blen; wa->procState = WA_STEPSTATE_PROCESS; /* uncomment next line to split into two steps */ /* return PICODATA_PU_ATOMIC; */ } else if (PICO_EOF == rv) { /* there was no item in the char buffer */ return PICODATA_PU_IDLE; } else if ((PICO_EXC_BUF_UNDERFLOW == rv) || (PICO_EXC_BUF_OVERFLOW == rv)) { PICODBG_ERROR(("problem getting item")); picoos_emRaiseException(this->common->em, rv, NULL, NULL); return PICODATA_PU_ERROR; } else { PICODBG_ERROR(("problem getting item, unhandled")); picoos_emRaiseException(this->common->em, rv, NULL, NULL); return PICODATA_PU_ERROR; } } else { /* there already is an item in the input buffer */ PICODBG_WARN(("item already in input buffer")); picoos_emRaiseWarning(this->common->em, PICO_WARN_PU_IRREG_ITEM, NULL, NULL); wa->procState = WA_STEPSTATE_PROCESS; /* uncomment next to split into two steps */ /* return PICODATA_PU_ATOMIC; */ } break; /* process state: process item in internal inBuf and put * result in internal outBuf */ case WA_STEPSTATE_PROCESS: /* ensure there is an item in inBuf and it is valid */ if ((wa->inLen > 0) && picodata_is_valid_item(wa->inBuf, wa->inLen)) { picodata_itemhead_t ihead; picoos_uint8 *icontent; pico_status_t rvP = PICO_OK; rv = picodata_get_iteminfo(wa->inBuf, wa->inLen, &ihead, &icontent); if (PICO_OK == rv) { switch (ihead.type) { case PICODATA_ITEM_WORDGRAPH: if (0 < ihead.len) { rvP = waProcessWordgraph(this, wa, &ihead, icontent); } else { /* else ignore empty WORDGRAPH */ wa->inLen = 0; wa->procState = WA_STEPSTATE_COLLECT; return PICODATA_PU_BUSY; } break; case PICODATA_ITEM_OTHER: /* skip item */ rvP = PICO_WARN_PU_DISCARD_BUF; break; default: /* copy item unmodified */ rvP = picodata_copy_item(wa->inBuf, wa->inLen, wa->outBuf, wa->outBufSize, &wa->outLen); break; } if (PICO_OK == rvP) { wa->inLen = 0; wa->procState = WA_STEPSTATE_FEED; /* uncomment next to split into two steps */ /* return PICODATA_PU_ATOMIC; */ } else if (PICO_WARN_PU_DISCARD_BUF == rvP) { /* discard input buffer and get a new item */ PICODBG_INFO(("skipping OTHER item")); /* picoos_emRaiseWarning(this->common->em, PICO_WARN_PU_DISCARD_BUF, NULL, NULL); */ wa->inLen = 0; wa->procState = WA_STEPSTATE_COLLECT; return PICODATA_PU_BUSY; } else { /* PICO_EXC_BUF_OVERFLOW <- overflow in outbuf PICO_ERR_OTHER <- no valid item in inbuf or return from processWordgraph */ PICODBG_ERROR(("problem processing item", rvP)); picoos_emRaiseException(this->common->em, rvP, NULL, NULL); return PICODATA_PU_ERROR; } } else { /* could not get iteminfo */ /* PICO_EXC_BUF_OVERFLOW <- overflow in outbuf PICO_ERR_OTHER <- no valid item in inbuf */ PICODBG_ERROR(("problem getting item info, " "discard buffer content")); wa->inLen = 0; wa->procState = WA_STEPSTATE_COLLECT; picoos_emRaiseException(this->common->em, rv, NULL, NULL); return PICODATA_PU_ERROR; } } else if (wa->inLen == 0) { /* no item in inBuf */ PICODBG_INFO(("no item in inBuf")); /* wa->inLen = 0;*/ wa->procState = WA_STEPSTATE_COLLECT; return PICODATA_PU_BUSY; } else { /* no valid item in inBuf */ /* bad state/item, discard buffer content */ PICODBG_WARN(("no valid item, discard buffer content")); picoos_emRaiseWarning(this->common->em, PICO_WARN_PU_IRREG_ITEM, NULL, NULL); picoos_emRaiseWarning(this->common->em, PICO_WARN_PU_DISCARD_BUF, NULL, NULL); wa->inLen = 0; wa->procState = WA_STEPSTATE_COLLECT; return PICODATA_PU_BUSY; } break; /* feed state: copy item in internal outBuf to output charBuf */ case WA_STEPSTATE_FEED: /* check that item fits in cb should not be needed */ rv = picodata_cbPutItem(this->cbOut, wa->outBuf, wa->outLen, numBytesOutput); PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG], (picoos_uint8 *)"wana: ", wa->outBuf, wa->outLen); PICODBG_DEBUG(("put item, status: %d", rv)); if (PICO_OK == rv) { wa->outLen = 0; wa->procState = WA_STEPSTATE_COLLECT; return PICODATA_PU_BUSY; } else if (PICO_EXC_BUF_OVERFLOW == rv) { PICODBG_INFO(("feeding, overflow, PICODATA_PU_OUT_FULL")); return PICODATA_PU_OUT_FULL; } else if ((PICO_EXC_BUF_UNDERFLOW == rv) || (PICO_ERR_OTHER == rv)) { PICODBG_WARN(("feeding problem, discarding item")); wa->outLen = 0; wa->procState = WA_STEPSTATE_COLLECT; picoos_emRaiseWarning(this->common->em, rv, NULL,NULL); return PICODATA_PU_BUSY; } break; default: break; } /* switch */ } /* while */ /* should be never reached */ PICODBG_ERROR(("reached end of function")); picoos_emRaiseException(this->common->em, PICO_ERR_OTHER, NULL, NULL); return PICODATA_PU_ERROR; } #ifdef __cplusplus } #endif /* end */