/***************************************************************************
cvoicecontrol.c - a simple speech recognizer
-------------------
begin : Sat Feb 12 2000
copyright : (C) 2000 by Daniel Kiecza
email : daniel@kiecza.de
***************************************************************************/
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
#define MAIN_C
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<math.h>
#include<float.h>
#include<pthread.h>
#include<unistd.h>
#include<fcntl.h>
#include<sys/ioctl.h>
#include<sys/time.h>
#include<sys/types.h>
#include<sys/soundcard.h>
#ifdef _AIX
#include<sys/select.h>
#endif
#include "cvoicecontrol.h"
#include "model.h"
#include "configuration.h"
#include "queue.h"
#include "semaphore.h"
#include "score.h"
#include "bb_queue.h"
#include "audio.h"
#include "mixer.h"
#include "preprocess.h"
Model *model; /***** speaker model */
Queue queue1; /***** thread-safe queue used to hand data from 'recording' to 'preprocessing' */
Queue queue2; /***** thread-safe queue used to hand data from 'preprocessing' to 'recognition' */
ScoreQueue score_queue;
/*****
* forward declaration:
* each of these three functions is run in a separate thread
* these threads are connected via thread safe data queues.
* Together the three threads form the whole recognizer
*****/
void record(void);
void preprocess(void);
void recognize(void);
/*****
* status of the audio recording thread (plus mutex variables)
*****/
enum AudioStatus {A_invalid, A_off, A_prefetching,
A_recording, A_aborting,
A_exiting} A_status = A_invalid;
pthread_mutex_t mutex_A_status; /***** ensure thread-safe access to A_status */
Semaphore auto_recording; /***** used to switch audio thread to auto recording mode */
/*****
* indicator whether preprocessing of current utterance is finished
* (used to switch to faster branch&bound decoding at the right time)
* plus mutex for thread-safe access
*****/
int P_done;
pthread_mutex_t mutex_P_done;
/***** terminates server program, if set to 0 */
int running = 1;
/*****
* if set, program exits after one successful recognition
* pass. Can be set via command line option (--once)
******/
int run_once = 0;
/***** ID of the recognition result. For use in shell scripts */
int result_id = 0;
void beep()
{
/* fprintf(stderr, "BEEP\n"); */
}
/********************************************************************************
* thread safe changing of the status of the audio recording thread
********************************************************************************/
enum AudioStatus setAudioStatus(enum AudioStatus s)
{
int retval = A_invalid; /***** return value */
pthread_mutex_lock( &mutex_A_status ); /***** get exclusive access to audio status variable */
/*****
* reaction depends on
* - what state the caller requests (via 's') AND
* - what state the audio thread is currently in!
*****/
switch (s)
{
case A_off: /* ok ? */
A_status = A_off;
retval = A_off;
break;
case A_prefetching: /***** switch to prefetching only if the audio thread is in a passive state currently*/
if (A_status == A_off || A_status == A_invalid)
{
A_status = A_prefetching;
retval = A_prefetching;
semaphore_up( &auto_recording);
}
else /***** ... otherwise just return 'invalid' */
retval = A_invalid;
break;
case A_recording: /***** switch to recording only if the audio thread was in prefetching mode before */
if (A_status == A_prefetching)
{
A_status = A_recording;
retval = A_recording;
}
else /***** ... otherwise just return 'invalid' */
retval = A_invalid;
break;
case A_aborting: /***** ignore request for aborting if thread is passive already */
if (A_status == A_off || A_status == A_invalid)
{
retval = A_off;
}
else /***** otherwise switch to aborting mode */
{
A_status = A_aborting;
retval = A_aborting;
}
break;
case A_exiting:
retval = A_exiting;
A_status = A_exiting;
semaphore_up( &auto_recording);
break;
case A_invalid: /***** invalid: change nothing! */
retval = A_invalid;
break;
}
pthread_mutex_unlock( &mutex_A_status ); /***** give up exclusive access to audio status */
return retval;
}
/********************************************************************************
* thread safe reading of the audio status
********************************************************************************/
enum AudioStatus getAudioStatus()
{
enum AudioStatus retval; /***** return value */
pthread_mutex_lock( &mutex_A_status ); /***** get exclusive access to audio status variable */
retval = A_status; /***** put status of audio thread in return variable */
pthread_mutex_unlock( &mutex_A_status ); /***** give up exclusive access to audio status */
return retval;
}
/********************************************************************************
* thread safe changing of the variable P_done
********************************************************************************/
void setPDone(int val)
{
pthread_mutex_lock( &mutex_P_done ); /***** get exclusive access to audio status variable */
/*****
* ignore illegal values for P_done,
* otherwise set P_done to requested value
*****/
if (val != 0 && val != 1)
fprintf(stderr, "Illegal value for P_done: %d\n", val);
else
P_done = val;
pthread_mutex_unlock( &mutex_P_done ); /***** give up exclusive access to audio status */
}
/********************************************************************************
* thread safe reading of the variable P_done
********************************************************************************/
int getPDone()
{
int retval; /***** return value */
pthread_mutex_lock( &mutex_P_done ); /***** get exclusive access to audio status variable */
retval = P_done; /***** put value of P_done in return variable */
pthread_mutex_unlock( &mutex_P_done ); /***** give up exclusive access to audio status */
return retval;
}
/********************************************************************************
* block audio recording thread until 'auto recording' is requested
********************************************************************************/
void waitForAutoRecordingRequest()
{
/*****
* if semaphore is > 0, decrease it and continue,
* otherwise the calling thread blocks here until semaphore > 0
*****/
semaphore_down( &auto_recording );
}
/********************************************************************************
* main
********************************************************************************/
int main(int argc, char *argv[])
{
/***** thread variables */
pthread_t record_t;
pthread_t preprocess_t;
pthread_t recognize_t;
char *model_file;
if ((argc != 2 && argc != 3) ||
(argc == 3 && strcmp(argv[1], "--once") != 0))
{
printf("Usage: %s [--once] <speakermodel.cvc>\n", argv[0]);
printf("If --runonce is specified, the program will exit after the first successful\n");
printf("recognition run. In this case the exit code will be the id of the recognized\n");
printf("speaker model item. This feature is provided to allow for speech prompts in\n");
printf("shell scripts.\n");
exit(-1);
}
else if (argc == 3 && strcmp(argv[1], "--once") == 0)
{
run_once = 1;
model_file = argv[2];
}
else if (argc == 2)
{
model_file = argv[1];
}
/*****
* load configuration from ~/.cvoicecontrol/config :
* see configuration.h for more information on what
* configuration data is loaded ...
*****/
if (loadConfiguration() == 0)
{
fprintf(stderr, "Couldn't load configuration!");
fprintf(stderr, "Please run microphone_config before using this application!\n");
exit(-1);
}
/***** setup speaker model */
model = (Model *) malloc(sizeof(Model));
initModel(model);
if (loadModel(model, model_file, 0) == 0)
{
fprintf(stderr, "Failed to load speaker model: %s !\n", model_file);
exit(-1);
}
/*****
* initialize the two thread-safe queues that are
* used to "connect" the three main threads
*****/
initQueue(&queue1, "char");
initQueue(&queue2, "float");
/***** initialize mutex and semaphore variables */
pthread_mutex_init(&mutex_A_status, NULL);
semaphore_init( &auto_recording);
semaphore_down( &auto_recording); /***** init semaphore to '0' */
pthread_mutex_init(&mutex_P_done, NULL);
setPDone(0);
/***** create the three main threads */
pthread_create( &record_t, NULL, (void*)&record, NULL);
pthread_create( &preprocess_t, NULL, (void*)&preprocess, NULL);
pthread_create( &recognize_t, NULL, (void*)&recognize, NULL);
while (getAudioStatus() != A_off)
;
setAudioStatus(A_prefetching); /* for now: switch to auto recording after start up */
/***** join the threads here */
pthread_join(record_t, NULL);
pthread_join(preprocess_t, NULL);
pthread_join(recognize_t, NULL);
/***** cleanup stuff *****/
resetModel(model);
free(model);
if (run_once) /***** set exit code to the ID of the recognition result */
exit(result_id);
else /***** leave program with exit code 'ok' */
exit(0);
}
/********************************************************************************
* calculate euklidian distance of two feature vectors
********************************************************************************/
inline float euklid_distance(float *a, float *b)
{
float result = 0; /***** resulting distance */
int i;
/*****
* sum up the squares of the differences of the vector's components
* the result is the square root of this value
*****/
for (i = 0; i < FEAT_VEC_SIZE; i++)
result += (a[i]-b[i])*(a[i]-b[i]);
result = sqrt(result);
return result;
}
/********************************************************************************
* recognizer thread
********************************************************************************/
void recognize(void)
{
/*****
* frame is actual feature vector,
* last_frame is feature vector at (pos-1)
*****/
float frame[FEAT_VEC_SIZE];
float last_frame[FEAT_VEC_SIZE];
/*****
* pos is the number of feature vectors
* that have been processed
*****/
int pos = 0;
/***** initialize the status of the recognition thread to 'Q_invalid' */
enum QStatus R_status = Q_invalid;
/***** loop variables */
int i, j, k;
/*****
* set to 1 if we are waiting for an 'abort'
* to arrive through the queues
*****/
int abort_requested = 0;
/*****
* counter variables for
* a) list of references,
* b) list of sample utterances
*****/
int ref, samp;
/*****
* temporary pointer to a sample utterance
* used during recognition
*****/
ModelItemSample *sample;
float act_dist; /***** (euklid) distance at current DTW position */
float column_min_dist; /***** minimum distance in current DTW column */
float tmp_dist; /***** temp. variable */
/***** branch&bound related stuff */
/*****
* indicates whether recognition is done with B&B method
*****/
int do_branchNbound = 0;
/*****
* position in test utterance where recognizer switched to B&B
*****/
int bNb_start_pos = 0;
/*****
* buffer that holds the preprocessed data of a test utterance
* when using B&B method, (plus length of utterance)
******/
float **test_utterance = NULL;
int test_utt_length = 0;
/*****
* indicate whether recognition run has been finished
* successfully
*****/
int recognition_done = 0;
/*****
* queue that holds reference utterances sorted by their
* recognition score so far.
* used in B&B method
*****/
BBQueue bb_queue;
initBBQueue(&bb_queue);
/*****
* some recognizer-specific variables:
* their meaning is described in server.h
*****/
/* adjust_win_width and score_threshold should be put in a config file type of place */
adjust_window_width = 90;
sloppy_corner = 4;
/* score_threshold = 18; */
float_max = FLT_MAX*0.0001;
/*****
* initialize score_queue:
* this is a queue that holds a sorted (by score) list of recognition hypothesis
* from which the final recognition result is derived
*****/
initScoreQueue(&score_queue);
/***** main loop of recognition thread */
while (running)
{
/*****
* report results and reset score queue if recognition
* finished successfully.
*****/
if (recognition_done)
{
int id = getResultID(&score_queue);
recognition_done = 0;
if (id >= 0) /***** something recognized! */
{
if (run_once || strcmp((getModelItem(model, id))->command,
"cvoicecontrol_off") == 0)
{
result_id = id;
/***** exit!! */
/*fprintf(stderr, "Exit ID: %d\n", id);*/
setAudioStatus(A_exiting);
break;
}
else
{
/***** execute command */
/*fprintf(stderr, "%s\n", (getModelItem(model, id))->label);*/
system((getModelItem(model, id))->command);
}
}
/***** free the space occupied by score_queue */
resetScoreQueue(&score_queue);
beep();
setAudioStatus(A_prefetching);
}
/*****
* time-synchronous calculation of the DTW matrices of all reference utterances
* at column 'pos'
*****/
if (!do_branchNbound)
{
float *tmp_data;
/*****
* If an abort was requested by the last frame that was extracted
* from the queue from the preprocessing thread,
* we empty the queue until an 'end'- or an 'abort'-type frame
* comes through the queue, then the recognizer is reset to
* wait for the next utterance ...
*****/
if (abort_requested)
{
/*****
* number of remaining frames in queue after
* the preprocessing of the current utterance has finished
*****/
int remaining_frames;
enum QStatus tmp_status; /***** temporary variable */
int i; /***** counter variable */
/*****
* wait until preprocessing has finished,
* then reset P_done to 0 ...
*****/
while(!getPDone())
;
setPDone(0);
/***** ... and empty the queue */
remaining_frames = numberOfElements(&queue2);
for (i = 0; i < remaining_frames; i++)
dequeue(&queue2, &tmp_status);
abort_requested = 0; /***** reset 'abort_requested' to 0 */
/* if negative recognition answer is desired, output one here!! */
/* implement any further abort functionality !!!!!? */
while (getAudioStatus() != A_off)
;
beep();
setAudioStatus(A_prefetching); /* switch audio thread back to 'auto recording' */
continue;
}
/*****
* get next feature vector from the head of queue2
* last frame is moved to last_frame
* (the dequeue function blocks the current thread while the queue is empty)
*****/
tmp_data = dequeue(&queue2, &R_status);
memcpy(last_frame, frame, sizeof(float)*FEAT_VEC_SIZE);
memcpy(frame, tmp_data, sizeof(float)*FEAT_VEC_SIZE);
free(tmp_data);
/*****
* check whether switching to B&B makes sense
* if yes, prepare for and switch to B&B method
*
* this check is only performed if the first couple of DTW columns,
* that need 'special treatment', have already been calculated AND
* if preprocessing has already been finished
* the latter condition is required as we need to know the actual length
* of the currently incoming utterance to be able to use the B&B method!
*****/
if (getPDone() && pos >= sloppy_corner+1)
{
/*****
* remaining frames to evaluate with B&B = frames in the queue + last_frame + (current) frame !
*****/
int remaining_frames = numberOfElements(&queue2) + 2;
int i; /***** counter variable */
setPDone(0); /***** reset P_done to 0 ... */
/*****
* if at least 30 frames are left to evaluate and abort has not been requested,
* switch to B&B method
*****/
if (remaining_frames >= 30 && !abort_requested)
{
test_utt_length = remaining_frames + pos; /***** total length of test utterance */
/***** switch to branch and bound method at DTW column 'pos' */
do_branchNbound = 1;
bNb_start_pos = pos;
/***** retrieve all remaining frames from queue and put them in an array */
test_utterance = (float **)malloc(sizeof(float **) * remaining_frames);
test_utterance[0] = last_frame;
test_utterance[1] = frame;
for (i = 2; i < remaining_frames; i++)
test_utterance[i] = dequeue(&queue2, &R_status);
/*****
* setup B&B queue:
* put all sample utterances in the queue sorted by increasing score,
* deactivate all utterances that don't meet the required constraints
* (this can be determined now, as the length of the test utterance is known at this point)
*****/
for (i = 0; i < model->total_number_of_sample_utterances; i++)
{
int I = model->direct[i]->length; /***** length of sample utterance */
float tmp_dist = float_max, tmp_dist2; /***** temporary variables */
int j; /****** counter variable */
if (!(model->direct[i]->isActive) || /***** item still active */
((test_utt_length-1)*2 + (sloppy_corner-1) < I-1) || /***** min slope is 0.5 */
((test_utt_length-sloppy_corner)/2 > I-1) || /***** max slope is 2 */
(I + adjust_window_width < test_utt_length) || /***** adjustment window right side */
(I - adjust_window_width > test_utt_length)) /***** adjustment window left side */
continue;
/***** get minimum distance in actual column */
for (j = 0; j < I; j++)
{
tmp_dist2 = model->direct[i]->matrix[pos%3][j]/((pos+1)+(j+1));
if (tmp_dist2 < tmp_dist)
tmp_dist = tmp_dist2;
}
/***** insert item into B&B queue */
insertIntoBBQueue(&bb_queue, pos+1, tmp_dist, i);
}
continue; /***** enter next 'while' step in B&B mode */
}
}
/***** react to status of current queue item */
switch (R_status)
{
case Q_invalid: /***** this case should not occur! */
break;
case Q_start: /***** start of a new utterance */
pos = 0; /***** start at position 'pos' */
resetScoreQueue(&score_queue); /***** make sure, the score queue is empty */
activateAllSamples(model); /***** activate all model items! */
break;
case Q_data: /***** data or end-type frame*/
case Q_end:
pos++; /***** advance to next position in DTW matrix */
/*abort_requested = 0;*/
break;
case Q_abort: /***** start next while loop, abort is handled there */
continue;
break;
case Q_exit: /***** exit program */
continue;
break;
}
/***** loop all sample utterances */
for (samp = 0; samp < model->total_number_of_sample_utterances; samp++)
{
/***** - skip if sample is inactive */
if (!model->direct[samp]->isActive)
continue;
/*****
* ref and sample are pointers to the current reference item and
* sample utterance respectively, just to increase code clarity.
*****/
ref = model->direct_map2ref[samp];
sample = model->direct[samp];
/*****
* deactivate sample if it is too short to be aligned with the current
* adjust_window_width, if no more samples are left, request abort!
*****/
if (pos - adjust_window_width > sample->length)
{
model->direct[samp]->isActive = 0; /***** deactivate sample */
/***** all samples deactivated!! request abort! */
model->number_of_active_sample_utterances--;
if (model->number_of_active_sample_utterances <= 0)
{
abort_requested = 1;
setAudioStatus(A_aborting); /* what would happen if (audioStatus == A_off) at this point? */
break; /***** break for loop, -> enter next while loop */
}
else
continue; /***** sample deactivated, continue with next sample */
}
/*****
* after all items in a column are calculated, this variable contains the minimum
* distance value in the current column
*****/
column_min_dist = float_max;
/***** fill actual DTW matrix column */
if (pos == 0) /***** at pos == 0, we initialize the first matrix column */
{
/***** initialize current DTW matrix column, i.e. set all values to 'infinity' */
for (k = 0; k < sample->length ; k++)
sample->matrix[0][k] = float_max;
/***** ccalculate the first <sloppy_corner> items in the current column */
sample->matrix[0][0] = 2*euklid_distance(sample->data[0], frame);
column_min_dist = sample->matrix[0][0]/((0+1)+(0+1));
for (i = 1; i < sloppy_corner ; i++)
{
sample->matrix[0][i] = sample->matrix[0][i-1] + euklid_distance(sample->data[i], frame);
tmp_dist = sample->matrix[0][i]/((0+1)+(i+1));
if (tmp_dist < column_min_dist)
column_min_dist = tmp_dist;
}
}
else if (pos == 1)
{
/*****
* at pos == 1, we use a special (shorter) warping function
* as the history (of one matrix column) does not allow
* for the application of the full warping function yet
*****/
/***** initialize current DTW matrix column, i.e. set all values to 'infinity' */
for (k = 0; k < sample->length ; k++)
sample->matrix[1][k] = float_max;
/***** calculate the first <sloppy_corner+1> elements */
sample->matrix[1][0] = sample->matrix[0][0] + euklid_distance(sample->data[0], frame);
column_min_dist = sample->matrix[1][0]/((1+1)+(0+1));
sample->matrix[1][1] = MIN3(sample->matrix[0][1] + act_dist,
sample->matrix[1][0] + act_dist,
sample->matrix[0][0] + 2*act_dist);
tmp_dist = sample->matrix[1][1]/((1+1)+(1+1));
if (tmp_dist < column_min_dist)
column_min_dist = tmp_dist;
for (i = 2; i < sloppy_corner+1 ; i++)
{
act_dist = euklid_distance(sample->data[i], frame);
sample->matrix[1][i] = MIN3(sample->matrix[0][i] + act_dist,
sample->matrix[0][i-1] + 2*act_dist,
sample->matrix[0][i-2] +
2*euklid_distance(sample->data[i-1], frame) + act_dist);
tmp_dist = sample->matrix[1][i]/((1+1)+(i+1));
if (tmp_dist < column_min_dist)
column_min_dist = tmp_dist;
}
}
else if (pos > 1)
{
/*****
* beyond pos == 1, the warping function lies inside the matrix
* and can be calculated completely.
*****/
/***** initialize current DTW matrix column, i.e. set all values to 'infinity' */
for (k = 0; k < sample->length ; k++)
sample->matrix[pos%3][k] = float_max;
/***** take care of sloppy start */
if (pos < sloppy_corner) /***** element in first row of DTW matrix */
{
sample->matrix[pos%3][0] = sample->matrix[(pos-1)%3][0] + euklid_distance(sample->data[0], frame);
column_min_dist = sample->matrix[pos%3][0]/((pos+1)+(0+1));
}
if (pos < sloppy_corner+1) /***** element in second row of DTW matrix */
{
act_dist = euklid_distance(sample->data[1], frame);
/***** use a simpler, smaller warping function that fits into the DTW matrix */
sample->matrix[pos%3][1] = MIN3(sample->matrix[pos%3][0] + act_dist,
sample->matrix[(pos-1)%3][0] + 2*act_dist,
sample->matrix[(pos-2)%3][0] +
2* euklid_distance(sample->data[1], last_frame) + act_dist);
tmp_dist = sample->matrix[pos%3][1]/((pos+1)+(1+1));
if (tmp_dist < column_min_dist)
column_min_dist = tmp_dist;
}
/*****
* loop rows in current DTW column within
* - range of adjust_window,
* - range of warping function
* - sample length
*****/
for (j = MAX3(2, pos-adjust_window_width, (pos-2)/2); j < MIN3(sloppy_corner+1+(pos-1)*2,
sample->length,
pos+adjust_window_width); j++)
{
act_dist = euklid_distance(sample->data[j], frame);
/*****
* apply warping function to calculate current DTW matrix element
* (see server.h for description of warping function)
*****/
if (sample->matrix[(pos-1)%3][j-1] < float_max ||
sample->matrix[(pos-1)%3][j-2] < float_max ||
sample->matrix[(pos-2)%3][j-1] < float_max)
{
sample->matrix[pos%3][j] = MIN3(sample->matrix[(pos-1)%3][j-1] + 2*act_dist,
sample->matrix[(pos-1)%3][j-2] +
2*euklid_distance(sample->data[j-1], frame) + act_dist,
sample->matrix[(pos-2)%3][j-1] +
2*euklid_distance(sample->data[j], last_frame) + act_dist);
tmp_dist = sample->matrix[pos%3][j]/((pos+1)+(j+1));
if (tmp_dist < column_min_dist)
column_min_dist = tmp_dist;
}
else
sample->matrix[pos%3][j] = float_max;
}
/*****
* if the minimum distance in the current row exceeds the
* overall threshold, deactivate this sample utterance!
*****/
if (column_min_dist > score_threshold)
{
model->direct[samp]->isActive = 0; /***** deactivate sample */
/***** all samples deactivated!! request abort! */
model->number_of_active_sample_utterances--;
if (model->number_of_active_sample_utterances <= 0)
{
abort_requested = 1;
setAudioStatus(A_aborting); /* what would happen if (audioStatus == A_off) at this point? */
break; /***** break for loop, -> enter next while loop */
}
else
continue;
}
if (R_status == Q_end)
{
/***** calculate score from upper right corner of DTW matrix */
int s;
float tmp_score = sample->matrix[pos%3][sample->length-1]/(pos + sample->length);
for (s = 1; s < sloppy_corner; s++)
{
tmp_dist = sample->matrix[pos%3][sample->length-1-s]/(pos + sample->length - s);
if (tmp_dist < tmp_score)
tmp_score = tmp_dist;
}
/*****
* if the final score is below the score_threshold
* enqueue the pair (utterance/score) into the ScoreQueue
* (sorted by increasing recognition score)
*****/
if (tmp_score <= score_threshold)
insertInScoreQueue(&score_queue, tmp_score, ref);
}
/***** if (R_status == Q_end) */
}
/***** else if (pos > 1) */
}
/***** for (samp ...) */
/*****
* if all sample utterances have been processed at final position
* retrieve recognition result from ScoreQueue
*****/
if (R_status == Q_end && !abort_requested)
{
/***** ready for the next recording session */
while (getAudioStatus() != A_off)
; //fprintf(stderr, ".");
recognition_done = 1;
}
}
else /***** B&B mode! */
{
BBQueueItem *item = 0;
int nbest = 6; /***** find the 'nbest' best hypotheses using B&B the method */
int nbest_found = 0;
/***** do B&B until 'nbest' hypotheses have been found or B&B queue is empty */
while (nbest_found < nbest && bb_queue.length > 0)
{
/***** remove first element (having minimum score) from list */
item = headBBQueue(&bb_queue);
/*****
* if (at right corner) -> best item found
* (increase N-best counter, if ==0, stop)
*****/
if (item->pos == test_utt_length-1)
{
/***** insert results of current hypothesis into the score queue */
insertInScoreQueue(&score_queue, item->score, model->direct_map2ref[item->sample_index]);
nbest_found++;
free(item);
}
else
{
/*****
* else expand the DTW matrix calculation of
* the current sample utterance by one column ...
*****/
int pos = item->pos;
ModelItemSample *sample = model->direct[item->sample_index];
float column_min_dist = float_max;
float tmp_dist;
int bottom, top;
for (j = 0; j < sample->length ; j++)
sample->matrix[pos%3][j] = float_max;
if (pos < test_utt_length-1) /***** on right edge, just evaluate sloppy corner items */
{
bottom = MAX3(2, pos-adjust_window_width, (pos-2)/2);
top = MIN3(sloppy_corner+1+(pos-1)*2, sample->length, pos+adjust_window_width);
}
else /***** otherwise follow the general constraints within the DTW matrix */
{
bottom = MAX3(sample->length - sloppy_corner, pos-adjust_window_width, (pos-2)/2);
top = sample->length;
}
/***** calculate relevant entries in the DTW matrix */
for (j = bottom; j < top; j++)
{
act_dist = euklid_distance(sample->data[j], test_utterance[pos-bNb_start_pos]);
/*****
* apply warping function to calculate current DTW matrix element
* (see server.h for a detailed description of the warping function)
*****/
if (sample->matrix[(pos-1)%3][j-1] < float_max ||
sample->matrix[(pos-1)%3][j-2] < float_max ||
sample->matrix[(pos-2)%3][j-1] < float_max)
{
sample->matrix[pos%3][j] = MIN3(sample->matrix[(pos-1)%3][j-1] + 2*act_dist,
sample->matrix[(pos-1)%3][j-2] +
2*euklid_distance(sample->data[j-1],
test_utterance[pos-bNb_start_pos]) + act_dist,
sample->matrix[(pos-2)%3][j-1] +
2*euklid_distance(sample->data[j],
test_utterance[pos-bNb_start_pos-1]) + act_dist);
tmp_dist = sample->matrix[pos%3][j]/((pos+1)+(j+1));
if (tmp_dist < column_min_dist)
column_min_dist = tmp_dist;
}
else
sample->matrix[pos%3][j] = float_max;
}
/***** reinsert the item into the BBQueue if the score is still below the threshold */
if (column_min_dist <= score_threshold)
{
item->pos++;
item->score = column_min_dist;
insertItemIntoBBQueue(&bb_queue, item);
}
}
}
/***** ready for next recording session */
while (getAudioStatus() != A_off)
; //fprintf(stderr, ",");
/***** B&B search done, report results */
resetBBQueue(&bb_queue); /***** reset B&B related variables */
for (i = 2; i < test_utt_length - bNb_start_pos; i++)
free(test_utterance[i]);
free(test_utterance);
test_utterance = NULL;
do_branchNbound = 0;
recognition_done = 1;
}
}
}
/********************************************************************************
* preprocessing thread
********************************************************************************/
void preprocess(void)
{
/***** data container used for fft calculation */
float frame[FFT_SIZE];
/***** holds current buffer of waveform data plus remainders of last buffer */
unsigned char buffer[FRAG_SIZE+OFFSET];
/***** amount of valid chars in buffer, 0 .. frag_size+offset */
int buffer_counter = 0;
/***** number of whole frames that can be extracted from current waveform buffer */
int frames_N;
/***** counter variables */
int frameI, i;
/***** remainder */
int remainder;
/***** status of current queue item */
enum QStatus P_status;
/***** init melscale buffer ... */
float feat_vector[FEAT_VEC_SIZE];
/***** initialize preprocessing */
initPreprocess();
/***** main loop of the preprocessing thread */
while (running)
{
/*****
* get head data and status from queue1
* (the dequeue function blocks the preprocessing thread if the queue is empty!)
*****/
char *tmp_data = (char *)dequeue(&queue1, &P_status);
memcpy(buffer+buffer_counter, tmp_data, FRAG_SIZE);
free(tmp_data);
/***** react to status of current queue item */
switch (P_status)
{
case Q_invalid: /***** this case should not occur! */
break;
case Q_start: /***** start of a new utterance */
buffer_counter = 0; /***** set buffer_counter to the beginning of the buffer */
while (getPDone() != 0)
setPDone(0); /***** make sure P_done is set to 0 */
break;
case Q_data: /***** nothing special to do at this point, just process data below*/
case Q_end:
break;
case Q_abort: /***** don't process this frame */
/***** insert an 'abort' marked frame into the queue to signal 'aborting'! */
enqueue(&queue2, feat_vector, FEAT_VEC_SIZE, Q_abort);
setPDone(1);
continue; /***** skip the current step of the while loop */
break;
case Q_exit:
enqueue(&queue2, feat_vector, FEAT_VEC_SIZE, Q_exit);
continue;
break;
}
/***** make the buffer_counter point behind the currently received data */
buffer_counter += FRAG_SIZE;
/*****
* number of frames that can be extracted from the current amount
* of available audio data
*****/
frames_N = (buffer_counter-FFT_SIZE_CHAR)/OFFSET + 1;
/***** extract these frames: */
for (frameI = 0; frameI < frames_N; frameI++)
{
/***** prepare a hamming windowed frame from the audio data ... */
for (i = 0; i < FFT_SIZE; i++)
frame[i] = ((float)((signed short)(buffer[frameI*OFFSET+2*i]|(buffer[frameI*OFFSET+2*i+1]<<8)))) *
hamming_window[i];
preprocessFrame(frame, feat_vector); /* ... and have it preprocessed */
/***** put the resulting feature vector in queue2 using the proper status flag*/
if (P_status == Q_start)
{
enqueue(&queue2, feat_vector, FEAT_VEC_SIZE, Q_start);
//fprintf(stderr, "Start preprocessing!\n");
P_status = Q_data;
}
else if (P_status == Q_end && frameI == frames_N - 1)
{
enqueue(&queue2, feat_vector, FEAT_VEC_SIZE, Q_end);
setPDone(1);
//fprintf(stderr, "Done preprocessing!\n");
}
else
{
enqueue(&queue2, feat_vector, FEAT_VEC_SIZE, Q_data);
}
}
/***** there may be remaining data that did not fit in the last frame */
remainder = buffer_counter - frames_N*OFFSET;
/*****
* move this remaining data to the front of the buffer and adjust the buffer_counter
*
* use memcpy, if the two memory regions do not overlap, otherwise use for
*****/
if (remainder < frames_N*OFFSET)
memcpy(buffer, buffer+frames_N*OFFSET, remainder);
else
for (i = 0; i < remainder; i++)
buffer[i] = buffer[i+frames_N*OFFSET];
buffer_counter = remainder;
}
endPreprocess(); /***** free any allocated memory etc. */
}
/********************************************************************************
* audio recording thread
********************************************************************************/
void record(void)
{
/*****
* needed by select() function which is used to retrieve
* incoming data from the microphone
*****/
fd_set reads;
/*****
* number of incoming blocks that need to be
* considered '[not] containing speech' before recording is started
*****/
int count = 0;
/***** set prefetch buffer size to 5, and allocate the required memory */
int prefetch_N = 5;
int prefetch_pos = 0;
unsigned char prefetch_buf[FRAG_SIZE*prefetch_N];
void *prefetch = prefetch_buf;
/*****
* a buffer of size 'FRAG_SIZE' that contains (raw) audio data which
* was recorded from the sound card.
*****/
unsigned char buffer_raw[FRAG_SIZE];
int new_start = 1; /***** first if statement inside while loop initializes the recording */
struct audio_buf_info info;
signed short max = 0;
signed short value;
int i;
/***** needed for select() */
struct timeval time;
time.tv_sec = 1;
time.tv_usec= 0;
/***** main loop of the audio recording thread */
while (running)
{
if (getAudioStatus() == A_exiting)
{
running = 0;
enqueue(&queue1, buffer_raw, FRAG_SIZE, Q_exit); /***** enqueue an 'abort'-type frame into queue1 */
break;
}
if (getAudioStatus() == A_aborting)
{
/*****
* got message from recognition thread that the current recording
* wouldn't match any of the hypotheses, -> abort!
*****/
enqueue(&queue1, buffer_raw, FRAG_SIZE, Q_abort); /***** enqueue an 'abort'-type frame into queue1 */
/***** wait for audio signal to fall back to silence!! */
count = 0;
while (count < CONSECUTIVE_NONSPEECH_BLOCKS_THRESHOLD)
{
if (read(fd_audio, buffer_raw, FRAG_SIZE) != FRAG_SIZE)
{
fprintf(stderr, "audio device read error!\n");
exit(-1);
}
max = 0;
for (i = 0; i < FRAG_SIZE/2 - 1; i += 2)
{
value = abs((signed short)(buffer_raw[i]|(buffer_raw[i+1]<<8)));
if (value > max)
max = value;
}
//fprintf(stderr, "WAITING MAX: %d\n", max);
if (max <= stop_level)
count++;
else
count = 0;
}
new_start = 1;
}
if (new_start)
{
FD_CLR(fd_audio, &reads);
closeAudio();
count = 0; /***** reset count */
prefetch_pos = 0; /***** reset position in audio prefetch buffer */
setAudioStatus(A_off); /***** set status to A_off */
memset (prefetch_buf, 0x00, FRAG_SIZE*prefetch_N);
waitForAutoRecordingRequest(); /***** wait at a semaphore for 'auto recording' request */
/*****
* actually, opening the audio device here
* should be repeated a specified number of times
* in case it fails, (perhaps with a somewhat more
* professional delay than just using a for-loop ;-) )
* if opening fails a couple of times, turn off
* auto recording!!!!!
*****/
while (openAudio() == AUDIO_ERR) /***** connect to microphone */
{
int i;
fprintf(stderr, "AUDIO_ERR\n");
for (i=0; i<10000000;i++);
}
new_start = 0;
}
/***** use 'select()' to connect to the audio device */
/***** if there is incoming audio data ... */
if (FD_ISSET(fd_audio, &reads))
{
info.bytes = 0;
while (info.bytes < FRAG_SIZE)
{
if (ioctl(fd_audio, SNDCTL_DSP_GETISPACE, &info)==-1)
{
fprintf(stderr, "audio device read error!\n");
exit(-1);
}
}
/***** ... read the data from the device */
if (read(fd_audio, buffer_raw, FRAG_SIZE) != FRAG_SIZE)
{
fprintf(stderr, "audio device read error!\n");
exit(-1);
}
if (getAudioStatus() == A_prefetching)
{
/***** prefetch data into a circular buffer ... */
memcpy(prefetch+prefetch_pos*(FRAG_SIZE), buffer_raw, FRAG_SIZE);
prefetch_pos = (prefetch_pos+1)%prefetch_N;
/***** and check it for speech content */
max = 0;
for (i = 0; i < FRAG_SIZE/2 - 1; i += 2)
{
value = abs((signed short)(buffer_raw[i]|(buffer_raw[i+1]<<8)));
if (value > max)
max = value;
}
//fprintf(stderr, "MAX: %d\n", max);
if (max >= rec_level)
count++;
else
count = 0;
if (count >= CONSECUTIVE_SPEECH_BLOCKS_THRESHOLD) /***** if speech detected ... */
{
count = 0;
//fprintf(stderr, "Recording\n");
/***** ... start recording, ... */
setAudioStatus(A_recording);
/***** ... extract the data from the prefetch buffer and insert it into queue1 */
for (i = prefetch_pos; i < prefetch_pos+prefetch_N; i++)
enqueue(&queue1, prefetch+(i%prefetch_N)*FRAG_SIZE,
FRAG_SIZE, (i==prefetch_pos?Q_start:Q_data));
}
}
else if (getAudioStatus() == A_recording) /***** currently recording audio data ... */
{
/***** check whether no more speech signal, then stop recording */
max = 0;
for (i = 0; i < FRAG_SIZE/2 - 1; i += 2)
{
value = abs((signed short)(buffer_raw[i]|(buffer_raw[i+1]<<8)));
if (value > max)
max = value;
}
//fprintf(stderr, "MAX: %d\n", max);
if (max <= stop_level)
count++;
else
count = 0;
/*****
* recording will be stopped,
* if more than a predefined number of non-speech blocks come in a row
*****/
if (count >= CONSECUTIVE_NONSPEECH_BLOCKS_THRESHOLD)
{
/***** here we insert the last data chunk into queue1 */
/* fprintf(stderr, "Stopped\n"); */
enqueue(&queue1, buffer_raw, FRAG_SIZE, Q_end);
count = 0; /***** reset count */
/***** turn off recognition */
//fprintf(stderr, "Stopped\n");
new_start = 1;
}
else
{
/***** insert the current chunk of data into queue1 */
enqueue(&queue1, buffer_raw, FRAG_SIZE, Q_data);
}
}
}
else
{
FD_SET(fd_audio, &reads);
if (select(fd_audio+1, &reads, NULL, NULL, &time) == -1)
{
fprintf(stderr, "select\n");
exit(-1);
}
}
}
}
