Processing Phoneme Data using a Vizeme Mapping

Extending the mxj object from step 2 and introducing a new object, the map file, enables phoneme values to drive the talking face. The map file consists of a list of entries, one for each phoneme, specifying a PC vector representing the canonical facial expression or "vizeme" corresponding to the phoneme. The original map file used, facevectors.txt, is shown below:

Each phoneme is associated to eight values, one for each Principal Component governing the shape of the talking face. Thus, if the face looks as though it is saying "EE", it is being sent the parameter values -50 100 -50 100 0 -25 0 0. In mapping terms, a probability of 1 for EE is mapped to the values -50 100 -50 100 0 -25 0 0. If the face is halfway between "EE" and "F", each vector is weighted according to the phoneme's probability, and the vectors are added, yielding 25 62.5 -50 62.5 0 -12.5 0 0.

In general, the face parameter vector is calculated from phoneme probabilities first by multiplying each vector in the map file by its corresponding phoneme's probability, and then adding the vectors. This produces a face which represents a sort of average of different vizemes at their relative weights.

The file loading and calculation, at this step of the process, is performed by the MXJ object sendToFace, here shown with the "facevectors.txt" message to load the map file. Other inlets, from left to right, receive the vowel/consonant decision variable, list of vowel probabilities, and list of consonant probabilities.

sendToFace performs the following tasks (inlets numbered from the left):

i) receives lists of vowel and consonant probabilities in the third and fourth inlets, storing the most recent values
ii) upon receiving a filename message in the first inlet,
.....a) reads the file
.....b) loads values to create a mapping of phonemes to PC vectors
.....c) attempts a socket connection
iii) upon receiving a value for the decision variable in the second inlet,
.....a) calculates PC vector using method described above
.....b) send PC vector through the socket to Artisynth's KuraFace

The sendToFace code is shown below, along with comments detailing its behaviour

	import com.cycling74.max.*;
	import java.io.*;
	import java.net.Socket;
	import java.net.UnknownHostException;
	import java.util.HashMap;
	import java.util.Set;
	import java.util.Iterator;

	public class sendToFace extends MaxObject {


		public sendToFace() {

			declareTypedIO( "mfll", "ffffffff");

			float f = 0;

			//initialize for safety

			for( int i=0; i<vowels.length; i++) {

				vowelweights.put( vowels[i], f);
			}

			for( int i=0; i<consonants.length; i++) {

				consonantweights.put( consonants[i], f);
			}


		}

		public void anything( String filename, Atom[] args ) {

Receives filename, inlet 0

			// filename inlet
			if( getInlet() == 0 ) {

				// try to load the file

				BufferedReader in = null;
				String newline = "";
				String splitline[] = new String[9];

				try {

					in = new BufferedReader( new FileReader( filename.toString() ));

				}
				catch( FileNotFoundException fnfe ) {

					System.out.println( "Could not locate the file " + filename.toString() + ": " + fnfe);
					return;
				}

attempts to load PC vectors from the file

				try{

					while( ( newline = in.readLine()) != null) {

						if( !newline.matches( "[A-Z]{1,2} (-?\\d+\\.?\\d* ){7}-?\\d+\\.?\\d*" )) {

							System.out.println( "Invalid line:\n " + newline + "\n  -- each line should consist of:\n"
									+ "[vowel or consonant] [float] [float] [float] [float] [float] [float] [float] [float]\n"
									+ " where vowel or consonant is for example EE I E or V F DH");
							return;
						}
						else {

							splitline = newline.split( " " );

							if( splitline[0].matches( "[AEIOU][A-Z]?" )) {

								vowelvectors.put( splitline[0], new float[8] );
								for( int i=1; i<9; i++ ) {

									vowelvectors.get( splitline[0])[i - 1] = Float.parseFloat( splitline[i] );
								}
							}
							else {

								consonantvectors.put( splitline[0], new float[8] );
								for( int i=1; i<9; i++ ) {

									consonantvectors.get( splitline[0])[i - 1] = Float.parseFloat( splitline[i] );
								}
							}
						}
					}
				}
				catch( IOException io ) {

					System.out.println( "could not read from file " + filename + ": " + io);
					return;
				}

				String missing = "";

				for( int i=0; i<vowels.length; i++) {

					if( !vowelvectors.containsKey( vowels[i] )) {

						missing += vowels[i] + " ";
					}
				}

				for( int i=0; i<consonants.length; i++) {

					if( !consonantvectors.containsKey( consonants[i] )) {

						missing += consonants[i] + " ";
					}
				}

notifies the user if the file does not contain a complete mapping ie: one or more phonemes are missing

				if( !missing.equals("")) {

					System.out.println( "PC vector file is missing the following vowels/consonants:\n" + missing );
					return;
				}
				else {

					isloaded = true;
					System.out.println( "Loaded vowels and consonants from file: " + filename );

					// if you loaded the file, try a socket connection

if a complete mapping was successfully loaded, attempts a socket connection to Artisynth

			        try {

			            kkSocket = new Socket( hostname, port);
			            out = new PrintWriter(kkSocket.getOutputStream(), true);
			            in = new BufferedReader(new InputStreamReader(kkSocket.getInputStream()));

			            issending = true;
			            System.out.println( "connected on port " + port );
			        }

			        catch (UnknownHostException e) {

			            System.err.println("Don't know about host: " + hostname);
			        }

			        catch (IOException e) {

			            System.err.println("Couldn't get I/O for the connection to: " + hostname);
			        }
				}
			}
		}

		public void inlet( float vc ) {

			if( getInlet() == 1) {

				vprob = vc;
				cprob = 1 - vc;
			}

checks that file is loaded, and that there is a socket connection

			// do calculations here //
			if( !isloaded ) {

				System.out.println( "PC vector file not loaded - cannot calculate principal components");
				return;
			}
			if( !issending ) {

				System.out.println( "Warning: Not connected to server socket - PC values will not be sent to server");
			}

			for( int i=0; i<8; i++) {

				PCs[i] = 0;
			}

calculates PC values, first for vowels, then for consonants

			// vowels first
			Set vowellist = vowelweights.keySet();
			Iterator voweliter = vowellist.iterator();

			while( voweliter.hasNext() ) {

				String vowel = voweliter.next().toString();
				float weight = ((float) vowelweights.get( vowel ));

				for( int i=0; i<8; i++ ) {

					float PCvalue = ((float[]) vowelvectors.get( vowel ))[i];
					PCs[i] += vprob * weight * PCvalue; 
				}
			}

			// then consonants
			Set consonantlist = consonantweights.keySet();
			Iterator consonantiter = consonantlist.iterator();

			while( consonantiter.hasNext() ) {

				String consonant = consonantiter.next().toString();
				float weight = ((float) consonantweights.get( consonant ));

				for( int i=0; i<8; i++ ) {

					float PCvalue = ((float[]) consonantvectors.get( consonant ))[i];
					PCs[i] += cprob * weight * PCvalue; 
				}
			}

			// print PC's out for viewing purposes
			System.out.println( PCs[0] + " " + PCs[1] + " " + PCs[2]
			                     + " " + PCs[3] + " " + PCs[4]
			                     + " " + PCs[5] + " " + PCs[6]
			                     + " " + PCs[7]);


			// send PC values to outlets for viewing purposes
			for( int i=0; i<8; i++) {

				outlet(i, PCs[i]);
			}

			String outputLine = "";

			for( int i=0; i<8; i++ ) {

				outputLine += PCs[i] + ";";
			}

sends PC values through the socket connection to Artisynth, to drive the KuraFace

			// then through the socket!
			out.println( outputLine );

	/*		try {

				String fromServer = in.readLine();
				System.out.println( fromServer );
			}

			catch(IOException io ) {

				System.out.println("could not read from server socket:" + io);
			}
	*/
		}

method for parsing list of vowel and consonant probabilities arriving through the inlets

		public void list( float[] weights ) {

			float nrmlztnsum = 0;

			// then normalize for safety's sake
			for( int i=0; i<weights.length; i++ ) {

				nrmlztnsum += weights[i];
			}

			if( getInlet() == 3 || testwich.equals("consonants")) {

				if( weights.length != 15 ) {

					System.out.println( "Inlet 2 received wrong number of consonants -- "
							+ "should receive list of 15 floats");
					return;
				}
				else {

					for( int i=0; i<consonants.length; i++ ) {

						float nrmlzweight = weights[i] / nrmlztnsum;
						consonantweights.put( consonants[i], nrmlzweight );
					}
				}
			}

			else if( getInlet() == 2 || testwich.equals("vowels")) {

				if( weights.length != 11 ) {

					System.out.println( "Inlet 2 received wrong number of vowels -- "
							+ "should receive list of 11 floats");
					return;
				}
				else {

					for( int i=0; i<vowels.length; i++ ) {

						float nrmlzweight = weights[i] / nrmlztnsum;
						vowelweights.put( vowels[i], nrmlzweight );
					}
				}
			}
		}

		String filename = "";
		boolean isloaded = false;

		HashMap< String, Float> vowelweights = new HashMap< String, Float>();
		HashMap< String, Float> consonantweights = new HashMap< String, Float>();

		// HashMap of vowel, consonant string (ex:  EE I E, V F DH) to an array of 8 PC float values
		HashMap< String, float[]> vowelvectors = new HashMap< String, float[]>();
		HashMap< String, float[]> consonantvectors = new HashMap< String, float[]>();

		String vowels[] = {"EE", "I", "E", "AA", "U", "AR", "O", "AW", "OO", "UU", "A"};
		String consonants[] = {"V", "F", "DH", "TH", "Z", "S", "ZH", "SH", "H", "W", "L", "R", "M", "N", "NG"};

		float vprob = 0;
		float cprob = 1 - vprob;

		float PCs[] = new float[8];

		// socket stuff
		boolean issending = false;

	    Socket kkSocket = null;
	    PrintWriter out = null;
	    BufferedReader in = null;
		int port = 4444;
	    String hostname = "BrahmsOSX.local";

	    String testwich = "";
	}