vpr/SRC/util/token.c

/**
 * Jason Luu
 * July 22, 2009
 * Tokenizer
 */

#include <string.h>
#include <assert.h>
#include "util.h"
#include "token.h"
#include "ezxml.h"
#include "read_xml_util.h"

enum e_token_type GetTokenTypeFromChar(INP enum e_token_type cur_token_type, INP char cur);

/* Returns a token list of the text for a given string. */ 
t_token *GetTokensFromString(INP const char* inString, OUTP int * num_tokens)
{
	const char *cur;
	t_token * tokens;
	int i, in_string_index, prev_in_string_index;
	boolean has_null;
	enum e_token_type cur_token_type, new_token_type;

	*num_tokens = i = 0;
	cur_token_type = TOKEN_NULL;

	if(inString == NULL) { return NULL; };

	cur = inString;	

	/* Count number of tokens */
	while(*cur)
	{
  		new_token_type = GetTokenTypeFromChar(cur_token_type, *cur);
		if(new_token_type != cur_token_type) {
			cur_token_type = new_token_type;
			if(new_token_type != TOKEN_NULL) {
				i++;
			}
		}
	    ++cur;
	}
	*num_tokens = i;

	if(*num_tokens > 0) {
		tokens = my_calloc(*num_tokens + 1, sizeof(t_token));
	} else {
		return NULL;
	}

	/* populate tokens */
	i = 0;
	in_string_index = 0;
	has_null = TRUE;
	prev_in_string_index = 0;
	cur_token_type = TOKEN_NULL;

	cur = inString;	

	while(*cur)
	{
	    
		new_token_type = GetTokenTypeFromChar(cur_token_type, *cur);
		if(new_token_type != cur_token_type) {
			if(!has_null) {
				tokens[i-1].data[in_string_index - prev_in_string_index] = '\0'; /* NULL the end of the data string */
				has_null = TRUE;
			}
			if(new_token_type != TOKEN_NULL) {
				tokens[i].type = new_token_type;
				tokens[i].data = my_strdup(inString + in_string_index);
				prev_in_string_index = in_string_index;
				has_null = FALSE;				
				i++;
			}
			cur_token_type = new_token_type;
		}
		++cur;
		in_string_index++;
	}

	assert(i == *num_tokens);

	tokens[*num_tokens].type = TOKEN_NULL;
	tokens[*num_tokens].data = NULL;

    
	/* Return the list */ 
	return tokens;
}

void freeTokens(INP t_token *tokens, INP int num_tokens) {
	int i;
	for(i = 0; i < num_tokens; i++) {
		free(tokens[i].data);
	}
	free(tokens);
}


enum e_token_type GetTokenTypeFromChar(INP enum e_token_type cur_token_type, INP char cur) {
	if(IsWhitespace(cur)) {
		return TOKEN_NULL;
	} else {
		if(cur == '[') {
			return TOKEN_OPEN_SQUARE_BRACKET;
		} else if(cur == ']') {
			return TOKEN_CLOSE_SQUARE_BRACKET;
		} else if(cur == '{') {
			return TOKEN_OPEN_SQUIG_BRACKET;
		} else if(cur == '}') {
			return TOKEN_CLOSE_SQUIG_BRACKET;
		} else if(cur == ':') {
			return TOKEN_COLON;
		} else if(cur == '.') {
			return TOKEN_DOT;
		} else if(cur >= '0' && cur <= '9' && cur_token_type != TOKEN_STRING) {
			return TOKEN_INT;
		} else {
			return TOKEN_STRING;
		}
	}
}

boolean checkTokenType(INP t_token token, OUTP enum e_token_type token_type) {
	if(token.type != token_type) {
		return FALSE;
	}
	return TRUE;
}




void my_atof_2D(INOUTP float **matrix, INP int max_i, INP int max_j, INP char *instring) {
	int i, j;
	char *cur, *cur2, *copy, *final;
	int string_count;

	copy = my_strdup(instring);
	final = copy;
	while(*final != '\0') {
		final++;
	}


	cur = copy;
	i = j = 0;
	while(cur != final) {
		string_count = 0;
		while(IsWhitespace(*cur) && cur != final) {
			if(j == max_j) {
				i++;
				j = 0;
			}
			cur++;
		}
		if(cur == final) {
			break;
		}
		cur2 = cur;
		while(!IsWhitespace(*cur2) && cur2 != final) {
			cur2++;
		}
		*cur2 = '\0';
		assert(i < max_i && j < max_j);
		matrix[i][j] = atof(cur);
		j++;
		cur = cur2;
		*cur = ' ';
	}

	assert ((i == max_i && j == 0) || (i == max_i - 1 && j == max_j));
	
	free(copy);
}