vpr/SRC/util/token.c

/**
 * Jason Luu
 * July 22, 2009
 * Tokenizer
 */

#include <string.h>
#include <assert.h>
#include "util.h"
#include "token.h"
#include "ezxml.h"
#include "read_xml_util.h"

enum e_token_type GetTokenTypeFromChar(INP enum e_token_type cur_token_type,
		INP char cur);

/* Returns a token list of the text for a given string. */
t_token *GetTokensFromString(INP const char* inString, OUTP int * num_tokens) {
	const char *cur;
	t_token * tokens;
	int i, in_string_index, prev_in_string_index;
	boolean has_null;
	enum e_token_type cur_token_type, new_token_type;

	*num_tokens = i = 0;
	cur_token_type = TOKEN_NULL;

	if (inString == NULL) {
		return NULL;
	};

	cur = inString;

	/* Count number of tokens */
	while (*cur) {
		new_token_type = GetTokenTypeFromChar(cur_token_type, *cur);
		if (new_token_type != cur_token_type) {
			cur_token_type = new_token_type;
			if (new_token_type != TOKEN_NULL) {
				i++;
			}
		}
		++cur;
	}
	*num_tokens = i;

	if (*num_tokens > 0) {
		tokens = (t_token*)my_calloc(*num_tokens + 1, sizeof(t_token));
	} else {
		return NULL;
	}

	/* populate tokens */
	i = 0;
	in_string_index = 0;
	has_null = TRUE;
	prev_in_string_index = 0;
	cur_token_type = TOKEN_NULL;

	cur = inString;

	while (*cur) {

		new_token_type = GetTokenTypeFromChar(cur_token_type, *cur);
		if (new_token_type != cur_token_type) {
			if (!has_null) {
				tokens[i - 1].data[in_string_index - prev_in_string_index] =
						'\0'; /* NULL the end of the data string */
				has_null = TRUE;
			}
			if (new_token_type != TOKEN_NULL) {
				tokens[i].type = new_token_type;
				tokens[i].data = my_strdup(inString + in_string_index);
				prev_in_string_index = in_string_index;
				has_null = FALSE;
				i++;
			}
			cur_token_type = new_token_type;
		}
		++cur;
		in_string_index++;
	}

	assert(i == *num_tokens);

	tokens[*num_tokens].type = TOKEN_NULL;
	tokens[*num_tokens].data = NULL;

	/* Return the list */
	return tokens;
}

void freeTokens(INP t_token *tokens, INP int num_tokens) {
	int i;
	for (i = 0; i < num_tokens; i++) {
		free(tokens[i].data);
	}
	free(tokens);
}

enum e_token_type GetTokenTypeFromChar(INP enum e_token_type cur_token_type,
		INP char cur) {
	if (IsWhitespace(cur)) {
		return TOKEN_NULL;
	} else {
		if (cur == '[') {
			return TOKEN_OPEN_SQUARE_BRACKET;
		} else if (cur == ']') {
			return TOKEN_CLOSE_SQUARE_BRACKET;
		} else if (cur == '{') {
			return TOKEN_OPEN_SQUIG_BRACKET;
		} else if (cur == '}') {
			return TOKEN_CLOSE_SQUIG_BRACKET;
		} else if (cur == ':') {
			return TOKEN_COLON;
		} else if (cur == '.') {
			return TOKEN_DOT;
		} else if (cur >= '0' && cur <= '9' && cur_token_type != TOKEN_STRING) {
			return TOKEN_INT;
		} else {
			return TOKEN_STRING;
		}
	}
}

boolean checkTokenType(INP t_token token, OUTP enum e_token_type token_type) {
	if (token.type != token_type) {
		return FALSE;
	}
	return TRUE;
}

void my_atof_2D(INOUTP float **matrix, INP int max_i, INP int max_j,
		INP char *instring) {
	int i, j;
	char *cur, *cur2, *copy, *final;

	copy = my_strdup(instring);
	final = copy;
	while (*final != '\0') {
		final++;
	}

	cur = copy;
	i = j = 0;
	while (cur != final) {
		while (IsWhitespace(*cur) && cur != final) {
			if (j == max_j) {
				i++;
				j = 0;
			}
			cur++;
		}
		if (cur == final) {
			break;
		}
		cur2 = cur;
		while (!IsWhitespace(*cur2) && cur2 != final) {
			cur2++;
		}
		*cur2 = '\0';
		assert(i < max_i && j < max_j);
		matrix[i][j] = atof(cur);
		j++;
		cur = cur2;
		*cur = ' ';
	}

	assert((i == max_i && j == 0) || (i == max_i - 1 && j == max_j));

	free(copy);
}