Lines Matching refs:tokens
351 struct token *tokens; in tokenise() local
358 token_list = tokens = calloc((end - buffer) / 2, sizeof(struct token)); in tokenise()
359 if (!tokens) { in tokenise()
410 tokens[tix].line = lineno; in tokenise()
411 tokens[tix].value = p; in tokenise()
423 tokens[tix].size = q - p; in tokenise()
429 if (islower(tokens[tix].value[0])) { in tokenise()
430 tokens[tix++].token_type = TOKEN_ELEMENT_NAME; in tokenise()
437 dir = bsearch(&tokens[tix], directives, in tokenise()
442 tokens[tix++].token_type = dir - directives; in tokenise()
446 tokens[tix++].token_type = TOKEN_TYPE_NAME; in tokenise()
456 tokens[tix].size = q - p; in tokenise()
458 tokens[tix++].token_type = TOKEN_NUMBER; in tokenise()
465 tokens[tix].size = 3; in tokenise()
466 tokens[tix++].token_type = TOKEN_ASSIGNMENT; in tokenise()
474 tokens[tix].size = 2; in tokenise()
475 tokens[tix++].token_type = TOKEN_OPEN_ACTION; in tokenise()
480 tokens[tix].size = 2; in tokenise()
481 tokens[tix++].token_type = TOKEN_CLOSE_ACTION; in tokenise()
487 tokens[tix].size = 1; in tokenise()
491 tokens[tix++].token_type = TOKEN_OPEN_CURLY; in tokenise()
495 tokens[tix++].token_type = TOKEN_CLOSE_CURLY; in tokenise()
499 tokens[tix++].token_type = TOKEN_OPEN_SQUARE; in tokenise()
503 tokens[tix++].token_type = TOKEN_CLOSE_SQUARE; in tokenise()
507 tokens[tix++].token_type = TOKEN_COMMA; in tokenise()