yenru0/cval
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

minor implement lexer

Browse Source
This commit is contained in:
yenru0
2025-11-15 08:01:34 +09:00
parent 171006117e
commit 682cbdc216
10 changed files with 407 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
src/lex.c
View File

@@ -1,6 +1,11 @@
#include "lex.h"
Lexer *new_lexer() {
#include "util.h"
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
Lexer *lexer_new() {
Lexer *lexer = malloc(sizeof(Lexer));
lexer->linepos = 0;
@@ -13,6 +18,8 @@ Lexer *new_lexer() {
lexer->buffer = calloc(LEX_BUF_SIZE, sizeof(char));
lexer->curr = lexer->buffer;
lexer->buffer[0] = '\0';
return lexer;
}
void lexer_set_source(Lexer *lexer, FILE *source) {
@@ -20,6 +27,9 @@ void lexer_set_source(Lexer *lexer, FILE *source) {
lexer->linepos = 0;
lexer->source = source;
lexer_fill_buffer(lexer);
}
void lexer_fill_buffer(Lexer *lexer) {
@@ -56,6 +66,13 @@ void lexer_consume(Lexer *lexer) {
if (read == LEX_BUF_SIZE - 2) {
lexer_fill_buffer(lexer);
} else {
if (lexer_curr(lexer) == '\n') {
lexer->lineno += 1;
lexer->linepos = 0;
} else {
lexer->linepos++;
}
lexer->curr++;
}
}
@@ -69,10 +86,200 @@ char lexer_peek(Lexer *lexer) {
}
Token lexer_next_token(Lexer *lexer) {
// TODO: implement
char c = lexer_curr(lexer);
if (c == '\0' && lexer->flag_EOF) {
Token eof_tok;
eof_tok.type = EOF_TOKEN;
eof_tok.line = lexer->lineno;
return eof_tok;
}
Token tok;
char *tmp = NULL;
while (is_whitespace(c)) {
lexer_consume(lexer);
c = lexer_curr(lexer);
}
switch (c) {
case '[':
tok.type = LBRACK;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case ']':
tok.type = RBRACK;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case '{':
tok.type = LCURLY;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case '}':
tok.type = RCURLY;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case '(':
tok.type = LPAREN;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case ')':
tok.type = RPAREN;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case ';':
tok.type = SEMI;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case ',':
tok.type = COMMA;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case '*':
tok.type = STAR;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case '&':
tok.type = ANDREF;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case '/':
if (lexer_peek(lexer) == '/') {
while (lexer_curr(lexer) != '\n' && lexer_curr(lexer) != '\0') {
lexer_consume(lexer);
}
if (lexer_curr(lexer) == '\n') {
lexer_consume(lexer);
}
return lexer_next_token(lexer);
} else {
tok.type = ERROR;
tok.line = lexer->lineno;
}
break;
case '$':
tok.type = DOLLAR;
tok.line = lexer->lineno;
lexer_consume(lexer);
break;
case '-':
if (lexer_peek(lexer) == '>') {
tok.type = ARROW;
tok.line = lexer->lineno;
lexer_consume(lexer);
lexer_consume(lexer);
} else {
tok.type = ERROR;
tok.line = lexer->lineno;
}
break;
case '\"':
tmp = calloc(4096, sizeof(char));
size_t i = 0;
char to_be_buff;
lexer_consume(lexer);
to_be_buff = lexer_curr(lexer);
while (to_be_buff != '\"') {
tmp[i] = to_be_buff;
i++;
lexer_consume(lexer);
to_be_buff = lexer_curr(lexer);
if (to_be_buff == '\n') {
tok.type = ERROR;
tok.line = lexer->lineno;
lexer_consume(lexer);
goto final;
}
}
tok.type = STRING_LITERAL;
tok.line = lexer->lineno;
tok.data.len = i;
tok.data.string = calloc(i + 1, sizeof(char));
strncpy(tok.data.string, tmp, i + 1);
tok.data.string[i] = '\0';
lexer_consume(lexer);
break;
case '\0':
tok.type = ERROR;
tok.line = lexer->lineno;
break;
default:
if (is_digit(c)) {
tmp = calloc(4096, sizeof(char));
size_t i = 0;
char tbb;
tmp[i] = c;
i++;
lexer_consume(lexer);
tbb = lexer_curr(lexer);
while (is_digit(tbb)) {
tmp[i] = tbb;
i++;
lexer_consume(lexer);
tbb = lexer_curr(lexer);
}
tok.type = NUM;
tok.line = lexer->lineno;
tok.data.len = i;
tok.data.string = calloc(i + 1, sizeof(char));
tok.data.string[i] = '\0';
strncpy(tok.data.string, tmp, i + 1);
} else if (is_alpha(c)) {
tmp = calloc(4096, sizeof(char));
size_t i = 0;
char tbb;
tmp[i] = c;
i++;
lexer_consume(lexer);
tbb = lexer_curr(lexer);
while (is_alpha_digit(tbb)) {
tmp[i] = tbb;
i++;
lexer_consume(lexer);
tbb = lexer_curr(lexer);
}
tmp[i] = '\0';
// Check for keywords
tok.line = lexer->lineno;
if (strcmp(tmp, "val") == 0) {
tok.type = VAL;// VAL token type not in enum, treating as ID
} else if (strcmp(tmp, "return") == 0) {
tok.type = RETURN;
} else if (strcmp(tmp, "if") == 0) {
tok.type = IF;
} else if (strcmp(tmp, "else") == 0) {
tok.type = ELSE;
} else {
tok.type = ID;
tok.data.len = i;
tok.data.string = calloc(i + 1, sizeof(char));
strncpy(tok.data.string, tmp, i + 1);
tok.data.string[i] = '\0';
}
} else {
tok.type = ERROR;
tok.line = lexer->lineno;
}
break;
}
final:
if (tmp != NULL) {
free(tmp);
}
return tok;
}
void free_lexer(Lexer *lexer) {
void lexer_free(Lexer *lexer) {
free(lexer->buffer);
free(lexer);
}

27
src/main.c Normal file
View File

@@ -0,0 +1,27 @@
#include "globals.h"
#include "lex.h"
#include "util.h"
int main() {
Lexer *lexer = lexer_new();
FILE *f = fopen("test.cval", "r");
if (f == NULL) {
perror("Failed to open file");
return 1;
}
lexer_set_source(lexer, f);
Token tok;
do {
tok = lexer_next_token(lexer);
print_token(tok);
} while(tok.type != EOF_TOKEN && tok.type != ERROR);
fclose(f);
return 0;
}

107
src/util.c Normal file
View File

@@ -0,0 +1,107 @@
#include "util.h"
#include <stdio.h>
#include <string.h>
int is_whitespace(char c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\v' || c == '\f';
}
int is_alpha(char c) {
char tmp_lower = c - 'a';
char tmp_upper = c - 'A';
return (tmp_lower <= ('z' - 'a') && 0 <= tmp_lower) || (tmp_upper <= ('Z' - 'A') && 0 <= tmp_upper);
}
int is_digit(char c) {
char tmp = c - '0';
return 0 <= tmp && tmp <= 9;
}
int is_alpha_digit(char c) {
return is_digit(c) || is_alpha(c) || c == '_';
}
void print_token(Token tok) {
char token_name[20];
switch (tok.type) {
case LBRACK:
strcpy(token_name, "LBRACK");
break;
case RBRACK:
strcpy(token_name, "RBRACK");
break;
case LCURLY:
strcpy(token_name, "LCURLY");
break;
case RCURLY:
strcpy(token_name, "RCURLY");
break;
case LPAREN:
strcpy(token_name, "LPAREN");
break;
case RPAREN:
strcpy(token_name, "RPAREN");
break;
case SEMI:
strcpy(token_name, "SEMI");
break;
case COMMA:
strcpy(token_name, "COMMA");
break;
case ARROW:
strcpy(token_name, "ARROW");
break;
case STAR:
strcpy(token_name, "STAR");
break;
case ANDREF:
strcpy(token_name, "ANDREF");
break;
case DOLLAR:
strcpy(token_name, "DOLLAR");
break;
case COMMENT:
strcpy(token_name, "COMMENT");
break;
case ID:
strcpy(token_name, "ID");
break;
case NUM:
strcpy(token_name, "NUM");
break;
case STRING_LITERAL:
strcpy(token_name, "STRING_LITERAL");
break;
case VAL:
strcpy(token_name, "VAL");
break;
case RETURN:
strcpy(token_name, "RETURN");
break;
case IF:
strcpy(token_name, "IF");
break;
case ELSE:
strcpy(token_name, "ELSE");
break;
case EOF_TOKEN:
strcpy(token_name, "EOF_TOKEN");
break;
case ERROR:
strcpy(token_name, "ERROR");
break;
default:
strcpy(token_name, "UNKNOWN");
break;
}
printf("Token Name: %s ", token_name);
printf("Line: %u ", tok.line);
if (tok.type == ID || tok.type == NUM || tok.type == STRING_LITERAL) {
printf("Data: %s", tok.data.string);
}
printf("\n");
}