borked the shitty lexer, tests, new entities

tokenizer.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Feed struct {
+	URL string
+	Description string
+	Tags []string
+}
+
+// Return the final feed string, depending on either the link has a description, tags or not
+func (f Feed) String() string {
+	var ff string
+	var tags string
+
+	if len(f.Tags) > 0 {
+		for i := range f.Tags {
+			tags += " " +  f.Tags[i]
+		}
+	}
+
+	if f.Description == "" {
+		ff = fmt.Sprintf("%s %s", f.URL, tags)
+	} else {
+		ff = fmt.Sprintf("%s %s # %s", f.URL, f.Description, tags)
+	}
+	
+	return strings.TrimSpace(ff)
+}
+
+
+type Token struct {
+	Type  string
+	Value string
+}
+
+func (t Token) String() string {
+	return fmt.Sprintf("%s : '%s'", t.Type, t.Value)
+}
+
+// Lexer embbed a temporary buffer to store "content"
+// (url, description, tags) and an array of tokens
+type Lexer struct {
+	Buf    string
+	Tokens []Token
+}
+
+func (l *Lexer) AddToken(t string, s string) {
+	l.Tokens = append(l.Tokens, Token{t, s})
+}
+
+// Return last tokenized item, useful to determine context for a symbol
+func (l Lexer) LastToken() Token {
+	var lastToken Token
+	if len(l.Tokens) > 0 {
+		lastToken = l.Tokens[len(l.Tokens)-1]
+	}
+	return lastToken
+}
+
+func (l Lexer) CountToken(t string) int {
+	var counter int
+	for i := range l.Tokens {
+		if l.Tokens[i].Value == t {
+			counter ++
+		}
+	}
+	return counter
+}
+
+func (l Lexer) IdentifyContent() string {
+	var tokenType string
+	if l.CountToken("[") == 2 {
+		tokenType = "URL"
+	}
+	if l.CountToken("[") == 3 {
+		tokenType = "DESC"
+	} else {
+		
+	}
+	return tokenType
+}
+
+// Tokenize a given line s from the org file
+func (l Lexer) Process(s string) []Token {
+	if strings.Count(s, "[") <= 0 {
+		ss := strings.Split(s, " ")
+		fmt.Println(ss)
+		for w := range ss {
+			if w == 0 {
+				l.AddToken("URL", ss[w])
+			} else {
+				l.AddToken("TAG", ss[w])
+			}
+		}
+		
+		return l.Tokens
+	}
+	
+	for i := range s {
+		char := string(s[i])
+		switch char {
+		case "*":
+			l.AddToken("HEADER", char)
+
+		case "[":
+			l.AddToken("OBRACKET", char)
+
+		case "]":
+			// non-empty buffer and closing bracket means
+			// current state is out of "content" context and buffer can be tokenized
+			if len(l.Buf) > 0 {
+				var tokenType string = l.IdentifyContent()
+				l.AddToken("CBRACKET", char)
+				l.AddToken(tokenType, l.Buf)
+				l.Buf = ""
+			}
+
+			// whitespaces have different meaning given the context :
+			// Either separator or part of a content string
+		case " ":
+			lt := l.LastToken()
+			if len(l.Buf) > 0 {
+				l.AddToken(l.IdentifyContent(), l.Buf)
+				l.Buf = ""
+				l.AddToken("WHITESPACE", char)
+			}
+
+			if i > 0 {
+				if string(s[i-1]) == " " {
+					break
+				}
+			}
+
+			if lt.Type != "WHITESPACE" {
+				if len(l.Buf) == 0 {
+					l.AddToken("WHITESPACE", char)
+				} else {
+					l.Buf += char
+				}
+			}
+
+		default:
+			l.Buf += char
+		}
+	}
+
+	if len(l.Buf) > 0 {
+		l.AddToken(l.IdentifyContent(), l.Buf)
+	}
+
+	return l.Tokens
+}
+
+// Only retrieve content tokens, ignores uneeded separators and brackets
+func Parse(t []Token) Feed {
+	var f Feed
+	for i := range t {
+		token := t[i]
+		if token.Type == "URL" {
+			f.URL = token.Value
+		}
+		
+		if token.Type == "DESC" {
+			f.Description = token.Value
+		}
+		
+		if token.Type == "TAG" {
+			f.Tags = append(f.Tags, token.Value)
+		}
+	}
+
+	return f
+}