removing useless buffer, changing token type to int constants

lexer.go

@@ -5,14 +5,12 @@ import (
 	"strings"
 )
 
-// Lexer embbed a temporary buffer to store "content"
-// (url, description, tags) and an array of tokens
+// Lexer embbed an array of token to retrieve them later for building urls
 type Lexer struct {
-	Buf    string
 	Tokens []Token
 }
 
-func (l *Lexer) AddToken(t string, s string) {
+func (l *Lexer) AddToken(t int, s string) {
 	l.Tokens = append(l.Tokens, Token{t, s})
 }
 
@@ -40,11 +38,11 @@ func (l *Lexer) ProcessSimpleLink(s string) {
 	for w := range ss {
 		if w == 0 {
 			url := strings.TrimSpace(ss[w])
-			l.AddToken("URL", url)
+			l.AddToken(URL, url)
 		} else {
 			if ss[w] != "" && ss[w] != " " {
 				tag := strings.ReplaceAll(ss[w], ":", "")
-				l.AddToken("TAG", tag)
+				l.AddToken(TAG, tag)
 			}
 		}
 	}
@@ -60,17 +58,17 @@ func (l Lexer) Process(s string) []Token {
 	re := regexp.MustCompile(`(?:\[\[)(?P<url>\S+)(?:\]\[)(?P<desc>.+)(?:\]\])(?P<tags>.+)?`)
 	matches := re.FindStringSubmatch(s)
 	if len(matches) > 1 {
-		l.AddToken("URL", strings.TrimSpace(matches[1]))
+		l.AddToken(URL, strings.TrimSpace(matches[1]))
 	}
 	if len(matches) > 2 {
-		l.AddToken("DESC", strings.TrimSpace(matches[2]))
+		l.AddToken(DESCRIPTION, strings.TrimSpace(matches[2]))
 	}
 	if len(matches) > 3 {
 		tags := strings.Split(matches[3], " ")
 		for t := range tags {
 			if tags[t] != "" && tags[t] != " " {
 				tag := strings.ReplaceAll(tags[t], ":", "")
-				l.AddToken("TAG", strings.TrimSpace(tag))
+				l.AddToken(TAG, strings.TrimSpace(tag))
 			}
 		}
 	}

token.go

@@ -4,28 +4,34 @@ import (
 	"fmt"
 )
 
+const (
+	URL = iota
+	TAG
+	DESCRIPTION
+)
+
 type Token struct {
-	Type  string
+	Type  int
 	Value string
 }
 
 func (t Token) String() string {
-	return fmt.Sprintf("%s : '%s'", t.Type, t.Value)
+	return fmt.Sprintf("%v : '%s'", t.Type, t.Value)
 }
 
 func Parse(t []Token) Feed {
 	var f Feed
 	for i := range t {
 		token := t[i]
-		if token.Type == "URL" {
+		if token.Type == URL {
 			f.URL = token.Value
 		}
 		
-		if token.Type == "DESC" {
+		if token.Type == DESCRIPTION {
 			f.Description = token.Value
 		}
 		
-		if token.Type == "TAG" {
+		if token.Type == TAG {
 			f.Tags = append(f.Tags, token.Value)
 		}
 	}