adminoo/org2newsboat
1
0
Fork 0
Code Releases Activity

regex after all heh

Browse Source
This commit is contained in:
adminoo
2021-03-04 17:04:17 +01:00
parent 70adeedf39
commit f2347b8801
2 changed files with 38 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
main_test.go
View File

@ -26,8 +26,8 @@ func TestLinkNoTag(t *testing.T) {
}
func TestLinkTag(t *testing.T) {
var message string = "** https://pleroma.social/announcements/feed.xml :software:"
var expected string = "https://pleroma.social/announcements/feed.xml software"
var message string = "** https://pleroma.social/announcements/feed.xml :software: :social:"
var expected string = "https://pleroma.social/announcements/feed.xml software social"
result := LexerTestWrapper(message, expected)
if result != expected {
LexerTestWrapperFail(expected, result)
@ -36,8 +36,8 @@ func TestLinkTag(t *testing.T) {
}
func TestLinkDescTag(t *testing.T) {
var message string = "** [[https://pleroma.social/announcements/feed.xml][Pleroma Social]] :software:"
var expected string = "https://pleroma.social/announcements/feed.xml # Pleroma Social"
var message string = "** [[https://pleroma.social/announcements/feed.xml][Pleroma Social]] :software:"
var expected string = "https://pleroma.social/announcements/feed.xml software # Pleroma Social"
var result string = LexerTestWrapper(message, expected)
if result != expected {
LexerTestWrapperFail(expected, result)

115
tokenizer.go
View File

@ -2,6 +2,7 @@ package main
import (
"fmt"
"regexp"
"strings"
)
@ -14,18 +15,12 @@ type Feed struct {
// Return the final feed string, depending on either the link has a description, tags or not
func (f Feed) String() string {
var ff string
var tags string
var tags string = strings.TrimSpace(strings.Join(f.Tags, " "))
if len(f.Tags) > 0 {
for i := range f.Tags {
tags += " " + f.Tags[i]
}
}
ff = fmt.Sprintf("%s %s", f.URL, tags)
if f.Description == "" {
ff = fmt.Sprintf("%s %s", f.URL, tags)
} else {
ff = fmt.Sprintf("%s %s # %s", f.URL, f.Description, tags)
if f.Description != "" {
ff = fmt.Sprintf("%s # %s", strings.TrimSpace(ff), f.Description)
}
return strings.TrimSpace(ff)
@ -71,91 +66,49 @@ func (l Lexer) CountToken(t string) int {
return counter
}
func (l Lexer) IdentifyContent() string {
var tokenType string
if l.CountToken("[") == 2 {
tokenType = "URL"
func (l *Lexer) ProcessSimpleLink(s string) {
ss := strings.Split(s[3:], " ")
for w := range ss {
if w == 0 {
url := strings.TrimSpace(ss[w])
l.AddToken("URL", url)
} else {
if ss[w] != "" && ss[w] != " " {
tag := strings.ReplaceAll(ss[w], ":", "")
l.AddToken("TAG", tag)
}
}
}
if l.CountToken("[") == 3 {
tokenType = "DESC"
} else {
}
return tokenType
}
// Tokenize a given line s from the org file
func (l Lexer) Process(s string) []Token {
if strings.Count(s, "[") <= 0 {
ss := strings.Split(s, " ")
fmt.Println(ss)
for w := range ss {
if w == 0 {
l.AddToken("URL", ss[w])
} else {
l.AddToken("TAG", ss[w])
}
}
if s[3] != '[' {
l.ProcessSimpleLink(s)
return l.Tokens
}
for i := range s {
char := string(s[i])
switch char {
case "*":
l.AddToken("HEADER", char)
case "[":
l.AddToken("OBRACKET", char)
case "]":
// non-empty buffer and closing bracket means
// current state is out of "content" context and buffer can be tokenized
if len(l.Buf) > 0 {
var tokenType string = l.IdentifyContent()
l.AddToken("CBRACKET", char)
l.AddToken(tokenType, l.Buf)
l.Buf = ""
re := regexp.MustCompile(`(?:\[\[)(?P<url>\S+)(?:\]\[)(?P<desc>.+)(?:\]\])(?P<tags>.+)?`)
matches := re.FindStringSubmatch(s)
if len(matches) > 1 {
l.AddToken("URL", strings.TrimSpace(matches[1]))
}
if len(matches) > 2 {
l.AddToken("DESC", strings.TrimSpace(matches[2]))
}
if len(matches) > 3 {
tags := strings.Split(matches[3], " ")
for t := range tags {
if tags[t] != "" && tags[t] != " " {
tag := strings.ReplaceAll(tags[t], ":", "")
l.AddToken("TAG", strings.TrimSpace(tag))
}
// whitespaces have different meaning given the context :
// Either separator or part of a content string
case " ":
lt := l.LastToken()
if len(l.Buf) > 0 {
l.AddToken(l.IdentifyContent(), l.Buf)
l.Buf = ""
l.AddToken("WHITESPACE", char)
}
if i > 0 {
if string(s[i-1]) == " " {
break
}
}
if lt.Type != "WHITESPACE" {
if len(l.Buf) == 0 {
l.AddToken("WHITESPACE", char)
} else {
l.Buf += char
}
}
default:
l.Buf += char
}
}
if len(l.Buf) > 0 {
l.AddToken(l.IdentifyContent(), l.Buf)
}
return l.Tokens
}
// Only retrieve content tokens, ignores uneeded separators and brackets
func Parse(t []Token) Feed {
var f Feed
for i := range t {