Some basic structures
This commit is contained in:
33
mmql/engine/definitions.go
Normal file
33
mmql/engine/definitions.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package engine
|
||||
|
||||
const (
|
||||
K_SPACES = "\t\n\r "
|
||||
// Must be defined in pairs with
|
||||
// open bracket on left and close bracket on right
|
||||
K_BRACKETS = "()[]{}"
|
||||
K_QUOTES = "'\"`"
|
||||
)
|
||||
|
||||
const K_STATEMENT_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"
|
||||
|
||||
var KEYWORDS = []string {
|
||||
"ANYWHERE",
|
||||
"BOUGHT",
|
||||
"BUY",
|
||||
"CREATE",
|
||||
"EVERY",
|
||||
"EXCHANGE",
|
||||
"FOR",
|
||||
"FROM",
|
||||
"GET",
|
||||
"LIMIT",
|
||||
"LIST",
|
||||
"OF",
|
||||
"SELL",
|
||||
"SOLD",
|
||||
"TRIGGER",
|
||||
"TRIGGERS",
|
||||
"TYPE",
|
||||
"UPDATE",
|
||||
"WITH",
|
||||
}
|
||||
311
mmql/engine/lexer.go
Normal file
311
mmql/engine/lexer.go
Normal file
@@ -0,0 +1,311 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"strconv"
|
||||
|
||||
stmtd "github.com/tgckpg/mmqlengine/mmql/statements"
|
||||
)
|
||||
|
||||
type LexerExpect struct {
|
||||
Statements bool
|
||||
Keywords bool
|
||||
Brackets bool
|
||||
Quotes bool
|
||||
Key string
|
||||
}
|
||||
|
||||
type Lexer struct {
|
||||
SupportedStmts *map[string] func( *Lexer ) ( stmtd.IStatement, error )
|
||||
reader *strings.Reader
|
||||
readToken bool
|
||||
}
|
||||
|
||||
func ( this *Lexer ) Parse( line string ) ( stmtd.IStatement, error ) {
|
||||
|
||||
this.reader = strings.NewReader( line )
|
||||
|
||||
ex := LexerExpect{
|
||||
Statements: true,
|
||||
Keywords: false,
|
||||
Brackets: true,
|
||||
Quotes: true,
|
||||
}
|
||||
|
||||
return this.ReadStatement( &ex )
|
||||
}
|
||||
|
||||
func ( this *Lexer ) SetReader( reader *strings.Reader ) {
|
||||
this.reader = reader
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads
|
||||
* 123,456,789.00001
|
||||
* 1234.567
|
||||
* 1234
|
||||
* 10.5k
|
||||
*/
|
||||
func ( this *Lexer ) ReadDecimal() ( f float64, err error ) {
|
||||
reader := this.reader
|
||||
var b strings.Builder
|
||||
|
||||
dot := false
|
||||
comma := false
|
||||
k := false
|
||||
end := false
|
||||
for {
|
||||
r, _, _err := reader.ReadRune()
|
||||
err = _err
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if comma {
|
||||
err = fmt.Errorf( "Unexpected ',' char" )
|
||||
return
|
||||
}
|
||||
if b.Len() == 0 {
|
||||
err = fmt.Errorf( "Nothing left to read" )
|
||||
return
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if end && !strings.ContainsRune( K_SPACES, r ) {
|
||||
if k {
|
||||
b.WriteRune( 'k' )
|
||||
}
|
||||
b.WriteRune( r )
|
||||
err = fmt.Errorf( "Cannot parse '%s'", b.String() )
|
||||
return
|
||||
}
|
||||
|
||||
if r == ',' {
|
||||
if comma || dot {
|
||||
err = fmt.Errorf( "Unexpected ',' char" )
|
||||
return
|
||||
}
|
||||
comma = true
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.ContainsRune( K_SPACES, r ) {
|
||||
if comma {
|
||||
err = fmt.Errorf( "Unexpected ',' char" )
|
||||
return
|
||||
}
|
||||
if 0 < b.Len() {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if r == '.' {
|
||||
if !dot {
|
||||
b.WriteRune( r )
|
||||
dot = true
|
||||
continue
|
||||
}
|
||||
err = fmt.Errorf( "Unexpected '.' char" )
|
||||
return
|
||||
}
|
||||
|
||||
if r == 'k' {
|
||||
k = true
|
||||
end = true
|
||||
continue
|
||||
}
|
||||
|
||||
if '0' <= r && r <= '9' {
|
||||
b.WriteRune( r )
|
||||
comma = false
|
||||
} else {
|
||||
b.WriteRune( r )
|
||||
err = fmt.Errorf( "Cannot parse '%s'", b.String() )
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
f, err = strconv.ParseFloat( b.String(), 64 )
|
||||
if k {
|
||||
f *= 1000
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func ( this *Lexer ) ReadAlpha() ( s string, err error ) {
|
||||
reader := this.reader
|
||||
|
||||
var b strings.Builder
|
||||
|
||||
for {
|
||||
r, _, _err := reader.ReadRune()
|
||||
err = _err
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if 0 < b.Len() {
|
||||
err = nil
|
||||
} else {
|
||||
err = fmt.Errorf( "Nothing left to read" )
|
||||
return
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if strings.ContainsRune( K_SPACES, r ) {
|
||||
if 0 < b.Len() {
|
||||
break
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if ( 'A' <= r && r <= 'Z' ) || ( 'a' <= r && r <= 'z' ) {
|
||||
b.WriteRune( r )
|
||||
}
|
||||
}
|
||||
s = b.String()
|
||||
return
|
||||
}
|
||||
|
||||
func ( this *Lexer ) ReadStatement( expecting *LexerExpect ) ( stmtd.IStatement, error ) {
|
||||
reader := this.reader
|
||||
|
||||
var stmt stmtd.IStatement
|
||||
|
||||
for {
|
||||
r, _, err := reader.ReadRune()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if expecting.Quotes && strings.ContainsRune( K_QUOTES, r ) {
|
||||
s, err := this.ReadUntilClose( r, true, '\\' )
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stmt = stmtd.QuotedValue{
|
||||
RawStatement: &stmtd.RawStatement{ Value: s },
|
||||
Quote: string( r ),
|
||||
}
|
||||
return stmt, nil
|
||||
}
|
||||
|
||||
bracketIndex := strings.IndexRune( K_BRACKETS, r )
|
||||
if expecting.Brackets && bracketIndex % 2 == 0 {
|
||||
cl := rune( K_BRACKETS[ bracketIndex + 1 ] )
|
||||
s, err := this.ReadUntilClose( cl, false, 'A' )
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteRune( r )
|
||||
b.WriteRune( cl )
|
||||
|
||||
stmt = stmtd.BracketedValue{
|
||||
RawStatement: &stmtd.RawStatement{ Value: s },
|
||||
Brackets: b.String(),
|
||||
}
|
||||
return stmt, nil
|
||||
}
|
||||
|
||||
s := expecting.Key
|
||||
if s == "" && strings.ContainsRune( K_STATEMENT_CHARS, r ) {
|
||||
s = this.ReadUntilNot( K_STATEMENT_CHARS, true, r )
|
||||
}
|
||||
|
||||
if expecting.Statements && s != "" {
|
||||
|
||||
if this.SupportedStmts == nil {
|
||||
return nil, fmt.Errorf( "Statement not supported: %s", s )
|
||||
}
|
||||
|
||||
if f, ok := (*this.SupportedStmts)[ strings.ToUpper( s ) ]; ok {
|
||||
|
||||
// If we are using key, we need to move 1 rune back
|
||||
if expecting.Key != "" {
|
||||
err = reader.UnreadRune()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return f( this )
|
||||
} else {
|
||||
var supported strings.Builder
|
||||
comma := false
|
||||
for k := range *this.SupportedStmts {
|
||||
if comma {
|
||||
supported.WriteString( ", " )
|
||||
} else {
|
||||
comma = true
|
||||
}
|
||||
supported.WriteString( k )
|
||||
}
|
||||
return nil, fmt.Errorf( "Statement not supported: %s, supported statements are: %s", s, supported.String() )
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf( "Nothing to read" )
|
||||
}
|
||||
|
||||
func ( this *Lexer ) ReadUntilNot( charRange string, writeHead bool, head rune ) string {
|
||||
var b strings.Builder
|
||||
|
||||
if writeHead {
|
||||
b.WriteRune( head )
|
||||
}
|
||||
|
||||
reader := this.reader
|
||||
for {
|
||||
r, _, err := reader.ReadRune()
|
||||
|
||||
if err == nil && strings.ContainsRune( charRange, r ) {
|
||||
b.WriteRune( r )
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func ( this *Lexer ) ReadUntilClose( closing rune, canEsc bool, escRune rune ) ( string, error ) {
|
||||
var b strings.Builder
|
||||
|
||||
reader := this.reader
|
||||
for {
|
||||
r, _, err := reader.ReadRune()
|
||||
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
return "", fmt.Errorf( "Unexpected end of data after opening %s", string( closing ) )
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
if r == closing {
|
||||
break
|
||||
} else if canEsc && r == escRune {
|
||||
r2, _, err := reader.ReadRune()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if r2 != closing {
|
||||
b.WriteRune( r )
|
||||
b.WriteRune( r2 )
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
b.WriteRune( r )
|
||||
}
|
||||
|
||||
return b.String(), nil
|
||||
}
|
||||
119
mmql/engine/lexer_test.go
Normal file
119
mmql/engine/lexer_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
// "fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
stmtd "github.com/tgckpg/mmqlengine/mmql/statements"
|
||||
)
|
||||
|
||||
func TestParse( t *testing.T ) {
|
||||
lexer := Lexer{}
|
||||
s, err := lexer.Parse( "\"ABC\"" )
|
||||
if err != nil {
|
||||
t.Error( err )
|
||||
}
|
||||
|
||||
got := (*any( s ).( stmtd.QuotedValue ).RawStatement).Value
|
||||
if got != "ABC" {
|
||||
t.Errorf( "Expected ABC, got %s", got )
|
||||
}
|
||||
|
||||
s, err = lexer.Parse( "\"ABC" )
|
||||
if err == nil {
|
||||
t.Errorf( "Expected error, got value %s", s )
|
||||
}
|
||||
|
||||
s, err = lexer.Parse( "( ABC )" )
|
||||
|
||||
got = (*any( s ).( stmtd.BracketedValue ).RawStatement).Value
|
||||
if got != " ABC " {
|
||||
t.Errorf( "Expected ABC, got %s", got )
|
||||
}
|
||||
|
||||
s, err = lexer.Parse( "( ABC" )
|
||||
if err == nil {
|
||||
t.Errorf( "Expected error, got value %s", s )
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecimal( t *testing.T ) {
|
||||
lexer := Lexer{}
|
||||
read_oks := map[string] float64 {
|
||||
"1,000k": 1000000,
|
||||
"10.8": 10.8,
|
||||
"10,000.60": 10000.6,
|
||||
"10,000.60 this_string_should_not_be_read": 10000.6,
|
||||
"10,000 this_string_should_not_be_read": 10000,
|
||||
"10.": 10,
|
||||
".2": 0.2,
|
||||
".2k": 200,
|
||||
"1 g": 1,
|
||||
" 1 g": 1,
|
||||
}
|
||||
|
||||
for k := range read_oks {
|
||||
lexer.SetReader( strings.NewReader( k ) )
|
||||
f, err := lexer.ReadDecimal()
|
||||
if err != nil {
|
||||
t.Errorf( "Expected number for %s, got %s", k, err )
|
||||
continue
|
||||
}
|
||||
|
||||
expects, _ := read_oks[ k ]
|
||||
if f != expects {
|
||||
t.Errorf( "Expected %f, got %f", expects, f )
|
||||
}
|
||||
}
|
||||
|
||||
read_fails := []string {
|
||||
"1,000ki",
|
||||
"10,000.60s",
|
||||
"10,,0",
|
||||
"100s",
|
||||
"10, 000",
|
||||
"10,",
|
||||
}
|
||||
|
||||
for _, v := range read_fails {
|
||||
lexer.SetReader( strings.NewReader( v ) )
|
||||
f, err := lexer.ReadDecimal()
|
||||
if err == nil {
|
||||
t.Errorf( "Expected error for %s, got %f", v, f )
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpha( t *testing.T ) {
|
||||
lexer := Lexer{}
|
||||
read_oks := map[string] string {
|
||||
"abcd efgh": "abcd",
|
||||
" abcd": "abcd",
|
||||
}
|
||||
|
||||
for k := range read_oks {
|
||||
lexer.SetReader( strings.NewReader( k ) )
|
||||
f, err := lexer.ReadAlpha()
|
||||
if err != nil {
|
||||
t.Errorf( "Expected string for \"%s\", got %s", k, err )
|
||||
continue
|
||||
}
|
||||
|
||||
expects, _ := read_oks[ k ]
|
||||
if f != expects {
|
||||
t.Errorf( "Expected %s, got %s", expects, f )
|
||||
}
|
||||
}
|
||||
|
||||
read_fails := []string {
|
||||
"1", " ",
|
||||
}
|
||||
|
||||
for _, v := range read_fails {
|
||||
lexer.SetReader( strings.NewReader( v ) )
|
||||
f, err := lexer.ReadAlpha()
|
||||
if err == nil {
|
||||
t.Errorf( "Expected error for %s, got %s", v, f )
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user