Some basic structures

mmql/engine/definitions.go

@@ -1,11 +1,4 @@
-package mmql
+package engine
-
-type TokenType int
-
-type Token struct {
-    Type TokenType
-    Value string
-}
 
 const (
     K_SPACES = "\t\n\r "
@@ -15,16 +8,7 @@ const (
     K_QUOTES = "'\"`"
 )
 
-const (
+const K_STATEMENT_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"
-    UnknownToken TokenType = iota
-    KeywordToken
-    BracketToken
-    SquareBracketToken
-    CurlyBracketToken
-    SingleQuote
-    DoubleQuote
-    AccuteAccentQuote
-)
 
 var KEYWORDS = []string {
     "ANYWHERE",
@@ -47,7 +31,3 @@ var KEYWORDS = []string {
     "UPDATE",
     "WITH",
 }
-
-var STATEMENTS = map[string] func( *Lexer ) ( IStatement, error ) {
-    "BUY": _buyStatement,
-}

mmql/engine/lexer.go

@@ -0,0 +1,311 @@
+package engine
+
+import (
+    "fmt"
+    "io"
+    "strings"
+    "strconv"
+
+    stmtd "github.com/tgckpg/mmqlengine/mmql/statements"
+)
+
+type LexerExpect struct {
+    Statements bool
+    Keywords bool
+    Brackets bool
+    Quotes bool
+    Key string
+}
+
+type Lexer struct {
+    SupportedStmts *map[string] func( *Lexer ) ( stmtd.IStatement, error ) 
+    reader *strings.Reader
+    readToken bool
+}
+
+func ( this *Lexer ) Parse( line string ) ( stmtd.IStatement, error ) {
+
+    this.reader = strings.NewReader( line )
+
+    ex := LexerExpect{
+        Statements: true,
+        Keywords: false,
+        Brackets: true,
+        Quotes: true,
+    }
+
+    return this.ReadStatement( &ex )
+}
+
+func ( this *Lexer ) SetReader( reader *strings.Reader ) {
+    this.reader = reader
+}
+
+/**
+ *  Reads
+ *    123,456,789.00001
+ *    1234.567
+ *    1234
+ *    10.5k
+ */
+func ( this *Lexer ) ReadDecimal() ( f float64, err error ) {
+    reader := this.reader
+    var b strings.Builder
+
+    dot := false
+    comma := false
+    k := false
+    end := false
+    for {
+        r, _, _err := reader.ReadRune()
+        err = _err
+        if err != nil {
+            if err == io.EOF {
+                if comma {
+                    err = fmt.Errorf( "Unexpected ',' char" )
+                    return
+                }
+                if b.Len() == 0 {
+                    err = fmt.Errorf( "Nothing left to read" )
+                    return
+                }
+            }
+            break
+        }
+
+        if end && !strings.ContainsRune( K_SPACES, r ) {
+            if k {
+                b.WriteRune( 'k' )
+            }
+            b.WriteRune( r )
+            err = fmt.Errorf( "Cannot parse '%s'", b.String() )
+            return
+        }
+
+        if r == ',' {
+            if comma || dot {
+                err = fmt.Errorf( "Unexpected ',' char" )
+                return
+            }
+            comma = true
+            continue
+        }
+
+        if strings.ContainsRune( K_SPACES, r ) {
+            if comma {
+                err = fmt.Errorf( "Unexpected ',' char" )
+                return
+            }
+            if 0 < b.Len() {
+                break
+            }
+            continue
+        }
+
+        if r == '.' {
+            if !dot {
+                b.WriteRune( r )
+                dot = true
+                continue
+            }
+            err = fmt.Errorf( "Unexpected '.' char" )
+            return
+        }
+
+        if r == 'k' {
+            k = true
+            end = true
+            continue
+        }
+
+        if '0' <= r && r <= '9' {
+            b.WriteRune( r )
+            comma = false
+        } else {
+            b.WriteRune( r )
+            err = fmt.Errorf( "Cannot parse '%s'", b.String() )
+            return
+        }
+    }
+
+    f, err = strconv.ParseFloat( b.String(), 64 )
+    if k {
+        f *= 1000
+    }
+    return
+}
+
+func ( this *Lexer ) ReadAlpha() ( s string, err error ) {
+    reader := this.reader
+
+    var b strings.Builder
+
+    for {
+        r, _, _err := reader.ReadRune()
+        err = _err
+        if err != nil {
+            if err == io.EOF {
+                if 0 < b.Len() {
+                    err = nil
+                } else {
+                    err = fmt.Errorf( "Nothing left to read" )
+                    return
+                }
+            }
+            break
+        }
+
+        if strings.ContainsRune( K_SPACES, r ) {
+            if 0 < b.Len() {
+                break
+            }
+            continue
+        }
+
+        if ( 'A' <= r && r <= 'Z' ) || ( 'a' <= r && r <= 'z' ) {
+            b.WriteRune( r )
+        }
+    }
+    s = b.String()
+    return
+}
+
+func ( this *Lexer ) ReadStatement( expecting *LexerExpect ) ( stmtd.IStatement, error ) {
+    reader := this.reader
+
+    var stmt stmtd.IStatement
+
+    for {
+        r, _, err := reader.ReadRune()
+        if err != nil {
+            return nil, err
+        }
+
+        if expecting.Quotes && strings.ContainsRune( K_QUOTES, r ) {
+            s, err := this.ReadUntilClose( r, true, '\\' )
+            if err != nil {
+                return nil, err
+            }
+
+            stmt = stmtd.QuotedValue{
+                RawStatement: &stmtd.RawStatement{ Value: s },
+                Quote: string( r ),
+            }
+            return stmt, nil
+        }
+
+        bracketIndex := strings.IndexRune( K_BRACKETS, r )
+        if expecting.Brackets && bracketIndex % 2 == 0 {
+            cl := rune( K_BRACKETS[ bracketIndex + 1 ] )
+            s, err := this.ReadUntilClose( cl, false, 'A' )
+            if err != nil {
+                return nil, err
+            }
+
+            var b strings.Builder
+            b.WriteRune( r )
+            b.WriteRune( cl )
+
+            stmt = stmtd.BracketedValue{
+                RawStatement: &stmtd.RawStatement{ Value: s },
+                Brackets: b.String(),
+            }
+            return stmt, nil
+        }
+
+        s := expecting.Key
+        if s == "" && strings.ContainsRune( K_STATEMENT_CHARS, r ) {
+            s = this.ReadUntilNot( K_STATEMENT_CHARS, true, r )
+        }
+
+        if expecting.Statements && s != "" {
+
+            if this.SupportedStmts == nil {
+                return nil, fmt.Errorf( "Statement not supported: %s", s )
+            }
+
+            if f, ok := (*this.SupportedStmts)[ strings.ToUpper( s ) ]; ok {
+
+                // If we are using key, we need to move 1 rune back
+                if expecting.Key != "" {
+                    err = reader.UnreadRune()
+                    if err != nil {
+                        return nil, err
+                    }
+                }
+
+                return f( this )
+            } else {
+                var supported strings.Builder
+                comma := false
+                for k := range *this.SupportedStmts {
+                    if comma {
+                        supported.WriteString( ", " )
+                    } else {
+                        comma = true
+                    }
+                    supported.WriteString( k )
+                }
+                return nil, fmt.Errorf( "Statement not supported: %s, supported statements are: %s", s, supported.String() )
+            }
+        }
+    }
+
+    return nil, fmt.Errorf( "Nothing to read" )
+}
+
+func ( this *Lexer ) ReadUntilNot( charRange string, writeHead bool, head rune ) string {
+    var b strings.Builder
+
+    if writeHead {
+        b.WriteRune( head )
+    }
+
+    reader := this.reader
+    for {
+        r, _, err := reader.ReadRune()
+
+        if err == nil && strings.ContainsRune( charRange, r ) {
+            b.WriteRune( r )
+        } else {
+            break
+        }
+    }
+
+    return b.String()
+}
+
+func ( this *Lexer ) ReadUntilClose( closing rune, canEsc bool, escRune rune ) ( string, error ) {
+    var b strings.Builder
+
+    reader := this.reader
+    for {
+        r, _, err := reader.ReadRune()
+
+        if err != nil {
+            if err == io.EOF {
+                return "", fmt.Errorf( "Unexpected end of data after opening %s", string( closing ) )
+            }
+            return "", err
+        }
+
+        if r == closing {
+            break
+        } else if canEsc && r == escRune {
+            r2, _, err := reader.ReadRune()
+            if err != nil {
+                return "", err
+            }
+
+            if r2 != closing {
+                b.WriteRune( r )
+                b.WriteRune( r2 )
+                continue
+            }
+        }
+
+        b.WriteRune( r )
+    }
+
+    return b.String(), nil
+}

mmql/engine/lexer_test.go

@@ -0,0 +1,119 @@
+package engine
+
+import (
+    // "fmt"
+    "strings"
+    "testing"
+    stmtd "github.com/tgckpg/mmqlengine/mmql/statements"
+)
+
+func TestParse( t *testing.T ) {
+    lexer := Lexer{}
+    s, err := lexer.Parse( "\"ABC\"" )
+    if err != nil {
+        t.Error( err )
+    }
+
+    got := (*any( s ).( stmtd.QuotedValue ).RawStatement).Value
+    if got != "ABC" {
+        t.Errorf( "Expected ABC, got %s", got )
+    }
+
+    s, err = lexer.Parse( "\"ABC" )
+    if err == nil {
+        t.Errorf( "Expected error, got value %s", s )
+    }
+
+    s, err = lexer.Parse( "( ABC )" )
+
+    got = (*any( s ).( stmtd.BracketedValue ).RawStatement).Value
+    if got != " ABC " {
+        t.Errorf( "Expected ABC, got %s", got )
+    }
+
+    s, err = lexer.Parse( "( ABC" )
+    if err == nil {
+        t.Errorf( "Expected error, got value %s", s )
+    }
+}
+
+func TestDecimal( t *testing.T ) {
+    lexer := Lexer{}
+    read_oks := map[string] float64 {
+        "1,000k": 1000000,
+        "10.8": 10.8,
+        "10,000.60": 10000.6,
+        "10,000.60 this_string_should_not_be_read": 10000.6,
+        "10,000 this_string_should_not_be_read": 10000,
+        "10.": 10,
+        ".2": 0.2,
+        ".2k": 200,
+        "1 g": 1,
+        " 1 g": 1,
+    }
+
+    for k := range read_oks {
+        lexer.SetReader( strings.NewReader( k ) )
+        f, err := lexer.ReadDecimal()
+        if err != nil {
+            t.Errorf( "Expected number for %s, got %s", k, err )
+            continue
+        }
+
+        expects, _ := read_oks[ k ]
+        if f != expects {
+            t.Errorf( "Expected %f, got %f", expects, f )
+        }
+    }
+
+    read_fails := []string {
+        "1,000ki",
+        "10,000.60s",
+        "10,,0",
+        "100s",
+        "10, 000",
+        "10,",
+    }
+
+    for _, v := range read_fails {
+        lexer.SetReader( strings.NewReader( v ) )
+        f, err := lexer.ReadDecimal()
+        if err == nil {
+            t.Errorf( "Expected error for %s, got %f", v, f )
+        }
+    }
+}
+
+func TestAlpha( t *testing.T ) {
+    lexer := Lexer{}
+    read_oks := map[string] string {
+        "abcd efgh": "abcd",
+        " abcd": "abcd",
+    }
+
+    for k := range read_oks {
+        lexer.SetReader( strings.NewReader( k ) )
+        f, err := lexer.ReadAlpha()
+        if err != nil {
+            t.Errorf( "Expected string for \"%s\", got %s", k, err )
+            continue
+        }
+
+        expects, _ := read_oks[ k ]
+        if f != expects {
+            t.Errorf( "Expected %s, got %s", expects, f )
+        }
+    }
+
+    read_fails := []string {
+        "1", " ",
+    }
+
+    for _, v := range read_fails {
+        lexer.SetReader( strings.NewReader( v ) )
+        f, err := lexer.ReadAlpha()
+        if err == nil {
+            t.Errorf( "Expected error for %s, got %s", v, f )
+        }
+    }
+}

mmql/lexer.go

@@ -1,119 +0,0 @@
-package mmql
-
-import (
-    "fmt"
-    "io"
-    "strings"
-)
-
-type LexerExpect struct {
-    statements bool
-    keywords bool
-    brackets bool
-    quotes bool
-    key string
-}
-
-type Lexer struct {
-    reader *strings.Reader
-    readToken bool
-}
-
-func ( this *Lexer ) Parse( line string ) ( IStatement, error ) {
-
-    this.reader = strings.NewReader( line )
-
-    ex := LexerExpect{
-        statements: true,
-        keywords: false,
-        brackets: true,
-        quotes: true,
-    }
-
-    return this.ReadStatement( &ex )
-}
-
-func ( this *Lexer ) ReadStatement( expecting *LexerExpect ) ( IStatement, error ) {
-    reader := this.reader
-
-    var stmt IStatement
-
-    for {
-        r, _, err := reader.ReadRune()
-        if err != nil {
-            return nil, err
-        }
-
-        if expecting.quotes && strings.ContainsRune( K_QUOTES, r ) {
-            s, err := this.ReadUntilClose( r, true, '\\' )
-            if err != nil {
-                return nil, err
-            }
-
-            stmt = QuotedValue{
-                RawStatement: &RawStatement{ Value: s },
-                Quote: string( r ),
-            }
-            return stmt, nil
-        }
-
-        bracketIndex := strings.IndexRune( K_BRACKETS, r )
-        if expecting.brackets && bracketIndex % 2 == 0 {
-            cl := rune( K_BRACKETS[ bracketIndex + 1 ] )
-            s, err := this.ReadUntilClose( cl, false, 'A' )
-            if err != nil {
-                return nil, err
-            }
-
-            var b strings.Builder
-            b.WriteRune( r )
-            b.WriteRune( cl )
-
-            stmt = BracketedValue{
-                RawStatement: &RawStatement{ Value: s },
-                Brackets: b.String(),
-            }
-            return stmt, nil
-        }
-
-        if expecting.statements {
-        }
-    }
-
-    return nil, fmt.Errorf( "Nothing to read" )
-}
-
-func ( this *Lexer ) ReadUntilClose( closing rune, canEsc bool, escRune rune ) ( string, error ) {
-    var b strings.Builder
-
-    reader := this.reader
-    for {
-        r, _, err := reader.ReadRune()
-
-        if err != nil {
-            if err == io.EOF {
-                return "", fmt.Errorf( "Unexpected end of data after opening %s", string( closing ) )
-            }
-            return "", err
-        }
-
-        if r == closing {
-            break
-        } else if canEsc && r == escRune {
-            r2, _, err := reader.ReadRune()
-            if err != nil {
-                return "", err
-            }
-
-            if r2 != closing {
-                b.WriteRune( r )
-                b.WriteRune( r2 )
-                continue
-            }
-        }
-
-        b.WriteRune( r )
-    }
-
-    return b.String(), nil
-}

mmql/lexer_test.go

@@ -1,49 +0,0 @@
-package mmql
-
-import (
-    // "fmt"
-    "testing"
-)
-
-func TestParse( t *testing.T ) {
-    lexer := Lexer{}
-    s, err := lexer.Parse( "\"ABC\"" )
-    if err != nil {
-        t.Error( err )
-    }
-
-    got := (*any( s ).( QuotedValue ).RawStatement).Value
-    if got != "ABC" {
-        t.Errorf( "Expected ABC, got %s", got )
-    }
-
-    s, err = lexer.Parse( "\"ABC" )
-    if err == nil {
-        t.Errorf( "Expected error, got value %s", s )
-    }
-
-    s, err = lexer.Parse( "( ABC )" )
-
-    got = (*any( s ).( BracketedValue ).RawStatement).Value
-    if got != " ABC " {
-        t.Errorf( "Expected ABC, got %s", got )
-    }
-
-    s, err = lexer.Parse( "( ABC" )
-    if err == nil {
-        t.Errorf( "Expected error, got value %s", s )
-    }
-
-    s, err = lexer.Parse( `
-BUY 1 SHARES OF BTC_ETF
-  FOR 10 USD
-  FROM "MyBrokerAccount"
-  WITH LIMIT OF PURCHASING_POWER( "MyBrokerAccount", "QQQ" )
-  FOR EVERY
-    1 BTC OF USD_BTC SOLD
-        FROM "CoinBase"
-` )
-    if err != nil {
-        t.Error( err )
-    }
-}

mmql/statements/actions/amount.go

@@ -0,0 +1,32 @@
+package actions
+
+import (
+    // "fmt"
+    engine "github.com/tgckpg/mmqlengine/mmql/engine"
+    stmtd "github.com/tgckpg/mmqlengine/mmql/statements"
+)
+
+/*
+EXPECT: [Number] [Units]
+OR
+EXPECT: [FUNCTION]( ...params )
+*/
+
+func AmountStatement( lexer *engine.Lexer ) ( istmt stmtd.IStatement, err error ) {
+    stmt := stmtd.AmountStatement{}
+
+    val, err := lexer.ReadDecimal()
+    if err != nil {
+        return
+    }
+    stmt.Value = val
+
+    unit, err := lexer.ReadAlpha()
+    if err != nil {
+        return
+    }
+    stmt.Unit = unit
+
+    istmt = stmt
+    return
+}

mmql/statements/actions/buy.go

@@ -0,0 +1,41 @@
+package actions
+
+import (
+    engine "github.com/tgckpg/mmqlengine/mmql/engine"
+    stmtd "github.com/tgckpg/mmqlengine/mmql/statements"
+)
+
+/*
+EXPECT: BUY
+THEN EXPECT: [AmountStatement] OF [ProductStatement]
+THEN EXPECT: FOR [AmountStatement]
+THEN EXPECT: FROM [ExchangeType]
+THEN OPT EXPECT: [ExchangeType]
+THEN OPT EXPECT: [AND|OR]
+    THEN EXPECT: [ActionExpression]
+THEN OPT EXPECT: FOR EVERY
+    THEN EXPECT: [AmountType] OF [ProductType]
+    THEN EXPECT: [SOLD|BOUGHT] FROM [ExchangeType]
+*/
+
+func BuyStatement( lexer *engine.Lexer ) ( stmtd.IStatement, error ) {
+    orderStmt := stmtd.OrderStatement{}
+    orderStmt.Action = "BUY"
+
+    readAmount := engine.LexerExpect{
+        Statements: true,
+        Keywords: false,
+        Brackets: false,
+        Quotes: false,
+        Key: "AMOUNT",
+    }
+
+    amountStmt, err := lexer.ReadStatement( &readAmount )
+    if err != nil {
+        return nil, err
+    }
+    orderStmt.For( amountStmt )
+
+    var istmt stmtd.IStatement = orderStmt
+    return istmt, err
+}

mmql/statements/actions/buy_test.go

@@ -0,0 +1,30 @@
+package actions
+
+import (
+    // "fmt"
+    "testing"
+    engine "github.com/tgckpg/mmqlengine/mmql/engine"
+    stmtd "github.com/tgckpg/mmqlengine/mmql/statements"
+)
+
+func TestBuyStatement( t *testing.T ) {
+    lexer := engine.Lexer{}
+    supportedStmts := map[string] func( *engine.Lexer ) ( stmtd.IStatement, error ) {
+        "BUY": BuyStatement,
+        "AMOUNT": AmountStatement,
+    }
+    lexer.SupportedStmts = &supportedStmts
+
+    _, err := lexer.Parse( `
+BUY 1 SHARES OF BTC_ETF
+  FOR 10 USD
+  FROM "MyBrokerAccount"
+  WITH LIMIT OF PURCHASING_POWER( "MyBrokerAccount", "QQQ" )
+  FOR EVERY
+    1 BTC OF USD_BTC SOLD
+        FROM "CoinBase"
+` )
+    if err != nil {
+        t.Error( err )
+    }
+}

mmql/statements/types.go

@@ -1,4 +1,4 @@
-package mmql
+package statements
 
 type IStatement interface { }
 
@@ -19,12 +19,18 @@ type BracketedValue struct {
     Brackets string
 }
 
-type ActionStatement struct {
+type AmountStatement struct {
     IStatement
+    Value float64
+    Unit string
 }
 
+type IActionStatement interface { }
+
 type OrderStatement struct {
-    *ActionStatement
+    IStatement
+    IActionStatement
+    Action string
 }
 
 func ( this *OrderStatement ) For( stmt IStatement ) {

mmql/stmt_buy.go

@@ -1,35 +0,0 @@
-package mmql
-
-/*
-EXPECT: BUY
-THEN EXPECT: [AmountType] OF [ProductType]
-THEN EXPECT: FOR [AmountType]
-THEN EXPECT: FROM [ExchangeType]
-THEN OPT EXPECT: [ExchangeType]
-THEN OPT EXPECT: [AND|OR]
-    THEN EXPECT: [ActionExpression]
-THEN OPT EXPECT: FOR EVERY
-    THEN EXPECT: [AmountType] OF [ProductType]
-    THEN EXPECT: [SOLD|BOUGHT] FROM [ExchangeType]
-*/
-
-func _buyStatement( lexer *Lexer ) ( IStatement, error ) {
-    a := OrderStatement{}
-
-    ex := LexerExpect{
-        statements: true,
-        keywords: false,
-        brackets: false,
-        quotes: false,
-        key: "AMOUNT",
-    }
-
-    stmt, err := lexer.ReadStatement( &ex )
-    if err != nil {
-        return nil, err
-    }
-    a.For( stmt )
-
-    var b IStatement = a
-    return b, nil
-}