subtest token

token
----
rulebasedscanner.String{} false false ""


token
  # Just a comment.
----
rulebasedscanner.String{} false false ""

token
a b
----
rulebasedscanner.String{Value:"a", Quoted:false} false false " b"

token
a,b
----
rulebasedscanner.String{Value:"a", Quoted:false} true false "b"

token
a, b
----
rulebasedscanner.String{Value:"a", Quoted:false} true false " b"


token
a  ,b
----
rulebasedscanner.String{Value:"a", Quoted:false} false false "  ,b"

token
abc,def
----
rulebasedscanner.String{Value:"abc", Quoted:false} true false "def"

token
"abc",def
----
rulebasedscanner.String{Value:"abc", Quoted:true} true false "def"

token
"abc"def
----
rulebasedscanner.String{Value:"abc", Quoted:true} false false "def"

token
# abc,def
----
rulebasedscanner.String{} false false ""

token
# "abc
----
rulebasedscanner.String{} false false ""


token
"abc
----
error: unterminated quoted string

token
"abc=def"ghi
----
rulebasedscanner.String{Value:"abc=def", Quoted:true} false false "ghi"

token
abc="def"ghi
----
rulebasedscanner.String{Value:"abc=", Quoted:false} false true "\"def\"ghi"

token
abc= "def"
----
rulebasedscanner.String{Value:"abc=", Quoted:false} false true " \"def\""

token
"abc= def
----
error: unterminated quoted string

token
abc=def,ghi
----
rulebasedscanner.String{Value:"abc=def", Quoted:false} true false "ghi"

subtest end

subtest field

field
----
[]
""

field
  # Just a comment.
----
[]
""

field
a b c
----
[a]
" b c"

field
  a  b # c d e
----
[a]
"  b # c d e"

field
   a,b    
----
[a b]
""

field
   a,b    c
----
[a b]
"    c"

field
   a,b,    c
----
[a b c]
""

field
   a,b    ,c
----
[a b]
"    ,c"

field
a,"b",c  d
----
[a "b" c]
"  d"

field
"a",b,c   d
----
["a" b c]
"   d"

field
"a","b","c"   d
----
["a" "b" "c"]
"   d"

field
"a, b ,c"   d
----
["a, b ,c"]
"   d"

field
all,abc
----
[all abc]
""

field
# all,abc
----
[]
""

field
abc=def
----
[abc=def]
""

field
"abc=def"
----
["abc=def"]
""

field
"abc= def "
----
["abc= def "]
""

field
abc=" def "
----
[abc= " def "]
""

field
abc= " def "
----
[abc= " def "]
""

field
abc="def=ghi"
----
[abc= "def=ghi"]
""

field
abc= "def=ghi"
----
[abc= "def=ghi"]
""

field
abc=def,ghi
----
[abc=def ghi]
""

field
abc=def, ghi
----
[abc=def ghi]
""

field
abc="def
----
error: unterminated quoted string

field
all,"abc
----
error: unterminated quoted string

subtest end

subtest whitespace

file
----
rulebasedscanner.ScannedInput{}

file
# Just a comment.
----
rulebasedscanner.ScannedInput{}

file
#

    # comment and empty lines

#
----
rulebasedscanner.ScannedInput{}

subtest end

subtest whitespace_around_lines

file
#

a     

#
----
rulebasedscanner.ScannedInput{
    Lines: {
        {
            Input:  "a",
            Tokens: {
                {
                    {Value:"a", Quoted:false},
                },
            },
        },
    },
    Linenos: {3},
}

file
#

      a

#
----
rulebasedscanner.ScannedInput{
    Lines: {
        {
            Input:  "a",
            Tokens: {
                {
                    {Value:"a", Quoted:false},
                },
            },
        },
    },
    Linenos: {3},
}

file
#

      a      

b # tailing comment + whitespace   

#
----
rulebasedscanner.ScannedInput{
    Lines: {
        {
            Input:  "a",
            Tokens: {
                {
                    {Value:"a", Quoted:false},
                },
            },
        },
        {
            Input:  "b # tailing comment + whitespace",
            Tokens: {
                {
                    {Value:"b", Quoted:false},
                },
            },
        },
    },
    Linenos: {3, 5},
}

subtest end

subtest fields

file
a b c
----
rulebasedscanner.ScannedInput{
    Lines: {
        {
            Input:  "a b c",
            Tokens: {
                {
                    {Value:"a", Quoted:false},
                },
                {
                    {Value:"b", Quoted:false},
                },
                {
                    {Value:"c", Quoted:false},
                },
            },
        },
    },
    Linenos: {1},
}

file
a b c
d e
----
rulebasedscanner.ScannedInput{
    Lines: {
        {
            Input:  "a b c",
            Tokens: {
                {
                    {Value:"a", Quoted:false},
                },
                {
                    {Value:"b", Quoted:false},
                },
                {
                    {Value:"c", Quoted:false},
                },
            },
        },
        {
            Input:  "d e",
            Tokens: {
                {
                    {Value:"d", Quoted:false},
                },
                {
                    {Value:"e", Quoted:false},
                },
            },
        },
    },
    Linenos: {1, 2},
}

file
#

  a  b # c d e

 d  e # b c

#
----
rulebasedscanner.ScannedInput{
    Lines: {
        {
            Input:  "a  b # c d e",
            Tokens: {
                {
                    {Value:"a", Quoted:false},
                },
                {
                    {Value:"b", Quoted:false},
                },
            },
        },
        {
            Input:  "d  e # b c",
            Tokens: {
                {
                    {Value:"d", Quoted:false},
                },
                {
                    {Value:"e", Quoted:false},
                },
            },
        },
    },
    Linenos: {3, 5},
}

file
#

  a  "b=c" # c d e

 d  e="f" # b c

   f  a="e=c" c= a f= b, c # d b

#
----
rulebasedscanner.ScannedInput{
    Lines: {
        {
            Input:  "a  \"b=c\" # c d e",
            Tokens: {
                {
                    {Value:"a", Quoted:false},
                },
                {
                    {Value:"b=c", Quoted:true},
                },
            },
        },
        {
            Input:  "d  e=\"f\" # b c",
            Tokens: {
                {
                    {Value:"d", Quoted:false},
                },
                {
                    {Value:"e=", Quoted:false},
                    {Value:"f", Quoted:true},
                },
            },
        },
        {
            Input:  "f  a=\"e=c\" c= a f= b, c # d b",
            Tokens: {
                {
                    {Value:"f", Quoted:false},
                },
                {
                    {Value:"a=", Quoted:false},
                    {Value:"e=c", Quoted:true},
                },
                {
                    {Value:"c=", Quoted:false},
                    {Value:"a", Quoted:false},
                },
                {
                    {Value:"f=", Quoted:false},
                    {Value:"b", Quoted:false},
                    {Value:"c", Quoted:false},
                },
            },
        },
    },
    Linenos: {3, 5, 7},
}

file
d  a e="f # b c
----
error: line 1: unterminated quoted string


subtest end
