implement select from time quantum columns (fb-1654) (#2282)

* first time quantum queries working

* implement select from timequantum columns

* skip a dax test

* make linter happy

* addressed review feedback

* reverted over eager test elimination

Author: paddyjok

dax/table.go

@@ -73,14 +73,16 @@ const PrefixTable = "tbl"
 
 // Base types.
 const (
-	BaseTypeBool      = "bool"      //
-	BaseTypeDecimal   = "decimal"   //
-	BaseTypeID        = "id"        // non-keyed mutex
-	BaseTypeIDSet     = "idset"     // non-keyed set
-	BaseTypeInt       = "int"       //
-	BaseTypeString    = "string"    // keyed mutex
-	BaseTypeStringSet = "stringset" // keyed set
-	BaseTypeTimestamp = "timestamp" //
+	BaseTypeBool       = "bool"       //
+	BaseTypeDecimal    = "decimal"    //
+	BaseTypeID         = "id"         // non-keyed mutex
+	BaseTypeIDSet      = "idset"      // non-keyed set
+	BaseTypeIDSetQ     = "idsetq"     // non-keyed set timequantum
+	BaseTypeInt        = "int"        //
+	BaseTypeString     = "string"     // keyed mutex
+	BaseTypeStringSet  = "stringset"  // keyed set
+	BaseTypeStringSetQ = "stringsetq" // keyed set timequantum
+	BaseTypeTimestamp  = "timestamp"  //
 
 	DefaultPartitionN = 256
 
@@ -679,9 +681,11 @@ func BaseTypeFromString(s string) (BaseType, error) {
 		BaseTypeDecimal,
 		BaseTypeID,
 		BaseTypeIDSet,
+		BaseTypeIDSetQ,
 		BaseTypeInt,
 		BaseTypeString,
 		BaseTypeStringSet,
+		BaseTypeStringSetQ,
 		BaseTypeTimestamp:
 		return BaseType(lowered), nil
 	default:
@@ -706,7 +710,7 @@ func (f *Field) String() string {
 // StringKeys returns true if the field uses string keys.
 func (f *Field) StringKeys() bool {
 	switch f.Type {
-	case BaseTypeString, BaseTypeStringSet:
+	case BaseTypeString, BaseTypeStringSet, BaseTypeStringSetQ:
 		return true
 	}
 	return false

dax/test/dax/dax_test.go

@@ -143,6 +143,7 @@ func TestDAXIntegration(t *testing.T) {
 			"viewtests/drop-view", // drop view does a delete
 			"viewtests/drop-view-if-exists-after-drop",
 			"viewtests/select-view-after-drop",
+			"time_quantum_insert/test-12", // orchestrator currently does not support to,from args on Rows()
 		}
 
 		doSkip := func(name string) bool {

schema.go

@@ -243,9 +243,9 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field {
 		fieldType = dax.BaseTypeBool
 	case FieldTypeTime:
 		if fo.Keys {
-			fieldType = dax.BaseTypeStringSet
+			fieldType = dax.BaseTypeStringSetQ
 		} else {
-			fieldType = dax.BaseTypeIDSet
+			fieldType = dax.BaseTypeIDSetQ
 		}
 		timeQuantum = dax.TimeQuantum(fo.TimeQuantum)
 	default:
@@ -397,11 +397,13 @@ func fieldToFieldType(f *dax.Field) string {
 			return string(f.Type)
 		}
 		return "mutex"
+
 	case dax.BaseTypeIDSet, dax.BaseTypeStringSet:
-		if f.Options.TimeQuantum != "" {
-			return "time"
-		}
 		return "set"
+
+	case dax.BaseTypeIDSetQ, dax.BaseTypeStringSetQ:
+		return "time"
+
 	default:
 		return string(f.Type)
 	}
@@ -449,15 +451,13 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) {
 			OptFieldTypeMutex(cacheType, cacheSize),
 		)
 	case dax.BaseTypeIDSet:
-		if fld.Options.TimeQuantum != "" {
-			opts = append(opts,
-				OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()),
-			)
-		} else {
-			opts = append(opts,
-				OptFieldTypeSet(cacheType, cacheSize),
-			)
-		}
+		opts = append(opts,
+			OptFieldTypeSet(cacheType, cacheSize),
+		)
+	case dax.BaseTypeIDSetQ:
+		opts = append(opts,
+			OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()),
+		)
 	case dax.BaseTypeInt:
 		opts = append(opts,
 			OptFieldTypeInt(fld.Options.Min.ToInt64(0), fld.Options.Max.ToInt64(0)),
@@ -468,17 +468,15 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) {
 			OptFieldKeys(),
 		)
 	case dax.BaseTypeStringSet:
-		if fld.Options.TimeQuantum != "" {
-			opts = append(opts,
-				OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()),
-				OptFieldKeys(),
-			)
-		} else {
-			opts = append(opts,
-				OptFieldTypeSet(cacheType, cacheSize),
-				OptFieldKeys(),
-			)
-		}
+		opts = append(opts,
+			OptFieldTypeSet(cacheType, cacheSize),
+			OptFieldKeys(),
+		)
+	case dax.BaseTypeStringSetQ:
+		opts = append(opts,
+			OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()),
+			OptFieldKeys(),
+		)
 	case dax.BaseTypeTimestamp:
 		opts = append(opts,
 			OptFieldTypeTimestamp(fld.Options.Epoch, fld.Options.TimeUnit),

sql3/errors.go

@@ -37,6 +37,7 @@ const (
 	ErrIntOrDecimalOrTimestampOrStringExpressionExpected errors.Code = "ErrIntOrDecimalOrTimestampOrStringExpressionExpected"
 	ErrStringExpressionExpected                          errors.Code = "ErrStringExpressionExpected"
 	ErrSetExpressionExpected                             errors.Code = "ErrSetExpressionExpected"
+	ErrTimeQuantumExpressionExpected                     errors.Code = "ErrTimeQuantumExpressionExpected"
 	ErrSingleRowExpected                                 errors.Code = "ErrSingleRowExpected"
 
 	// type related errors
@@ -58,7 +59,8 @@ const (
 	ErrInvalidColumnInFilterExpression  errors.Code = "ErrInvalidColumnInFilterExpression"
 	ErrInvalidTypeInFilterExpression    errors.Code = "ErrInvalidTypeInFilterExpression"
 
-	ErrTypeAssignmentIncompatible errors.Code = "ErrTypeAssignmentIncompatible"
+	ErrTypeAssignmentIncompatible              errors.Code = "ErrTypeAssignmentIncompatible"
+	ErrTypeAssignmentToTimeQuantumIncompatible errors.Code = "ErrTypeAssignmentToTimeQuantumIncompatible"
 
 	ErrInvalidUngroupedColumnReference         errors.Code = "ErrInvalidUngroupedColumnReference"
 	ErrInvalidUngroupedColumnReferenceInHaving errors.Code = "ErrInvalidUngroupedColumnReferenceInHaving"
@@ -132,6 +134,10 @@ const (
 	ErrValueOutOfRange          errors.Code = "ErrValueOutOfRange"
 	ErrStringLengthMismatch     errors.Code = "ErrStringLengthMismatch"
 	ErrUnexpectedTypeConversion errors.Code = "ErrUnexpectedTypeConversion"
+
+	// time quantum function eval
+	ErrQRangeFromAndToTimeCannotBeBothNull errors.Code = "ErrQRangeFromAndToTimeCannotBeBothNull"
+	ErrQRangeInvalidUse                    errors.Code = "ErrQRangeInvalidUse"
 )
 
 func NewErrDuplicateColumn(line int, col int, column string) error {
@@ -235,6 +241,13 @@ func NewErrInvalidTypeCoercion(line, col int, from, to string) error {
 	)
 }
 
+func NewErrTypeAssignmentToTimeQuantumIncompatible(line, col int, type1 string) error {
+	return errors.New(
+		ErrTypeAssignmentToTimeQuantumIncompatible,
+		fmt.Sprintf("[%d:%d] an expression of type '%s' cannot be assigned to a timequantum", line, col, type1),
+	)
+}
+
 func NewErrLiteralExpected(line, col int) error {
 	return errors.New(
 		ErrLiteralExpected,
@@ -445,6 +458,13 @@ func NewErrSetExpressionExpected(line, col int) error {
 	)
 }
 
+func NewErrTimeQuantumExpressionExpected(line, col int) error {
+	return errors.New(
+		ErrTimeQuantumExpressionExpected,
+		fmt.Sprintf("[%d:%d] time quantum expression expected", line, col),
+	)
+}
+
 func NewErrSingleRowExpected(line, col int) error {
 	return errors.New(
 		ErrSingleRowExpected,
@@ -816,3 +836,19 @@ func NewErrUnexpectedTypeConversion(line, col int, val interface{}) error {
 		NewErrInternalf("unexpected type conversion %T", val).Error(),
 	)
 }
+
+// time quantum function evaluation
+
+func NewErrQRangeFromAndToTimeCannotBeBothNull(line, col int) error {
+	return errors.New(
+		ErrQRangeFromAndToTimeCannotBeBothNull,
+		fmt.Sprintf("[%d:%d] calling ranqeq() 'from' and 'to' parameters cannot both be null", line, col),
+	)
+}
+
+func NewErrQRangeInvalidUse(line, col int) error {
+	return errors.New(
+		ErrQRangeInvalidUse,
+		fmt.Sprintf("[%d:%d] calling ranqeq() usage invalid", line, col),
+	)
+}

sql3/parser/astdatatype.go

@@ -13,9 +13,11 @@ func IsValidTypeName(typeName string) bool {
 		dax.BaseTypeDecimal,
 		dax.BaseTypeID,
 		dax.BaseTypeIDSet,
+		dax.BaseTypeIDSetQ,
 		dax.BaseTypeInt,
 		dax.BaseTypeString,
 		dax.BaseTypeStringSet,
+		dax.BaseTypeStringSetQ,
 		dax.BaseTypeTimestamp:
 		return true
 	default:
@@ -238,7 +240,6 @@ func (*DataTypeIDSet) TypeInfo() map[string]interface{} {
 	return nil
 }
 
-// TODO (pok) should time quantum be it's own type and not a constraint?
 type DataTypeIDSetQuantum struct {
 }
 
@@ -247,7 +248,7 @@ func NewDataTypeIDSetQuantum() *DataTypeIDSetQuantum {
 }
 
 func (*DataTypeIDSetQuantum) BaseTypeName() string {
-	return dax.BaseTypeIDSet
+	return dax.BaseTypeIDSetQ
 }
 
 func (dt *DataTypeIDSetQuantum) TypeDescription() string {
@@ -323,7 +324,7 @@ func NewDataTypeStringSetQuantum() *DataTypeStringSetQuantum {
 }
 
 func (*DataTypeStringSetQuantum) BaseTypeName() string {
-	return dax.BaseTypeStringSet
+	return dax.BaseTypeStringSetQ
 }
 
 func (dt *DataTypeStringSetQuantum) TypeDescription() string {

sql3/planner/compilecreatetable.go

@@ -196,19 +196,17 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column
 	switch strings.ToLower(typeName) {
 	case dax.BaseTypeBool:
 		column.fos = append(column.fos, pilosa.OptFieldTypeBool())
-	case dax.BaseTypeDecimal:
 
+	case dax.BaseTypeDecimal:
 		// if we don't have a scale, it's an error
 		if col.Type.Scale == nil {
 			return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column)
 		}
-
 		// get the scale value
 		scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64)
 		if err != nil {
 			return nil, err
 		}
-
 		// Adjust min/max to fit within the scaled min/max.
 		scaledMin, scaledMax := pql.MinMax(scale)
 		if scaledMax.LessThan(max) {
@@ -217,30 +215,35 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column
 		if scaledMin.GreaterThan(min) {
 			min = scaledMin
 		}
-
 		column.fos = append(column.fos, pilosa.OptFieldTypeDecimal(scale, min, max))
+
 	case dax.BaseTypeID:
 		column.fos = append(column.fos, pilosa.OptFieldTypeMutex(cacheType, cacheSize))
+
 	case dax.BaseTypeIDSet:
-		if timeQuantum != "" {
-			column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl))
-		} else {
-			column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize))
-		}
+		column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize))
+
+	case dax.BaseTypeIDSetQ:
+		column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl))
+
 	case dax.BaseTypeInt:
 		column.fos = append(column.fos, pilosa.OptFieldTypeInt(min.ToInt64(0), max.ToInt64(0)))
+
 	case dax.BaseTypeString:
 		column.fos = append(column.fos, pilosa.OptFieldTypeMutex(cacheType, cacheSize))
 		column.fos = append(column.fos, pilosa.OptFieldKeys())
+
 	case dax.BaseTypeStringSet:
-		if timeQuantum != "" {
-			column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl))
-		} else {
-			column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize))
-		}
+		column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize))
 		column.fos = append(column.fos, pilosa.OptFieldKeys())
+
+	case dax.BaseTypeStringSetQ:
+		column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl))
+		column.fos = append(column.fos, pilosa.OptFieldKeys())
+
 	case dax.BaseTypeTimestamp:
 		column.fos = append(column.fos, pilosa.OptFieldTypeTimestamp(epoch, timeUnit))
+
 	}
 	return column, nil
 }
@@ -392,8 +395,8 @@ func (p *ExecutionPlanner) analyzeColumn(typeName string, col *parser.ColumnDefi
 			handledConstraints[parser.TIMEUNIT] = struct{}{}
 
 		case *parser.TimeQuantumConstraint:
-			//make sure we have a set type
-			if !(strings.EqualFold(typeName, dax.BaseTypeStringSet) || strings.EqualFold(typeName, dax.BaseTypeIDSet)) {
+			//make sure we have one of the time quantum types
+			if !(strings.EqualFold(typeName, dax.BaseTypeStringSetQ) || strings.EqualFold(typeName, dax.BaseTypeIDSetQ)) {
 				return sql3.NewErrBadColumnConstraint(col.Name.NamePos.Line, col.Name.NamePos.Column, "TIMEQUANTUM", typeName)
 			}
 			//check the type of the expression

sql3/planner/compileselect.go

@@ -573,35 +573,7 @@ func (p *ExecutionPlanner) analyzeSource(ctx context.Context, source parser.Sour
 		return source, nil
 
 	case *parser.TableValuedFunction:
-		// check it actually is a table valued function - we only support one right now; subtable()
-		switch strings.ToUpper(source.Name.Name) {
-		case "SUBTABLE":
-			_, err := p.analyzeCallExpression(ctx, source.Call, scope)
-			if err != nil {
-				return nil, err
-			}
-
-			tvfResultType, ok := source.Call.ResultDataType.(*parser.DataTypeSubtable)
-			if !ok {
-				return nil, sql3.NewErrInternalf("unexepected tvf return type")
-			}
-
-			// populate the output columns from the source
-			for idx, member := range tvfResultType.Columns {
-				soc := &parser.SourceOutputColumn{
-					TableName:   "", // TODO (pok) use the tq column actually referenced as the table name
-					ColumnName:  member.Name,
-					ColumnIndex: idx,
-					Datatype:    member.DataType,
-				}
-				source.OutputColumns = append(source.OutputColumns, soc)
-			}
-
-		default:
-			return nil, sql3.NewErrInternalf("table valued function expected")
-		}
-
-		return source, nil
+		return nil, sql3.NewErrInternalf("table valued function expected")
 
 	case *parser.SelectStatement:
 		expr, err := p.analyzeSelectStatement(ctx, source)

sql3/planner/expression.go

@@ -1576,6 +1576,9 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er
 		return n.EvaluateToTimestamp(currentRow)
 	case "STR":
 		return n.EvaluateStr(currentRow)
+		// time quantum functions
+	case "RANGEQ":
+		return n.EvaluateRangeQ(currentRow)
 	default:
 		return nil, sql3.NewErrInternalf("unhandled function name '%s'", n.name)
 	}
@@ -2063,6 +2066,19 @@ func (n *stringLiteralPlanExpression) WithChildren(children ...types.PlanExpress
 	return n, nil
 }
 
+func (expr *stringLiteralPlanExpression) ConvertToTimestamp() *time.Time {
+	//try to coerce to a date
+	if tm, err := time.ParseInLocation(time.RFC3339Nano, expr.value, time.UTC); err == nil {
+		return &tm
+	} else if tm, err := time.ParseInLocation(time.RFC3339, expr.value, time.UTC); err == nil {
+		return &tm
+	} else if tm, err := time.ParseInLocation("2006-01-02", expr.value, time.UTC); err == nil {
+		return &tm
+	} else {
+		return nil
+	}
+}
+
 // castPlanExpressionis a cast op
 type castPlanExpression struct {
 	lhs        types.PlanExpression