index: support sorted index, add basic method Floor and Ceiling for treemap

cmd/protoc-gen-cpp-tableau-loader/embed/hub.pc.h

@@ -3,10 +3,12 @@
 #include <google/protobuf/util/json_util.h>
 #include <tableau/protobuf/tableau.pb.h>
 
+#include <algorithm>
 #include <chrono>
 #include <cstddef>
 #include <ctime>
 #include <functional>
+#include <map>
 #include <mutex>
 #include <string>
 #include <thread>

cmd/protoc-gen-cpp-tableau-loader/index.go

@@ -141,6 +141,30 @@ func genOneCppIndexLoader(g *protogen.GeneratedFile, depth int, descriptor *inde
 			generateOneCppMulticolumnIndex(g, depth, descriptor, parentDataName, keys)
 		}
 	}
+	if depth == 1 && len(descriptor.KeyFields) != 0 {
+		g.P(strings.Repeat("  ", depth), "for (auto&& item : ", indexContainerName, ") {")
+		g.P(strings.Repeat("  ", depth+1), "std::sort(item.second.begin(), item.second.end(),")
+		g.P(strings.Repeat("  ", depth+6), "[](const ", helper.ParseCppClassType(descriptor.MD), "* a, const ", helper.ParseCppClassType(descriptor.MD), "* b) {")
+		for i, field := range descriptor.KeyFields {
+			fieldName := ""
+			for i, leveledFd := range field.LeveledFDList {
+				accessOperator := "."
+				if i == 0 {
+					accessOperator = "->"
+				}
+				fieldName += accessOperator + helper.ParseIndexFieldName(leveledFd) + "()"
+			}
+			if i == len(descriptor.KeyFields)-1 {
+				g.P(strings.Repeat("  ", depth+7), "return a", fieldName, " < b", fieldName, ";")
+			} else {
+				g.P(strings.Repeat("  ", depth+7), "if (a", fieldName, " != b", fieldName, ") {")
+				g.P(strings.Repeat("  ", depth+8), "return a", fieldName, " < b", fieldName, ";")
+				g.P(strings.Repeat("  ", depth+7), "}")
+			}
+		}
+		g.P(strings.Repeat("  ", depth+6), "});")
+		g.P(strings.Repeat("  ", depth), "}")
+	}
 }
 
 func generateOneCppMulticolumnIndex(g *protogen.GeneratedFile, depth int, descriptor *index.IndexDescriptor, parentDataName string, keys []string) []string {

cmd/protoc-gen-go-tableau-loader/index.go

@@ -98,6 +98,25 @@ func genOneIndexLoader(gen *protogen.Plugin, g *protogen.GeneratedFile, depth in
 			generateOneMulticolumnIndex(gen, g, depth, descriptor, parentDataName, keys)
 		}
 	}
+	if depth == 1 && len(descriptor.KeyFields) != 0 {
+		g.P("for _, item := range x.", indexContainerName, " {")
+		g.P(sortPackage.Ident("Slice"), "(item, func(i, j int) bool {")
+		for i, field := range descriptor.KeyFields {
+			fieldName := ""
+			for _, leveledFd := range field.LeveledFDList {
+				fieldName += ".Get" + helper.ParseIndexFieldName(gen, leveledFd) + "()"
+			}
+			if i == len(descriptor.KeyFields)-1 {
+				g.P("return item[i]", fieldName, " < item[j]", fieldName)
+			} else {
+				g.P("if item[i]", fieldName, " != item[j]", fieldName, " {")
+				g.P("return item[i]", fieldName, " < item[j]", fieldName)
+				g.P("}")
+			}
+		}
+		g.P("})")
+		g.P("}")
+	}
 }
 
 func generateOneMulticolumnIndex(gen *protogen.Plugin, g *protogen.GeneratedFile,

cmd/protoc-gen-go-tableau-loader/messager.go

@@ -22,6 +22,7 @@ const (
 	treeMapPackage = protogen.GoImportPath("github.com/tableauio/loader/pkg/treemap")
 	pairPackage    = protogen.GoImportPath("github.com/tableauio/loader/pkg/pair")
 	timePackage    = protogen.GoImportPath("time")
+	sortPackage    = protogen.GoImportPath("sort")
 	protoPackage   = protogen.GoImportPath("google.golang.org/protobuf/proto")
 )
 

internal/index/descriptor.go

@@ -13,9 +13,11 @@ import (
 type IndexDescriptor struct {
 	*Index
 
-	MD     protoreflect.MessageDescriptor // deepest level message descriptor
-	Name   string                         // index name, e.g.: name of (ID, Name)@ItemInfo is "ItemInfo"
-	Fields []*LevelField                  // index fields in the same struct (protobuf message), refer to the deepest level message's Fields.
+	MD   protoreflect.MessageDescriptor // deepest level message descriptor
+	Name string                         // index name, e.g.: name of (ID, Name)@ItemInfo is "ItemInfo"
+
+	Fields    []*LevelField // index fields in the same struct (protobuf message), refer to the deepest level message's Fields.
+	KeyFields []*LevelField // key fields in the same struct (protobuf message), refer to the deepest level message's Fields.
 
 	LevelMessage *LevelMessage // message hierarchy to the deepest level message which contains all index fields.
 }
@@ -59,10 +61,14 @@ type LevelMessage struct {
 	// Deepest level message fields corresponding to index fields
 	// NOTE: Fields is valid only when this level is the deepest level.
 	Fields []*LevelField
+
+	// Deepest level message fields corresponding to key fields
+	// NOTE: Fields is valid only when this level is the deepest level.
+	KeyFields []*LevelField
 }
 
-// ParseRecursively parses multi-column index related info.
-func ParseRecursively(gen *protogen.Plugin, cols []string, prefix string, md protoreflect.MessageDescriptor) *LevelMessage {
+// parseRecursively parses multi-column index related info.
+func parseRecursively(gen *protogen.Plugin, cols, keys []string, prefix string, md protoreflect.MessageDescriptor) *LevelMessage {
 	levelInfo := &LevelMessage{}
 	for i := 0; i < md.Fields().Len(); i++ {
 		fd := md.Fields().Get(i)
@@ -71,27 +77,28 @@ func ParseRecursively(gen *protogen.Plugin, cols []string, prefix string, md pro
 		fdOpts := proto.GetExtension(opts, tableaupb.E_Field).(*tableaupb.FieldOptions)
 		fieldOptName := fdOpts.GetName()
 		if fd.IsMap() && fd.MapValue().Kind() == protoreflect.MessageKind {
-			levelInfo.NextLevel = ParseRecursively(gen, cols, prefix+fieldOptName, fd.MapValue().Message())
+			levelInfo.NextLevel = parseRecursively(gen, cols, keys, prefix+fieldOptName, fd.MapValue().Message())
 			if levelInfo.NextLevel != nil {
 				levelInfo.FD = fd
 				return levelInfo
 			}
 		} else if fd.IsList() && fd.Kind() == protoreflect.MessageKind {
-			levelInfo.NextLevel = ParseRecursively(gen, cols, prefix+fieldOptName, fd.Message())
+			levelInfo.NextLevel = parseRecursively(gen, cols, keys, prefix+fieldOptName, fd.Message())
 			if levelInfo.NextLevel != nil {
 				levelInfo.FD = fd
 				return levelInfo
 			}
 		}
 	}
-	levelInfo.Fields = ParseInSameLevel(gen, cols, prefix, md, nil)
+	levelInfo.Fields = parseInSameLevel(gen, cols, prefix, md, nil)
+	levelInfo.KeyFields = parseInSameLevel(gen, keys, prefix, md, nil)
 	if len(levelInfo.Fields) != 0 {
 		return levelInfo
 	}
 	return nil
 }
 
-func ParseInSameLevel(gen *protogen.Plugin, cols []string, prefix string, md protoreflect.MessageDescriptor, leveledFDList []protoreflect.FieldDescriptor) []*LevelField {
+func parseInSameLevel(gen *protogen.Plugin, cols []string, prefix string, md protoreflect.MessageDescriptor, leveledFDList []protoreflect.FieldDescriptor) []*LevelField {
 	levelFields := []*LevelField{}
 	for i := 0; i < md.Fields().Len(); i++ {
 		fd := md.Fields().Get(i)
@@ -110,7 +117,7 @@ func ParseInSameLevel(gen *protogen.Plugin, cols []string, prefix string, md pro
 				break
 			} else if fd.Kind() == protoreflect.MessageKind && strings.HasPrefix(columnName, prefix+fieldOptName) {
 				levelFields = append(levelFields,
-					ParseInSameLevel(
+					parseInSameLevel(
 						gen, cols, prefix+fieldOptName, fd.Message(),
 						append(leveledFDList, fd),
 					)...,
@@ -128,7 +135,7 @@ func ParseIndexDescriptor(gen *protogen.Plugin, md protoreflect.MessageDescripto
 		if len(index.Cols) == 0 {
 			continue
 		}
-		levelInfo := ParseRecursively(gen, index.Cols, "", md)
+		levelInfo := parseRecursively(gen, index.Cols, index.Keys, "", md)
 		if levelInfo == nil {
 			continue
 		}
@@ -146,6 +153,7 @@ func ParseIndexDescriptor(gen *protogen.Plugin, md protoreflect.MessageDescripto
 			deepestLevelMessage = deepestLevelMessage.NextLevel
 		}
 		descriptor.Fields = deepestLevelMessage.Fields
+		descriptor.KeyFields = deepestLevelMessage.KeyFields
 		descriptor.Name = index.Name
 		if descriptor.Name == "" {
 			// use index field's parent message name if not set.

internal/index/index.go

@@ -1,6 +1,7 @@
 package index
 
 import (
+	"regexp"
 	"strings"
 
 	"github.com/tableauio/tableau/proto/tableaupb"
@@ -9,21 +10,27 @@ import (
 	"google.golang.org/protobuf/types/descriptorpb"
 )
 
-// Single-column index:
-//	- ID
-//	- ID@Item
-//
-// Multi-column index (composite index):
-//  - (ID, Name)
-//  - (ID, Name)@Item
+var indexRegexp *regexp.Regexp
 
-const colAndNameSep = "@"
-const multiColSep = ","
-const multiColGroupCutset = "()"
+func init() {
+	// Single-column index:
+	//  - ID
+	//  - ID@Item
+	//  - ID<Key>@Item
+	//  - ID<Key, Key2>@Item
+	//
+	// Multi-column index (composite index):
+	//  - (ID, Name)
+	//  - (ID, Name)@Item
+	//  - (ID, Name)<Key>@Item
+	//  - (ID, Name)<Key1, Key2>@Item
+	indexRegexp = regexp.MustCompile(`^(?P<cols>\([^)]+\)|[^<@]+)?(<(?P<keys>[^>]+)>)?(@(?P<name>.+))?$`)
+}
 
 type Index struct {
 	Cols []string // column names in CamelCase (single-column or multi-column)
 	Name string   // index name in CamelCase
+	Keys []string // key names in CamelCase (single-column or multi-column)
 }
 
 func (index *Index) String() string {
@@ -34,10 +41,12 @@ func (index *Index) String() string {
 			syntax += ","
 		}
 	}
-
 	if len(index.Cols) > 1 {
 		syntax = "(" + syntax + ")"
 	}
+	if len(index.Keys) != 0 {
+		syntax += "<" + strings.Join(index.Keys, ",") + ">"
+	}
 	if index.Name != "" {
 		syntax += "@" + index.Name
 	}
@@ -51,33 +60,50 @@ func parseWSOptionIndex(md protoreflect.MessageDescriptor) []*Index {
 	return parseIndexFrom(wsOpts.Index)
 }
 
-func parseColsFrom(multiColGroup string) []string {
-	trimmedStr := strings.Trim(multiColGroup, multiColGroupCutset)
-	cols := strings.Split(trimmedStr, multiColSep)
-	for i, col := range cols {
-		cols[i] = strings.TrimSpace(col)
-	}
-	return cols
-}
-
 func parseIndex(indexStr string) *Index {
-	var cols []string
-	var name string
-	splits := strings.SplitN(indexStr, colAndNameSep, 2)
-	switch len(splits) {
-	case 1:
-		cols = parseColsFrom(splits[0])
-	case 2:
-		cols = parseColsFrom(splits[0])
-		name = splits[1]
-	default:
+	index := &Index{}
+	matches := indexRegexp.FindStringSubmatch(indexStr)
+	// Extract columns
+	if cols := matches[indexRegexp.SubexpIndex("cols")]; cols != "" {
+		if strings.HasPrefix(cols, "(") && strings.HasSuffix(cols, ")") {
+			// Multi-column index
+			cols = cols[1 : len(cols)-1]
+			splitCols := strings.Split(cols, ",")
+			if len(splitCols) <= 1 {
+				return nil
+			}
+			for _, col := range splitCols {
+				col = strings.TrimSpace(col)
+				if col != "" {
+					index.Cols = append(index.Cols, col)
+				}
+			}
+		} else {
+			// Single-column index
+			if len(strings.Split(cols, ",")) > 1 {
+				return nil
+			}
+			col := strings.TrimSpace(cols)
+			if col != "" {
+				index.Cols = append(index.Cols, col)
+			}
+		}
+	}
+	if len(index.Cols) == 0 {
 		return nil
 	}
-
-	return &Index{
-		Cols: cols,
-		Name: name,
+	// Extract keys
+	if keys := matches[indexRegexp.SubexpIndex("keys")]; keys != "" {
+		index.Keys = strings.Split(keys, ",")
+		for i, key := range index.Keys {
+			index.Keys[i] = strings.TrimSpace(key)
+		}
+	}
+	// Extract name
+	if name := matches[indexRegexp.SubexpIndex("name")]; name != "" {
+		index.Name = name
 	}
+	return index
 }
 
 func parseIndexFrom(indexList []string) []*Index {