diff --git a/README.md b/README.md index 574ccf52..8003233b 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ Stencil is a schema registry that provides schema mangement and validation to en Discover why users choose Stencil as their main schema registry - **Version history** Stencil stores versioned history of proto descriptor file on specified namespace and name -- **Backward compatibility** enforce backward compatability check on upload by default -- **Flexbility** ability to skip some of the backward compatability checks while upload +- **Backward compatibility** enforce backward compatibility check on upload by default +- **Flexbility** ability to skip some of the backward compatibility checks while upload - **Descriptor fetch** ability to download proto descriptor files - **Metadata** provides metadata API to retrieve latest version number given a name and namespace - **Clients in multiple languages** Stencil provides clients in GO, Java, JS languages to interact with Stencil server and deserialize messages using dynamic schema @@ -90,7 +90,7 @@ Stencil has three major components. Server, CLI and clients. Stencil server and **Server** -Stencil server provides a way to store and fetch schemas and enforce compatability rules. Run `stencil server --help` to see instructions to manage Stencil server. +Stencil server provides a way to store and fetch schemas and enforce compatibility rules. Run `stencil server --help` to see instructions to manage Stencil server. Stencil server also provides a fully-featured GRPC and HTTP API to interact with Stencil server. Both APIs adheres to a set of standards that are rigidly followed. Please refer to [proton](https://github.com/odpf/proton/tree/main/odpf/stencil/v1beta1) for GRPC API definitions. diff --git a/cmd/cdk.go b/cmd/cdk.go new file mode 100644 index 00000000..50b779f1 --- /dev/null +++ b/cmd/cdk.go @@ -0,0 +1,24 @@ +package cmd + +var dict = map[string]string{ + "COMPATIBILITY_BACKWARD": "backward", + "COMPATIBILITY_FORWARD": "forward", + "COMPATIBILITY_FULL": "full", + "FORMAT_PROTOBUF": "protobuf", + "FORMAT_JSON": "json", + "FORMAT_AVRO": "avro", +} + +var ( + formats = []string{ + "FORMAT_JSON", + "FORMAT_PROTOBUF", + "FORMAT_AVRO", + } + + comps = []string{ + "COMPATIBILITY_BACKWARD", + "COMPATIBILITY_FORWARD", + "COMPATIBILITY_FULL", + } +) diff --git a/cmd/check.go b/cmd/check.go new file mode 100644 index 00000000..25d2a164 --- /dev/null +++ b/cmd/check.go @@ -0,0 +1,78 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/printer" + "github.com/odpf/salt/term" + stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" + "github.com/spf13/cobra" + "google.golang.org/grpc/status" +) + +func checkSchemaCmd() *cobra.Command { + var host, comp, file, namespaceID string + var req stencilv1beta1.CheckCompatibilityRequest + + cmd := &cobra.Command{ + Use: "check ", + Args: cobra.ExactArgs(1), + Short: "Check schema compatibility", + Long: heredoc.Doc(` + Check schema compatibility of a local schema + against a remote schema(against) on stencil server.`), + Example: heredoc.Doc(` + $ stencil schema check -n odpf -c COMPATIBILITY_BACKWARD -F ./booking.desc + `), + RunE: func(cmd *cobra.Command, args []string) error { + spinner := printer.Spin("") + defer spinner.Stop() + + fileData, err := os.ReadFile(file) + if err != nil { + return err + } + + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + schemaID := args[0] + + req.Data = fileData + req.NamespaceId = namespaceID + req.SchemaId = schemaID + req.Compatibility = stencilv1beta1.Schema_Compatibility(stencilv1beta1.Schema_Compatibility_value[comp]) + + _, err = client.CheckCompatibility(context.Background(), &req) + if err != nil { + errStatus := status.Convert(err) + return errors.New(errStatus.Message()) + } + + spinner.Stop() + fmt.Printf("\n%s Schema is compatible.\n", term.Green(term.SuccessIcon())) + return nil + }, + } + + cmd.Flags().StringVar(&host, "host", "", "Server host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + + cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Parent namespace ID") + cmd.MarkFlagRequired("namespace") + + cmd.Flags().StringVarP(&comp, "comp", "c", "", "Schema compatibility") + cmd.MarkFlagRequired("comp") + + cmd.Flags().StringVarP(&file, "file", "F", "", "Path to the schema file") + cmd.MarkFlagRequired("file") + + return cmd +} diff --git a/cmd/create.go b/cmd/create.go new file mode 100644 index 00000000..7b977c32 --- /dev/null +++ b/cmd/create.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/printer" + "github.com/odpf/salt/term" + stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" + "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func createSchemaCmd() *cobra.Command { + var host, format, comp, file, namespaceID string + var req stencilv1beta1.CreateSchemaRequest + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a schema", + Args: cobra.ExactArgs(1), + Example: heredoc.Doc(` + $ stencil schema create booking -n odpf –F booking.json + $ stencil schema create booking -n odpf -f FORMAT_JSON –c COMPATIBILITY_BACKWARD –F ./booking.json + `), + RunE: func(cmd *cobra.Command, args []string) error { + fileData, err := os.ReadFile(file) + if err != nil { + return err + } + req.Data = fileData + + spinner := printer.Spin("") + defer spinner.Stop() + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + schemaID := args[0] + req.NamespaceId = namespaceID + req.SchemaId = schemaID + req.Format = stencilv1beta1.Schema_Format(stencilv1beta1.Schema_Format_value[format]) + req.Compatibility = stencilv1beta1.Schema_Compatibility(stencilv1beta1.Schema_Compatibility_value[comp]) + + res, err := client.CreateSchema(context.Background(), &req) + if err != nil { + errStatus := status.Convert(err) + if codes.AlreadyExists == errStatus.Code() { + fmt.Printf("\n%s Schema with id '%s' already exist.\n", term.FailureIcon(), args[0]) + return nil + } + return errors.New(errStatus.Message()) + } + + id := res.GetId() + + spinner.Stop() + fmt.Printf("\n%s Created schema with id %s.\n", term.Green(term.SuccessIcon()), term.Cyan(id)) + return nil + }, + } + + cmd.Flags().StringVar(&host, "host", "", "Stencil host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + + cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Namespace ID") + cmd.MarkFlagRequired("namespace") + + cmd.Flags().StringVarP(&format, "format", "f", "", "Schema format") + + cmd.Flags().StringVarP(&comp, "comp", "c", "", "Schema compatibility") + + cmd.Flags().StringVarP(&file, "file", "F", "", "Path to the schema file") + cmd.MarkFlagRequired("file") + + return cmd +} diff --git a/cmd/delete.go b/cmd/delete.go new file mode 100644 index 00000000..a4faf819 --- /dev/null +++ b/cmd/delete.go @@ -0,0 +1,72 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/printer" + stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" + "github.com/spf13/cobra" +) + +func deleteSchemaCmd() *cobra.Command { + var host, namespaceID string + var req stencilv1beta1.DeleteSchemaRequest + var reqVer stencilv1beta1.DeleteVersionRequest + var version int32 + + cmd := &cobra.Command{ + Use: "delete ", + Short: "Delete a schema", + Args: cobra.ExactArgs(1), + Example: heredoc.Doc(` + $ stencil schema delete booking -n odpf + `), + RunE: func(cmd *cobra.Command, args []string) error { + spinner := printer.Spin("") + defer spinner.Stop() + + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + schemaID := args[0] + + if version == 0 { + req.NamespaceId = namespaceID + req.SchemaId = schemaID + + _, err = client.DeleteSchema(context.Background(), &req) + if err != nil { + return err + } + } else { + reqVer.NamespaceId = namespaceID + reqVer.SchemaId = schemaID + reqVer.VersionId = version + + _, err = client.DeleteVersion(context.Background(), &reqVer) + if err != nil { + return err + } + } + + spinner.Stop() + fmt.Printf("Schema successfully deleted") + return nil + }, + } + + cmd.Flags().StringVar(&host, "host", "", "Stencil host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + + cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Parent namespace ID") + cmd.MarkFlagRequired("namespace") + + cmd.Flags().Int32VarP(&version, "version", "v", 0, "Particular version to be deleted") + + return cmd +} diff --git a/cmd/diff.go b/cmd/diff.go new file mode 100644 index 00000000..738a15a5 --- /dev/null +++ b/cmd/diff.go @@ -0,0 +1,159 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/printer" + stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" + "github.com/spf13/cobra" + "github.com/yudai/gojsondiff" + "github.com/yudai/gojsondiff/formatter" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/descriptorpb" +) + +func diffSchemaCmd() *cobra.Command { + var fullname string + var host string + var namespace string + var earlierVersion int32 + var laterVersion int32 + + var schemaFetcher = func(req *stencilv1beta1.GetSchemaRequest, client stencilv1beta1.StencilServiceClient) ([]byte, error) { + res, err := client.GetSchema(context.Background(), req) + if err != nil { + return nil, err + } + return res.Data, nil + } + var protoSchemaFetcher = func(req *stencilv1beta1.GetSchemaRequest, client stencilv1beta1.StencilServiceClient) ([]byte, error) { + if fullname == "" { + return nil, fmt.Errorf("fullname flag is mandator for FORMAT_PROTO") + } + res, err := client.GetSchema(context.Background(), req) + if err != nil { + return nil, err + } + fds := &descriptorpb.FileDescriptorSet{} + if err := proto.Unmarshal(res.Data, fds); err != nil { + return nil, fmt.Errorf("descriptor set file is not valid. %w", err) + } + files, err := protodesc.NewFiles(fds) + if err != nil { + return nil, fmt.Errorf("file is not fully contained descriptor file. hint: generate file descriptorset with --include_imports option. %w", err) + } + desc, err := files.FindDescriptorByName(protoreflect.FullName(fullname)) + if err != nil { + return nil, fmt.Errorf("unable to find message. %w", err) + } + mDesc, ok := desc.(protoreflect.MessageDescriptor) + if !ok { + return nil, fmt.Errorf("not a message desc") + } + jsonByte, err := protojson.Marshal(protodesc.ToDescriptorProto(mDesc)) + if err != nil { + return nil, fmt.Errorf("fail to convert json. %w", err) + } + return jsonByte, nil + } + + cmd := &cobra.Command{ + Use: "diff", + Short: "Diff(s) of two schema versions", + Args: cobra.ExactArgs(1), + Example: heredoc.Doc(` + $ stencil schema diff booking -n=odpf --later-version=2 --earlier-version=1 --fullname= + `), + RunE: func(cmd *cobra.Command, args []string) error { + spinner := printer.Spin("") + defer spinner.Stop() + + schemaID := args[0] + + metaReq := stencilv1beta1.GetSchemaMetadataRequest{ + NamespaceId: namespace, + SchemaId: schemaID, + } + eReq := &stencilv1beta1.GetSchemaRequest{ + NamespaceId: namespace, + SchemaId: schemaID, + VersionId: earlierVersion, + } + lReq := &stencilv1beta1.GetSchemaRequest{ + NamespaceId: namespace, + SchemaId: schemaID, + VersionId: laterVersion, + } + + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + meta, err := client.GetSchemaMetadata(context.Background(), &metaReq) + if err != nil { + return err + } + + var getSchema = schemaFetcher + if meta.Format == *stencilv1beta1.Schema_FORMAT_PROTOBUF.Enum() { + getSchema = protoSchemaFetcher + } + + eJson, err := getSchema(eReq, client) + if err != nil { + return err + } + + lJson, err := getSchema(lReq, client) + if err != nil { + return err + } + + d, err := gojsondiff.New().Compare(eJson, lJson) + if err != nil { + return err + } + + var placeholder map[string]interface{} + json.Unmarshal(eJson, &placeholder) + config := formatter.AsciiFormatterConfig{ + ShowArrayIndex: true, + Coloring: true, + } + + formatter := formatter.NewAsciiFormatter(placeholder, config) + diffString, err := formatter.Format(d) + if err != nil { + return err + } + + spinner.Stop() + if !d.Modified() { + fmt.Print("No diff!") + return nil + } + fmt.Print(diffString) + + return nil + }, + } + + cmd.Flags().StringVar(&host, "host", "", "Stencil host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "Parent namespace ID") + cmd.MarkFlagRequired("namespace") + cmd.Flags().Int32Var(&earlierVersion, "earlier-version", 0, "Earlier version of the schema") + cmd.MarkFlagRequired("earlier-version") + cmd.Flags().Int32Var(&laterVersion, "later-version", 0, "Later version of the schema") + cmd.MarkFlagRequired("later-version") + cmd.Flags().StringVar(&fullname, "fullname", "", "Only applicable for FORMAT_PROTO. fullname of proto schema eg: odpf.common.v1.Version") + return cmd +} diff --git a/cmd/download.go b/cmd/download.go new file mode 100644 index 00000000..ec8fa290 --- /dev/null +++ b/cmd/download.go @@ -0,0 +1,61 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/printer" + "github.com/odpf/salt/term" + "github.com/spf13/cobra" +) + +func downloadSchemaCmd() *cobra.Command { + var host, output, namespaceID string + var version int32 + var data []byte + + cmd := &cobra.Command{ + Use: "download ", + Short: "Download a schema", + Args: cobra.ExactArgs(1), + Example: heredoc.Doc(` + $ stencil schema download customer -n=odpf --version 1 + `), + RunE: func(cmd *cobra.Command, args []string) error { + spinner := printer.Spin("") + defer spinner.Stop() + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + data, _, err = fetchSchemaAndMeta(client, version, namespaceID, args[0]) + if err != nil { + return err + } + spinner.Stop() + + err = os.WriteFile(output, data, 0666) + if err != nil { + return err + } + + fmt.Printf("%s Schema successfully written to %s\n", term.Green(term.SuccessIcon()), output) + return nil + }, + } + cmd.Flags().StringVar(&host, "host", "", "Stencil host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + + cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Parent namespace ID") + cmd.MarkFlagRequired("namespace") + + cmd.Flags().Int32VarP(&version, "version", "v", 0, "Version of the schema") + + cmd.Flags().StringVarP(&output, "output", "o", "", "Path to the output file") + cmd.MarkFlagRequired("output") + + return cmd +} diff --git a/cmd/edit.go b/cmd/edit.go new file mode 100644 index 00000000..173578e5 --- /dev/null +++ b/cmd/edit.go @@ -0,0 +1,61 @@ +package cmd + +import ( + "context" + "fmt" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/printer" + stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" + "github.com/spf13/cobra" +) + +func editSchemaCmd() *cobra.Command { + var host, comp, namespaceID string + var req stencilv1beta1.UpdateSchemaMetadataRequest + + cmd := &cobra.Command{ + Use: "edit", + Short: "Edit a schema", + Args: cobra.ExactArgs(1), + Example: heredoc.Doc(` + $ stencil schema edit booking -n odpf -c COMPATIBILITY_BACKWARD + `), + RunE: func(cmd *cobra.Command, args []string) error { + spinner := printer.Spin("") + defer spinner.Stop() + + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + schemaID := args[0] + + req.NamespaceId = namespaceID + req.SchemaId = schemaID + req.Compatibility = stencilv1beta1.Schema_Compatibility(stencilv1beta1.Schema_Compatibility_value[comp]) + + _, err = client.UpdateSchemaMetadata(context.Background(), &req) + if err != nil { + return err + } + + spinner.Stop() + fmt.Printf("Schema successfully updated") + return nil + }, + } + + cmd.Flags().StringVar(&host, "host", "", "Server host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + + cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Parent namespace ID") + cmd.MarkFlagRequired("namespace") + + cmd.Flags().StringVarP(&comp, "comp", "c", "", "Schema compatibility") + cmd.MarkFlagRequired("comp") + + return cmd +} diff --git a/cmd/graph.go b/cmd/graph.go new file mode 100644 index 00000000..ef01c7a9 --- /dev/null +++ b/cmd/graph.go @@ -0,0 +1,77 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/stencil/pkg/graph" + stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" + "github.com/spf13/cobra" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" +) + +func graphSchemaCmd() *cobra.Command { + var host, output, namespaceID string + var version int32 + + cmd := &cobra.Command{ + Use: "graph", + Aliases: []string{"g"}, + Short: "View schema dependencies graph", + Args: cobra.ExactArgs(1), + Example: heredoc.Doc(` + $ stencil schema graph booking -n odpf -v 1 -o ./vis.dot + `), + RunE: func(cmd *cobra.Command, args []string) error { + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + schemaID := args[0] + + data, resMetadata, err := fetchSchemaAndMeta(client, version, namespaceID, schemaID) + if err != nil { + return err + } + + format := stencilv1beta1.Schema_Format_name[int32(resMetadata.GetFormat())] + if format != "FORMAT_PROTOBUF" { + fmt.Printf("Graph is not supported for %s", format) + return nil + } + + msg := &descriptorpb.FileDescriptorSet{} + err = proto.Unmarshal(data, msg) + if err != nil { + return fmt.Errorf("invalid file descriptorset file. %w", err) + } + + graph, err := graph.GetProtoFileDependencyGraph(msg) + if err != nil { + return err + } + if err = os.WriteFile(output, []byte(graph.String()), 0666); err != nil { + return err + } + + fmt.Println("Created graph file at", output) + return nil + }, + } + + cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + + cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "provide namespace/group or entity name") + cmd.MarkFlagRequired("namespace") + + cmd.Flags().Int32VarP(&version, "version", "v", 0, "provide version number") + + cmd.Flags().StringVarP(&output, "output", "o", "./proto_vis.dot", "write to .dot file") + + return cmd +} diff --git a/cmd/info.go b/cmd/info.go new file mode 100644 index 00000000..d5acae0b --- /dev/null +++ b/cmd/info.go @@ -0,0 +1,133 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "strconv" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/printer" + "github.com/odpf/salt/term" + stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" + "github.com/spf13/cobra" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func infoSchemaCmd() *cobra.Command { + var host, namespace string + + cmd := &cobra.Command{ + Use: "info ", + Short: "View schema information", + Long: "Display the information about a schema.", + Args: cobra.ExactArgs(1), + Example: heredoc.Doc(` + $ stencil schema info events -n odpf + `), + RunE: func(cmd *cobra.Command, args []string) error { + spinner := printer.Spin("") + defer spinner.Stop() + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + req := stencilv1beta1.GetSchemaMetadataRequest{ + NamespaceId: namespace, + SchemaId: args[0], + } + info, err := client.GetSchemaMetadata(cmd.Context(), &req) + spinner.Stop() + if err != nil { + errStatus, _ := status.FromError(err) + if codes.NotFound == errStatus.Code() { + fmt.Printf("%s Schema with id '%s' not found.\n", term.Red(term.FailureIcon()), args[0]) + return nil + } + return err + } + + fmt.Printf("\n%s\n", term.Blue(args[0])) + fmt.Printf("\n%s\n\n", term.Grey("No description provided")) + fmt.Printf("%s \t %s \n", term.Grey("Namespace:"), namespace) + fmt.Printf("%s \t %s \n", term.Grey("Format:"), dict[info.GetFormat().String()]) + fmt.Printf("%s \t %s \n", term.Grey("Compatibility:"), dict[info.GetCompatibility().String()]) + fmt.Printf("%s \t %s \n\n", term.Grey("Authority:"), dict[info.GetAuthority()]) + return nil + }, + } + + cmd.Flags().StringVar(&host, "host", "", "Stencil host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + + cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "Provide schema namespace") + cmd.MarkFlagRequired("namespace") + + return cmd +} + +func versionSchemaCmd() *cobra.Command { + var host, namespaceID string + var req stencilv1beta1.ListVersionsRequest + + cmd := &cobra.Command{ + Use: "version", + Short: "View versions of a schema", + Args: cobra.ExactArgs(1), + Example: heredoc.Doc(` + $ stencil schema version booking -n odpf + `), + RunE: func(cmd *cobra.Command, args []string) error { + spinner := printer.Spin("") + defer spinner.Stop() + + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + schemaID := args[0] + req.NamespaceId = namespaceID + req.SchemaId = schemaID + + res, err := client.ListVersions(context.Background(), &req) + if err != nil { + return err + } + + versions := res.GetVersions() + spinner.Stop() + + if len(versions) == 0 { + fmt.Printf("No version found for %s in %s", schemaID, namespaceID) + return nil + } + + report := [][]string{} + report = append(report, []string{"VERSION", "CREATED", "MESSAGE"}) + + for _, v := range versions { + report = append(report, []string{ + term.Greenf("#%v", strconv.FormatInt(int64(v), 10)), + "-", + "-", + }) + } + fmt.Printf("\nShowing %[1]d of %[1]d versions for %s\n \n", len(versions), schemaID) + printer.Table(os.Stdout, report) + return nil + }, + } + + cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") + cmd.MarkFlagRequired("host") + + cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "parent namespace ID") + cmd.MarkFlagRequired("namespace") + + return cmd +} diff --git a/cmd/list.go b/cmd/list.go new file mode 100644 index 00000000..771899a4 --- /dev/null +++ b/cmd/list.go @@ -0,0 +1,89 @@ +package cmd + +import ( + "context" + "fmt" + "os" + + "github.com/MakeNowJust/heredoc" + "github.com/odpf/salt/printer" + "github.com/odpf/salt/term" + stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" + "github.com/spf13/cobra" +) + +func listSchemaCmd() *cobra.Command { + var host, namespace string + var req stencilv1beta1.ListSchemasRequest + + cmd := &cobra.Command{ + Use: "list", + Short: "List all schemas", + Long: heredoc.Doc(` + List schemas in a namespace. + `), + Args: cobra.ExactArgs(0), + Example: heredoc.Doc(` + $ stencil schema list -n odpf + `), + RunE: func(cmd *cobra.Command, args []string) error { + spinner := printer.Spin("") + defer spinner.Stop() + + client, cancel, err := createClient(cmd) + if err != nil { + return err + } + defer cancel() + + req.Id = namespace + res, err := client.ListSchemas(context.Background(), &req) + if err != nil { + return err + } + + schemas := res.GetSchemas() + + // TODO(Ravi): List schemas should also handle namespace not found + if len(schemas) == 0 { + spinner.Stop() + fmt.Printf("No schema found in namespace %s\n", namespace) + return nil + } + + report := [][]string{} + index := 1 + report = append(report, []string{ + term.Bold("INDEX"), + term.Bold("NAME"), + term.Bold("FORMAT"), + term.Bold("COMPATIBILITY"), + term.Bold("AUTHORITY"), + }) + for _, s := range schemas { + meta, _ := fetchMeta(client, namespace, s) + c := meta.GetCompatibility().String() + f := meta.GetFormat().String() + a := meta.GetAuthority() + + if a == "" { + a = "-" + } + report = append(report, []string{term.Greenf("#%d", index), s, dict[f], dict[c], a}) + index++ + } + + spinner.Stop() + fmt.Printf("\nShowing %d of %d schemas in %s\n\n", len(schemas), len(schemas), namespace) + printer.Table(os.Stdout, report) + return nil + }, + } + + cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "Namespace ID") + cmd.MarkFlagRequired("namespace") + + cmd.Flags().StringVar(&host, "host", "", "Stencil host address eg: localhost:8000") + + return cmd +} diff --git a/cmd/namespace.go b/cmd/namespace.go index 7eca6f0c..a8e17c62 100644 --- a/cmd/namespace.go +++ b/cmd/namespace.go @@ -64,19 +64,20 @@ func listNamespaceCmd() *cobra.Command { return err } - report := [][]string{} - namespaces := res.GetNamespaces() - spinner.Stop() - fmt.Printf("\nShowing %[1]d of %[1]d namespaces \n \n", len(namespaces)) + if len(namespaces) == 0 { + fmt.Println("No namespace found") + return nil + } - report = append(report, []string{"INDEX", "NAMESPACE", "FORMAT", "COMPATIBILITY", "DESCRIPTION"}) + fmt.Printf("\nShowing %[1]d of %[1]d namespaces \n \n", len(namespaces)) + report := [][]string{} index := 1 - + report = append(report, []string{"INDEX", "NAMESPACE", "FORMAT", "COMPATIBILITY", "DESCRIPTION"}) for _, n := range namespaces { - report = append(report, []string{term.Greenf("#%02d", index), n, "-", "-", "-"}) + report = append(report, []string{term.Greenf("#%d", index), n, "-", "-", "-"}) index++ } printer.Table(os.Stdout, report) @@ -113,13 +114,11 @@ func createNamespaceCmd() *cobra.Command { } if format == "" { - formats := []string{"FORMAT_JSON", "FORMAT_PROTOBUF", "FORMAT_AVRO"} formatAnswer, _ := prompter.Select("Select a default schema format for this namespace:", formats[0], formats) format = formats[formatAnswer] } if comp == "" { - comps := []string{"COMPATIBILITY_BACKWARD", "COMPATIBILITY_FORWARD", "COMPATIBILITY_FULL"} formatAnswer, _ := prompter.Select("Select a default compatibility for this namespace:", comps[0], comps) fmt.Println() comp = comps[formatAnswer] @@ -178,7 +177,7 @@ func editNamespaceCmd() *cobra.Command { Short: "Edit a namespace", Args: cobra.ExactArgs(1), Example: heredoc.Doc(` - $ stencil namespace edit odpf -f FORMAT_JSON -c COMPATABILITY_BACKWARD -d "Hello message" + $ stencil namespace edit odpf -f FORMAT_JSON -c COMPATIBILITY_BACKWARD -d "Hello message" `), RunE: func(cmd *cobra.Command, args []string) error { spinner := printer.Spin("") @@ -297,7 +296,7 @@ func deleteNamespaceCmd() *cobra.Command { id := args[0] prompter := prompt.New() - confirm, _ := prompter.Input(fmt.Sprintf("You're going to delete namespace `%s`. To confirm, type the namespace id:", id), "") + confirm, _ := prompter.Input(fmt.Sprintf("Deleting namespace `%s`. To confirm, type the namespace id:", id), "") if id != confirm { fmt.Printf("\n%s Namespace id '%s' did not match.\n", term.WarningIcon(), confirm) return nil @@ -340,10 +339,15 @@ func deleteNamespaceCmd() *cobra.Command { } func printNamespace(namespace *stencilv1beta1.Namespace) { - fmt.Printf("%s \t\t %s \n", term.Bold("Name:"), namespace.GetId()) - fmt.Printf("%s \t %s \n", term.Bold("Format:"), namespace.GetFormat().String()) - fmt.Printf("%s \t %s \n", term.Bold("Compatibility:"), namespace.GetCompatibility().String()) - fmt.Printf("%s \t %s \n\n", term.Bold("Description:"), namespace.GetDescription()) - fmt.Printf("%s %s, ", term.Grey("Created"), humanize.Time(namespace.GetCreatedAt().AsTime())) - fmt.Printf("%s %s \n\n", term.Grey("last updated"), humanize.Time(namespace.GetCreatedAt().AsTime())) + desc := namespace.GetDescription() + if desc == "" { + desc = "No description provided" + } + + fmt.Printf("\n%s\n", term.Blue(namespace.GetId())) + fmt.Printf("\n%s.\n\n", term.Grey(desc)) + fmt.Printf("%s \t %s \n", term.Grey("Format:"), namespace.GetFormat().String()) + fmt.Printf("%s \t %s \n", term.Grey("Compatibility:"), namespace.GetCompatibility().String()) + fmt.Printf("\n%s %s, ", term.Grey("Created"), humanize.Time(namespace.GetCreatedAt().AsTime())) + fmt.Printf("%s %s \n\n", term.Grey("last updated"), humanize.Time(namespace.GetUpdatedAt().AsTime())) } diff --git a/cmd/print.go b/cmd/print.go index 89f3e5e9..b0af2547 100644 --- a/cmd/print.go +++ b/cmd/print.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "os" "strings" "github.com/MakeNowJust/heredoc" @@ -17,17 +16,18 @@ import ( "google.golang.org/protobuf/types/descriptorpb" ) -func printCmd() *cobra.Command { - var output, filterPathPrefix, host, namespaceID string +func printSchemaCmd() *cobra.Command { + var filter, host, namespaceID string var version int32 cmd := &cobra.Command{ - Use: "print ", - Short: "Print a given schema snapshot", - Args: cobra.ExactArgs(1), + Use: "view ", + Short: "Print snapshot of a schema", + Args: cobra.ExactArgs(1), + Aliases: []string{"print"}, Example: heredoc.Doc(` - $ stencil schema print events -n odpf - $ stencil schema print events -n odpf -v 2 -o ./schema + $ stencil schema view booking -n odpf + $ stencil schema view booking -n odpf -v 2 `), RunE: func(cmd *cobra.Command, args []string) error { spinner := printer.Spin("") @@ -38,27 +38,24 @@ func printCmd() *cobra.Command { } defer cancel() - schemaID := args[0] - - data, meta, err := fetchSchemaAndMetadata(client, version, namespaceID, schemaID) + data, meta, err := fetchSchemaAndMeta(client, version, namespaceID, args[0]) if err != nil { return err } spinner.Stop() format := stencilv1beta1.Schema_Format_name[int32(meta.GetFormat())] - switch format { case "FORMAT_AVRO": - if err := printSchema(data, output); err != nil { + if err := printSchema(data); err != nil { return err } case "FORMAT_JSON": - if err := printSchema(data, output); err != nil { + if err := printSchema(data); err != nil { return err } case "FORMAT_PROTOBUF": - printProtoSchema(data, filterPathPrefix, output) + printProtoSchema(data, filter) default: fmt.Printf("%s Unknown schema format: %s\n", term.Red(term.FailureIcon()), format) } @@ -66,29 +63,19 @@ func printCmd() *cobra.Command { }, } - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") + cmd.Flags().StringVar(&host, "host", "", "Server host address eg: localhost:8000") cmd.MarkFlagRequired("host") - cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "provide namespace/group or entity name") + cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Provide namespace/group or entity name") cmd.MarkFlagRequired("namespace") - cmd.Flags().Int32VarP(&version, "version", "v", 0, "provide version number") - - cmd.Flags().StringVarP(&output, "output", "o", "", "the directory path to write the descriptor files, default is to print on stdout") - - cmd.Flags().StringVar(&filterPathPrefix, "filter-path", "", "filter protocol buffer files by path prefix, e.g., --filter-path=google/protobuf") + cmd.Flags().Int32VarP(&version, "version", "v", 0, "Provide version number") + cmd.Flags().StringVar(&filter, "filter", "", "Filter schema files by path prefix, e.g., --filter=google/protobuf") return cmd } -func printSchema(data []byte, output string) error { - if output != "" { - if err := os.WriteFile(output, data, 0666); err != nil { - return err - } - return nil - } - +func printSchema(data []byte) error { page := term.New() page.Start() defer page.Stop() @@ -100,7 +87,7 @@ func printSchema(data []byte, output string) error { return nil } -func printProtoSchema(data []byte, filterPathPrefix string, output string) error { +func printProtoSchema(data []byte, filter string) error { fds := &descriptorpb.FileDescriptorSet{} if err := proto.Unmarshal(data, fds); err != nil { return fmt.Errorf("descriptor set file is not valid. %w", err) @@ -111,7 +98,7 @@ func printProtoSchema(data []byte, filterPathPrefix string, output string) error } var filteredFds []*desc.FileDescriptor for fdName, fd := range fdsMap { - if filterPathPrefix != "" && !strings.HasPrefix(fdName, filterPathPrefix) { + if filter != "" && !strings.HasPrefix(fdName, filter) { continue } filteredFds = append(filteredFds, fd) @@ -119,13 +106,6 @@ func printProtoSchema(data []byte, filterPathPrefix string, output string) error protoPrinter := &protoprint.Printer{} - if output != "" { - if err := protoPrinter.PrintProtosToFileSystem(filteredFds, output); err != nil { - return err - } - return nil - } - var schema string for _, fd := range filteredFds { @@ -144,6 +124,5 @@ func printProtoSchema(data []byte, filterPathPrefix string, output string) error if err != nil { fmt.Fprint(page.Out, schema) } - return nil } diff --git a/cmd/schema.go b/cmd/schema.go index 82fd92a9..ff933dc6 100644 --- a/cmd/schema.go +++ b/cmd/schema.go @@ -2,27 +2,9 @@ package cmd import ( "context" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "os" - "strconv" - "github.com/MakeNowJust/heredoc" - "github.com/odpf/salt/printer" - "github.com/odpf/salt/term" - "github.com/odpf/stencil/pkg/graph" stencilv1beta1 "github.com/odpf/stencil/proto/odpf/stencil/v1beta1" "github.com/spf13/cobra" - "github.com/yudai/gojsondiff" - "github.com/yudai/gojsondiff/formatter" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protodesc" - "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/types/descriptorpb" ) func SchemaCmd() *cobra.Command { @@ -30,693 +12,39 @@ func SchemaCmd() *cobra.Command { Use: "schema", Aliases: []string{"schemas"}, Short: "Manage schemas", - Long: heredoc.Doc(` - Work with schemas. - `), - Example: heredoc.Doc(` - $ stencil schema list - $ stencil schema create - $ stencil schema view - $ stencil schema edit - $ stencil schema delete - $ stencil schema version - $ stencil schema graph - $ stencil schema print - $ stencil schema check - `), + Long: "Work with schemas.", Annotations: map[string]string{ "group": "core", }, } cmd.AddCommand(createSchemaCmd()) - cmd.AddCommand(checkSchemaCmd()) cmd.AddCommand(listSchemaCmd()) - cmd.AddCommand(getSchemaCmd()) - cmd.AddCommand(updateSchemaCmd()) + cmd.AddCommand(infoSchemaCmd()) + cmd.AddCommand(versionSchemaCmd()) + cmd.AddCommand(printSchemaCmd()) + cmd.AddCommand(downloadSchemaCmd()) + cmd.AddCommand(checkSchemaCmd()) + cmd.AddCommand(editSchemaCmd()) cmd.AddCommand(deleteSchemaCmd()) cmd.AddCommand(diffSchemaCmd()) - cmd.AddCommand(versionSchemaCmd()) - cmd.AddCommand(printCmd()) - cmd.AddCommand(graphCmd()) - - return cmd -} - -func listSchemaCmd() *cobra.Command { - var host, namespace string - var req stencilv1beta1.ListSchemasRequest - - cmd := &cobra.Command{ - Use: "list", - Short: "List all schemas", - Args: cobra.ExactArgs(0), - Example: heredoc.Doc(` - $ stencil schema list -n odpf - `), - RunE: func(cmd *cobra.Command, args []string) error { - spinner := printer.Spin("") - defer spinner.Stop() - - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - req.Id = namespace - res, err := client.ListSchemas(context.Background(), &req) - if err != nil { - return err - } - - report := [][]string{} - - schemas := res.GetSchemas() - - spinner.Stop() - - // TODO(Ravi): List schemas should also handle namespace not found - if len(schemas) == 0 { - fmt.Printf("No schema found in namespace %s.\n", term.Blue(namespace)) - return nil - } - - fmt.Printf("\nShowing %d of %d schemas \n\n", len(schemas), len(schemas)) - index := 1 - - for _, s := range schemas { - report = append(report, []string{term.Greenf("#%02d", index), s}) - index++ - } - printer.Table(os.Stdout, report) - return nil - }, - } - - // TODO(Ravi): Namespace should be optional. - cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("namespace") - - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - - return cmd -} - -func createSchemaCmd() *cobra.Command { - var host, format, comp, filePath, namespaceID string - var req stencilv1beta1.CreateSchemaRequest - - cmd := &cobra.Command{ - Use: "create", - Short: "Create a schema", - Args: cobra.ExactArgs(1), - Example: heredoc.Doc(` - $ stencil schema create --namespace= --format= –-comp= –-filePath= - `), - RunE: func(cmd *cobra.Command, args []string) error { - spinner := printer.Spin("") - defer spinner.Stop() - - fileData, err := ioutil.ReadFile(filePath) - if err != nil { - return err - } - req.Data = fileData - - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - schemaID := args[0] - - req.NamespaceId = namespaceID - req.SchemaId = schemaID - req.Format = stencilv1beta1.Schema_Format(stencilv1beta1.Schema_Format_value[format]) - req.Compatibility = stencilv1beta1.Schema_Compatibility(stencilv1beta1.Schema_Compatibility_value[comp]) - - res, err := client.CreateSchema(context.Background(), &req) - if err != nil { - errStatus := status.Convert(err) - return errors.New(errStatus.Message()) - } - - id := res.GetId() - - spinner.Stop() - fmt.Printf("\n%s Created schema with id %s.\n", term.Green(term.SuccessIcon()), term.Cyan(id)) - return nil - }, - } - - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("host") - - cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "Namespace ID") - cmd.MarkFlagRequired("namespace") - - cmd.Flags().StringVarP(&format, "format", "f", "", "schema format") - cmd.MarkFlagRequired("format") - - cmd.Flags().StringVarP(&comp, "comp", "c", "", "schema compatibility") - cmd.MarkFlagRequired("comp") - - cmd.Flags().StringVarP(&filePath, "filePath", "F", "", "path to the schema file") - cmd.MarkFlagRequired("filePath") - - return cmd -} - -func checkSchemaCmd() *cobra.Command { - var host, comp, filePath, namespaceID string - var req stencilv1beta1.CheckCompatibilityRequest - - cmd := &cobra.Command{ - Use: "check", - Short: "Check schema compatibility", - Args: cobra.ExactArgs(1), - Example: heredoc.Doc(` - $ stencil schema check --namespace= comp= filePath= - `), - RunE: func(cmd *cobra.Command, args []string) error { - spinner := printer.Spin("") - defer spinner.Stop() - - fileData, err := ioutil.ReadFile(filePath) - if err != nil { - return err - } - req.Data = fileData - - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - schemaID := args[0] - - req.NamespaceId = namespaceID - req.SchemaId = schemaID - req.Compatibility = stencilv1beta1.Schema_Compatibility(stencilv1beta1.Schema_Compatibility_value[comp]) - - _, err = client.CheckCompatibility(context.Background(), &req) - if err != nil { - errStatus := status.Convert(err) - return errors.New(errStatus.Message()) - } - - spinner.Stop() - fmt.Println("schema is compatible") - fmt.Printf("\n%s Schema is compatible.\n", term.Green(term.SuccessIcon())) - return nil - }, - } - - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("host") - - cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "parent namespace ID") - cmd.MarkFlagRequired("namespace") - - cmd.Flags().StringVarP(&comp, "comp", "c", "", "schema compatibility") - cmd.MarkFlagRequired("comp") - - cmd.Flags().StringVarP(&filePath, "filePath", "F", "", "path to the schema file") - cmd.MarkFlagRequired("filePath") - - return cmd -} - -func updateSchemaCmd() *cobra.Command { - var host, comp, namespaceID string - var req stencilv1beta1.UpdateSchemaMetadataRequest - - cmd := &cobra.Command{ - Use: "edit", - Short: "Edit a schema", - Args: cobra.ExactArgs(1), - Example: heredoc.Doc(` - $ stencil schema edit --namespace= --comp= - `), - RunE: func(cmd *cobra.Command, args []string) error { - spinner := printer.Spin("") - defer spinner.Stop() - - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - schemaID := args[0] - - req.NamespaceId = namespaceID - req.SchemaId = schemaID - req.Compatibility = stencilv1beta1.Schema_Compatibility(stencilv1beta1.Schema_Compatibility_value[comp]) - - _, err = client.UpdateSchemaMetadata(context.Background(), &req) - if err != nil { - return err - } - - spinner.Stop() - - fmt.Printf("Schema successfully updated") - return nil - }, - } - - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("host") - - cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "parent namespace ID") - cmd.MarkFlagRequired("namespace") - - cmd.Flags().StringVarP(&comp, "comp", "c", "", "schema compatibility") - cmd.MarkFlagRequired("comp") + cmd.AddCommand(graphSchemaCmd()) return cmd } -func getSchemaCmd() *cobra.Command { - var host, output, namespaceID string - var version int32 - var metadata bool - var data []byte - var resMetadata *stencilv1beta1.GetSchemaMetadataResponse - - cmd := &cobra.Command{ - Use: "view", - Short: "View a schema", - Args: cobra.ExactArgs(1), - Example: heredoc.Doc(` - $ stencil schema view --namespace= --version --metadata - `), - RunE: func(cmd *cobra.Command, args []string) error { - spinner := printer.Spin("") - defer spinner.Stop() - - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - schemaID := args[0] - - data, resMetadata, err = fetchSchemaAndMetadata(client, version, namespaceID, schemaID) - if err != nil { - return err - } - spinner.Stop() - - err = os.WriteFile(output, data, 0666) - if err != nil { - return err - } - - fmt.Printf("Schema successfully written to %s\n", output) - - if resMetadata == nil || !metadata { - return nil - } - - report := [][]string{} - - fmt.Printf("\nMETADATA\n") - report = append(report, []string{"FORMAT", "COMPATIBILITY", "AUTHORITY"}) - - report = append(report, []string{ - stencilv1beta1.Schema_Format_name[int32(resMetadata.GetFormat())], - stencilv1beta1.Schema_Compatibility_name[int32(resMetadata.GetCompatibility())], - resMetadata.GetAuthority(), - }) - - printer.Table(os.Stdout, report) - - return nil - }, - } - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("host") - - cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "parent namespace ID") - cmd.MarkFlagRequired("namespace") - - cmd.Flags().Int32VarP(&version, "version", "v", 0, "version of the schema") - - cmd.Flags().BoolVarP(&metadata, "metadata", "m", false, "set this flag to get metadata") - cmd.MarkFlagRequired("metadata") - - cmd.Flags().StringVarP(&output, "output", "o", "", "path to the output file") - cmd.MarkFlagRequired("output") - - return cmd -} - -func deleteSchemaCmd() *cobra.Command { - var host, namespaceID string - var req stencilv1beta1.DeleteSchemaRequest - var reqVer stencilv1beta1.DeleteVersionRequest - var version int32 - - cmd := &cobra.Command{ - Use: "delete", - Short: "Delete a schema", - Args: cobra.ExactArgs(1), - Example: heredoc.Doc(` - $ stencil schema delete --namespace= - `), - RunE: func(cmd *cobra.Command, args []string) error { - spinner := printer.Spin("") - defer spinner.Stop() - - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - schemaID := args[0] - - if version == 0 { - req.NamespaceId = namespaceID - req.SchemaId = schemaID - - _, err = client.DeleteSchema(context.Background(), &req) - if err != nil { - return err - } - } else { - reqVer.NamespaceId = namespaceID - reqVer.SchemaId = schemaID - reqVer.VersionId = version - - _, err = client.DeleteVersion(context.Background(), &reqVer) - if err != nil { - return err - } - } - - spinner.Stop() - - fmt.Printf("schema successfully deleted") - - return nil - }, - } - - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("host") - - cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "parent namespace ID") - cmd.MarkFlagRequired("namespace") - - cmd.Flags().Int32VarP(&version, "version", "v", 0, "particular version to be deleted") - - return cmd -} - -func diffSchemaCmd() *cobra.Command { - var fullname string - var host string - var namespace string - var earlierVersion int32 - var laterVersion int32 - - var schemaFetcher = func(req *stencilv1beta1.GetSchemaRequest, client stencilv1beta1.StencilServiceClient) ([]byte, error) { - res, err := client.GetSchema(context.Background(), req) - if err != nil { - return nil, err - } - return res.Data, nil - } - var protoSchemaFetcher = func(req *stencilv1beta1.GetSchemaRequest, client stencilv1beta1.StencilServiceClient) ([]byte, error) { - if fullname == "" { - return nil, fmt.Errorf("fullname flag is mandator for FORMAT_PROTO") - } - res, err := client.GetSchema(context.Background(), req) - if err != nil { - return nil, err - } - fds := &descriptorpb.FileDescriptorSet{} - if err := proto.Unmarshal(res.Data, fds); err != nil { - return nil, fmt.Errorf("descriptor set file is not valid. %w", err) - } - files, err := protodesc.NewFiles(fds) - if err != nil { - return nil, fmt.Errorf("file is not fully contained descriptor file. hint: generate file descriptorset with --include_imports option. %w", err) - } - desc, err := files.FindDescriptorByName(protoreflect.FullName(fullname)) - if err != nil { - return nil, fmt.Errorf("unable to find message. %w", err) - } - mDesc, ok := desc.(protoreflect.MessageDescriptor) - if !ok { - return nil, fmt.Errorf("not a message desc") - } - jsonByte, err := protojson.Marshal(protodesc.ToDescriptorProto(mDesc)) - if err != nil { - return nil, fmt.Errorf("fail to convert json. %w", err) - } - return jsonByte, nil - } - - cmd := &cobra.Command{ - Use: "diff", - Short: "Diff(s) of two schema versions", - Args: cobra.ExactArgs(1), - Example: heredoc.Doc(` - $ stencil schema diff --namespace= --later-version= --earlier-version= --fullname= - `), - RunE: func(cmd *cobra.Command, args []string) error { - spinner := printer.Spin("") - defer spinner.Stop() - - schemaID := args[0] - - metaReq := stencilv1beta1.GetSchemaMetadataRequest{ - NamespaceId: namespace, - SchemaId: schemaID, - } - eReq := &stencilv1beta1.GetSchemaRequest{ - NamespaceId: namespace, - SchemaId: schemaID, - VersionId: earlierVersion, - } - lReq := &stencilv1beta1.GetSchemaRequest{ - NamespaceId: namespace, - SchemaId: schemaID, - VersionId: laterVersion, - } - - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - meta, err := client.GetSchemaMetadata(context.Background(), &metaReq) - if err != nil { - return err - } - - var getSchema = schemaFetcher - if meta.Format == *stencilv1beta1.Schema_FORMAT_PROTOBUF.Enum() { - getSchema = protoSchemaFetcher - } - - eJson, err := getSchema(eReq, client) - if err != nil { - return err - } - - lJson, err := getSchema(lReq, client) - if err != nil { - return err - } - - d, err := gojsondiff.New().Compare(eJson, lJson) - if err != nil { - return err - } - - var placeholder map[string]interface{} - json.Unmarshal(eJson, &placeholder) - config := formatter.AsciiFormatterConfig{ - ShowArrayIndex: true, - Coloring: true, - } - - formatter := formatter.NewAsciiFormatter(placeholder, config) - diffString, err := formatter.Format(d) - if err != nil { - return err - } - - spinner.Stop() - if !d.Modified() { - fmt.Print("No diff!") - return nil - } - fmt.Print(diffString) - - return nil - }, - } - - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("host") - cmd.Flags().StringVarP(&namespace, "namespace", "n", "", "parent namespace ID") - cmd.MarkFlagRequired("namespace") - cmd.Flags().Int32Var(&earlierVersion, "earlier-version", 0, "earlier version of the schema") - cmd.MarkFlagRequired("earlier-version") - cmd.Flags().Int32Var(&laterVersion, "later-version", 0, "later version of the schema") - cmd.MarkFlagRequired("later-version") - cmd.Flags().StringVar(&fullname, "fullname", "", "only required for FORMAT_PROTO. fullname of proto schema eg: odpf.common.v1.Version") - return cmd -} - -func versionSchemaCmd() *cobra.Command { - var host, namespaceID string - var req stencilv1beta1.ListVersionsRequest - - cmd := &cobra.Command{ - Use: "version", - Short: "Version(s) of a schema", - Args: cobra.ExactArgs(1), - Example: heredoc.Doc(` - $ stencil schema version --namespace= - `), - RunE: func(cmd *cobra.Command, args []string) error { - spinner := printer.Spin("") - defer spinner.Stop() - - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - schemaID := args[0] - - req.NamespaceId = namespaceID - req.SchemaId = schemaID - - res, err := client.ListVersions(context.Background(), &req) - if err != nil { - return err - } - - report := [][]string{} - versions := res.GetVersions() - - spinner.Stop() - - if len(versions) == 0 { - fmt.Printf("%s has no versions in %s", schemaID, namespaceID) - return nil - } - - report = append(report, []string{"VERSIONS(s)"}) - - for _, v := range versions { - report = append(report, []string{ - strconv.FormatInt(int64(v), 10), - }) - } - printer.Table(os.Stdout, report) - return nil - }, - } - - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("host") - - cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "parent namespace ID") - cmd.MarkFlagRequired("namespace") - - return cmd -} - -func graphCmd() *cobra.Command { - var host, output, namespaceID string - var version int32 - - cmd := &cobra.Command{ - Use: "graph", - Aliases: []string{"g"}, - Short: "Generate file descriptorset dependencies graph", - Args: cobra.ExactArgs(1), - Example: heredoc.Doc(` - $ stencil schema graph --namespace= --version= --output= - `), - RunE: func(cmd *cobra.Command, args []string) error { - client, cancel, err := createClient(cmd) - if err != nil { - return err - } - defer cancel() - - schemaID := args[0] - - data, resMetadata, err := fetchSchemaAndMetadata(client, version, namespaceID, schemaID) - if err != nil { - return err - } - - format := stencilv1beta1.Schema_Format_name[int32(resMetadata.GetFormat())] - if format != "FORMAT_PROTOBUF" { - fmt.Printf("cannot create graph for %s", format) - return nil - } - - msg := &descriptorpb.FileDescriptorSet{} - err = proto.Unmarshal(data, msg) - if err != nil { - return fmt.Errorf("invalid file descriptorset file. %w", err) - } - - graph, err := graph.GetProtoFileDependencyGraph(msg) - if err != nil { - return err - } - if err = os.WriteFile(output, []byte(graph.String()), 0666); err != nil { - return err - } - - fmt.Println(".dot file has been created in", output) - return nil - }, - } - - cmd.Flags().StringVar(&host, "host", "", "stencil host address eg: localhost:8000") - cmd.MarkFlagRequired("host") - - cmd.Flags().StringVarP(&namespaceID, "namespace", "n", "", "provide namespace/group or entity name") - cmd.MarkFlagRequired("namespace") - - cmd.Flags().Int32VarP(&version, "version", "v", 0, "provide version number") - - cmd.Flags().StringVarP(&output, "output", "o", "./proto_vis.dot", "write to .dot file") - - return cmd -} - -func fetchSchemaAndMetadata(client stencilv1beta1.StencilServiceClient, version int32, namespaceID, schemaID string) ([]byte, *stencilv1beta1.GetSchemaMetadataResponse, error) { +func fetchSchemaAndMeta(client stencilv1beta1.StencilServiceClient, version int32, namespaceID, schemaID string) ([]byte, *stencilv1beta1.GetSchemaMetadataResponse, error) { var req stencilv1beta1.GetSchemaRequest var reqLatest stencilv1beta1.GetLatestSchemaRequest - var reqMetadata stencilv1beta1.GetSchemaMetadataRequest var data []byte + ctx := context.Background() + if version != 0 { req.NamespaceId = namespaceID req.SchemaId = schemaID req.VersionId = version - res, err := client.GetSchema(context.Background(), &req) + res, err := client.GetSchema(ctx, &req) if err != nil { return nil, nil, err } @@ -724,19 +52,30 @@ func fetchSchemaAndMetadata(client stencilv1beta1.StencilServiceClient, version } else { reqLatest.NamespaceId = namespaceID reqLatest.SchemaId = schemaID - res, err := client.GetLatestSchema(context.Background(), &reqLatest) + res, err := client.GetLatestSchema(ctx, &reqLatest) if err != nil { return nil, nil, err } data = res.GetData() } - reqMetadata.NamespaceId = namespaceID - reqMetadata.SchemaId = schemaID - resMetadata, err := client.GetSchemaMetadata(context.Background(), &reqMetadata) + reqMeta := stencilv1beta1.GetSchemaMetadataRequest{ + NamespaceId: namespaceID, + SchemaId: schemaID, + } + meta, err := client.GetSchemaMetadata(context.Background(), &reqMeta) + if err != nil { - return data, nil, err + return nil, nil, err } - return data, resMetadata, nil + return data, meta, nil +} + +func fetchMeta(client stencilv1beta1.StencilServiceClient, namespace string, schema string) (*stencilv1beta1.GetSchemaMetadataResponse, error) { + req := stencilv1beta1.GetSchemaMetadataRequest{ + NamespaceId: namespace, + SchemaId: schema, + } + return client.GetSchemaMetadata(context.Background(), &req) } diff --git a/docs/docs/server/overview.md b/docs/docs/server/overview.md index fc58d1b8..9444b972 100644 --- a/docs/docs/server/overview.md +++ b/docs/docs/server/overview.md @@ -5,8 +5,8 @@ Stencil is dynamic protobuf schema registry. It provides REST interface for stor ## Features - stores versioned history of proto descriptor file on specified namespace and name -- enforce backward compatability check on upload by default -- ability to skip some of the backward compatability checks while upload +- enforce backward compatibility check on upload by default +- ability to skip some of the backward compatibility checks while upload - ability to download fully contained proto descriptor file for specified proto message [fullName](https://pkg.go.dev/google.golang.org/protobuf@v1.27.1/reflect/protoreflect#FullName) - provides metadata API to retrieve latest version number given a name and namespace diff --git a/docs/docs/server/rules.md b/docs/docs/server/rules.md index cf6fb09f..d529bc76 100644 --- a/docs/docs/server/rules.md +++ b/docs/docs/server/rules.md @@ -1,55 +1,50 @@ # Compatability rules Stencil server provides following compatibility rules. -- BACKWARD_COMPATABILITY -- FORWARD_COMPATABILITY -- FULL_COMPATABILITY + +- BACKWARD_COMPATIBILITY +- FORWARD_COMPATIBILITY +- FULL_COMPATIBILITY Stencil currently supports protobuf, avro and json schema formats. Compatibility rules for each schema format has been built separately considering each schema format's features. ## Feature support matrix -| Compatability rule | Protobuf | Avro | JSON | -| ------------------ | --- | --- | --- | -| BACKWARD_COMPATABILITY | Yes | Yes | No | -| FORWARD_COMPATABILITY | Yes | Yes | No | -| FULL_COMPATABILITY | Yes | Yes | No | +| Compatability rule | Protobuf | Avro | JSON | +| ---------------------- | -------- | ---- | ---- | +| BACKWARD_COMPATIBILITY | Yes | Yes | No | +| FORWARD_COMPATIBILITY | Yes | Yes | No | +| FULL_COMPATIBILITY | Yes | Yes | No | ## Protobuf compatibility rules -Protobuf compatability rules composed of [compatability checks](#list-of-checks). +Protobuf compatibility rules composed of [compatibility checks](#list-of-checks). ### Rules -| Compatibility name | List of checks | -| ------------------ | -------------- | -| BACKWARD_COMPATABILITY | SYNTAX_CHANGE, MESSAGE_DELETE, NON_INCLUSIVE_RESERVED_RANGE, NON_INCLUSIVE_RESERVED_NAMES, FIELD_DELETE, FIELD_JSON_NAME_CHANGE, FIELD_LABEL_CHANGE, FIELD_KIND_CHANGE, FIELD_TYPE_CHANGE, ENUM_DELETE, ENUM_VALUE_DELETE, ENUM_VALUE_NUMBER_CHANGE | -| FORWARD_COMPATABILITY | SYNTAX_CHANGE, MESSAGE_DELETE, NON_INCLUSIVE_RESERVED_RANGE, NON_INCLUSIVE_RESERVED_NAMES, FIELD_JSON_NAME_CHANGE, FIELD_LABEL_CHANGE, FIELD_KIND_CHANGE, FIELD_TYPE_CHANGE, FIELD_DELETE_WITHOUT_RESERVED_NUMBER, FIELD_DELETE_WITHOUT_RESERVED_NAME, ENUM_DELETE, ENUM_VALUE_NUMBER_CHANGE, ENUM_VALUE_DELETE_WITHOUT_RESERVEDNUMBER, ENUM_VALUE_DELETE_WITHOUT_RESERVEDNAME | -| FULL_COMPATABILITY | SYNTAX_CHANGE, MESSAGE_DELETE, NON_INCLUSIVE_RESERVED_RANGE, NON_INCLUSIVE_RESERVED_NAMES, FIELD_DELETE, FIELD_JSON_NAME_CHANGE, FIELD_LABEL_CHANGE, FIELD_KIND_CHANGE, FIELD_TYPE_CHANGE, ENUM_DELETE, ENUM_VALUE_DELETE, ENUM_VALUE_NUMBER_CHANGE | +| Compatibility name | List of checks | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| BACKWARD_COMPATIBILITY | SYNTAX_CHANGE, MESSAGE_DELETE, NON_INCLUSIVE_RESERVED_RANGE, NON_INCLUSIVE_RESERVED_NAMES, FIELD_DELETE, FIELD_JSON_NAME_CHANGE, FIELD_LABEL_CHANGE, FIELD_KIND_CHANGE, FIELD_TYPE_CHANGE, ENUM_DELETE, ENUM_VALUE_DELETE, ENUM_VALUE_NUMBER_CHANGE | +| FORWARD_COMPATIBILITY | SYNTAX_CHANGE, MESSAGE_DELETE, NON_INCLUSIVE_RESERVED_RANGE, NON_INCLUSIVE_RESERVED_NAMES, FIELD_JSON_NAME_CHANGE, FIELD_LABEL_CHANGE, FIELD_KIND_CHANGE, FIELD_TYPE_CHANGE, FIELD_DELETE_WITHOUT_RESERVED_NUMBER, FIELD_DELETE_WITHOUT_RESERVED_NAME, ENUM_DELETE, ENUM_VALUE_NUMBER_CHANGE, ENUM_VALUE_DELETE_WITHOUT_RESERVEDNUMBER, ENUM_VALUE_DELETE_WITHOUT_RESERVEDNAME | +| FULL_COMPATIBILITY | SYNTAX_CHANGE, MESSAGE_DELETE, NON_INCLUSIVE_RESERVED_RANGE, NON_INCLUSIVE_RESERVED_NAMES, FIELD_DELETE, FIELD_JSON_NAME_CHANGE, FIELD_LABEL_CHANGE, FIELD_KIND_CHANGE, FIELD_TYPE_CHANGE, ENUM_DELETE, ENUM_VALUE_DELETE, ENUM_VALUE_NUMBER_CHANGE | ### List of Checks -| Check | Description | -| ---- | ------------ | -| SYNTAX_CHANGE | checks if proto file syntax does not switch between proto2 and proto3, including going to/from unset (which assumes proto2) to set to proto3. Changing the syntax results in differences in generated code for many languages. | -| MESSAGE_DELETE | checks that messages are deleted from a given file. Deleting a message will delete the corresponding generated type, which could be referenced in source code. Instead of deleting these types, deprecate them using [`deprecated` option](https://developers.google.com/protocol-buffers/docs/proto3#options).| -| NON_INCLUSIVE_RESERVED_NAMES | Checks if current reserved names contains all previous reserved names. | -| NON_INCLUSIVE_RESERVED_RANGE | Checks if current reserve range inclusive of previous reserved range. This check ensures previous reserved tag numbers haven't been deleted | -| FIELD_DELETE | checks that no message field is deleted. Deleting message field will result in the field being deleted from the generated source code, which could be referenced. Instead of deleting these, deprecate them using [`deprecated` option](https://developers.google.com/protocol-buffers/docs/proto3#options). | -| FIELD_JSON_NAME_CHANGE | Checks if the json_name for field does not change, which would break JSON compatibility. | -| FIELD_LABEL_CHANGE | checks that no field changes it's label, i.e. `optional`, `required`, `repeated`. Changing to/from optional/required and repeated will be a generated source code and JSON breaking change. Changing to/from optional and repeated is actually not a wire-breaking change, however changing to/from optional and required is. Given that it's unlikely to be advisable in any situation to change your label, and that there is only one exception, we find it best to just outlaw this entirely. | -| FIELD_KIND_CHANGE | checks that a field has the same type. Changing the type of a field can affect the type in the generated source code, wire compatibility, and JSON compatibility. | -| FIELD_TYPE_CHANGE | Checks if message/enum field it's message/enum type has changed from previous version. This rule only applies to message kind and enum kind. | -| FIELD_DELETE_WITHOUT_RESERVED_NUMBER | Checks if field is deleted, it's tag number should be added to reserved numbers. This will ensure deleted field tag number won't be used in future. | -| FIELD_DELETE_WITHOUT_RESERVED_NAME | Checks if field is deleted, it's tag name should be added to reserved names. This will help to keep the JSON compatibility. | -| ENUM_DELETE | Checks that no enum is deleted from current version. Deleting an enum will delete the corresponding generated type, which could be referenced in source code. Instead of deleting these types, deprecate them using `deprecated` option. | -| ENUM_VALUE_DELETE | Checks that no enum value is deleted. Deleting an enum value will result in the corresponding value being deleted from the generated source code, which could be referenced. Instead of deleting these, deprecate them. | -| ENUM_VALUE_DELETE_WITHOUT_RESERVEDNUMBER | Checks if enum value deleted, it's enum number should be added to reserved numbers. | -| ENUM_VALUE_DELETE_WITHOUT_RESERVEDNAME | Checks if enum value deleted, it's enum name should be added to reserved names. This will help to keep the JSON compatability | -| ENUM_VALUE_NUMBER_CHANGE | Check if enum number has changed between current, previous versions. For example You cannot change FOO_ONE = 1 to FOO_ONE = 2. Doing so will result in potential JSON incompatibilites and broken source code. | - - - - - - +| Check | Description | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| SYNTAX_CHANGE | checks if proto file syntax does not switch between proto2 and proto3, including going to/from unset (which assumes proto2) to set to proto3. Changing the syntax results in differences in generated code for many languages. | +| MESSAGE_DELETE | checks that messages are deleted from a given file. Deleting a message will delete the corresponding generated type, which could be referenced in source code. Instead of deleting these types, deprecate them using [`deprecated` option](https://developers.google.com/protocol-buffers/docs/proto3#options). | +| NON_INCLUSIVE_RESERVED_NAMES | Checks if current reserved names contains all previous reserved names. | +| NON_INCLUSIVE_RESERVED_RANGE | Checks if current reserve range inclusive of previous reserved range. This check ensures previous reserved tag numbers haven't been deleted | +| FIELD_DELETE | checks that no message field is deleted. Deleting message field will result in the field being deleted from the generated source code, which could be referenced. Instead of deleting these, deprecate them using [`deprecated` option](https://developers.google.com/protocol-buffers/docs/proto3#options). | +| FIELD_JSON_NAME_CHANGE | Checks if the json_name for field does not change, which would break JSON compatibility. | +| FIELD_LABEL_CHANGE | checks that no field changes it's label, i.e. `optional`, `required`, `repeated`. Changing to/from optional/required and repeated will be a generated source code and JSON breaking change. Changing to/from optional and repeated is actually not a wire-breaking change, however changing to/from optional and required is. Given that it's unlikely to be advisable in any situation to change your label, and that there is only one exception, we find it best to just outlaw this entirely. | +| FIELD_KIND_CHANGE | checks that a field has the same type. Changing the type of a field can affect the type in the generated source code, wire compatibility, and JSON compatibility. | +| FIELD_TYPE_CHANGE | Checks if message/enum field it's message/enum type has changed from previous version. This rule only applies to message kind and enum kind. | +| FIELD_DELETE_WITHOUT_RESERVED_NUMBER | Checks if field is deleted, it's tag number should be added to reserved numbers. This will ensure deleted field tag number won't be used in future. | +| FIELD_DELETE_WITHOUT_RESERVED_NAME | Checks if field is deleted, it's tag name should be added to reserved names. This will help to keep the JSON compatibility. | +| ENUM_DELETE | Checks that no enum is deleted from current version. Deleting an enum will delete the corresponding generated type, which could be referenced in source code. Instead of deleting these types, deprecate them using `deprecated` option. | +| ENUM_VALUE_DELETE | Checks that no enum value is deleted. Deleting an enum value will result in the corresponding value being deleted from the generated source code, which could be referenced. Instead of deleting these, deprecate them. | +| ENUM_VALUE_DELETE_WITHOUT_RESERVEDNUMBER | Checks if enum value deleted, it's enum number should be added to reserved numbers. | +| ENUM_VALUE_DELETE_WITHOUT_RESERVEDNAME | Checks if enum value deleted, it's enum name should be added to reserved names. This will help to keep the JSON compatibility | +| ENUM_VALUE_NUMBER_CHANGE | Check if enum number has changed between current, previous versions. For example You cannot change FOO_ONE = 1 to FOO_ONE = 2. Doing so will result in potential JSON incompatibilites and broken source code. | diff --git a/docs/src/pages/index.js b/docs/src/pages/index.js index 7da44652..7891962e 100644 --- a/docs/src/pages/index.js +++ b/docs/src/pages/index.js @@ -60,7 +60,7 @@ export default function Home() { title: 'Backward compatibility', content: (
- Enforce backward compatability check on upload by default. + Enforce backward compatibility check on upload by default.
), }, @@ -68,7 +68,7 @@ export default function Home() { title: 'Flexbility', content: (
- Ability to skip some of the backward compatability checks while upload. + Ability to skip some of the backward compatibility checks while upload.
), }, diff --git a/go.mod b/go.mod index f3aa91c6..db7aba70 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/odpf/stencil go 1.16 require ( - github.com/AlecAivazis/survey/v2 v2.3.5 github.com/MakeNowJust/heredoc v1.0.0 github.com/alecthomas/chroma v0.8.2 github.com/dgraph-io/ristretto v0.1.0