feat: added utils cli

cli/cmd/proto.go

@@ -0,0 +1,99 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+var protoCmd = &cobra.Command{
+	Use:   "proto",
+	Short: "Compile proto files to Python pb files",
+	Run:   runProto,
+}
+
+var (
+	protoDir  string
+	outputDir string
+)
+
+func init() {
+	protoCmd.Flags().StringVarP(&protoDir, "proto-dir", "p", "grpc/proto", "Directory containing proto files")
+	protoCmd.Flags().StringVarP(&outputDir, "output-dir", "o", "python/crawlab/grpc", "Output directory for Python pb files")
+	RootCmd.AddCommand(protoCmd)
+}
+
+func runProto(cmd *cobra.Command, args []string) {
+	// Ensure protoc is installed
+	if _, err := exec.LookPath("protoc"); err != nil {
+		fmt.Println("Error: protoc is not installed. Please install Protocol Buffers compiler first.")
+		os.Exit(1)
+	}
+
+	// Create output directory if it doesn't exist
+	if err := os.MkdirAll(outputDir, 0755); err != nil {
+		fmt.Printf("Error creating output directory: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Find all proto files
+	protoFiles, err := filepath.Glob(filepath.Join(protoDir, "**/*.proto"))
+	if err != nil {
+		fmt.Printf("Error finding proto files: %v\n", err)
+		os.Exit(1)
+	}
+
+	for _, protoFile := range protoFiles {
+		relPath, _ := filepath.Rel(protoDir, protoFile)
+		fmt.Printf("Compiling: %s\n", relPath)
+
+		args := []string{
+			"--proto_path=" + protoDir,
+			"--python_out=" + outputDir,
+			"--grpc_python_out=" + outputDir,
+			protoFile,
+		}
+
+		cmd := exec.Command("protoc", args...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+
+		if err := cmd.Run(); err != nil {
+			fmt.Printf("Error compiling %s: %v\n", relPath, err)
+			os.Exit(1)
+		}
+
+		// Fix Python imports
+		fixPythonImports(outputDir)
+	}
+
+	fmt.Println("Successfully compiled all proto files to Python")
+}
+
+func fixPythonImports(dir string) error {
+	pbFiles, err := filepath.Glob(filepath.Join(dir, "**/*_pb2*.py"))
+	if err != nil {
+		return err
+	}
+
+	for _, file := range pbFiles {
+		content, err := os.ReadFile(file)
+		if err != nil {
+			return err
+		}
+
+		// Replace import statements
+		newContent := strings.ReplaceAll(string(content),
+			"from grpc.proto",
+			"from crawlab.grpc")
+
+		if err := os.WriteFile(file, []byte(newContent), 0644); err != nil {
+			return err
+		}
+	}
+	return nil
+}

cli/cmd/root.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var RootCmd = &cobra.Command{
+	Use:   "crawlab",
+	Short: "Crawlab CLI tool",
+	Long:  `Command line interface for Crawlab, the distributed web crawler admin platform`,
+}
+
+func init() {
+	RootCmd.AddCommand(protoCmd)
+}

cli/go.mod

@@ -0,0 +1,10 @@
+module github.com/crawlab-team/crawlab/cli
+
+go 1.22
+
+require github.com/spf13/cobra v1.8.1
+
+require (
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+)

cli/go.sum

@@ -0,0 +1,10 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

cli/main.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/crawlab-team/crawlab/cli/cmd"
+)
+
+func main() {
+	if err := cmd.RootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}