diff --git a/cli/cmd/proto.go b/cli/cmd/proto.go new file mode 100644 index 00000000..41d23427 --- /dev/null +++ b/cli/cmd/proto.go @@ -0,0 +1,99 @@ +package cmd + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/spf13/cobra" +) + +var protoCmd = &cobra.Command{ + Use: "proto", + Short: "Compile proto files to Python pb files", + Run: runProto, +} + +var ( + protoDir string + outputDir string +) + +func init() { + protoCmd.Flags().StringVarP(&protoDir, "proto-dir", "p", "grpc/proto", "Directory containing proto files") + protoCmd.Flags().StringVarP(&outputDir, "output-dir", "o", "python/crawlab/grpc", "Output directory for Python pb files") + RootCmd.AddCommand(protoCmd) +} + +func runProto(cmd *cobra.Command, args []string) { + // Ensure protoc is installed + if _, err := exec.LookPath("protoc"); err != nil { + fmt.Println("Error: protoc is not installed. Please install Protocol Buffers compiler first.") + os.Exit(1) + } + + // Create output directory if it doesn't exist + if err := os.MkdirAll(outputDir, 0755); err != nil { + fmt.Printf("Error creating output directory: %v\n", err) + os.Exit(1) + } + + // Find all proto files + protoFiles, err := filepath.Glob(filepath.Join(protoDir, "**/*.proto")) + if err != nil { + fmt.Printf("Error finding proto files: %v\n", err) + os.Exit(1) + } + + for _, protoFile := range protoFiles { + relPath, _ := filepath.Rel(protoDir, protoFile) + fmt.Printf("Compiling: %s\n", relPath) + + args := []string{ + "--proto_path=" + protoDir, + "--python_out=" + outputDir, + "--grpc_python_out=" + outputDir, + protoFile, + } + + cmd := exec.Command("protoc", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + fmt.Printf("Error compiling %s: %v\n", relPath, err) + os.Exit(1) + } + + // Fix Python imports + fixPythonImports(outputDir) + } + + fmt.Println("Successfully compiled all proto files to Python") +} + +func fixPythonImports(dir string) error { + pbFiles, err := filepath.Glob(filepath.Join(dir, "**/*_pb2*.py")) + if err != nil { + return err + } + + for _, file := range pbFiles { + content, err := os.ReadFile(file) + if err != nil { + return err + } + + // Replace import statements + newContent := strings.ReplaceAll(string(content), + "from grpc.proto", + "from crawlab.grpc") + + if err := os.WriteFile(file, []byte(newContent), 0644); err != nil { + return err + } + } + return nil +} diff --git a/cli/cmd/root.go b/cli/cmd/root.go new file mode 100644 index 00000000..5b547095 --- /dev/null +++ b/cli/cmd/root.go @@ -0,0 +1,15 @@ +package cmd + +import ( + "github.com/spf13/cobra" +) + +var RootCmd = &cobra.Command{ + Use: "crawlab", + Short: "Crawlab CLI tool", + Long: `Command line interface for Crawlab, the distributed web crawler admin platform`, +} + +func init() { + RootCmd.AddCommand(protoCmd) +} diff --git a/cli/go.mod b/cli/go.mod new file mode 100644 index 00000000..d83c5ca6 --- /dev/null +++ b/cli/go.mod @@ -0,0 +1,10 @@ +module github.com/crawlab-team/crawlab/cli + +go 1.22 + +require github.com/spf13/cobra v1.8.1 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/cli/go.sum b/cli/go.sum new file mode 100644 index 00000000..912390a7 --- /dev/null +++ b/cli/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cli/main.go b/cli/main.go new file mode 100644 index 00000000..f839f060 --- /dev/null +++ b/cli/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + "os" + + "github.com/crawlab-team/crawlab/cli/cmd" +) + +func main() { + if err := cmd.RootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/go.work b/go.work index d8f3b724..ae1ada4a 100644 --- a/go.work +++ b/go.work @@ -7,4 +7,5 @@ use ( trace vcs backend + cli ) diff --git a/go.work.sum b/go.work.sum index 74868798..fe863180 100644 --- a/go.work.sum +++ b/go.work.sum @@ -802,9 +802,9 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/crawlab-team/crawlab/core v0.0.0-20240614095218-7b4ee8399ab0/go.mod h1:IcyK/SxfyYooua6xBC2sZ5+EfCId18Dfjh0di7ZQk1U= github.com/crawlab-team/crawlab/core v0.0.0-20241016121324-eb56598d93dc/go.mod h1:XBHHT5heUI7N0fbdO/U6fKOcbC5Ow5S7EYhzW+OC3Iw= -github.com/crawlab-team/crawlab/core v0.0.0-20241119094044-98a9c907c9cb/go.mod h1:wcICnYeY/1K5Lfe3qe1esVSv+2qs2qcftBhQF1h3d1w= github.com/crawlab-team/crawlab/fs v0.0.0-20240731075841-7fe770ae9d15/go.mod h1:LbFPUanR5Bm/C1ST45WtCLV1hiN+o0JSV2lwUQ/qIKA= github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0/go.mod h1:mUFNHqYwOHNQwmeWvNPK0S7Z0dWtr3z+obNwg+dO77o= github.com/crawlab-team/crawlab/template-parser v0.0.0-20241016121324-eb56598d93dc/go.mod h1:lKaowLGJrCwvFJnW/KYbdz0Pj69My34yus6IsLKDMro= @@ -852,7 +852,6 @@ github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomarkdown/markdown v0.0.0-20241105142532-d03b89096d81/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=