git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: added utils cli

Browse Source
This commit is contained in:
Marvin Zhang
2024-11-20 15:47:30 +08:00
parent 18f3a8e264
commit db6a379095
7 changed files with 151 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
cli/cmd/proto.go Normal file
View File

@@ -0,0 +1,99 @@
package cmd
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/spf13/cobra"
)
var protoCmd = &cobra.Command{
Use: "proto",
Short: "Compile proto files to Python pb files",
Run: runProto,
}
var (
protoDir string
outputDir string
)
func init() {
protoCmd.Flags().StringVarP(&protoDir, "proto-dir", "p", "grpc/proto", "Directory containing proto files")
protoCmd.Flags().StringVarP(&outputDir, "output-dir", "o", "python/crawlab/grpc", "Output directory for Python pb files")
RootCmd.AddCommand(protoCmd)
}
func runProto(cmd *cobra.Command, args []string) {
// Ensure protoc is installed
if _, err := exec.LookPath("protoc"); err != nil {
fmt.Println("Error: protoc is not installed. Please install Protocol Buffers compiler first.")
os.Exit(1)
}
// Create output directory if it doesn't exist
if err := os.MkdirAll(outputDir, 0755); err != nil {
fmt.Printf("Error creating output directory: %v\n", err)
os.Exit(1)
}
// Find all proto files
protoFiles, err := filepath.Glob(filepath.Join(protoDir, "**/*.proto"))
if err != nil {
fmt.Printf("Error finding proto files: %v\n", err)
os.Exit(1)
}
for _, protoFile := range protoFiles {
relPath, _ := filepath.Rel(protoDir, protoFile)
fmt.Printf("Compiling: %s\n", relPath)
args := []string{
"--proto_path=" + protoDir,
"--python_out=" + outputDir,
"--grpc_python_out=" + outputDir,
protoFile,
}
cmd := exec.Command("protoc", args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
fmt.Printf("Error compiling %s: %v\n", relPath, err)
os.Exit(1)
}
// Fix Python imports
fixPythonImports(outputDir)
}
fmt.Println("Successfully compiled all proto files to Python")
}
func fixPythonImports(dir string) error {
pbFiles, err := filepath.Glob(filepath.Join(dir, "**/*_pb2*.py"))
if err != nil {
return err
}
for _, file := range pbFiles {
content, err := os.ReadFile(file)
if err != nil {
return err
}
// Replace import statements
newContent := strings.ReplaceAll(string(content),
"from grpc.proto",
"from crawlab.grpc")
if err := os.WriteFile(file, []byte(newContent), 0644); err != nil {
return err
}
}
return nil
}

15
cli/cmd/root.go Normal file
View File

@@ -0,0 +1,15 @@
package cmd
import (
"github.com/spf13/cobra"
)
var RootCmd = &cobra.Command{
Use: "crawlab",
Short: "Crawlab CLI tool",
Long: `Command line interface for Crawlab, the distributed web crawler admin platform`,
}
func init() {
RootCmd.AddCommand(protoCmd)
}

10
cli/go.mod Normal file
View File

@@ -0,0 +1,10 @@
module github.com/crawlab-team/crawlab/cli
go 1.22
require github.com/spf13/cobra v1.8.1
require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
)

10
cli/go.sum Normal file
View File

@@ -0,0 +1,10 @@
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

15
cli/main.go Normal file
View File

@@ -0,0 +1,15 @@
package main
import (
"fmt"
"os"
"github.com/crawlab-team/crawlab/cli/cmd"
)
func main() {
if err := cmd.RootCmd.Execute(); err != nil {
fmt.Println(err)
os.Exit(1)
}
}

1
go.work
View File

@@ -7,4 +7,5 @@ use (
trace
vcs
backend
cli
)

3
go.work.sum
View File

@@ -802,9 +802,9 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+
github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/crawlab-team/crawlab/core v0.0.0-20240614095218-7b4ee8399ab0/go.mod h1:IcyK/SxfyYooua6xBC2sZ5+EfCId18Dfjh0di7ZQk1U=
github.com/crawlab-team/crawlab/core v0.0.0-20241016121324-eb56598d93dc/go.mod h1:XBHHT5heUI7N0fbdO/U6fKOcbC5Ow5S7EYhzW+OC3Iw=
github.com/crawlab-team/crawlab/core v0.0.0-20241119094044-98a9c907c9cb/go.mod h1:wcICnYeY/1K5Lfe3qe1esVSv+2qs2qcftBhQF1h3d1w=
github.com/crawlab-team/crawlab/fs v0.0.0-20240731075841-7fe770ae9d15/go.mod h1:LbFPUanR5Bm/C1ST45WtCLV1hiN+o0JSV2lwUQ/qIKA=
github.com/crawlab-team/crawlab/template-parser v0.0.0-20240614095218-7b4ee8399ab0/go.mod h1:mUFNHqYwOHNQwmeWvNPK0S7Z0dWtr3z+obNwg+dO77o=
github.com/crawlab-team/crawlab/template-parser v0.0.0-20241016121324-eb56598d93dc/go.mod h1:lKaowLGJrCwvFJnW/KYbdz0Pj69My34yus6IsLKDMro=
@@ -852,7 +852,6 @@ github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwm
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomarkdown/markdown v0.0.0-20241105142532-d03b89096d81/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=