crawlab/core/task/handler/runner_cleanup.go

/*
 * Copyright (c) 2025. Core Digital Limited
 * 版权所有 (c) 2025. 重庆科锐数研科技有限公司 (Core Digital Limited)
 * All rights reserved. 保留所有权利。
 *
 * 该软件由 重庆科锐数研科技有限公司 (Core Digital Limited) 开发，未经明确书面许可，任何人不得使用、复制、修改或分发该软件的任何部分。
 * This software is developed by Core Digital Limited. No one is permitted to use, copy, modify, or distribute this software without explicit written permission.
 *
 * 许可证：
 * 该软件仅供授权使用。授权用户有权在授权范围内使用、复制、修改和分发该软件。
 * License:
 * This software is for authorized use only. Authorized users are permitted to use, copy, modify, and distribute this software within the scope of their authorization.
 *
 * 免责声明：
 * 该软件按"原样"提供，不附带任何明示或暗示的担保，包括但不限于对适销性和适用于特定目的的担保。在任何情况下，版权持有者或其许可方对因使用该软件而产生的任何损害或其他责任概不负责。
 * Disclaimer:
 * This software is provided "as is," without any express or implied warranties, including but not limited to warranties of merchantability and fitness for a particular purpose. In no event shall the copyright holder or its licensors be liable for any damages or other liability arising from the use of this software.
 */

package handler

import (
	"runtime"
	"syscall"
	"time"

	"github.com/crawlab-team/crawlab/core/utils"
	"github.com/shirou/gopsutil/process"
)

// cleanupOrphanedProcesses attempts to clean up any orphaned processes related to this task
func (r *Runner) cleanupOrphanedProcesses() {
	r.Warnf("cleaning up orphaned processes for task %s (PID: %d)", r.tid.Hex(), r.pid)

	if r.pid <= 0 {
		r.Debugf("no PID to clean up")
		return
	}

	// Try to kill the process group if it exists
	if runtime.GOOS != "windows" {
		r.killProcessGroup()
	}

	// Force kill the main process if it still exists
	if utils.ProcessIdExists(r.pid) {
		r.Warnf("forcefully killing remaining process %d", r.pid)
		if r.cmd != nil && r.cmd.Process != nil {
			if err := utils.KillProcess(r.cmd, true); err != nil {
				r.Errorf("failed to force kill process: %v", err)
			}
		}
	}

	// Scan for any remaining child processes and kill them
	r.scanAndKillChildProcesses()
}

// killProcessGroup kills the entire process group on Unix systems
func (r *Runner) killProcessGroup() {
	if r.pid <= 0 {
		return
	}

	r.Debugf("attempting to kill process group for PID %d", r.pid)

	// Kill the process group (negative PID kills the group)
	err := syscall.Kill(-r.pid, syscall.SIGTERM)
	if err != nil {
		r.Debugf("failed to send SIGTERM to process group: %v", err)
		// Try SIGKILL as last resort
		err = syscall.Kill(-r.pid, syscall.SIGKILL)
		if err != nil {
			r.Debugf("failed to send SIGKILL to process group: %v", err)
		}
	} else {
		r.Debugf("successfully sent SIGTERM to process group %d", r.pid)
	}
}

// scanAndKillChildProcesses scans for and kills any remaining child processes
func (r *Runner) scanAndKillChildProcesses() {
	r.Debugf("scanning for orphaned child processes of task %s", r.tid.Hex())

	processes, err := utils.GetProcesses()
	if err != nil {
		r.Errorf("failed to get process list: %v", err)
		return
	}

	taskIdEnv := "CRAWLAB_TASK_ID=" + r.tid.Hex()
	killedCount := 0

	for _, proc := range processes {
		// Check if this process has our task ID in its environment
		if r.isTaskRelatedProcess(proc, taskIdEnv) {
			pid := int(proc.Pid)
			r.Warnf("found orphaned task process PID %d, killing it", pid)

			// Kill the orphaned process
			if err := proc.Kill(); err != nil {
				r.Errorf("failed to kill orphaned process %d: %v", pid, err)
			} else {
				killedCount++
				r.Infof("successfully killed orphaned process %d", pid)
			}
		}
	}

	if killedCount > 0 {
		r.Infof("cleaned up %d orphaned processes for task %s", killedCount, r.tid.Hex())
	} else {
		r.Debugf("no orphaned processes found for task %s", r.tid.Hex())
	}
}

// isTaskRelatedProcess checks if a process is related to this task
func (r *Runner) isTaskRelatedProcess(proc *process.Process, taskIdEnv string) bool {
	// Get process environment variables
	environ, err := proc.Environ()
	if err != nil {
		// If we can't read environment, skip this process
		return false
	}

	// Check if this process has our task ID
	for _, env := range environ {
		if env == taskIdEnv {
			return true
		}
	}

	return false
}

// startZombieMonitor starts a background goroutine to monitor for zombie processes
func (r *Runner) startZombieMonitor() {
	r.wg.Add(1)
	go func() {
		defer r.wg.Done()

		// Check for zombies every 5 minutes
		ticker := time.NewTicker(5 * time.Minute)
		defer ticker.Stop()

		for {
			select {
			case <-r.ctx.Done():
				return
			case <-ticker.C:
				r.checkForZombieProcesses()
			}
		}
	}()
}

// checkForZombieProcesses periodically checks for and cleans up zombie processes
func (r *Runner) checkForZombieProcesses() {
	r.Debugf("checking for zombie processes related to task %s", r.tid.Hex())

	// Check if our main process still exists and is in the expected state
	if r.pid > 0 && utils.ProcessIdExists(r.pid) {
		// Process exists, check if it's a zombie
		if proc, err := process.NewProcess(int32(r.pid)); err == nil {
			if status, err := proc.Status(); err == nil {
				if status == "Z" || status == "zombie" {
					r.Warnf("detected zombie process %d for task %s", r.pid, r.tid.Hex())
					go r.cleanupOrphanedProcesses()
				}
			}
		}
	}
}

// performPeriodicCleanup runs periodic cleanup for all tasks
func (r *Runner) performPeriodicCleanup() {
	r.wg.Add(1)
	defer r.wg.Done()

	// Cleanup every 10 minutes for all tasks
	r.resourceCleanup = time.NewTicker(10 * time.Minute)
	defer r.resourceCleanup.Stop()

	for {
		select {
		case <-r.ctx.Done():
			return
		case <-r.resourceCleanup.C:
			r.runPeriodicCleanup()
		}
	}
}

// runPeriodicCleanup performs memory and resource cleanup
func (r *Runner) runPeriodicCleanup() {
	r.Debugf("performing periodic cleanup for task")

	// Force garbage collection for memory management
	runtime.GC()

	// Log current resource usage
	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	r.Debugf("memory usage - alloc: %d KB, sys: %d KB, num_gc: %d",
		m.Alloc/1024, m.Sys/1024, m.NumGC)

	// Check if IPC channel is getting full
	if r.ipcChan != nil {
		select {
		case <-r.ipcChan:
			r.Debugf("drained stale IPC message during cleanup")
		default:
			// Channel is not full, good
		}
	}
}