Web browsing automation with chromedp, Go and vfbserver

In this tutorial, we will be using the Go chromedp library. While it supports headless mode, I have not been able to make it work, so I have opted to launch the binary within a reduced graphical server: xorg-vfbserver

Let’s install Chrome and xorg-vfbserver:

pkg install chromium xorg-vfbserver

Create a RC startup script:

vi /usr/local/etc/rc.d/vfbserver

#! /bin/sh
#
# $FreeBSD$
#

# PROVIDE: vfbserver
# REQUIRE: DAEMON
# KEYWORD: shutdown 

. /etc/rc.subr

name="vfbserver"
extra_commands="status"
username="kr0m"

start_cmd="${name}_start"
stop_cmd="${name}_stop"
status_cmd="${name}_status"

vfbserver_start(){
    PID=$(ps -U ${username}|grep Xvfb|grep -v grep|awk '{print$1}')
    if [ -z $PID ]; then
        echo "Starting service: Xvfb"
        /bin/rm /tmp/.X*-lock 2>/dev/null
        /usr/bin/su -l ${username} -c '/usr/bin/nohup /usr/local/bin/Xvfb :0 > /dev/null 2>&1 &' 1>/dev/null
    else
        echo "Service: Xvfb already started"
    fi
}

vfbserver_stop(){
    PID=$(ps -U ${username}|grep Xvfb|grep -v grep|awk '{print$1}')
    if [ -z $PID ]; then
        echo "It appears Xvfb is not running."
    else
        echo "Stopping service: Xvfb/X11vnc/XFCE"
        kill -s INT $PID
        sleep 3
    fi
}

vfbserver_status(){
    PID=$(ps -U ${username}|grep Xvfb|grep -v grep|awk '{print$1}')
    if [ -z $PID ]; then
        echo "Xvfb is not running."
    else
        echo "Xvfb running with PID: $PID"
    fi
}

load_rc_config ${name}
run_rc_command "$1"

Assign the required grants:

chmod 555 /usr/local/etc/rc.d/vfbserver
chown root:wheel /usr/local/etc/rc.d/vfbserver

Enable the service and start it:

sysrc vfbserver_enable="YES"
service vfbserver start

Create a directory for the Go code, besides we install chromedp library:

mkdir chromeDP
cd chromeDP
go mod init chromeDP
go mod tidy
go install github.com/chromedp/chromedp@latest
go mod tidy

In this example the program will make a login, click in one button and send a Telegram message:

vi chromeDP.go

package main

import (
    "context"
    "log"
    "fmt"
    "time"
    "math/rand"
    "bytes"
    "net/http"
    "encoding/json"

    "github.com/chromedp/chromedp"
    "github.com/chromedp/chromedp/kb"
)

func send_telegram(text string) {
	bot := "botTOKEN"
	chat_id := "CHATID"
	request_url := "https://api.telegram.org/" + bot + "/sendMessage"
	client := &http.Client{}
	values := map[string]string{"text": text, "chat_id": chat_id }
	json_paramaters, _ := json.Marshal(values)
	req, _:= http.NewRequest("POST", request_url, bytes.NewBuffer(json_paramaters))
	req.Header.Set("Content-Type", "application/json")
	res, err := client.Do(req)
	if (err != nil) {
		fmt.Println(err)
	} else {
		fmt.Println(res.Status)
		defer res.Body.Close()
	}
}

func main() {
    // Random sleep to make it more human-like
    rand.Seed(time.Now().UnixNano())
    // 5min: 60*5 = 300s
    n := rand.Intn(300)
    //fmt.Printf("Sleeping %d seconds...\n", n)
    time.Sleep(time.Duration(n)*time.Second)

    // Create chrome instance
    ctx, cancel := chromedp.NewContext(
        context.Background(),
        //chromedp.WithDebugf(log.Printf),
    )
    defer cancel()

    // Create a timeout
    ctx, cancel = context.WithTimeout(ctx, 15*time.Second)
    defer cancel()

    // Navigate to a page
    err := chromedp.Run(ctx,
        chromedp.Navigate(`https://auth.alfaexploit.com/login`),
        // USERNAME:
        chromedp.WaitVisible(`#mat-input-0`, chromedp.ByID),
        chromedp.SendKeys(`#mat-input-0`, kb.End+"kr0m@alfaexploit.com", chromedp.ByID),

        // PASSWORD:
        chromedp.WaitVisible(`#mat-input-1`, chromedp.ByID),
        chromedp.SendKeys(`#mat-input-1`, kb.End+"PASSWORD", chromedp.ByID),

        // XPath LOGIN
        chromedp.WaitVisible(`/html/body/app-root/div/app-page-login/div/div/div/div/form/button/span[2]`, chromedp.BySearch),
        chromedp.Click(`/html/body/app-root/div/app-page-login/div/div/div/div/form/button/span[2]`, chromedp.BySearch),
        //chromedp.Sleep(time.Second*2),

        // XPath START/STOP
        chromedp.WaitVisible(`/html/body/app-root/div/app-layout/section/div/div/div/section/app-dashboard-tasks/app-signup/div/div/div[2]/div/button/span[4]`, chromedp.BySearch),
        chromedp.Click(`/html/body/app-root/div/app-layout/section/div/div/div/section/app-dashboard-tasks/app-signup/div/div/div[2]/div/button/span[4]`, chromedp.BySearch),
        chromedp.Sleep(time.Second*2),
    )
    if err != nil {
        log.Fatal(err)
        send_telegram(err.Error())
    } else {
        send_telegram("Chromedp executed")
    }
}

NOTE: XPATHs were retrieved using the chrome debug console: Inspect -> Copy: Copy complete XPath

Launch the program manually:

DISPLAY=:0 go run chromeDP.go

Generate the binary:

go build

Crontab the binary execution:

crontab -e

# MONDAY - THURSDAY
00 08 * * 1,2,3,4 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
30 11 * * 1,2,3,4 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
00 12 * * 1,2,3,4 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
00 14 * * 1,2,3,4 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
00 15 * * 1,2,3,4 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
00 17 * * 1,2,3,4 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
# FRIDAY
00 08 * * 5 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
30 11 * * 5 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
00 12 * * 5 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP
00 15 * * 5 DISPLAY=:0 /home/kr0m/.scripts/chromeDP/chromeDP

We can monitor binary execution looking at CRON log file:

tail -f /var/log/cron|grep chromeDP