xpcool/service.xpcool.com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(recruitment): 爬虫修复与部署打通 - Scan no-rows 入库修复 / title 属性兜底 / 双条件过滤 / 空日期插 NULL / crawl_log 截断 / fetchHTML curl 兜底 / 人社局改 https / 新增 config.yaml

Browse Source
This commit is contained in:
夏犀麟 2026-08-27 02:12:03 +08:00
parent 1e28e02dc1
commit 8b11893d35
1 changed files with 42 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
internal/service/recruitment/crawler.go
View File

@ -6,6 +6,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"net/url" "net/url"
"os/exec"
"regexp" "regexp"
"strings" "strings"
"time" "time"
@ -189,20 +190,60 @@ func walkJSON(node any, out *[]dto.RecruitmentInput) {
} }
// fetchHTML 带超时与浏览器 UA 的请求，降低被反爬概率。 // fetchHTML 带超时与浏览器 UA 的请求，降低被反爬概率。
// 部分政府站对 Go HTTP 客户端做 TLS 指纹级拦截（返回无锚点页面），
// 此时回退系统 curl 重试（容器镜像内置 curl，已验证可拿到完整列表页）。
func fetchHTML(ctx context.Context, u string) (string, error) { func fetchHTML(ctx context.Context, u string) (string, error) {
client := g.Client() client := g.Client()
client.SetTimeout(15 * time.Second) client.SetTimeout(15 * time.Second)
client.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36") client.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
client.SetHeader("Accept", "text/html,application/xhtml+xml,*/*") client.SetHeader("Accept", "text/html,application/xhtml+xml,*/*")
client.SetHeader("Accept-Language", "zh-CN,zh;q=0.9")
resp, err := client.Get(ctx, u) resp, err := client.Get(ctx, u)
if err != nil { if err != nil {
// 网络层失败：尝试 curl 兜底，避免单点失败。
if body, e := fetchWithCurl(u); e == nil {
return body, nil
}
return "", gerror.Wrapf(err, "fetch %s", u) return "", gerror.Wrapf(err, "fetch %s", u)
} }
defer resp.Close() defer resp.Close()
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
// 非 200（如 302/403）：回退 curl 重试（curl 自动跟随重定向）。
if body, e := fetchWithCurl(u); e == nil {
return body, nil
}
return "", gerror.Newf("HTTP %d for %s", resp.StatusCode, u) return "", gerror.Newf("HTTP %d for %s", resp.StatusCode, u)
} }
return resp.ReadAllString(), nil body := resp.ReadAllString()
// 内容可疑（疑似 WAF/JS 挑战页：无 html 特征或锚点过少）时回退 curl。
if !looksLikeHtml(body) || len(body) < 3000 {
if cb, e := fetchWithCurl(u); e == nil {
return cb, nil
}
}
return body, nil
}
// fetchWithCurl 调用系统 curl 抓取（自动跟随重定向，带浏览器 UA）。
func fetchWithCurl(u string) (string, error) {
args := []string{
"-s", "-L", "--max-time", "25",
"-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
"-H", "Accept: text/html,application/xhtml+xml,*/*",
"-H", "Accept-Language: zh-CN,zh;q=0.9",
u,
}
out, err := exec.Command("curl", args...).Output()
if err != nil {
return "", gerror.Wrapf(err, "curl %s", u)
}
return string(out), nil
}
// looksLikeHtml 粗略判断响应是否为正常 HTML 文档（含 <a 锚点与 <html 文档标记）。
func looksLikeHtml(s string) bool {
lower := strings.ToLower(s)
return strings.Contains(lower, "<html") && strings.Contains(lower, "<a")
} }
type anchor struct { type anchor struct {