From 8b11893d3596933f08c733efdafd955443a46ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=8F=E7=8A=80=E9=BA=9F?= Date: Thu, 27 Aug 2026 02:12:03 +0800 Subject: [PATCH] =?UTF-8?q?feat(recruitment):=20=E7=88=AC=E8=99=AB?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=8E=E9=83=A8=E7=BD=B2=E6=89=93=E9=80=9A?= =?UTF-8?q?=20-=20Scan=20no-rows=20=E5=85=A5=E5=BA=93=E4=BF=AE=E5=A4=8D=20?= =?UTF-8?q?/=20title=20=E5=B1=9E=E6=80=A7=E5=85=9C=E5=BA=95=20/=20?= =?UTF-8?q?=E5=8F=8C=E6=9D=A1=E4=BB=B6=E8=BF=87=E6=BB=A4=20/=20=E7=A9=BA?= =?UTF-8?q?=E6=97=A5=E6=9C=9F=E6=8F=92=20NULL=20/=20crawl=5Flog=20?= =?UTF-8?q?=E6=88=AA=E6=96=AD=20/=20fetchHTML=20curl=20=E5=85=9C=E5=BA=95?= =?UTF-8?q?=20/=20=E4=BA=BA=E7=A4=BE=E5=B1=80=E6=94=B9=20https=20/=20?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=20config.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/service/recruitment/crawler.go | 43 ++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/internal/service/recruitment/crawler.go b/internal/service/recruitment/crawler.go index b77d38e..6eb6817 100644 --- a/internal/service/recruitment/crawler.go +++ b/internal/service/recruitment/crawler.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "net/url" + "os/exec" "regexp" "strings" "time" @@ -189,20 +190,60 @@ func walkJSON(node any, out *[]dto.RecruitmentInput) { } // fetchHTML 带超时与浏览器 UA 的请求，降低被反爬概率。 +// 部分政府站对 Go HTTP 客户端做 TLS 指纹级拦截（返回无锚点页面）， +// 此时回退系统 curl 重试（容器镜像内置 curl，已验证可拿到完整列表页）。 func fetchHTML(ctx context.Context, u string) (string, error) { client := g.Client() client.SetTimeout(15 * time.Second) client.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36") client.SetHeader("Accept", "text/html,application/xhtml+xml,*/*") + client.SetHeader("Accept-Language", "zh-CN,zh;q=0.9") resp, err := client.Get(ctx, u) if err != nil { + // 网络层失败：尝试 curl 兜底，避免单点失败。 + if body, e := fetchWithCurl(u); e == nil { + return body, nil + } return "", gerror.Wrapf(err, "fetch %s", u) } defer resp.Close() if resp.StatusCode != 200 { + // 非 200（如 302/403）：回退 curl 重试（curl 自动跟随重定向）。 + if body, e := fetchWithCurl(u); e == nil { + return body, nil + } return "", gerror.Newf("HTTP %d for %s", resp.StatusCode, u) } - return resp.ReadAllString(), nil + body := resp.ReadAllString() + // 内容可疑（疑似 WAF/JS 挑战页：无 html 特征或锚点过少）时回退 curl。 + if !looksLikeHtml(body) || len(body) < 3000 { + if cb, e := fetchWithCurl(u); e == nil { + return cb, nil + } + } + return body, nil +} + +// fetchWithCurl 调用系统 curl 抓取（自动跟随重定向，带浏览器 UA）。 +func fetchWithCurl(u string) (string, error) { + args := []string{ + "-s", "-L", "--max-time", "25", + "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36", + "-H", "Accept: text/html,application/xhtml+xml,*/*", + "-H", "Accept-Language: zh-CN,zh;q=0.9", + u, + } + out, err := exec.Command("curl", args...).Output() + if err != nil { + return "", gerror.Wrapf(err, "curl %s", u) + } + return string(out), nil +} + +// looksLikeHtml 粗略判断响应是否为正常 HTML 文档（含