1 changed files with 5 additions and 49 deletions
--- a/internal/service/recruitment/crawler.go
+++ b/internal/service/recruitment/crawler.go
@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
-	"os/exec"
 	"regexp"
 	"strings"
 	"time"
@ -79,10 +78,10 @@ func genericStaticCrawl(ctx context.Context, src entity.CrawlSource, force bool)
 	if err != nil {
 		return nil, err
 	}
-	g.Log().Warningf(ctx, "[recruit-debug] src=%d listURL=%s htmlLen=%d", src.Id, listURL, len(html))
+	g.Log().Infof(ctx, "[recruit-debug] src=%d listURL=%s htmlLen=%d", src.Id, listURL, len(html))
 	raw := extractAnchors(html, listURL)
 	anchors := filterArticleAnchors(raw)
-	g.Log().Warningf(ctx, "[recruit-debug] src=%d rawAnchors=%d filtered=%d", src.Id, len(raw), len(anchors))
+	g.Log().Infof(ctx, "[recruit-debug] src=%d rawAnchors=%d filtered=%d", src.Id, len(raw), len(anchors))
 	var out []dto.RecruitmentInput
 	limit := 60
 	if force {
@ -189,22 +188,12 @@ func walkJSON(node any, out *[]dto.RecruitmentInput) {
 	}
 }

-// fetchHTML 抓取页面。优先用系统 curl（已实测：政府站在容器内对 curl 返回完整页面，
-// 而 Go 客户端可能被 TLS 指纹级 WAF 拦截返回无内容挑战页），失败再回退 Go 客户端。
+// fetchHTML 带超时与浏览器 UA 的请求，降低被反爬概率。
 func fetchHTML(ctx context.Context, u string) (string, error) {
-	if body, e := fetchWithCurl(u); e == nil && looksLikeHtml(body) && len(body) >= 3000 {
-		g.Log().Warningf(ctx, "[recruit-debug] curl ok len=%d url=%s", len(body), u)
-		return body, nil
-	} else if e != nil {
-		g.Log().Warningf(ctx, "[recruit-debug] curl failed: %v url=%s", e, u)
-	} else {
-		g.Log().Warningf(ctx, "[recruit-debug] curl suspicious len=%d url=%s", len(body), u)
-	}
 	client := g.Client()
 	client.SetTimeout(15 * time.Second)
 	client.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
 	client.SetHeader("Accept", "text/html,application/xhtml+xml,*/*")
-	client.SetHeader("Accept-Language", "zh-CN,zh;q=0.9")
 	resp, err := client.Get(ctx, u)
 	if err != nil {
 		return "", gerror.Wrapf(err, "fetch %s", u)
@ -213,35 +202,7 @@ func fetchHTML(ctx context.Context, u string) (string, error) {
 	if resp.StatusCode != 200 {
 		return "", gerror.Newf("HTTP %d for %s", resp.StatusCode, u)
 	}
-	body := resp.ReadAllString()
-	if !looksLikeHtml(body) || len(body) < 3000 {
-		g.Log().Warningf(ctx, "[recruit-debug] go-fallback suspicious len=%d url=%s", len(body), u)
-		return "", gerror.Newf("suspicious page (len=%d) for %s", len(body), u)
-	}
-	g.Log().Warningf(ctx, "[recruit-debug] go-fallback ok len=%d url=%s", len(body), u)
-	return body, nil
-}
-
-// fetchWithCurl 调用系统 curl 抓取（自动跟随重定向，带浏览器 UA）。
-func fetchWithCurl(u string) (string, error) {
-	args := []string{
-		"-s", "-L", "--max-time", "25",
-		"-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
-		"-H", "Accept: text/html,application/xhtml+xml,*/*",
-		"-H", "Accept-Language: zh-CN,zh;q=0.9",
-		u,
-	}
-	out, err := exec.Command("curl", args...).Output()
-	if err != nil {
-		return "", gerror.Wrapf(err, "curl %s", u)
-	}
-	return string(out), nil
-}
-
-// looksLikeHtml 粗略判断响应是否为正常 HTML 文档（含 <a 锚点与 <html 文档标记）。
-func looksLikeHtml(s string) bool {
-	lower := strings.ToLower(s)
-	return strings.Contains(lower, "<html") && strings.Contains(lower, "<a")
+	return resp.ReadAllString(), nil
 }

 type anchor struct {
@ -419,12 +380,7 @@ func resolveURL(base, href string) string {
 	if err != nil {
 		return ""
 	}
-	abs := u.ResolveReference(ref).String()
-	// 同一政府页面常同时含 http/https 两种文章链接，统一升级为 https 便于去重。
-	if strings.HasPrefix(base, "https://") && strings.HasPrefix(abs, "http://") {
-		abs = "https://" + strings.TrimPrefix(abs, "http://")
-	}
-	return abs
+	return u.ResolveReference(ref).String()
 }

 // joinURL 拼接基础域名与路径。